- deleted Qfunction

# Conflicts:
#	linalg/tensor.hpp

CHANGELOG

@@ -44,7 +44,12 @@ Discretization improvements
 - Added methods to estimate function extremum using piecewise linear bounds plus
   recursive subdivision.
 
+<<<<<<<<< Temporary merge branch 1
+- Added class VectorQuadratureSpace that represents a vector (multi-component)
+  version of the scalar (single-component) class QuadratureSpaceBase.
+=========
 - Extend FindPointsGSLIB to support surface meshes.
+>>>>>>>>> Temporary merge branch 2
 
 Meshing improvements
 --------------------

CMakeLists.txt

@@ -601,6 +601,13 @@ if (MFEM_USE_ENZYME)
     set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
 endif()
 
+if (MFEM_USE_PROTEUS)
+    enable_language(C)
+    find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
+    message(STATUS "${PROTEUS_DIR}/include")
+    include_directories("${PROTEUS_DIR}/include")
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -737,6 +744,16 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
+
+if (MFEM_USE_PROTEUS)
+    add_library(ClangProteusFlags INTERFACE IMPORTED)
+    set_target_properties(ClangProteusFlags PROPERTIES
+        INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
+    )
+    target_link_libraries(mfem PUBLIC ClangProteusFlags)
+    target_link_libraries(mfem PUBLIC proteus)
+endif()
+
 if (TPL_TARGETS)
   add_dependencies(mfem ${TPL_TARGETS})
 endif()
@@ -773,7 +790,7 @@ set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 # If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
 # inside the build directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
-  target_compile_definitions(mfem PRIVATE
+  target_compile_definitions(mfem PUBLIC
     "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
 endif()
 
@@ -833,16 +850,15 @@ if (MFEM_ENABLE_TESTING)
   add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
   add_subdirectory(tests EXCLUDE_FROM_ALL)
 
-  if (MFEM_USE_BENCHMARK)
-    # Create a target for all benchmarks and, optionally, enable it.
-    set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-    add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                    ${MFEM_ENABLE_BENCHMARKS})
-    if (MFEM_ENABLE_BENCHMARKS)
-      add_subdirectory(tests/benchmarks) #install benchmarks if enabled
-    else()
-      add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
-    endif()
+  # Create a target for all benchmarks and, optionally, enable it. Some simple
+  # timer-based benchmarks in tests/benchmarks do not require Google Benchmark.
+  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
+  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
+                  ${MFEM_ENABLE_BENCHMARKS})
+  if (MFEM_ENABLE_BENCHMARKS)
+    add_subdirectory(tests/benchmarks) #install benchmarks if enabled
+  else()
+    add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
   endif()
 endif()
 

config/config.hpp

@@ -18,9 +18,9 @@
 #define MFEM_CONFIG_HPP
 
 #ifdef MFEM_CONFIG_FILE
-#include MFEM_CONFIG_FILE
+#include MFEM_CONFIG_FILE // IWYU pragma: export
 #else
-#include "_config.hpp"
+#include "_config.hpp" // IWYU pragma: export
 #endif
 
 #include <cstdint>

examples/CMakeLists.txt

@@ -47,6 +47,7 @@ list(APPEND ALL_EXE_SRCS
   ex39.cpp
   ex40.cpp
   ex41.cpp
+  # jitplayground.cpp
   )
 
 if (MFEM_USE_MPI)

examples/jitplayground.cpp

@@ -0,0 +1,548 @@
+#include <mfem.hpp>
+
+#ifdef MFEM_USE_PROTEUS
+
+#include "../fem/dfem/util.hpp"
+
+#include <proteus/CppJitModule.h>
+
+#include "jitplayground.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cctype>
+#include <cmath>
+#include <fstream>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace util
+{
+constexpr std::string_view Dirname(std::string_view path)
+{
+   const size_t last_sep = path.find_last_of("/\\");
+   if (last_sep == std::string_view::npos) { return {}; }
+   return path.substr(0, last_sep);
+}
+
+constexpr std::string_view thisFileDir = Dirname(__FILE__);
+}
+
+template <typename T>
+static std::string TypeNameString()
+{
+   return std::string(mfem::future::get_type_name<T>());
+}
+
+template <typename Tuple, size_t... Is>
+static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
+{
+   return std::array<std::string, sizeof...(Is)>
+   {
+      TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
+   };
+}
+
+template <typename Tuple>
+static auto ParamTypeStrings()
+{
+   return ParamTypeStringsImpl<Tuple>(
+             std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
+}
+
+static std::string_view Trim(std::string_view s)
+{
+   size_t begin = 0;
+   while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
+   {
+      ++begin;
+   }
+   size_t end = s.size();
+   while (end > begin &&
+          std::isspace(static_cast<unsigned char>(s[end - 1])))
+   {
+      --end;
+   }
+   return s.substr(begin, end - begin);
+}
+
+static bool IsValidIdentifier(std::string_view s)
+{
+   if (s.empty()) { return false; }
+   const unsigned char c0 = static_cast<unsigned char>(s[0]);
+   if (!(std::isalpha(c0) || c0 == '_')) { return false; }
+   for (size_t i = 1; i < s.size(); ++i)
+   {
+      const unsigned char c = static_cast<unsigned char>(s[i]);
+      if (!(std::isalnum(c) || c == '_')) { return false; }
+   }
+   return true;
+}
+
+static bool ParseJitDirective(std::string_view line,
+                              std::string &type,
+                              std::string &var,
+                              std::string &kind)
+{
+   const size_t jit_pos = line.find("$JIT");
+   if (jit_pos == std::string_view::npos) { return false; }
+
+   const size_t open = line.find('[', jit_pos);
+   const size_t close = line.find(']', jit_pos);
+   MFEM_VERIFY(open != std::string_view::npos &&
+               close != std::string_view::npos &&
+               close > open,
+               "malformed $JIT directive (expected brackets): " << line);
+
+   const std::string_view payload = line.substr(open + 1, close - open - 1);
+   const size_t comma1 = payload.find(',');
+   const size_t comma2 = (comma1 == std::string_view::npos)
+                         ? std::string_view::npos
+                         : payload.find(',', comma1 + 1);
+   MFEM_VERIFY(comma1 != std::string_view::npos &&
+               comma2 != std::string_view::npos,
+               "malformed $JIT directive (expected 3 comma-separated fields): "
+               << line);
+
+   const std::string_view f0 = Trim(payload.substr(0, comma1));
+   const std::string_view f1 = Trim(payload.substr(comma1 + 1,
+                                                   comma2 - comma1 - 1));
+   const std::string_view f2 = Trim(payload.substr(comma2 + 1));
+   MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
+               "malformed $JIT directive (empty field): " << line);
+
+   type.assign(f0);
+   var.assign(f1);
+   kind.assign(f2);
+   return true;
+}
+
+static std::string ReadFileOrEmpty(const std::string &fn)
+{
+   std::ifstream file(fn);
+   if (!file.is_open())
+   {
+      std::cerr << "could not open file " << fn << "\n";
+      return {};
+   }
+   std::stringstream buffer;
+   buffer << file.rdbuf();
+   return buffer.str();
+}
+
+static std::vector<std::string> ExtractJitVarNames(const std::string
+                                                   &kernel_code)
+{
+   std::stringstream ss(kernel_code);
+   std::string line;
+   std::vector<std::string> var_names;
+   std::unordered_set<std::string> seen_vars;
+
+   while (std::getline(ss, line))
+   {
+      std::string type, var, kind;
+      if (ParseJitDirective(line, type, var, kind))
+      {
+         MFEM_VERIFY(IsValidIdentifier(var),
+                     "$JIT variable must be a valid identifier: " << var);
+         MFEM_VERIFY(seen_vars.insert(var).second,
+                     "duplicate $JIT variable name: " << var);
+         var_names.push_back(var);
+      }
+   }
+   return var_names;
+}
+
+static std::string RewriteKernelForJit(std::string kernel_code,
+                                       const std::vector<std::string> &jit_values)
+{
+   std::stringstream ss(kernel_code);
+   std::string line;
+
+   std::string out;
+   out.reserve(kernel_code.size() + 128);
+
+   bool have_pending = false;
+   size_t pending_index = 0;
+   std::string pending_type;
+   std::string pending_var;
+   std::unordered_set<std::string> seen_vars;
+
+   while (std::getline(ss, line))
+   {
+      line.push_back('\n');
+
+      if (have_pending)
+      {
+         MFEM_VERIFY(pending_index < jit_values.size(),
+                     "not enough JIT values provided");
+         const size_t indent_end = line.find_first_not_of(" \t");
+         const std::string indent =
+            (indent_end == std::string::npos) ? std::string() :
+            line.substr(0, indent_end);
+         out += indent + "const " + pending_type + " " + pending_var + " = " +
+                jit_values[pending_index] + ";\n";
+         have_pending = false;
+         ++pending_index;
+         continue;
+      }
+
+      std::string type, var, kind;
+      if (ParseJitDirective(line, type, var, kind))
+      {
+         MFEM_VERIFY(IsValidIdentifier(var),
+                     "$JIT variable must be a valid identifier: " << var);
+         MFEM_VERIFY(kind == "generic",
+                     "unsupported $JIT kind: " << kind);
+         MFEM_VERIFY(seen_vars.insert(var).second,
+                     "duplicate $JIT variable name: " << var);
+
+         pending_type = std::move(type);
+         pending_var = std::move(var);
+         have_pending = true;
+         continue; // drop directive line
+      }
+
+      out += line;
+   }
+
+   MFEM_VERIFY(!have_pending,
+               "$JIT directive must annotate a following line");
+   MFEM_VERIFY(jit_values.size() == pending_index,
+               "JIT value count must match number of $JIT directives");
+   return out;
+}
+
+static std::string GeneratedOutputPath(std::string_view original_path)
+{
+   const size_t last_sep = original_path.find_last_of("/\\");
+   const size_t dot = original_path.find_last_of('.');
+   const bool dot_in_filename =
+      (dot != std::string_view::npos) &&
+      (last_sep == std::string_view::npos || dot > last_sep);
+
+   const std::string_view base =
+      dot_in_filename ? original_path.substr(0, dot) : original_path;
+   return std::string(base) + "_generated.hpp";
+}
+
+static void WriteFileOrWarn(const std::string &path,
+                            const std::string &contents)
+{
+   std::ofstream out(path);
+   if (!out.is_open())
+   {
+      std::cerr << "could not write generated file " << path << "\n";
+      return;
+   }
+   out << contents;
+}
+
+class JitQFunction
+{
+public:
+   template <typename ImplT, size_t N>
+   JitQFunction(ImplT, const std::string &fn,
+                const std::array<bool, N> &activity_map)
+   {
+      using qf_signature = typename
+                           mfem::future::get_function_signature<
+                           decltype(&ImplT::operator())>::type;
+      using qf_param_ts = typename qf_signature::parameter_ts;
+      constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
+      static_assert(N == nparams, "activity_map size must match qfunc arity");
+
+      this->fn = fn;
+      this->nparams = nparams;
+      this->activity_map.reserve(N);
+      for (size_t i = 0; i < N; ++i)
+      {
+         this->activity_map.push_back(activity_map[i]);
+      }
+      {
+         const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
+         this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
+      }
+      this->return_type = TypeNameString<typename qf_signature::return_t>();
+      this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
+      this->impl_type_name = TypeNameString<ImplT>();
+      this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run(std::string_view name,
+               std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+               Args&&... args)
+   {
+      auto ordered_values = MatchJitValues(jit_values);
+      auto &mod = GetOrCreateModule(ordered_values);
+      auto &instance = mod.instantiate(std::string(name), std::string());
+      return instance.template run<ReturnT>(std::forward<Args>(args)...);
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run_primal(
+      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+      Args&&... args)
+   {
+      return run<ReturnT>(qfunc_name, jit_values,
+                          std::forward<Args>(args)...);
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run_derivative(
+      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+      Args&&... args)
+   {
+      return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
+                          std::forward<Args>(args)...);
+   }
+
+private:
+   std::vector<std::string_view> MatchJitValues(
+      std::initializer_list<std::pair<std::string_view, std::string_view>>
+      named_values) const
+   {
+      std::unordered_map<std::string_view, std::string_view> value_map;
+      for (const auto &[name, value] : named_values)
+      {
+         value_map[name] = value;
+      }
+
+      std::vector<std::string_view> ordered_values;
+      ordered_values.reserve(jit_var_names.size());
+      for (const auto &var_name : jit_var_names)
+      {
+         auto it = value_map.find(var_name);
+         MFEM_VERIFY(it != value_map.end(),
+                     "missing JIT value for variable: " << var_name);
+         ordered_values.push_back(it->second);
+      }
+
+      MFEM_VERIFY(ordered_values.size() == named_values.size(),
+                  "provided " << named_values.size() << " JIT values but expected "
+                  << jit_var_names.size());
+      return ordered_values;
+   }
+
+
+   std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
+   {
+      std::string module_code =
+         RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
+      module_code += "\n\n";
+      module_code += "// --- generated ---\n";
+      module_code +=
+         "template <typename return_type, typename... Args>\n"
+         "return_type __enzyme_fwddiff(Args...);\n"
+         "\n"
+         "extern int enzyme_const;\n"
+         "extern int enzyme_dup;\n"
+         "\n";
+
+      // Generate a primal wrapper with the requested symbol name, so the kernel
+      // header can just define the qfunc as a functor.
+      //
+      // Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
+      // when there are no user template args, so keep the wrapper itself a
+      // template (with a default parameter) while still doing literal `$JIT`
+      // replacements in the kernel code.
+      module_code += "template <typename = void>\n";
+      module_code += return_type + " " +
+                     std::string(qfunc_name) + "(";
+      bool first = true;
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (!first) { module_code += ", "; }
+         first = false;
+         module_code += param_types[i] + " Arg" + std::to_string(i);
+      }
+      module_code += ")\n";
+      module_code += "{\n";
+      module_code += "   " + impl_type_name + " qf;\n";
+      if (return_is_void)
+      {
+         module_code += "   ";
+      }
+      else
+      {
+         module_code += "   return ";
+      }
+      module_code += "qf(";
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (i) { module_code += ", "; }
+         module_code += "Arg" + std::to_string(i);
+      }
+      module_code += ");\n";
+      module_code += "}\n\n";
+
+      module_code += "template <typename = void>\n";
+      module_code += return_type + " " +
+                     std::string(qfunc_name) + "_fwddiff(";
+
+      first = true;
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (!first) { module_code += ", "; }
+         first = false;
+         module_code += param_types[i] + " Arg" + std::to_string(i);
+         if (activity_map[i])
+         {
+            module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
+         }
+      }
+      module_code += ")\n";
+      module_code += "{\n";
+      if (return_is_void)
+      {
+         module_code += "   __enzyme_fwddiff<void>(\n";
+      }
+      else
+      {
+         module_code += "   return __enzyme_fwddiff<" +
+                        return_type + ">(\n";
+      }
+      module_code += "      (void*)" + std::string(qfunc_name) + "<>";
+      module_code += ",\n";
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (activity_map[i])
+         {
+            module_code += "      enzyme_dup, Arg" + std::to_string(i) +
+                           ", dArg" + std::to_string(i);
+         }
+         else
+         {
+            module_code += "      enzyme_const, Arg" + std::to_string(i);
+         }
+         module_code += (i + 1 == nparams) ? ");\n" : ",\n";
+      }
+      module_code += "}\n";
+
+      WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
+      return module_code;
+   }
+
+   proteus::CppJitModule &GetOrCreateModule(
+      const std::vector<std::string_view> &jit_values)
+   {
+      std::string key;
+      for (const auto &val : jit_values)
+      {
+         if (!key.empty()) { key += ","; }
+         key += val;
+      }
+
+      auto it = modules.find(key);
+      if (it != modules.end())
+      {
+         return *it->second;
+      }
+
+      std::vector<std::string> values(jit_values.begin(), jit_values.end());
+      std::string code = BuildModuleCode(values);
+      auto mod = std::make_unique<proteus::CppJitModule>("host", code,
+                                                         DefaultExtraArgs());
+      auto [inserted, ok] = modules.emplace(key, std::move(mod));
+      MFEM_VERIFY(ok, "failed to cache JIT module");
+      return *inserted->second;
+   }
+
+   static std::vector<std::string> DefaultExtraArgs()
+   {
+      return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
+   }
+
+   std::string qfunc_name = "qfunc_wrapper";
+   std::string fn;
+   size_t nparams = 0;
+   std::vector<bool> activity_map;
+   std::vector<std::string> param_types;
+   std::string return_type;
+   bool return_is_void = false;
+   std::string impl_type_name;
+   std::vector<std::string> jit_var_names;
+   std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
+};
+
+int main()
+{
+   const size_t N = 4;
+   const size_t M = 5;
+   const double A = 123.4;
+
+   std::vector<double> X(N);
+   std::vector<double> Y(N);
+   for (size_t i = 0; i < N; ++i)
+   {
+      X[i] = static_cast<double>(i + 1);
+      Y[i] = static_cast<double>(N - i);
+   }
+
+   // // >>> user interface calls
+   // const std::string kernel_path = std::string(util::thisFileDir) +
+   //                                 "/jitplayground.hpp";
+   // JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
+   // // <<< user interface calls
+
+   // // this will happen internally in dFEM
+
+   daxpy_op op;
+   printf("\n\nfunction call\n");
+   op(&A, X.data(), Y.data(), &N);
+
+   // reset X for the derivative test
+   for (size_t i = 0; i < N; ++i)
+   {
+      X[i] = static_cast<double>(i + 1);
+      Y[i] = static_cast<double>(N - i);
+   }
+
+   std::vector<double> dX(N, 1.0);
+   printf("\n\nforward diff call\n");
+   daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
+
+   std::vector<double> dX_manual(N, A);
+
+   printf("\n\nderivative checks\n");
+   std::cout << "dX: ";
+   for (size_t i = 0; i < N; ++i)
+   {
+      std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
+   }
+
+   std::cout << "dX_manual: ";
+   for (size_t i = 0; i < N; ++i)
+   {
+      std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
+   }
+
+   double max_abs_err = 0.0;
+   for (size_t i = 0; i < N; ++i)
+   {
+      max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
+   }
+   std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
+
+   return 0;
+}
+
+#else
+
+int main()
+{
+   std::cerr << "Proteus is required to run this example.\n";
+   return EXIT_FAILURE;
+}
+
+#endif // MFEM_USE_PROTEUS

examples/jitplayground.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <cstddef>
+#include <vector>
+#include <type_traits>
+
+#include "proteus/JitInterface.h"
+
+struct daxpy_op
+{
+   void operator()(
+      const double *a,
+      double *x,
+      const double *y,
+      const size_t *N) const
+   {
+      const size_t n = *N;
+      auto lam = [=, n = proteus::jit_variable(n)]
+                 () __attribute__((annotate("jit")))
+      {
+         printf("N = %zu\n", n);
+         for (size_t i = 0; i < n; ++i)
+         {
+            printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
+            x[i] = *a * x[i] + y[i];
+            printf("updated x[%zu] = %f\n", i, x[i]);
+         }
+      };
+
+      proteus::register_lambda(lam);
+
+      lam();
+   }
+};
+
+template <typename return_type, typename... Args>
+return_type __enzyme_fwddiff(Args...);
+
+extern int enzyme_const;
+extern int enzyme_dup;
+
+void daxpy_op_wrapper(const double * Arg0, double * Arg1,
+                      const double * Arg2, const size_t *Arg3)
+{
+   daxpy_op qf;
+   qf(Arg0, Arg1, Arg2, Arg3);
+}
+
+void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
+                      double * dArg1, const double * Arg2, const size_t *Arg3)
+{
+   __enzyme_fwddiff<void>(
+      (void*)daxpy_op_wrapper,
+      enzyme_const, Arg0,
+      enzyme_dup, Arg1, dArg1,
+      enzyme_const, Arg2,
+      enzyme_const, Arg3);
+}

fem/CMakeLists.txt

@@ -65,6 +65,7 @@ set(SRCS
   dgmassinv.cpp
   doftrans.cpp
   dfem/doperator.cpp
+  dfem/backends/local_qf/kernels.cpp
   eltrans.cpp
   batchitrans.cpp
   estimators.cpp
@@ -121,6 +122,11 @@ set(SRCS
   qinterp/eval_hdiv.cpp
   qinterp/grad_by_nodes.cpp
   qinterp/grad_by_vdim.cpp
+  qinterp/grad_transpose.cpp
+  qinterp/grad_transpose_by_nodes.cpp
+  qinterp/grad_transpose_by_vdim.cpp
+  qinterp/eval_transpose.cpp
+  qinterp/eval_transpose_by_vdim.cpp
   qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
@@ -213,13 +219,31 @@ set(HDRS
   dgmassinv_kernels.hpp
   doftrans.hpp
   dfem/doperator.hpp
+  dfem/fielddescriptor.hpp
   dfem/fieldoperator.hpp
-  dfem/integrate.hpp
+  dfem/integrator_ctx.hpp
   dfem/parameterspace.hpp
-  dfem/qfunction_apply.hpp
-  dfem/qfunction_transform.hpp
   dfem/tuple.hpp
   dfem/util.hpp
+  dfem/backends/util.hpp
+  dfem/backends/global_qf/action.hpp
+  dfem/backends/global_qf/derivative_action.hpp
+  dfem/backends/global_qf/derivative_apply.hpp
+  dfem/backends/global_qf/derivative_apply_transpose.hpp
+  dfem/backends/global_qf/derivative_setup.hpp
+  dfem/backends/global_qf/prelude.hpp
+  dfem/backends/local_qf/action.hpp
+  dfem/backends/local_qf/derivative_action.hpp
+  dfem/backends/local_qf/derivative_apply.hpp
+  dfem/backends/local_qf/derivative_apply_transpose.hpp
+  dfem/backends/local_qf/derivative_assemble.hpp
+  dfem/backends/local_qf/derivative_assemble_diagonal.hpp
+  dfem/backends/local_qf/derivative_setup.hpp
+  dfem/backends/local_qf/kernels.hpp
+  dfem/backends/local_qf/kernels_ho.hpp
+  dfem/backends/local_qf/kernels_lo.hpp
+  dfem/backends/local_qf/prelude.hpp
+  dfem/backends/local_qf/util.hpp
   eltrans.hpp
   estimators.hpp
   fe.hpp
@@ -284,8 +308,10 @@ set(HDRS
   qfunction.hpp
   qinterp/det.hpp
   qinterp/eval.hpp
+  qinterp/eval_transpose.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp
+  qinterp/grad_transpose.hpp
   qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
@@ -320,36 +346,36 @@ set(HDRS
   )
 
 if (MFEM_USE_SIDRE)
-  list(APPEND SRCS sidredatacollection.cpp)
-  list(APPEND HDRS sidredatacollection.hpp)
+    list(APPEND SRCS sidredatacollection.cpp)
+    list(APPEND HDRS sidredatacollection.hpp)
 endif()
 
 if (MFEM_USE_CONDUIT)
-  list(APPEND SRCS conduitdatacollection.cpp)
-  list(APPEND HDRS conduitdatacollection.hpp)
+    list(APPEND SRCS conduitdatacollection.cpp)
+    list(APPEND HDRS conduitdatacollection.hpp)
 endif()
 
 if (MFEM_USE_ADIOS2)
-  list(APPEND SRCS adios2datacollection.cpp)
-  list(APPEND HDRS adios2datacollection.hpp)
+    list(APPEND SRCS adios2datacollection.cpp)
+    list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
 if (MFEM_USE_FMS)
-  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+    list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+    list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
 endif()
 
 if (MFEM_USE_MPI)
-  list(APPEND SRCS
+    list(APPEND SRCS
     pbilinearform.cpp
     pfespace.cpp
     pgridfunc.cpp
     plinearform.cpp
     pnonlinearform.cpp
     prestriction.cpp)
-  # If this list (HDRS -> HEADERS) is used for install, we probably want the
-  # headers added all the time.
-  list(APPEND HDRS
+    # If this list (HDRS -> HEADERS) is used for install, we probably want the
+    # headers added all the time.
+    list(APPEND HDRS
     pbilinearform.hpp
     pfespace.hpp
     pgridfunc.hpp

fem/dfem/assemble.hpp

@@ -1,403 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-/// @brief Assemble element matrix for three dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_t3d(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& q1d,
-   const int& td1d)
-{
-   constexpr int dimension = 3;
-
-   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-
-   // [num_test_dof, ...]
-   const auto num_test_dof = A.GetShape()[0];
-
-   for (int Jx = 0; Jx < td1d; Jx++)
-   {
-      for (int Jy = 0; Jy < td1d; Jy++)
-      {
-         for (int Jz = 0; Jz < td1d; Jz++)
-         {
-            const int J = Jx + td1d * (Jy + td1d * Jz);
-
-            for (int j = 0; j < trial_vdim; j++)
-            {
-               for (int tv = 0; tv < test_vdim; tv++)
-               {
-                  for (int tod = 0; tod < test_op_dim; tod++)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-                              fhat(tv, tod, q) = 0.0;
-                           }
-                        }
-                     }
-                  }
-               }
-
-               // MSVC lambda capture workaround
-               [[maybe_unused]] const auto& inputs_ref = inputs;
-
-               int m_offset = 0;
-               for_constexpr<num_inputs>([&](auto s)
-               {
-                  using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
-
-                  const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
-                  if (trial_op_dim == 0)
-                  {
-                     // This is inside a lambda so we have to return
-                     // instead of idiomatic 'continue'.
-                     return;
-                  }
-
-                  auto& B = input_dtqmaps[s].B;
-                  auto& G = input_dtqmaps[s].G;
-
-                  if constexpr (is_value_fop<fop_t>::value)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-
-                              for (int m = 0; m < trial_op_dim; m++)
-                              {
-                                 for (int i = 0; i < test_vdim; i++)
-                                 {
-                                    for (int k = 0; k < test_op_dim; k++)
-                                    {
-                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                       fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
-                                    }
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-                  else if constexpr (is_gradient_fop<fop_t>::value)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-                              for (int m = 0; m < trial_op_dim; m++)
-                              {
-                                 for (int i = 0; i < test_vdim; i++)
-                                 {
-                                    for (int k = 0; k < test_op_dim; k++)
-                                    {
-                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                       if (m == 0)
-                                       {
-                                          fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
-                                       }
-                                       else if (m == 1)
-                                       {
-                                          fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy) * B(qz, 0, Jz);
-                                       }
-                                       else if (m == 2)
-                                       {
-                                          fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * G(qz, 0, Jz);
-                                       }
-                                    }
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-                  else
-                  {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-                     MFEM_ABORT("sum factorized sparse matrix assemble routine "
-                                "not implemented for field operator");
-#endif
-                  }
-                  MFEM_SYNC_THREAD;
-                  m_offset += trial_op_dim;
-               });
-
-               auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
-               map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
-                                             scratch_shmem, dimension, true);
-            }
-         }
-      }
-   }
-}
-
-/// @brief Assemble element matrix for two dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_t2d(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& q1d,
-   const int& td1d)
-{
-   constexpr int dimension = 2;
-
-   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-
-   // [num_test_dof, ...]
-   const auto num_test_dof = A.GetShape()[0];
-
-   for (int Jx = 0; Jx < td1d; Jx++)
-   {
-      for (int Jy = 0; Jy < td1d; Jy++)
-      {
-         const int J = Jy + Jx * td1d;
-
-         for (int j = 0; j < trial_vdim; j++)
-         {
-            for (int tv = 0; tv < test_vdim; tv++)
-            {
-               for (int tod = 0; tod < test_op_dim; tod++)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        fhat(tv, tod, q) = 0.0;
-                     }
-                  }
-               }
-            }
-
-            // MSVC lambda capture workaround
-            [[maybe_unused]] const auto& inputs_ref = inputs;
-
-            int m_offset = 0;
-            for_constexpr<num_inputs>([&](auto s)
-            {
-               using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
-
-               const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
-               if (trial_op_dim == 0)
-               {
-                  // This is inside a lambda so we have to return
-                  // instead of idiomatic 'continue'.
-                  return;
-               }
-
-               auto& B = input_dtqmaps[s].B;
-               auto& G = input_dtqmaps[s].G;
-
-               if constexpr (is_value_fop<fop_t>::value)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           for (int i = 0; i < test_vdim; i++)
-                           {
-                              for (int k = 0; k < test_op_dim; k++)
-                              {
-                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                 fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy);
-                              }
-                           }
-                        }
-                     }
-                  }
-               }
-               else if constexpr (is_gradient_fop<fop_t>::value)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           for (int i = 0; i < test_vdim; i++)
-                           {
-                              for (int k = 0; k < test_op_dim; k++)
-                              {
-                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                 if (m == 0)
-                                 {
-                                    fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy);
-                                 }
-                                 else
-                                 {
-                                    fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy);
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-               }
-               else
-               {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-                  MFEM_ABORT("sum factorized sparse matrix assemble routine "
-                             "not implemented for field operator");
-#endif
-               }
-               MFEM_SYNC_THREAD;
-               m_offset += trial_op_dim;
-            });
-
-            auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
-            map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
-                                          scratch_shmem, dimension, true);
-         }
-      }
-   }
-}
-
-/// @brief Assemble element matrix for two or three dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param dimension The spatial dimension.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-/// @param use_sum_factorization Indicator if sum factorization is used.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_naive(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& dimension,
-   const int& q1d,
-   const int& td1d,
-   const bool& use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 2)
-      {
-         assemble_element_mat_t2d(A, fhat, qpdc, itod, inputs, output,
-                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
-      }
-      else if (dimension == 3)
-      {
-         assemble_element_mat_t3d(A, fhat, qpdc, itod, inputs, output,
-                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
-      }
-   }
-   else
-   {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-      MFEM_ABORT("element matrix assemble not implemented for non tensor "
-                 "product basis");
-#endif
-   }
-}
-
-} // namespace mfem::future

fem/dfem/backends/global_qf/action.hpp

@@ -0,0 +1,122 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../util.hpp"
+#include "../../integrator_ctx.hpp"
+
+#include <utility>
+
+namespace mfem::future
+{
+
+namespace GlobalQFImpl
+{
+
+template<
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = tuple_size<inputs_t>::value,
+   size_t noutputs = tuple_size<outputs_t>::value>
+struct Action
+{
+   Action(
+      IntegratorContext ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs) :
+      ctx(ctx),
+      qfunc(qfunc),
+      inputs(inputs),
+      outputs(outputs)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      gnqp = nqp * ctx.nentities;
+
+      // prepare xq and yq BlockVectors
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         const auto input = get<i>(inputs);
+         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
+      });
+      xq_offsets.PartialSum();
+      InitBlockVector(xq, xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto i)
+      {
+         const auto output = get<i>(outputs);
+         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
+      });
+      yq_offsets.PartialSum();
+      InitBlockVector(yq, yq_offsets);
+   }
+
+   void operator()(
+      const std::vector<Vector *> &xe,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      // E -> Q
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      // Q -> Q
+      static_assert(
+         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
+         "qfunc signature not supported by default backend Action");
+
+      detail::call_qfunc(
+         qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
+         std::make_index_sequence<ninputs> {},
+         std::make_index_sequence<noutputs> {});
+
+      // Q -> E
+      integrate(output_to_outfd, output_bases, yq, ye);
+   }
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs>  input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   Array<int> xq_offsets, yq_offsets;
+   mutable BlockVector xq, yq;
+};
+
+}
+}

fem/dfem/backends/global_qf/derivative_action.hpp

@@ -0,0 +1,180 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+#include <utility>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+template<
+   int derivative_id,
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = tuple_size<inputs_t>::value,
+   size_t noutputs = tuple_size<outputs_t>::value>
+struct DerivativeAction
+{
+   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
+
+   DerivativeAction(
+      IntegratorContext ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs):
+      ctx(ctx), qfunc(qfunc),
+      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
+      inputs(inputs), outputs(outputs)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(
+         outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      gnqp = nqp * ctx.nentities;
+
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         const auto input = get<i>(inputs);
+         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
+      });
+      xq_offsets.PartialSum();
+      InitBlockVector(xq, xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto i)
+      {
+         const auto output = get<i>(outputs);
+         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
+      });
+      yq_offsets.PartialSum();
+      InitBlockVector(yq, yq_offsets);
+
+      // Shadow blocks use the same offsets as xq so tensor_array views
+      shadow_xq_offsets.SetSize(xq_offsets.Size());
+      shadow_xq_offsets = xq_offsets;
+      InitBlockVector(shadow_xq, shadow_xq_offsets);
+
+      dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
+
+      const size_t direction_fd_idx = FindIdx(derivative_id, ctx.infds);
+      MFEM_ASSERT(direction_fd_idx != SIZE_MAX,
+                  "derivative direction field not found in infds");
+      direction_fd = ctx.infds[direction_fd_idx];
+   }
+
+   void operator()(
+      const std::vector<Vector *> &xe,
+      const Vector *de,
+      std::vector<Vector *> &ye)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      // E -> Q
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      constexpr auto input_active =
+         detail::make_activity_map<derivative_id>(inputs_t{});
+
+      MFEM_ASSERT(de != nullptr, "derivative direction vector is null");
+      restriction(direction_fd, direction_rcache, *de, direction_e,
+                  dof_ordering);
+
+      shadow_xq = 0.0;
+      shadow_xq.SyncToBlocks();
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         if (!input_active[i]) { return; }
+         input_bases[i].forward(direction_e, shadow_xq.GetBlock(i));
+      });
+
+      static_assert(detail::supports_tensor_array_qfunc<qfunc_t,
+                    inputs_t,
+                    outputs_t>::value,
+                    "qfunc signature not supported by default backend Action");
+
+      // Q -> Q
+      yq = 0.0;
+      yq.SyncToBlocks();
+      if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
+      {
+         detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t, inputs_t,
+                outputs_t>(
+                   qfunc,
+                   qfunc_shadow,
+                   xq,
+                   shadow_xq,
+                   yq,
+                   gnqp,
+                   input_qlayouts,
+                   output_qlayouts,
+                   std::make_index_sequence<ninputs> {},
+                   std::make_index_sequence<noutputs> {});
+      }
+      else
+      {
+         detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
+            qfunc,
+            xq,
+            shadow_xq,
+            yq,
+            gnqp,
+            input_qlayouts,
+            output_qlayouts,
+            std::make_index_sequence<ninputs> {},
+            std::make_index_sequence<noutputs> {});
+      }
+
+      // Q -> E
+      integrate(output_to_outfd, output_bases, yq, ye);
+   }
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   qfunc_shadow_t qfunc_shadow;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs> input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
+   mutable BlockVector xq, shadow_xq, yq;
+
+   FieldDescriptor direction_fd;
+   ElementDofOrdering dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   mutable Vector direction_e;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply.hpp

@@ -0,0 +1,242 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+
+#include <array>
+#include <utility>
+#include <vector>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+// Q-function-shape-agnostic cached forward apply (J·v)
+template<
+   int derivative_id,
+   typename inputs_t,
+   typename outputs_t>
+struct DerivativeApply
+{
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   DerivativeApply(
+      IntegratorContext ctx,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache):
+      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
+      qp_cache(qp_cache)
+   {
+      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(this->inputs, input_to_infd, ctx.infds);
+      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(
+         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(
+         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      const int nqp = ctx.ir.GetNPoints();
+      const int ne = ctx.nentities;
+      num_qp = nqp;
+      gnqp = nqp * ne;
+
+      // Precompute Q-space BlockVector layouts
+      dir_q_offsets.SetSize(n_inputs + 1);
+      dir_q_offsets[0] = 0;
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         dir_q_offsets[i + 1] =
+            dir_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(dir_q_local, dir_q_offsets);
+
+      result_q_offsets.SetSize(n_outputs + 1);
+      result_q_offsets[0] = 0;
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         result_q_offsets[i + 1] =
+            result_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(result_q_local, result_q_offsets);
+
+      // Cache layout metadata (must match DerivativeSetup)
+      residual_size_on_qp = 0;
+      trial_vdim = 0;
+      total_trial_op_dim = 0;
+
+      constexpr auto activity =
+         detail::make_activity_map<derivative_id>(inputs_t{});
+
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         if (!activity[i]) { return; }
+         const auto &fop = get<i>(this->inputs);
+         trial_vdim = fop.vdim;
+         total_trial_op_dim += fop.size_on_qp / fop.vdim;
+      });
+
+      constexpr_for<0, n_outputs>([&](auto i)
+      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
+      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
+   }
+
+   void operator()(
+      const std::vector<Vector *> & /*xe*/,
+      const Vector *direction_l,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "Global DerivativeApply: direction vector is null");
+
+      // Re-zero pre-allocated Q temporaries
+      dir_q_local = 0.0;
+      dir_q_local.SyncToBlocks();
+      result_q_local = 0.0;
+      result_q_local.SyncToBlocks();
+
+      // Restrict trial direction from the derivative field
+      size_t in_fd = SIZE_MAX;
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         if (get<i>(inputs).GetFieldId() == derivative_id)
+         {
+            in_fd = input_to_infd[i.value];
+         }
+      });
+      MFEM_ASSERT(in_fd != SIZE_MAX,
+                  "DerivativeApply: derivative field not found among inputs");
+
+      const auto &fd = ctx.infds[in_fd];
+
+      Vector dir_e;
+      restriction(fd, direction_rcache, *direction_l, dir_e,
+                  ElementDofOrdering::LEXICOGRAPHIC);
+
+      // Forward the trial direction into active input Q block
+      constexpr_for<0, n_inputs>([&](auto s)
+      {
+         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+         input_bases[s.value].forward(dir_e, dir_q_local.GetBlock(s.value));
+      });
+
+      const real_t *cache_ptr = qp_cache.Read();
+      const int res_sz = residual_size_on_qp;
+      const int gnqp_local = gnqp;
+      const int num_qp_local = num_qp;
+      const int trial_vdim_local = trial_vdim;
+      const int total_trial_op_dim_local = total_trial_op_dim;
+
+      constexpr_for<0, n_outputs>([&](auto o)
+      {
+         const int tv_o = get<o>(outputs).vdim;
+         const int to_o = get<o>(outputs).size_on_qp / tv_o;
+         const int out_base = [&]
+         {
+            int off = 0;
+            constexpr_for<0, o.value>([&](auto prev)
+            { off += get<prev>(outputs).size_on_qp; });
+            return off;
+         }();
+
+         real_t *res_o = result_q_local.GetBlock(o.value).ReadWrite();
+
+         int m_offset = 0;
+         constexpr_for<0, n_inputs>([&](auto s)
+         {
+            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+
+            const int tv = get<s>(inputs).vdim;
+            const int to = get<s>(inputs).size_on_qp / tv;
+            const real_t *dir_s = dir_q_local.GetBlock(s.value).Read();
+
+            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
+            {
+               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
+               // read adjacent addresses for a fixed cache_idx.
+               const int cache_base =
+                  (gq % num_qp_local) +
+                  num_qp_local * res_sz * (gq / num_qp_local);
+
+               for (int j = 0; j < tv; ++j)
+               {
+                  for (int m = 0; m < to; ++m)
+                  {
+                     const real_t v = dir_s[(j * to + m) + (tv * to) * gq];
+                     const int m_global = m + m_offset;
+
+                     for (int i = 0; i < tv_o; ++i)
+                     {
+                        for (int k = 0; k < to_o; ++k)
+                        {
+                           const int out_comp = out_base + i * to_o + k;
+
+                           const int cache_idx =
+                              out_comp * trial_vdim_local * total_trial_op_dim_local +
+                              j * total_trial_op_dim_local + m_global;
+
+                           const real_t c =
+                              cache_ptr[cache_base + num_qp_local * cache_idx];
+                           res_o[(i * to_o + k) + (tv_o * to_o) * gq] += c * v;
+                        }
+                     }
+                  }
+               }
+            });
+            m_offset += to;
+         });
+      });
+
+      result_q_local.SyncToBlocks();
+
+      // Map result Q back to output fields
+      constexpr_for<0, n_outputs>([&](auto o)
+      {
+         const size_t out_fd = output_to_outfd[o.value];
+         output_bases[o.value].transpose(result_q_local.GetBlock(o.value),
+                                         *ye[out_fd]);
+      });
+   }
+
+private:
+   IntegratorContext ctx;
+   inputs_t inputs;
+   outputs_t outputs;
+   const Vector &qp_cache;
+
+   std::array<size_t, n_inputs> input_to_infd;
+   std::array<size_t, n_outputs> output_to_outfd;
+
+   std::array<FieldBasis, n_inputs> input_bases;
+   std::array<FieldBasis, n_outputs> output_bases;
+
+   int gnqp = 0;
+   int num_qp = 0;
+
+   Array<int> dir_q_offsets;
+   Array<int> result_q_offsets;
+   mutable BlockVector dir_q_local;
+   mutable BlockVector result_q_local;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+
+   int residual_size_on_qp = 0;
+   int trial_vdim = 0;
+   int total_trial_op_dim = 0;
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply_transpose.hpp

@@ -0,0 +1,260 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+
+#include <array>
+#include <utility>
+#include <vector>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+// Q-function-shape-agnostic cached transpose apply (Jᵀ·w)
+template<
+   int derivative_id,
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t>
+struct DerivativeApplyTranspose
+{
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   DerivativeApplyTranspose(
+      IntegratorContext ctx,
+      qfunc_t /*qfunc*/,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache):
+      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
+      qp_cache(qp_cache), dir_out_l(n_outputs), dir_out_e(n_outputs)
+   {
+      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(this->inputs, input_to_infd, ctx.infds);
+      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(
+         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(
+         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      const int nqp = ctx.ir.GetNPoints();
+      const int ne = ctx.nentities;
+      num_qp = nqp;
+      gnqp = nqp * ne;
+
+      // Precompute Q-space BlockVector layouts
+      dir_q_offsets.SetSize(n_outputs + 1);
+      dir_q_offsets[0] = 0;
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         dir_q_offsets[i + 1] =
+            dir_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(dir_q_local, dir_q_offsets);
+
+      result_q_offsets.SetSize(n_inputs + 1);
+      result_q_offsets[0] = 0;
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         result_q_offsets[i + 1] =
+            result_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(result_q_local, result_q_offsets);
+
+      // Cache layout metadata
+      residual_size_on_qp = 0;
+      trial_vdim = 0;
+      total_trial_op_dim = 0;
+
+      constexpr auto activity =
+         detail::make_activity_map<derivative_id>(inputs_t{});
+
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         if (!activity[i]) { return; }
+         const auto &fop = get<i>(this->inputs);
+         trial_vdim = fop.vdim;
+         total_trial_op_dim += fop.size_on_qp / fop.vdim;
+      });
+
+      constexpr_for<0, n_outputs>([&](auto i)
+      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
+      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
+   }
+
+   void operator()(
+      const std::vector<Vector *> & /*xe*/,
+      const Vector *direction_l,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "Global DerivativeApplyTranspose: direction vector is null");
+
+      // Re-zero the pre-allocated Q temporaries
+      dir_q_local = 0.0;
+      result_q_local = 0.0;
+      dir_q_local.SyncToBlocks();
+      result_q_local.SyncToBlocks();
+
+      // Bring test cotangent to quadrature points
+      pull_output_cotangents_to_q(direction_l, dir_q_local);
+
+      // Contract qp_cache with test directions at quadrature points
+      const real_t *cache_ptr = qp_cache.Read();
+      const int res_sz = residual_size_on_qp;
+      const int gnqp_local = gnqp;
+      const int num_qp_local = num_qp;
+      const int trial_vdim_local = trial_vdim;
+      const int total_trial_op_dim_local = total_trial_op_dim;
+
+      constexpr_for<0, n_outputs>([&](auto o)
+      {
+         const int tv_o = get<o>(outputs).vdim;
+         const int to_o = get<o>(outputs).size_on_qp / tv_o;
+         const int out_base = [&]
+         {
+            int off = 0;
+            constexpr_for<0, o.value>([&](auto prev)
+            { off += get<prev>(outputs).size_on_qp; });
+            return off;
+         }();
+
+         const int size_o = get<o>(outputs).size_on_qp;
+         const real_t *dir_o = dir_q_local.GetBlock(o.value).Read();
+
+         int m_offset = 0;
+         constexpr_for<0, n_inputs>([&](auto s)
+         {
+            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+
+            const int size_s = get<s>(inputs).size_on_qp;
+            const int to_s = size_s / trial_vdim_local;
+            real_t *res_s = result_q_local.GetBlock(s.value).ReadWrite();
+
+            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
+            {
+               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
+               // read adjacent addresses for a fixed cache_idx.
+               const int cache_base =
+                  (gq % num_qp_local) +
+                  num_qp_local * res_sz * (gq / num_qp_local);
+
+               for (int i = 0; i < tv_o; ++i)
+               {
+                  for (int k = 0; k < to_o; ++k)
+                  {
+                     const int out_comp = out_base + i * to_o + k;
+                     const real_t w = dir_o[(i * to_o + k) + size_o * gq];
+
+                     for (int j = 0; j < trial_vdim_local; ++j)
+                     {
+                        for (int m = 0; m < to_s; ++m)
+                        {
+                           const int m_global = m + m_offset;
+                           const int cache_idx =
+                              out_comp * trial_vdim_local * total_trial_op_dim_local +
+                              j * total_trial_op_dim_local + m_global;
+
+                           const real_t c =
+                              cache_ptr[cache_base + num_qp_local * cache_idx];
+                           res_s[(j * to_s + m) + size_s * gq] += c * w;
+                        }
+                     }
+                  }
+               }
+            });
+            m_offset += to_s;
+         });
+      });
+
+      // Map result Q back to the trial (input) fields
+      constexpr_for<0, n_inputs>([&](auto s)
+      {
+         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+
+         const size_t in_fd = input_to_infd[s.value];
+         input_bases[s.value].transpose(
+            result_q_local.GetBlock(s.value), *ye[in_fd]);
+      });
+   }
+
+private:
+   IntegratorContext ctx;
+   inputs_t inputs;
+   outputs_t outputs;
+   const Vector &qp_cache;
+
+   std::array<size_t, n_inputs> input_to_infd;
+   std::array<size_t, n_outputs> output_to_outfd;
+
+   std::array<FieldBasis, n_inputs> input_bases;
+   std::array<FieldBasis, n_outputs> output_bases;
+
+   int gnqp = 0;
+   int num_qp = 0;
+
+   // Pre-allocated Q-space temporaries
+   Array<int> dir_q_offsets;
+   Array<int> result_q_offsets;
+   mutable BlockVector dir_q_local;
+   mutable BlockVector result_q_local;
+
+   // Pre-allocated owning storage for output cotangent temporaries
+   mutable std::array<Vector, n_outputs> dir_out_l_owned;
+   mutable std::array<Vector, n_outputs> dir_out_e_owned;
+   mutable std::vector<Vector *> dir_out_l;
+   mutable std::vector<Vector *> dir_out_e;
+   mutable RestrictionCache<Entity::Element> out_rcache;
+
+   int residual_size_on_qp = 0;
+   int trial_vdim = 0;
+   int total_trial_op_dim = 0;
+
+   /// Pull output cotangents from L-space into the pre-allocated Q BlockVector
+   void pull_output_cotangents_to_q(const Vector *direction_l,
+                                    BlockVector &dir_q) const
+   {
+      int l_offset = 0;
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         const size_t outfd = output_to_outfd[i];
+         const auto &fd = ctx.outfds[outfd];
+         const int l_size = GetVSize(fd);
+
+         dir_out_l_owned[i] =
+            Vector(*const_cast<Vector *>(direction_l), l_offset, l_size);
+         dir_out_e_owned[i].SetSize(0);
+         dir_out_e_owned[i].UseDevice(true);
+
+         dir_out_l[i] = &dir_out_l_owned[i];
+         dir_out_e[i] = &dir_out_e_owned[i];
+         l_offset += l_size;
+      });
+
+      restriction(ctx.outfds, out_rcache, dir_out_l, dir_out_e);
+
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         output_bases[i.value].forward(*dir_out_e[i], dir_q.GetBlock(i.value));
+      });
+      dir_q.SyncToBlocks();
+   }
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_setup.hpp

@@ -0,0 +1,245 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+#include <utility>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+template<
+   int derivative_id,
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = tuple_size<inputs_t>::value,
+   size_t noutputs = tuple_size<outputs_t>::value>
+struct DerivativeSetup
+{
+   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
+
+   DerivativeSetup(
+      IntegratorContext ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      Vector &qp_cache) :
+      ctx(ctx),
+      qfunc(qfunc),
+      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
+      inputs(inputs),
+      outputs(outputs),
+      qp_cache(qp_cache)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      num_qp = nqp;
+      nentities = ctx.nentities;
+      gnqp = nqp * nentities;
+
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         xq_offsets[i + 1] = nqp * get<i>(inputs).size_on_qp * nentities;
+      });
+      xq_offsets.PartialSum();
+      InitBlockVector(xq, xq_offsets);
+
+      shadow_xq_offsets.SetSize(xq_offsets.Size());
+      shadow_xq_offsets = xq_offsets;
+      InitBlockVector(shadow_xq, shadow_xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto o)
+      {
+         yq_offsets[o + 1] = nqp * get<o>(outputs).size_on_qp * nentities;
+      });
+      yq_offsets.PartialSum();
+      InitBlockVector(yq, yq_offsets);
+
+      total_out_size_on_qp = 0;
+      constexpr_for<0, noutputs>([&](auto o)
+      {
+         total_out_size_on_qp += get<o>(outputs).size_on_qp;
+         out_vdim[o] = get<o>(outputs).vdim;
+         out_op_dim[o] = get<o>(outputs).size_on_qp / get<o>(outputs).vdim;
+      });
+
+      activity_map = detail::make_activity_map<derivative_id>(inputs_t {});
+
+      trial_vdim = 0;
+      total_trial_op_dim = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         if (!activity_map[i]) { return; }
+         const auto inp = get<i>(inputs);
+         trial_vdim = inp.vdim;
+         total_trial_op_dim += inp.size_on_qp / inp.vdim;
+      });
+
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         input_size_on_qp_arr[i] = get<i>(inputs).size_on_qp;
+      });
+
+      residual_size_on_qp = total_out_size_on_qp * trial_vdim * total_trial_op_dim;
+      qp_cache.SetSize(residual_size_on_qp * num_qp * nentities);
+      qp_cache.UseDevice(true);
+   }
+
+   void operator()(const std::vector<Vector *> &xe)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      const int gnqp_local = gnqp;
+      const int num_qp_local = num_qp;
+      const int trial_vdim_local = trial_vdim;
+      const int total_trial_op_dim_local = total_trial_op_dim;
+      const int residual_size_local = residual_size_on_qp;
+
+      for (int j = 0; j < trial_vdim; j++)
+      {
+         int m_offset = 0;
+         constexpr_for<0, ninputs>([&](auto s)
+         {
+            if (!activity_map[s]) { return; }
+
+            const int input_vdim_s  = get<s>(inputs).vdim;
+            const int input_size_s  = input_size_on_qp_arr[s];
+            const int trial_op_dim_s = input_size_s / input_vdim_s;
+
+            for (int m = 0; m < trial_op_dim_s; m++)
+            {
+               shadow_xq = 0.0;
+               shadow_xq.SyncToBlocks();
+
+               // Set component (j + input_vdim_s * m) to 1 at all QPs
+               const int c_shadow = j + input_vdim_s * m;
+               real_t *shadow_ptr = shadow_xq.GetBlock(s.value).ReadWrite();
+               mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
+               {
+                  shadow_ptr[c_shadow + input_size_s * gq] = 1.0;
+               });
+
+               yq = 0.0;
+               yq.SyncToBlocks();
+
+               if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
+               {
+                  detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t,
+                         inputs_t, outputs_t>(
+                            qfunc, qfunc_shadow, xq, shadow_xq, yq, gnqp,
+                            input_qlayouts, output_qlayouts,
+                            std::make_index_sequence<ninputs> {},
+                            std::make_index_sequence<noutputs> {});
+               }
+               else
+               {
+                  detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
+                     qfunc, xq, shadow_xq, yq, gnqp,
+                     input_qlayouts, output_qlayouts,
+                     std::make_index_sequence<ninputs> {},
+                     std::make_index_sequence<noutputs> {});
+               }
+
+               yq.SyncToBlocks();
+               real_t *cache_d = qp_cache.ReadWrite();
+
+               // Write yq into the cache column
+               const int m_global = m + m_offset;
+               const int j_cur    = j;
+               int out_offset = 0;
+               constexpr_for<0, noutputs>([&](auto o)
+               {
+                  const int test_vdim_o   = out_vdim[o];
+                  const int test_op_dim_o = out_op_dim[o];
+                  const int yq_out_size   = test_vdim_o * test_op_dim_o;
+                  const int out_offset_o  = out_offset;
+                  const real_t *yq_d      = yq.GetBlock(o.value).Read();
+
+                  // The cache is (q, cache_idx, e) with the quadrature index
+                  // fastest, so gq is the fastest-varying thread index to keep
+                  // the stores coalesced.
+                  mfem::forall(gnqp_local * yq_out_size, [=] MFEM_HOST_DEVICE(int idx)
+                  {
+                     const int gq       = idx % gnqp_local;
+                     const int c_out    = idx / gnqp_local;
+                     const int q        = gq % num_qp_local;
+                     const int entity   = gq / num_qp_local;
+                     const int out_comp = out_offset_o + c_out;
+                     const int cache_idx =
+                        out_comp * trial_vdim_local * total_trial_op_dim_local +
+                        j_cur * total_trial_op_dim_local +
+                        m_global;
+                     cache_d[q + num_qp_local *
+                                 (cache_idx + residual_size_local * entity)] =
+                        yq_d[c_out + yq_out_size * gq];
+                  });
+                  out_offset += yq_out_size;
+               });
+            }
+            m_offset += trial_op_dim_s;
+         });
+      }
+   }
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   qfunc_shadow_t qfunc_shadow;
+   inputs_t inputs;
+   outputs_t outputs;
+   Vector &qp_cache;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs>  input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   int num_qp = 0;
+   int nentities = 0;
+
+   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
+   mutable BlockVector xq, shadow_xq, yq;
+
+   int total_out_size_on_qp = 0;
+   int trial_vdim = 0;
+   int total_trial_op_dim = 0;
+   int residual_size_on_qp = 0;
+
+   std::array<int, noutputs> out_vdim {};
+   std::array<int, noutputs> out_op_dim {};
+   std::array<int, ninputs>  input_size_on_qp_arr {};
+   std::array<bool, ninputs> activity_map {};
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/prelude.hpp

@@ -0,0 +1,219 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "action.hpp"
+#include "derivative_action.hpp"
+#include "derivative_setup.hpp"
+
+#include "derivative_apply_transpose.hpp"
+
+#include "../local_qf/derivative_apply.hpp"
+#include "../local_qf/derivative_assemble.hpp"
+#include "../local_qf/derivative_assemble_diagonal.hpp"
+
+#include "../scratch_bank.hpp"
+
+namespace mfem::future
+{
+
+namespace detail
+{
+
+template <typename T>
+struct LocalQFShapeArg
+{
+   using type = std::remove_const_t<T>&;
+};
+
+template <typename scalar_t, int ndims, int... tensor_sizes>
+struct LocalQFShapeArg<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
+{
+   using scalar_type = std::remove_const_t<scalar_t>;
+   using type = std::conditional_t<
+                   sizeof...(tensor_sizes) == 0,
+                   scalar_type,
+                   tensor<scalar_type, tensor_sizes...>>&;
+};
+
+template <typename scalar_t, int... tensor_sizes>
+struct LocalQFShapeArg<tensor<scalar_t, tensor_sizes...>>
+{
+   using scalar_type = std::remove_const_t<scalar_t>;
+   using type = std::conditional_t<
+                   sizeof...(tensor_sizes) == 0,
+                   scalar_type,
+                   tensor<scalar_type, tensor_sizes...>>&;
+};
+
+template <typename qf_param_ts>
+struct LocalQFShapeFunction;
+
+template <typename... qf_param_ts>
+struct LocalQFShapeFunction<tuple<qf_param_ts...>>
+{
+   void operator()(
+      typename LocalQFShapeArg<qf_param_decay_t<qf_param_ts>>::type...) const;
+};
+
+template <typename qfunc_t>
+using LocalQFShapeFunctionFor = LocalQFShapeFunction<
+   typename get_function_signature<qfunc_t>::type::parameter_ts>;
+
+} // namespace detail
+
+struct GlobalQFBackend
+{
+   /**
+    * @brief Make an action for a global Q-function.
+    *
+    * @param ctx The integrator context.
+    * @param args The arguments to the action.
+    * @return The action.
+    */
+   template<
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
+   }
+
+   /**
+    * @brief Make a derivative action for a global Q-function.
+    *
+    * @tparam derivative_id The id of the derivative.
+    * @param ctx The integrator context.
+    * @param args The arguments to the derivative action.
+    * @return The derivative action.
+    */
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return GlobalQFImpl::DerivativeAction<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs);
+   }
+
+   /**
+    * @brief Make a derivative setup for a global Q-function.
+    *
+    * @tparam derivative_id The id of the derivative.
+    * @param ctx The integrator context.
+    * @param args The arguments to the derivative setup.
+    * @return The derivative setup.
+    */
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeSetup(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      Vector &qp_cache)
+   {
+      return GlobalQFImpl::DerivativeSetup<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeApply(
+      const IntegratorContext &ctx,
+      const qfunc_t & /*qfunc*/,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeApply<
+             derivative_id,
+             detail::LocalQFShapeFunctionFor<qfunc_t>,
+             inputs_t,
+             outputs_t>(ctx,
+                        detail::LocalQFShapeFunctionFor<qfunc_t> {},
+                        inputs,
+                        outputs,
+                        qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeApplyTranspose(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return GlobalQFImpl::DerivativeApplyTranspose<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAssemble(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssemble<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAssembleDiagonal(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssembleDiagonal<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+};
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/action.hpp

@@ -0,0 +1,514 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "fem/dfem/fieldoperator.hpp"
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+
+namespace mfem::future::LocalQFImpl
+{
+
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+class Action
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   const qfunc_t qfunc;
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx; // input to field
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx; // output to field
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // other constants
+   const int dim, ne, nq, q1d;
+
+public:
+   ////////////////////////////////////////////////////////
+   Action() = delete;
+
+   Action(IntegratorContext ctx,
+          qfunc_t qfunc,
+          inputs_t inputs,
+          outputs_t outputs):
+      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
+      dtqs(make_dtqs(ctx)),
+      // inputs: dtq, idx, B, G, d1d, q1d, vdim
+      input_dtq(create_dtq_maps<Entity::Element>(
+                   inputs,
+                   dtqs,
+                   create_union_field_map_for_dtq(ctx, inputs),
+                   ctx.unionfds,
+                   ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      // outputs: dtq, idx, B, G, d1d, q1d, vdim
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      // other constants
+      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+      q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+   }
+
+   template<typename Backend>
+   void run_kernels(const std::vector<Vector *> &xe,
+                    std::vector<Vector *> &ye) const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   // arguments
+                   ctx,
+                   qfunc,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   // outputs
+                   output_idx,
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   // input and output vectors
+                   xe,
+                   ye,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   void operator()(const std::vector<Vector *> &xe,
+                   std::vector<Vector *> &ye) const
+   {
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<ActionLO>(xe, ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<ActionHO>(xe, ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   ////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   action_callback(const IntegratorContext &ctx,
+                   const qfunc_t &qfunc,
+                   // inputs: idx, B, G, vdim, d1d, q1d
+                   const std::array<size_t, n_inputs> &in_idx,
+                   const std::array<const real_t *, n_inputs> in_B,
+                   const std::array<const real_t *, n_inputs> in_G,
+                   const std::array<int, n_inputs> &in_vdim,
+                   const std::array<int, n_inputs> &in_d1d,
+                   const std::array<int, n_inputs> &in_q1d,
+                   // outputs: idx, B, G, vdim, d1d, q1d
+                   const std::array<size_t, n_outputs> &out_idx,
+                   const std::array<const real_t *, n_outputs> out_B,
+                   const std::array<const real_t *, n_outputs> out_G,
+                   const std::array<int, n_outputs> &out_vdim,
+                   const std::array<int, n_outputs> &out_d1d,
+                   const std::array<int, n_outputs> &out_q1d,
+                   const std::vector<Vector *> &xe,
+                   std::vector<Vector *> &ye,
+                   // fallback arguments
+                   const int dim,
+                   const int q1d)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+      MFEM_CONTRACT_VAR(dim);
+      MFEM_ASSERT(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::Q1D;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
+                        "tensor-product IR expected");
+            in_XE[i] = Reshape(
+                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
+      for_constexpr<n_outputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = out_idx[i];
+         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
+         using FOP = tuple_element_t<i, outputs_t>;
+         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP> || is_functionalvalue_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else
+         {
+            static_assert(false, "Unsupported FieldOperator");
+         }
+      });
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Inputs and outputs argument registers
+         // -----------------------------------------------
+         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+
+         // -----------------------------------------------
+         // Shared memory
+         // -----------------------------------------------
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load inputs
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            const auto &XE = in_XE[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            ;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &rarg = get<i>(rargs);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using XE_t = decltype(XE);
+               using rarg_t = decltype(rarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               backend_t::template LoadGradient<RNK, rarg_t, XE_t, qf_param_t>(
+                  smem, e, d, q, q1d, B, G, XE, rarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+               // qp values are read directly from in_XE / IR
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+
+         // -----------------------------------------------
+         // Evaluate the quadrature function
+         // Warning: no 'DIRECT' on the 'Z' direction,
+         // as one backend may need to iterate over it.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  args_tuple_t qargs;
+
+                  // --------------------------------------
+                  // Pulling arguments from registers to qargs tuple
+                  // --------------------------------------
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &qarg = get<i>(qargs);
+                     const auto &XE = in_XE[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           qarg = backend_t::template identity_qp_pull_dual<DT>(
+                              false, XE, XE, qx, qy, qz, e);
+                        }
+                        else
+                        {
+                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        qarg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        qarg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  // --------------------------------------
+                  // Call the quadrature function
+                  // --------------------------------------
+                  call_qfunc_no_move(qfunc, qargs);
+
+                  // --------------------------------------
+                  // Pushing arguments from qargs tuple to registers
+                  // --------------------------------------
+                  for_constexpr<n_outputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value, o = n_inputs + i;
+                     const auto &qarg = get<o>(qargs);
+                     const auto &YE = out_YE[i];
+                     using FOP = tuple_element_t<i, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP> ||
+                                   is_functionalvalue_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           backend_t::identity_qp_write_value(
+                              YE, qx, qy, qz, e, qarg);
+                        }
+                        else
+                        {
+                           as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
+                        }
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        auto &rarg = get<o>(rargs);
+                        backend_t::template qp_push<ARG>(
+                           rarg, qx, qy, qz, qarg);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate outputs
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value, o = n_inputs + i;
+            const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
+            const auto B = out_B[i], G = out_G[i];
+            const auto &YE = out_YE[i];
+            auto &rarg = get<o>(rargs);
+            using FOP = tuple_element_t<i, outputs_t>;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(rarg);
+               // Both the rank and the extents have to come from the output
+               // parameter slot o, not from the output index i.
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, Q1D, B, G, YE, rarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP> ||
+                               is_functionalvalue_fop_v<FOP>)
+            {
+               // nothing to do
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+   using KernelType = decltype(&Action::action_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionLO, KernelType, (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionHO, KernelType, (int, int) );
+};
+
+// Low Order kernels
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+template<int DIM, int Q1D>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Kernel()
+{
+   static_assert(Q1D <= LocalQFLOBackend<DIM>::MQ1);
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D>>;
+}
+
+// Low Order fallback
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Fallback(int dim, int q1d)
+{
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   using ActionLO = typename action_t::ActionLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<ActionLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<ActionLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+// High Order kernels
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+template<int DIM, int Q1D>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Kernel()
+{
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   return action_t::template action_callback<LocalQFHOBackend<DIM>, Q1D>;
+}
+
+// High Order fallback
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Fallback(int dim, int q1d)
+{
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   using ActionHO = typename action_t::ActionHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<ActionHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<ActionHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_action.hpp

@@ -0,0 +1,763 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+namespace mfem::future::LocalQFImpl
+{
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeAction
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   qfunc_t qfunc;
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx;
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // other constants
+   const int dim, ne, nq, q1d;
+
+   std::array<bool, n_inputs> input_is_dependent;
+   FieldDescriptor direction_fd;
+   mutable Vector direction_e;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+
+public:
+   //////////////////////////////////////////////////////////////////
+   DerivativeAction() = delete;
+
+   DerivativeAction(IntegratorContext ctx,
+                    qfunc_t qfunc,
+                    inputs_t inputs,
+                    outputs_t outputs):
+      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
+      dtqs(make_dtqs(ctx)),
+      // inputs: dtq, idx, B, G, d1d, q1d, vdim
+      input_dtq(create_dtq_maps<Entity::Element>(
+                   inputs,
+                   dtqs,
+                   create_union_field_map_for_dtq(ctx, inputs),
+                   ctx.unionfds,
+                   ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      // outputs: dtq, idx, B, G, d1d, q1d, vdim
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      // other constants
+      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+      q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+
+      // Determine which inputs are dependent on the derivative direction
+      auto dependency_map = make_dependency_map(inputs);
+      auto it = dependency_map.find(derivative_id);
+      MFEM_ASSERT(it != dependency_map.end(),
+                  "Derivative ID not found in dependency map");
+      input_is_dependent = it->second;
+
+      // Find direction field index
+      int direction_field_idx = -1;
+      for (size_t uf = 0; uf < nfields; uf++)
+      {
+         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
+         {
+            direction_field_idx = static_cast<int>(uf);
+            break;
+         }
+      }
+      MFEM_ASSERT(
+         direction_field_idx != -1,
+         "LocalQFBackend: derivative direction field not found in unionfds");
+      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(const std::vector<Vector *> &xe,
+                    std::vector<Vector *> &ye)
+   {
+      Backend::Run(dim,
+                   q1d,
+                   // arguments
+                   ctx,
+                   qfunc,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   // outputs
+                   output_idx,
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   // input and output vectors
+                   xe,
+                   ye,
+                   input_is_dependent,
+                   direction_e,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> &xe,
+                   const Vector *direction_l,
+                   std::vector<Vector *> &ye)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "LocalQF DerivativeAction: direction vector is null");
+
+      restriction(direction_fd,
+                  direction_rcache,
+                  *direction_l,
+                  direction_e,
+                  ElementDofOrdering::LEXICOGRAPHIC);
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeActionLO>(xe, ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeActionHO>(xe, ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   derivative_action_callback(const IntegratorContext &ctx,
+                              qfunc_t &qfunc,
+                              // inputs: idx, B, G, vdim, d1d, q1d
+                              const std::array<size_t, n_inputs> &in_idx,
+                              const std::array<const real_t *, n_inputs> in_B,
+                              const std::array<const real_t *, n_inputs> in_G,
+                              const std::array<int, n_inputs> &in_vdim,
+                              const std::array<int, n_inputs> &in_d1d,
+                              const std::array<int, n_inputs> &in_q1d,
+                              // outputs: idx, B, G, vdim, d1d, q1d
+                              const std::array<size_t, n_outputs> &out_idx,
+                              const std::array<const real_t *, n_outputs> out_B,
+                              const std::array<const real_t *, n_outputs> out_G,
+                              const std::array<int, n_outputs> &out_vdim,
+                              const std::array<int, n_outputs> &out_d1d,
+                              const std::array<int, n_outputs> &out_q1d,
+                              const std::vector<Vector *> &xe,
+                              std::vector<Vector *> &ye,
+                              const std::array<bool, n_inputs> &input_dep,
+                              const Vector &direction_e,
+                              // fallback arguments
+                              const int dim,
+                              const int q1d)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
+                        "tensor-product IR expected");
+            in_XE[i] = Reshape(
+                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      const auto d_direction = direction_e.Read();
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            if (input_dep[i])
+            {
+               MFEM_ASSERT(direction_e.Size() == xe[k]->Size(),
+                           "direction E-vector size mismatch for input " << i);
+               in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
+            }
+            else
+            {
+               in_XE_dir[i] = in_XE[i];
+            }
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            if (input_dep[i])
+            {
+               MFEM_VERIFY(direction_e.Size() == xe[k]->Size(),
+                           "direction E-vector size mismatch (identity input) "
+                           << i);
+               in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
+            }
+            else
+            {
+               in_XE_dir[i] = in_XE[i];
+            }
+         }
+         else if constexpr (is_weight_fop_v<FOP>) { in_XE_dir[i] = in_XE[i]; }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
+      for_constexpr<n_outputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = out_idx[i];
+         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
+         using FOP = tuple_element_t<i, outputs_t>;
+         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else
+         {
+            static_assert(false, "Unsupported FieldOperator");
+         }
+      });
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Inputs and outputs argument registers
+         // -----------------------------------------------
+         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1>
+         sargs; // shadow
+
+         // -----------------------------------------------
+         // Shared memory
+         // -----------------------------------------------
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load primal inputs (rargs)
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            const auto &XE = in_XE[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &rarg = get<i>(rargs);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(rarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, rarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+               // qp values are read directly from in_XE / IR
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+
+         // -----------------------------------------------
+         // Load tangent directions (sargs)
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            if (!input_dep[i]) { return; }
+            const auto &XE = in_XE_dir[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &sarg = get<i>(sargs); // shadow argument register
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(sarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+
+         // -----------------------------------------------
+         // Evaluate the quadrature function
+         // Warning: no 'DIRECT' on the 'Z' direction,
+         // as one backend may need to iterate over it.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+#ifdef MFEM_USE_ENZYME
+                  args_tuple_t primal_args {}, shadow_args {};
+
+                  // --------------------------------------
+                  // Pulling arguments from registers to primal and shadow
+                  // tuples
+                  // --------------------------------------
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &parg = get<i>(primal_args);
+                     auto &targ = get<i>(shadow_args);
+                     const auto &XE = in_XE[i];
+                     const auto &XEd = in_XE_dir[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        if (input_dep[i])
+                        {
+                           targ = as_tensor<ARG>(&XEd(0, qx, qy, qz, e));
+                        }
+                        else
+                        {
+                           targ = ARG{};
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        parg = XE(qx, qy, qz, 0, 0);
+                        targ = real_t(0.0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        parg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                        if (input_dep[i])
+                        {
+                           targ = backend_t::template qp_pull<ARG>(
+                              get<i>(sargs), qx, qy, qz);
+                        }
+                        else
+                        {
+                           targ = ARG{};
+                        }
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  // --------------------------------------
+                  // Call the quadrature function
+                  // --------------------------------------
+                  call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
+
+                  // --------------------------------------
+                  // Pushing arguments from enzyme_shadow tuple to registers
+                  // --------------------------------------
+                  for_constexpr<n_outputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value, o = n_inputs + i;
+                     const auto &qout = get<o>(shadow_args);
+                     auto &YE = out_YE[i];
+                     using FOP = tuple_element_t<i, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qout;
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        auto &rarg = get<o>(rargs);
+                        backend_t::template qp_push_tangent<ARG>(
+                           rarg, qx, qy, qz, qout);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+#else  // MFEM_USE_ENZYME
+                  args_tuple_t qargs;
+
+                  // --------------------------------------
+                  // Pulling arguments from registers to qargs tuple
+                  // --------------------------------------
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &qarg = get<i>(qargs);
+                     const auto &XE = in_XE[i];
+                     const auto &XEd = in_XE_dir[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           qarg = backend_t::template identity_qp_pull_dual<DT>(
+                              input_dep[i], XE, XEd, qx, qy, qz, e);
+                        }
+                        else
+                        {
+                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        qarg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        qarg = backend_t::template qp_pull_directional<ARG>(
+                           get<i>(rargs),
+                           get<i>(sargs),
+                           qx,
+                           qy,
+                           qz,
+                           input_dep[i]);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  // --------------------------------------
+                  // Call the quadrature function
+                  // --------------------------------------
+                  call_qfunc_no_move(qfunc, qargs);
+
+                  // --------------------------------------
+                  // Pushing arguments from qargs tuple to registers
+                  // --------------------------------------
+                  for_constexpr<n_outputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value, o = n_inputs + i;
+                     const auto &qarg = get<o>(qargs);
+                     const auto &YE = out_YE[i];
+                     using FOP = tuple_element_t<i, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           backend_t::identity_qp_write_tangent(
+                              YE, qx, qy, qz, e, qarg);
+                        }
+                        else
+                        {
+                           as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
+                        }
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        auto &rarg = get<o>(rargs);
+                        backend_t::template qp_push_tangent<ARG>(
+                           rarg, qx, qy, qz, qarg);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+#endif // MFEM_USE_ENZYME
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate outputs
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value, o = n_inputs + i;
+            const int d = out_d1d[i], q = out_q1d[i];
+            const auto B = out_B[i], G = out_G[i];
+            auto &YE = out_YE[i];
+            auto &rarg = get<o>(rargs);
+            using FOP = tuple_element_t<i, outputs_t>;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(rarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, q1d, B, G, YE, rarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP>)
+            {
+               // nothing to do
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+
+   using DerivativeKernelType =
+      decltype(&DerivativeAction::derivative_action_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionLO,
+                                     DerivativeKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionHO,
+                                     DerivativeKernelType,
+                                     (int, int) );
+};
+
+// Low Order kernels
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return derivative_action_t::template derivative_action_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+// Low Order fallback
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionLO::Fallback(int dim, int q1d)
+{
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeActionLO =
+      typename derivative_action_t::DerivativeActionLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeActionLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeActionLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+// High Order kernels
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionHO::Kernel()
+{
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return derivative_action_t::
+          template derivative_action_callback<LocalQFHOBackend<DIM>, Q1D>;
+}
+
+// High Order fallback
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionHO::Fallback(int dim, int q1d)
+{
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeActionHO = typename derivative_action_t::DerivativeActionHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeActionHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeActionHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply.hpp

@@ -0,0 +1,738 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+// Cached Jacobian apply: J·v from qp_cache filled by DerivativeSetup
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeApply
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const Vector &qp_cache;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx;
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // Jacobian cache metadata
+   const std::array<bool, n_inputs> input_is_dependent;
+   const std::array<int, n_inputs> input_size_on_qp;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int residual_size_on_qp;
+   // other constants
+   const int dim, ne, nq, q1d;
+   FieldDescriptor direction_fd;
+   mutable Vector direction_e;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+
+   template <std::size_t slot>
+   static constexpr int ParamRank()
+   {
+      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
+      return qf_param_shape<param_t>::rank;
+   }
+
+   template <std::size_t slot, int dim_idx>
+   static constexpr int ParamExtent()
+   {
+      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
+      return qf_param_shape<param_t>::extents[dim_idx];
+   }
+
+   template <typename fop_t, std::size_t slot>
+   static constexpr int StaticVDim()
+   {
+      constexpr int rank = ParamRank<slot>();
+      if constexpr (is_gradient_fop_v<fop_t>)
+      {
+         if constexpr (rank <= 1) { return 1; }
+         else { return ParamExtent<slot, 0>(); }
+      }
+      else
+      {
+         if constexpr (rank == 0) { return 1; }
+         else { return ParamExtent<slot, 0>(); }
+      }
+   }
+
+   template <typename fop_t, std::size_t slot>
+   static constexpr int StaticOpDim()
+   {
+      constexpr int rank = ParamRank<slot>();
+      if constexpr (is_gradient_fop_v<fop_t>)
+      {
+         if constexpr (rank == 0) { return 1; }
+         else if constexpr (rank == 1) { return ParamExtent<slot, 0>(); }
+         else { return ParamExtent<slot, 1>(); }
+      }
+      else
+      {
+         if constexpr (rank <= 1) { return 1; }
+         else { return ParamExtent<slot, 1>(); }
+      }
+   }
+
+   template <std::size_t input_slot>
+   static constexpr bool StaticInputDep()
+   {
+      using fop_t = tuple_element_t<input_slot, inputs_t>;
+      return fop_t::GetFieldId() == derivative_id;
+   }
+
+   template <std::size_t input_slot>
+   static constexpr int StaticInputVDim()
+   {
+      using fop_t = tuple_element_t<input_slot, inputs_t>;
+      return StaticVDim<fop_t, input_slot>();
+   }
+
+   template <std::size_t input_slot>
+   static constexpr int StaticInputOpDim()
+   {
+      using fop_t = tuple_element_t<input_slot, inputs_t>;
+      return StaticOpDim<fop_t, input_slot>();
+   }
+
+   template <std::size_t output_slot>
+   static constexpr int StaticOutputVDim()
+   {
+      using fop_t = tuple_element_t<output_slot, outputs_t>;
+      return StaticVDim<fop_t, n_inputs + output_slot>();
+   }
+
+   template <std::size_t output_slot>
+   static constexpr int StaticOutputOpDim()
+   {
+      using fop_t = tuple_element_t<output_slot, outputs_t>;
+      return StaticOpDim<fop_t, n_inputs + output_slot>();
+   }
+
+   template <std::size_t output_slot>
+   static constexpr int StaticOutputOffset()
+   {
+      int offset = 0;
+      for_constexpr<output_slot>([&](auto oc)
+      {
+         constexpr size_t o = oc.value;
+         offset += StaticOutputVDim<o>() * StaticOutputOpDim<o>();
+      });
+      return offset;
+   }
+
+   template <std::size_t input_slot>
+   static constexpr int StaticInputOpOffset()
+   {
+      int offset = 0;
+      for_constexpr<input_slot>([&](auto sc)
+      {
+         constexpr size_t s = sc.value;
+         if constexpr (StaticInputDep<s>()) { offset += StaticInputOpDim<s>(); }
+      });
+      return offset;
+   }
+
+   static constexpr int StaticTrialVDim()
+   {
+      int vdim = 1;
+      for_constexpr<n_inputs>([&](auto sc)
+      {
+         constexpr size_t s = sc.value;
+         if constexpr (StaticInputDep<s>()) { vdim = StaticInputVDim<s>(); }
+      });
+      return vdim;
+   }
+
+   static constexpr int StaticTotalTrialOpDim()
+   {
+      int op_dim = 0;
+      for_constexpr<n_inputs>([&](auto sc)
+      {
+         constexpr size_t s = sc.value;
+         if constexpr (StaticInputDep<s>()) { op_dim += StaticInputOpDim<s>(); }
+      });
+      return op_dim;
+   }
+
+public:
+   DerivativeApply() = delete;
+
+   DerivativeApply(IntegratorContext ctx,
+                   qfunc_t /*qfunc*/,
+                   inputs_t inputs,
+                   outputs_t outputs,
+                   const Vector &qp_cache_in):
+      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache_in),
+      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
+                                         inputs,
+                                         dtqs,
+                                         create_union_field_map_for_dtq(ctx, inputs),
+                                         ctx.unionfds,
+                                         ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+      input_size_on_qp(
+         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
+                           out_op_dim(compute_out_op_dim(outputs)),
+                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
+                           output_size_on_qp(
+                              [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs).size_on_qp; });
+      return s;
+   }()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
+   total_trial_op_dim(compute_total_trial_op_dim(
+                         inputs, input_is_dependent, input_size_on_qp)),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+   q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+
+      int direction_field_idx = -1;
+      for (size_t uf = 0; uf < nfields; uf++)
+      {
+         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
+         {
+            direction_field_idx = static_cast<int>(uf);
+            break;
+         }
+      }
+      MFEM_ASSERT(direction_field_idx != -1,
+                  "DerivativeApply: derivative direction field not found");
+
+      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(std::vector<Vector *> &ye) const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qp_cache,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   input_size_on_qp,
+                   input_is_dependent,
+                   // outputs
+                   output_idx,
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   out_op_dim,
+                   out_offsets,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   output_size_on_qp,
+                   direction_e,
+                   ye,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> &,
+                   const Vector *direction_l,
+                   std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "LocalQF DerivativeApply: direction vector is null");
+
+      restriction(direction_fd,
+                  direction_rcache,
+                  *direction_l,
+                  direction_e,
+                  ElementDofOrdering::LEXICOGRAPHIC);
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyLO>(ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyHO>(ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   derivative_apply_callback(const IntegratorContext &ctx,
+                             const Vector &qp_cache,
+                             // inputs: idx, B, G, vdim, d1d, q1d
+                             const std::array<size_t, n_inputs> & /*in_idx*/,
+                             const std::array<const real_t *, n_inputs> in_B,
+                             const std::array<const real_t *, n_inputs> in_G,
+                             const std::array<int, n_inputs> &in_vdim,
+                             const std::array<int, n_inputs> &in_d1d,
+                             const std::array<int, n_inputs> &in_q1d,
+                             const std::array<int, n_inputs> &in_size_on_qp,
+                             const std::array<bool, n_inputs> &input_dep,
+                             // outputs: idx, B, G, vdim, d1d, q1d
+                             const std::array<size_t, n_outputs> &out_idx,
+                             const std::array<const real_t *, n_outputs> out_B,
+                             const std::array<const real_t *, n_outputs> out_G,
+                             const std::array<int, n_outputs> &out_vdim,
+                             const std::array<int, n_outputs> &out_d1d,
+                             const std::array<int, n_outputs> &out_q1d,
+                             const std::array<int, n_outputs> &out_op_dim,
+                             const std::array<int, n_outputs> &out_offsets,
+                             const int trial_vdim,
+                             const int total_trial_op_dim,
+                             const int residual_size_on_qp,
+                             const int output_size_on_qp,
+                             const Vector &direction_e,
+                             std::vector<Vector *> &ye,
+                             // fallback arguments
+                             const int dim,
+                             const int q1d)
+   {
+      MFEM_CONTRACT_VAR(input_dep);
+      MFEM_CONTRACT_VAR(in_size_on_qp);
+      MFEM_CONTRACT_VAR(out_vdim);
+      MFEM_CONTRACT_VAR(out_op_dim);
+      MFEM_CONTRACT_VAR(out_offsets);
+      MFEM_CONTRACT_VAR(trial_vdim);
+      MFEM_CONTRACT_VAR(total_trial_op_dim);
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+      const int nq = ctx.ir.GetNPoints();
+      MFEM_CONTRACT_VAR(output_size_on_qp);
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // DIRECTION (trial): XE_dir for the dependent inputs
+      // --------------------------------------------------
+      const auto d_direction = direction_e.Read();
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if (!input_dep[i]) { return; }
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>) { /* never a direction */ }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
+      for_constexpr<n_outputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = out_idx[i];
+         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
+         using FOP = tuple_element_t<i, outputs_t>;
+         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else
+         {
+            static_assert(false, "Unsupported FieldOperator");
+         }
+      });
+
+      auto cache_tensor = DeviceTensor<3, const real_t>(
+                             qp_cache.Read(), nq, residual_size_on_qp, ne);
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Output integration registers, trial direction (shadow) registers
+         // and shared memory. `rargs` only ever holds test-function data, so
+         // it is an output-only bank: slot `o` is q-function parameter
+         // `n_inputs + o`.
+         // -----------------------------------------------
+         output_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> sargs;
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load trial direction (sargs) for the dependent inputs
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            if constexpr (!StaticInputDep<i>()) { return; }
+            const auto &XE = in_XE_dir[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &sarg = get<i>(sargs);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(sarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP> || is_weight_fop_v<FOP>)
+            {
+               // identity read at qp; weight is never a trial direction
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Contract the cached Jacobian with the trial direction at each
+         // quadrature point and push the result into the test registers.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+
+                  // The trial direction at this quadrature point is the same
+                  // for every test row (i, k), so pull each dependent input
+                  // slot out of the register bank once, here, instead of once
+                  // per row inside the contraction below.
+                  args_tuple_t dvecs {};
+                  for_constexpr<n_inputs>([&](auto sc)
+                  {
+                     constexpr size_t s = sc.value;
+                     if constexpr (StaticInputDep<s>())
+                     {
+                        using SARG =
+                           typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
+                        get<s>(dvecs) = backend_t::template qp_pull<SARG>(
+                                           get<s>(sargs), qx, qy, qz);
+                     }
+                  });
+
+                  for_constexpr<n_outputs>([&](auto oc)
+                  {
+                     constexpr size_t o = oc.value, ao = n_inputs + o;
+                     using FOP = tuple_element_t<o, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
+                     constexpr int tv = StaticOutputVDim<o>();
+                     constexpr int to = StaticOutputOpDim<o>();
+                     constexpr int offset_o = StaticOutputOffset<o>();
+                     constexpr int trial_vdim_ct = StaticTrialVDim();
+                     constexpr int total_trial_op_dim_ct = StaticTotalTrialOpDim();
+
+                     ARG fhat{};
+                     MFEM_UNROLL(tv)
+                     for (int i = 0; i < tv; i++)
+                     {
+                        MFEM_UNROLL(to)
+                        for (int k = 0; k < to; k++)
+                        {
+                           const int row = offset_o + i * to + k;
+                           const int cache_row =
+                              row * trial_vdim_ct * total_trial_op_dim_ct;
+                           real_t sum = 0.0;
+                           for_constexpr<n_inputs>([&](auto sc)
+                           {
+                              constexpr size_t s = sc.value;
+                              if constexpr (StaticInputDep<s>())
+                              {
+                                 constexpr int op_dim_s = StaticInputOpDim<s>();
+                                 constexpr int m_offset = StaticInputOpOffset<s>();
+                                 const auto &dvec = get<s>(dvecs);
+                                 MFEM_UNROLL(trial_vdim_ct)
+                                 for (int j = 0; j < trial_vdim_ct; j++)
+                                 {
+                                    MFEM_UNROLL(op_dim_s)
+                                    for (int m = 0; m < op_dim_s; m++)
+                                    {
+                                       const int cache_idx =
+                                          cache_row + j * total_trial_op_dim_ct +
+                                          (m + m_offset);
+                                       sum += cache_tensor(q, cache_idx, e) *
+                                              qf_value_at(dvec, j, m);
+                                    }
+                                 }
+                              }
+                           });
+                           qf_set_value_at(fhat, i, k, sum);
+                        }
+                     }
+
+                     auto &YE = out_YE[o];
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        MFEM_UNROLL(tv)
+                        for (int i = 0; i < tv; i++)
+                        {
+                           MFEM_UNROLL(to)
+                           for (int k = 0; k < to; k++)
+                           {
+                              YE(i + tv * k, qx, qy, qz, e) =
+                                 qf_value_at(fhat, i, k);
+                           }
+                        }
+                     }
+                     else
+                     {
+                        backend_t::template qp_push<ARG>(
+                           get<o>(rargs), qx, qy, qz, fhat);
+                     }
+                  });
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate value / gradient outputs to the test dofs
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value, o = n_inputs + i;
+            const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
+            const auto B = out_B[i], G = out_G[i];
+            auto &YE = out_YE[i];
+            auto &rarg = get<i>(rargs);
+            using FOP = tuple_element_t<i, outputs_t>;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(rarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, Q1D, B, G, YE, rarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP>) { /* written at qp */ }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+
+   using ApplyKernelType =
+      decltype(&DerivativeApply::derivative_apply_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyLO,
+                                     ApplyKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyHO,
+                                     ApplyKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return apply_t::template derivative_apply_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyLO::Fallback(int dim, int q1d)
+{
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyLO = typename apply_t::DerivativeApplyLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyHO::Kernel()
+{
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return apply_t::template derivative_apply_callback<LocalQFHOBackend<DIM>,
+                                                      Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyHO::Fallback(int dim, int q1d)
+{
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyHO = typename apply_t::DerivativeApplyHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply_transpose.hpp

@@ -0,0 +1,637 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+// Cached transposed Jacobian apply: Jᵀ·w from the qp_cache
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeApplyTranspose
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   // Input tuple slot referencing the derivative field (compile-time)
+   static constexpr size_t deriv_input_idx_ct = []() constexpr
+   {
+      size_t idx = SIZE_MAX;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         using FOP = tuple_element_t<i, inputs_t>;
+         if (FOP::GetFieldId() == derivative_id) { idx = i; }
+      });
+      return idx;
+   }();
+   static_assert(deriv_input_idx_ct < n_inputs,
+                 "DerivativeApplyTranspose: derivative input slot not found");
+
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const Vector &qp_cache; // Jacobian cache from DerivativeSetup
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, B, G, d1d, q1d, vdim (trial / derivative fields)
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim (test / cotangent fields)
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx;
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // Jacobian cache metadata
+   const std::array<bool, n_inputs> input_is_dependent;
+   const std::array<int, n_inputs> input_size_on_qp;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int residual_size_on_qp;
+   // other constants
+   const int dim, ne, nq, q1d;
+   const size_t deriv_infd_idx; // index of the derivative field in ye
+   // output cotangent restriction workspace (blocked by element)
+   std::array<int, n_outputs> out_elem_dof_size;
+   mutable Vector dir_out_e;
+   /// One restriction cache per output field, resolved on first use.
+   mutable std::array<RestrictionCache<Entity::Element>, n_outputs>
+   out_rcaches;
+
+public:
+   //////////////////////////////////////////////////////////////////
+   DerivativeApplyTranspose() = delete;
+
+   DerivativeApplyTranspose(IntegratorContext ctx,
+                            qfunc_t /*qfunc*/,
+                            inputs_t inputs,
+                            outputs_t outputs,
+                            const Vector &qp_cache):
+      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache),
+      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
+                                         inputs,
+                                         dtqs,
+                                         create_union_field_map_for_dtq(ctx, inputs),
+                                         ctx.unionfds,
+                                         ctx.ir)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+      input_size_on_qp(
+         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
+                           out_op_dim(compute_out_op_dim(outputs)),
+                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
+                           output_size_on_qp(
+                              [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs).size_on_qp; });
+      return s;
+   }()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
+   total_trial_op_dim(compute_total_trial_op_dim(
+                         inputs, input_is_dependent, input_size_on_qp)),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+   q1d(tensor_1d_size(nq, dim)),
+   deriv_infd_idx(find_infd_index(ctx, derivative_id)), out_elem_dof_size{}
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+      MFEM_ASSERT(
+         deriv_infd_idx != SIZE_MAX,
+         "DerivativeApplyTranspose: derivative field not found in infds");
+
+      // Size the workspace that holds the output cotangent(s) in element
+      // layout.
+      int total_dir_e_size = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      {
+         const int elem_sz = compute_element_dof_sz(
+            ctx.outfds[output_idx[o]], ne, ElementDofOrdering::LEXICOGRAPHIC);
+         out_elem_dof_size[o] = elem_sz;
+         total_dir_e_size += elem_sz;
+      });
+      dir_out_e.SetSize(total_dir_e_size * ne);
+      dir_out_e.UseDevice(true);
+      dir_out_e.Read();
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(std::vector<Vector *> &ye) const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qp_cache,
+                   dir_out_e,
+                   // inputs (integration target metadata)
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   input_size_on_qp,
+                   input_is_dependent,
+                   // outputs (direction interpolation metadata)
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   out_op_dim,
+                   out_offsets,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   output_size_on_qp,
+                   deriv_infd_idx,
+                   ye,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> & /*xe*/,
+                   const Vector *direction_l,
+                   std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "LocalQF DerivativeApplyTranspose: direction vector is null");
+
+      // Restrict output cotangent from L-vectors into element layout
+      // (dir_out_e).
+      int l_offset = 0;
+      int e_offset = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      {
+         const size_t outfd = output_idx[o];
+         const auto &fd = ctx.outfds[outfd];
+         const int l_size = GetVSize(fd);
+         Vector dir_o_l(*const_cast<Vector *>(direction_l), l_offset, l_size);
+         dir_o_l.UseDevice(true);
+         const int elem_sz = out_elem_dof_size[o];
+         Vector dir_o_e(dir_out_e, e_offset, elem_sz * ne);
+         dir_o_e.UseDevice(true);
+         restriction(fd, out_rcaches[o], dir_o_l, dir_o_e,
+                     ElementDofOrdering::LEXICOGRAPHIC);
+         l_offset += l_size;
+         e_offset += elem_sz * ne;
+      });
+
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyTransposeLO>(ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyTransposeHO>(ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void derivative_apply_transpose_callback(
+      const IntegratorContext &ctx,
+      const Vector &qp_cache,
+      const Vector &dir_e, // restricted, concatenated output cotangents
+      // inputs (integration target metadata)
+      const std::array<const real_t *, n_inputs> in_B,
+      const std::array<const real_t *, n_inputs> in_G,
+      const std::array<int, n_inputs> &in_vdim,
+      const std::array<int, n_inputs> &in_d1d,
+      const std::array<int, n_inputs> &in_q1d,
+      const std::array<int, n_inputs> &in_size_on_qp,
+      const std::array<bool, n_inputs> &input_dep,
+      // outputs (direction interpolation metadata)
+      const std::array<const real_t *, n_outputs> out_B,
+      const std::array<const real_t *, n_outputs> out_G,
+      const std::array<int, n_outputs> &out_vdim,
+      const std::array<int, n_outputs> &out_d1d,
+      const std::array<int, n_outputs> &out_q1d,
+      const std::array<int, n_outputs> &out_op_dim,
+      const std::array<int, n_outputs> &out_offsets,
+      const int trial_vdim,
+      const int total_trial_op_dim,
+      const int residual_size_on_qp,
+      const int output_size_on_qp,
+      const size_t deriv_infd_idx,
+      std::vector<Vector *> &ye,
+      // fallback arguments
+      const int dim,
+      const int q1d)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+      const int nq = ctx.ir.GetNPoints();
+      MFEM_CONTRACT_VAR(output_size_on_qp);
+      MFEM_CONTRACT_VAR(in_q1d);
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // DIRECTION (test cotangent): out_XE_dir, concatenated per output
+      // --------------------------------------------------
+      const auto d_dir = dir_e.Read();
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_outputs> out_XE_dir;
+      int e_offset = 0;
+      for_constexpr<n_outputs>([&](auto oc)
+      {
+         constexpr size_t o = oc.value;
+         const int d = out_d1d[o], q = out_q1d[o], v = out_vdim[o];
+         using FOP = tuple_element_t<o, outputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            out_XE_dir[o] = Reshape(d_dir + e_offset, d, d, B2D ? 1 : d, v, ne);
+            e_offset += k_dim(d) * v * ne;
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            out_XE_dir[o] = Reshape(d_dir + e_offset, v, q, q, B2D ? 1 : q, ne);
+            e_offset += k_dim(q) * v * ne;
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // DERIVATIVE TRIAL FIELD: ye_XE (accumulates Jᵀ w)
+      // --------------------------------------------------
+      const int d_in = in_d1d[deriv_input_idx_ct];
+      const int v_in = in_vdim[deriv_input_idx_ct];
+      auto ye_XE = Reshape(
+                      ye[deriv_infd_idx]->ReadWrite(), d_in, d_in, B2D ? 1 : d_in, v_in, ne);
+
+      auto cache_tensor = DeviceTensor<3, const real_t>(
+                             qp_cache.Read(), nq, residual_size_on_qp, ne);
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Output cotangent (direction) registers live in the output slots;
+         // the trial integration data is pushed into the input slots.
+         // -----------------------------------------------
+         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Interpolate the test cotangent to quadrature points (output slots)
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto oc)
+         {
+            constexpr size_t o = oc.value, ao = n_inputs + o;
+            using FOP = tuple_element_t<o, outputs_t>;
+            const auto &XE = out_XE_dir[o];
+            const int d = out_d1d[o], q = out_q1d[o], Q1D = q1d;
+            const real_t *B = out_B[o], *G = out_G[o];
+            auto &oarg = get<ao>(rargs);
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, oarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, ao>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, ao>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(oarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, oarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP>)
+            {
+               // identity cotangent is read directly at qp from out_XE_dir
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Contract the transposed cached Jacobian with the test cotangent at
+         // each quadrature point and push the trial result into the dependent
+         // input registers.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+
+                  // The test cotangent at this quadrature point is the same
+                  // for every trial column (j, m), so pull each interpolated
+                  // output slot out of the register bank once, here, instead
+                  // of once per column inside the contraction below. Identity
+                  // outputs have no register bank and are read from
+                  // out_XE_dir at the point of use.
+                  args_tuple_t wvecs {};
+                  for_constexpr<n_outputs>([&](auto oc)
+                  {
+                     constexpr size_t o = oc.value, ao = n_inputs + o;
+                     using OFOP = tuple_element_t<o, outputs_t>;
+                     if constexpr (is_value_fop_v<OFOP> ||
+                                   is_gradient_fop_v<OFOP>)
+                     {
+                        using OARG =
+                           typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
+                        get<ao>(wvecs) = backend_t::template qp_pull<OARG>(
+                                            get<ao>(rargs), qx, qy, qz);
+                     }
+                  });
+
+                  int m_offset = 0;
+                  for_constexpr<n_inputs>([&](auto sc)
+                  {
+                     constexpr size_t s = sc.value;
+                     if (!input_dep[s]) { return; }
+                     using SARG =
+                        typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
+                     const int vdim_s = in_vdim[s];
+                     const int op_dim_s = in_size_on_qp[s] / vdim_s;
+
+                     SARG fhat{};
+                     for (int j = 0; j < trial_vdim; j++)
+                     {
+                        for (int m = 0; m < op_dim_s; m++)
+                        {
+                           const int col =
+                              j * total_trial_op_dim + (m + m_offset);
+                           real_t sum = 0.0;
+                           for_constexpr<n_outputs>([&](auto oc)
+                           {
+                              constexpr size_t o = oc.value, ao = n_inputs + o;
+                              using OFOP = tuple_element_t<o, outputs_t>;
+                              const int tv = out_vdim[o], to = out_op_dim[o];
+                              const auto offset_o = out_offsets[o];
+                              const auto &cache = cache_tensor;
+                              if constexpr (is_value_fop_v<OFOP> ||
+                                            is_gradient_fop_v<OFOP>)
+                              {
+                                 const auto &wvec = get<ao>(wvecs);
+                                 for (int i = 0; i < tv; i++)
+                                 {
+                                    for (int k = 0; k < to; k++)
+                                    {
+                                       const int row = offset_o + i * to + k;
+                                       const int cache_idx =
+                                          row * trial_vdim *
+                                          total_trial_op_dim +
+                                          col;
+                                       sum += cache(q, cache_idx, e) *
+                                              qf_value_at(wvec, i, k);
+                                    }
+                                 }
+                              }
+                              else if constexpr (is_identity_fop_v<OFOP>)
+                              {
+                                 const auto &XEo = out_XE_dir[o];
+                                 for (int i = 0; i < tv; i++)
+                                 {
+                                    for (int k = 0; k < to; k++)
+                                    {
+                                       const int row = offset_o + i * to + k;
+                                       const int cache_idx =
+                                          row * trial_vdim *
+                                          total_trial_op_dim +
+                                          col;
+                                       sum += cache(q, cache_idx, e) *
+                                              XEo(i + tv * k, qx, qy, qz, e);
+                                    }
+                                 }
+                              }
+                           });
+                           qf_set_value_at(fhat, j, m, sum);
+                        }
+                     }
+                     backend_t::template qp_push<SARG>(
+                        get<s>(rargs), qx, qy, qz, fhat);
+                     m_offset += op_dim_s;
+                  });
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate the trial result into the derivative field dofs. Multiple
+         // dependent input slots (e.g. value and gradient of the same field)
+         // accumulate into ye_XE via the writers' '+=' semantics.
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto sc)
+         {
+            constexpr size_t s = sc.value;
+            if (!input_dep[s]) { return; }
+            using FOP = tuple_element_t<s, inputs_t>;
+            const int d = in_d1d[s], q = in_q1d[s], Q1D = q1d;
+            const real_t *B = in_B[s], *G = in_G[s];
+            auto &sarg = get<s>(rargs);
+            auto &YE = ye_XE;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, sarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(sarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, s>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, s>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, Q1D, B, G, YE, sarg);
+            }
+            else
+            {
+               // identity / weight derivative targets are not produced here
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+
+   using TransposeKernelType =
+      decltype(&DerivativeApplyTranspose::
+               derivative_apply_transpose_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeLO,
+                                     TransposeKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeHO,
+                                     TransposeKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return transpose_t::template derivative_apply_transpose_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeLO::Fallback(int dim, int q1d)
+{
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyTransposeLO =
+      typename transpose_t::DerivativeApplyTransposeLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeHO::Kernel()
+{
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return transpose_t::
+          template derivative_apply_transpose_callback<LocalQFHOBackend<DIM>, Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeHO::Fallback(int dim, int q1d)
+{
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyTransposeHO =
+      typename transpose_t::DerivativeApplyTransposeHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_assemble.hpp

@@ -0,0 +1,910 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../../../kernels.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+#include <type_traits>
+
+namespace ker = mfem::kernels::internal;
+
+namespace mfem::future::LocalQFImpl
+{
+
+namespace detail
+{
+
+template<int DIM>
+MFEM_HOST_DEVICE inline int tensor_idx(int x, int y, int z, int N)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   if constexpr (DIM == 2) { assert(z == 0); }
+   return x + N * (y + N * z);
+}
+
+template<int DIM>
+MFEM_HOST_DEVICE inline real_t
+trial_basis_weight_value(const DeviceTensor<3, const real_t> &B,
+                         const int qx,
+                         const int qy,
+                         const int qz,
+                         const int Jx,
+                         const int Jy,
+                         const int Jz)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   return B(qx, 0, Jx) * B(qy, 0, Jy) * ((DIM == 3) ? B(qz, 0, Jz) : 1.0);
+}
+
+template<int DIM>
+MFEM_HOST_DEVICE inline real_t
+trial_basis_weight_gradient(const DeviceTensor<3, const real_t> &B,
+                            const DeviceTensor<3, const real_t> &G,
+                            const int m,
+                            const int qx,
+                            const int qy,
+                            const int qz,
+                            const int Jx,
+                            const int Jy,
+                            const int Jz)
+{
+   const auto Gx = G(qx, 0, Jx), Gy = G(qy, 0, Jy);
+   const auto Bx = B(qx, 0, Jx), By = B(qy, 0, Jy);
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz & Jz);
+      return (m == 0) ? Gx * By : Bx * Gy;
+   }
+   else
+   {
+      const auto Bz = B(qz, 0, Jz), Gz = G(qz, 0, Jz);
+      return (m == 0)   ? Gx * By * Bz
+             : (m == 1) ? Bx * Gy * Bz
+             : (m == 2) ? Bx * By * Gz
+             : (assert(false), 0.0);
+   }
+}
+
+template<int DIM, int MQ1, typename Shared, typename output_t>
+MFEM_HOST_DEVICE void
+map_quadrature_data_to_fields(DeviceTensor<2, real_t> &y,
+                              const DeviceTensor<3, real_t> &f,
+                              const output_t &output,
+                              const DofToQuadMap &dtq,
+                              Shared &s,
+                              const int tv_dof = -1)
+{
+   using output_fop_t = std::decay_t<output_t>;
+   const auto B = dtq.B, G = dtq.G;
+   const bool f_slab = (tv_dof >= 0);
+   const int vdim = output.vdim;
+   const int vd_begin = f_slab ? tv_dof : 0;
+   const int vd_end = f_slab ? tv_dof + 1 : vdim;
+
+   if constexpr (is_value_fop_v<output_fop_t>)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      MFEM_CONTRACT_VAR(unused);
+      const int test_dim = output.size_on_qp / vdim;
+      MFEM_CONTRACT_VAR(test_dim);
+      const int f_vdim = f_slab ? 1 : vdim;
+
+      if constexpr (DIM == 2)
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::s_regs2d_t<MQ1> r_qp, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            { r_qp[qy][qx] = fqp(fi, 0, qx, qy); }
+            MFEM_SYNC_THREAD;
+            ker::Eval2d<MQ1, true>(d1d, q1d, s.M, s.B, r_qp, Y);
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            MFEM_FOREACH_THREAD(dx, x, d1d) { yd(dx, dy, vd) += Y[dy][dx]; }
+            MFEM_SYNC_THREAD;
+         }
+      }
+      else
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::s_regs3d_t<MQ1> f_qp, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               { f_qp[qz][qy][qx] = fqp(fi, 0, qx, qy, qz); }
+            }
+            MFEM_SYNC_THREAD;
+            ker::Eval3d<MQ1, true>(d1d, q1d, s.M, s.B, f_qp, Y);
+            for (int dz = 0; dz < d1d; dz++)
+            {
+               MFEM_FOREACH_THREAD(dy, y, d1d)
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               { yd(dx, dy, dz, vd) += Y[dz][dy][dx]; }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   }
+   else if constexpr (is_gradient_fop_v<output_fop_t>)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      MFEM_CONTRACT_VAR(unused);
+      const int test_dim = output.size_on_qp / vdim;
+      const int f_vdim = f_slab ? 1 : vdim;
+
+      if constexpr (DIM == 2)
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::LoadMatrix(d1d, q1d, G, s.G);
+         ker::vd_regs2d_t<1, DIM, MQ1> X, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            for (int k = 0; k < DIM; k++)
+            {
+               X[0][k][qy][qx] = fqp(fi, k, qx, qy);
+            }
+            MFEM_SYNC_THREAD;
+            ker::Grad2d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t u = 0.0;
+               for (int k = 0; k < DIM; k++) { u += Y[0][k][dy][dx]; }
+               yd(dx, dy, vd) += u;
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+      else
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::LoadMatrix(d1d, q1d, G, s.G);
+         ker::vd_regs3d_t<1, DIM, MQ1> X, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               for (int k = 0; k < DIM; k++)
+               {
+                  X[0][k][qz][qy][qx] = fqp(fi, k, qx, qy, qz);
+               }
+            }
+            MFEM_SYNC_THREAD;
+            ker::Grad3d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
+            for (int dz = 0; dz < d1d; dz++)
+            {
+               MFEM_FOREACH_THREAD(dy, y, d1d)
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t u = 0.0;
+                  for (int k = 0; k < DIM; k++) { u += Y[0][k][dz][dy][dx]; }
+                  yd(dx, dy, dz, vd) += u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   }
+   else if constexpr (is_identity_fop_v<output_fop_t>)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      MFEM_CONTRACT_VAR(unused);
+      MFEM_CONTRACT_VAR(d1d);
+
+      const int f_sq = f_slab ? 1 : output.size_on_qp;
+      const int sq_begin = f_slab ? tv_dof : 0;
+      const int sq_end = f_slab ? tv_dof + 1 : output.size_on_qp;
+
+      if constexpr (DIM == 2)
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d);
+         auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
+         for (int sq = sq_begin; sq < sq_end; sq++)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               int qz = 0;
+               MFEM_CONTRACT_VAR(qz);
+               yqp(sq, qx, qy) = fqp(0, qx, qy);
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+      else
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d, q1d);
+         auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
+         for (int sq = sq_begin; sq < sq_end; sq++)
+         {
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               { yqp(sq, qx, qy, qz) = fqp(0, qx, qy, qz); }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template<int DIM,
+         int MQ1,
+         typename Shared,
+         typename input_fop_ts,
+         std::size_t n_inputs,
+         typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_sumfact(
+   const DeviceTensor<5, real_t> &Ae,
+   const DeviceTensor<6, const real_t> &qpdc,
+   const int e,
+   const DeviceTensor<1, const real_t> &itod,
+   const input_fop_ts &inputs,
+   const output_fop_t &output,
+   const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
+   const DofToQuadMap &output_dtq,
+   const int q1d,
+   const int num_trial_dof_1d,
+   Shared &smem)
+{
+   static constexpr int MQN = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
+   // Slab must hold full (test_vdim, test_op_dim, nq) fhat
+   static constexpr int FHAT_SLAB_MAX = MQN * 4;
+
+   static constexpr bool grad_out = is_gradient_fop_v<output_fop_t>;
+   static constexpr bool ident_out = is_identity_fop_v<output_fop_t>;
+
+   // qpdc shape: (nq, total_trial_op_dim, trial_vdim, test_op_dim, test_vdim, ne)
+   const int test_vdim = qpdc.GetShape()[4];
+   const int test_op_dim = qpdc.GetShape()[3];
+   const int trial_vdim = qpdc.GetShape()[2];
+   const int num_test_dof = Ae.GetShape()[0];
+   const int nq = qpdc.GetShape()[0];
+   const int size_on_qp = output.size_on_qp;
+
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+   MFEM_VERIFY(test_op_dim <= DIM,
+               "DerivativeAssemble: test_op_dim exceeds spatial DIM");
+   MFEM_VERIFY(test_op_dim * nq <= FHAT_SLAB_MAX,
+               "DerivativeAssemble: fhat slab exceeds capacity");
+#endif
+
+   MFEM_SHARED real_t fhat_storage[FHAT_SLAB_MAX];
+
+   const auto &inputs_ref = inputs;
+
+   // Iterate quadrature points using the thread-block mapping
+   const auto foreach_qp = [&](auto &&body)
+   {
+      if constexpr (DIM == 2)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         MFEM_FOREACH_THREAD(qy, y, q1d) { body(qx, qy, 0); }
+      }
+      else
+      {
+         for (int qz = 0; qz < q1d; qz++)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD(qx, x, q1d) { body(qx, qy, qz); }
+         }
+      }
+   };
+
+   const auto zero_slab = [&](const int n_comp)
+   {
+      foreach_qp([&](const int qx, const int qy, const int qz)
+      {
+         const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+         for (int k = 0; k < n_comp; k++) { fhat_storage[k * nq + q] = 0.0; }
+      });
+      MFEM_SYNC_THREAD;
+   };
+
+   const auto accumulate_tv = [&](const int Jx,
+                                  const int Jy,
+                                  const int Jz,
+                                  const int j,
+                                  const int tv,
+                                  const int tod_only = -1)
+   {
+      int m_offset = 0;
+      for_constexpr<n_inputs>([&](auto inp)
+      {
+         using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
+
+         const int trial_op_dim = static_cast<int>(itod(static_cast<int>(inp)));
+         if (trial_op_dim == 0) { return; }
+
+         const auto &B = input_dtq_maps[inp].B;
+         const auto &G = input_dtq_maps[inp].G;
+
+         if constexpr (is_value_fop<fop_t>::value)
+         {
+            foreach_qp([&](const int qx, const int qy, const int qz)
+            {
+               const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+               const real_t w =
+                  trial_basis_weight_value<DIM>(B, qx, qy, qz, Jx, Jy, Jz);
+               for (int m = 0; m < trial_op_dim; m++)
+               {
+                  for (int k = 0; k < test_op_dim; k++)
+                  {
+                     if (tod_only >= 0 && k != tod_only) { continue; }
+                     const real_t f = qpdc(q, m + m_offset, j, k, tv, e);
+                     if constexpr (grad_out && !ident_out)
+                     {
+                        fhat_storage[k * nq + q] += f * w;
+                     }
+                     else
+                     {
+                        fhat_storage[q] += f * w;
+                     }
+                  }
+               }
+            });
+         }
+         else if constexpr (is_gradient_fop<fop_t>::value)
+         {
+            foreach_qp([&](const int qx, const int qy, const int qz)
+            {
+               const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+               for (int m = 0; m < trial_op_dim; m++)
+               {
+                  const real_t w = trial_basis_weight_gradient<DIM>(
+                                      B, G, m, qx, qy, qz, Jx, Jy, Jz);
+                  for (int k = 0; k < test_op_dim; k++)
+                  {
+                     if (tod_only >= 0 && k != tod_only) { continue; }
+                     const real_t f = qpdc(q, m + m_offset, j, k, tv, e);
+                     if constexpr (grad_out && !ident_out)
+                     {
+                        fhat_storage[k * nq + q] += f * w;
+                     }
+                     else
+                     {
+                        fhat_storage[q] += f * w;
+                     }
+                  }
+               }
+            });
+         }
+         else
+         {
+            MFEM_ABORT_KERNEL("sum factorized sparse matrix assemble routine "
+                              "not implemented for field operator");
+         }
+         MFEM_SYNC_THREAD;
+         m_offset += trial_op_dim;
+      });
+   };
+
+   for (int Jz = 0; Jz < ((DIM == 2) ? 1 : num_trial_dof_1d); Jz++)
+   {
+      for (int Jy = 0; Jy < num_trial_dof_1d; Jy++)
+      {
+         for (int Jx = 0; Jx < num_trial_dof_1d; Jx++)
+         {
+            const int J = tensor_idx<DIM>(Jx, Jy, Jz, num_trial_dof_1d);
+            for (int j = 0; j < trial_vdim; j++)
+            {
+               auto bvtfhat =
+                  Reshape(&Ae(0, 0, J, j, e), num_test_dof, test_vdim);
+               const int fhat_size = test_vdim * test_op_dim * nq;
+
+               if (fhat_size <= FHAT_SLAB_MAX)
+               {
+                  auto fhat =
+                     Reshape(&fhat_storage[0], test_vdim, test_op_dim, nq);
+                  for (int tv = 0; tv < test_vdim; tv++)
+                  {
+                     for (int tod = 0; tod < test_op_dim; tod++)
+                     {
+                        foreach_qp([&](const int qx, const int qy, const int qz)
+                        {
+                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+                           fhat(tv, tod, q) = 0.0;
+                        });
+                     }
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  int m_offset = 0;
+                  for_constexpr<n_inputs>([&](auto inp)
+                  {
+                     using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
+
+                     const int trial_op_dim =
+                        static_cast<int>(itod(static_cast<int>(inp)));
+                     if (trial_op_dim == 0) { return; }
+
+                     const auto &B = input_dtq_maps[inp].B;
+                     const auto &G = input_dtq_maps[inp].G;
+
+                     if constexpr (is_value_fop<fop_t>::value)
+                     {
+                        foreach_qp([&](const int qx, const int qy, const int qz)
+                        {
+                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+                           const real_t w = trial_basis_weight_value<DIM>(
+                                               B, qx, qy, qz, Jx, Jy, Jz);
+                           for (int m = 0; m < trial_op_dim; m++)
+                           {
+                              for (int i = 0; i < test_vdim; i++)
+                              {
+                                 for (int k = 0; k < test_op_dim; k++)
+                                 {
+                                    const real_t f =
+                                       qpdc(q, m + m_offset, j, k, i, e);
+                                    fhat(i, k, q) += f * w;
+                                 }
+                              }
+                           }
+                        });
+                     }
+                     else if constexpr (is_gradient_fop<fop_t>::value)
+                     {
+                        foreach_qp([&](const int qx, const int qy, const int qz)
+                        {
+                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+                           for (int m = 0; m < trial_op_dim; m++)
+                           {
+                              const real_t w = trial_basis_weight_gradient<DIM>(
+                                                  B, G, m, qx, qy, qz, Jx, Jy, Jz);
+                              for (int i = 0; i < test_vdim; i++)
+                              {
+                                 for (int k = 0; k < test_op_dim; k++)
+                                 {
+                                    const real_t f =
+                                       qpdc(q, m + m_offset, j, k, i, e);
+                                    fhat(i, k, q) += f * w;
+                                 }
+                              }
+                           }
+                        });
+                     }
+                     else
+                     {
+                        MFEM_ABORT_KERNEL(
+                           "sum factorized sparse matrix assemble routine "
+                           "not implemented for field operator");
+                     }
+                     MFEM_SYNC_THREAD;
+                     m_offset += trial_op_dim;
+                  });
+                  map_quadrature_data_to_fields<DIM, MQ1>(
+                     bvtfhat, fhat, output, output_dtq, smem);
+               }
+               else if constexpr (ident_out)
+               {
+                  for (int sq = 0; sq < size_on_qp; sq++)
+                  {
+                     const int tv = sq / test_op_dim;
+                     const int tod = sq % test_op_dim;
+                     zero_slab(1);
+                     accumulate_tv(Jx, Jy, Jz, j, tv, tod);
+                     auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
+                     map_quadrature_data_to_fields<DIM, MQ1>(
+                        bvtfhat, f_slab, output, output_dtq, smem, sq);
+                  }
+               }
+               else if constexpr (grad_out)
+               {
+                  for (int tv = 0; tv < test_vdim; tv++)
+                  {
+                     zero_slab(test_op_dim);
+                     accumulate_tv(Jx, Jy, Jz, j, tv);
+                     auto f_slab =
+                        Reshape(&fhat_storage[0], 1, test_op_dim, nq);
+                     map_quadrature_data_to_fields<DIM, MQ1>(
+                        bvtfhat, f_slab, output, output_dtq, smem, tv);
+                  }
+               }
+               else
+               {
+                  for (int tv = 0; tv < test_vdim; tv++)
+                  {
+                     zero_slab(1);
+                     accumulate_tv(Jx, Jy, Jz, j, tv);
+                     auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
+                     map_quadrature_data_to_fields<DIM, MQ1>(
+                        bvtfhat, f_slab, output, output_dtq, smem, tv);
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
+} // namespace detail
+
+// ────────────────────────────────────────────────────────────────────────────
+// Assemble sparse Jacobian from cached quadrature derivatives (tensor 2D/3D)
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeAssemble
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   const IntegratorContext ctx;
+   const Vector &qp_cache;
+   inputs_t inputs;
+   outputs_t outputs;
+   const bool use_sum_factorization;
+   const std::vector<const DofToQuad *> dtqs;
+   const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
+   const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
+   const std::array<bool, n_inputs> input_is_dependent;
+   const size_t trial_field_uf;
+   const size_t test_field_uf;
+   const ParFiniteElementSpace *test_fes;
+   const ParFiniteElementSpace *trial_fes;
+   const int test_vdim;
+   const int test_op_dim;
+   const int num_test_dof;
+   const int trial_vdim;
+   const int trial_op_dim;
+   const int num_trial_dof;
+   const int dim, ne, nq, q1d;
+   const int num_trial_dof_1d;
+   const int total_trial_op_dim;
+   mutable Vector inputs_trial_op_dim;
+   mutable Vector Ae_mem;
+
+public:
+   DerivativeAssemble() = delete;
+
+   DerivativeAssemble(IntegratorContext ctx_in,
+                      qfunc_t /*qfunc*/,
+                      inputs_t inputs_in,
+                      outputs_t outputs_in,
+                      const Vector &qp_cache_in):
+      ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
+      outputs(outputs_in), use_sum_factorization(
+         [&]
+   {
+      const Element::Type etype =
+         Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
+      return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
+   }()),
+   dtqs(
+      [&]
+   {
+      const DofToQuad::Mode dtq_mode = use_sum_factorization
+      ? DofToQuad::Mode::TENSOR
+      : DofToQuad::Mode::FULL;
+      std::vector<const DofToQuad *> maps;
+      maps.reserve(ctx_in.unionfds.size());
+      for (const auto &field : ctx_in.unionfds)
+      {
+         maps.emplace_back(
+            GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
+      }
+      return maps;
+   }()),
+   input_dtq_maps(create_dtq_maps<Entity::Element>(
+                     inputs,
+                     dtqs,
+                     create_union_field_map_for_dtq(ctx_in, inputs),
+                     ctx_in.unionfds,
+                     ctx_in.ir)),
+   output_dtq_maps(create_dtq_maps<Entity::Element>(
+                      outputs,
+                      dtqs,
+                      create_union_field_map_for_dtq(ctx_in, outputs),
+                      ctx_in.unionfds,
+                      ctx_in.ir)),
+   input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+   trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
+   test_field_uf(
+      find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
+   test_fes(
+      [&]
+   {
+      const auto *fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[test_field_uf].data);
+      MFEM_ASSERT(fes != nullptr && *fes != nullptr,
+                  "LocalQFBackend: test space is not a ParFiniteElementSpace");
+      return *fes;
+   }()),
+   trial_fes(
+      [&]
+   {
+      const auto *fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[trial_field_uf].data);
+      MFEM_ASSERT(fes != nullptr && *fes != nullptr,
+                  "LocalQFBackend: trial space is not a ParFiniteElementSpace");
+      return *fes;
+   }()),
+   test_vdim(get<0>(outputs).vdim),
+   test_op_dim(get<0>(outputs).size_on_qp / test_vdim),
+   num_test_dof(test_fes->GetFE(0)->GetDof()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)), trial_op_dim(
+      [&]
+   {
+      int top = 0;
+      for_constexpr<n_inputs>([&](auto i)
+      {
+         if (get<i>(inputs).GetFieldId() == derivative_id)
+         {
+            top = get<i>(inputs).size_on_qp / get<i>(inputs).vdim;
+         }
+      });
+      return top;
+   }()),
+   num_trial_dof(trial_fes->GetFE(0)->GetDof()),
+   dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
+   nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
+   num_trial_dof_1d(tensor_1d_size(num_trial_dof, dim)), total_trial_op_dim(
+      [&]
+   {
+      const auto in_qp_sizes =
+      get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
+      return compute_total_trial_op_dim(
+         inputs, input_is_dependent, in_qp_sizes);
+   }()),
+   inputs_trial_op_dim(), Ae_mem()
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+      MFEM_ASSERT(trial_field_uf != SIZE_MAX,
+                  "DerivativeAssemble: trial field not found in unionfds");
+      MFEM_ASSERT(test_field_uf != SIZE_MAX,
+                  "DerivativeAssemble: test field not found in unionfds");
+      MFEM_ASSERT(trial_vdim > 0,
+                  "LocalQFBackend: could not determine trial vdim");
+      MFEM_ASSERT(total_trial_op_dim > 0,
+                  "LocalQFBackend: no dependent inputs found");
+
+      inputs_trial_op_dim.UseDevice(true);
+      inputs_trial_op_dim.SetSize(n_inputs);
+      auto inputs_trial_op_dim_host = inputs_trial_op_dim.HostWrite();
+      for_constexpr<n_inputs>([&](auto i)
+      {
+         inputs_trial_op_dim_host[i] =
+            input_is_dependent[i]
+            ? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
+            : 0;
+      });
+
+      const int elem_mat_size =
+         num_test_dof * test_vdim * num_trial_dof * trial_vdim;
+      Ae_mem.SetSize(elem_mat_size * ne, Device::GetDeviceMemoryType());
+      Ae_mem.UseDevice(true);
+      Ae_mem = 0.0;
+   }
+
+   void operator()(SparseMatrix *&A) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      if (!(use_sum_factorization && (dim == 2 || dim == 3)))
+      {
+         MFEM_ABORT("DerivativeAssemble optimized path is implemented "
+                    "for tensor-product 2D/3D elements only");
+      }
+
+      DerivativeAssembleHO::Run(dim,
+                                q1d,
+                                ctx,
+                                qp_cache,
+                                Ae_mem,
+                                inputs,
+                                outputs,
+                                input_dtq_maps,
+                                output_dtq_maps[0],
+                                inputs_trial_op_dim,
+                                test_vdim,
+                                test_op_dim,
+                                num_test_dof,
+                                num_trial_dof,
+                                num_trial_dof_1d,
+                                trial_vdim,
+                                total_trial_op_dim,
+                                nq,
+                                ne,
+                                q1d,
+                                dim);
+
+      A = new SparseMatrix;
+      A->OverrideSize(test_fes->GetVSize(), trial_fes->GetVSize());
+
+      const auto *test_restr = dynamic_cast<const ElementRestriction *>(
+                                  test_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
+      const auto *trial_restr = dynamic_cast<const ElementRestriction *>(
+                                   trial_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
+      MFEM_VERIFY(test_restr != nullptr && trial_restr != nullptr,
+                  "DerivativeAssemble SparseMatrix assembly requires "
+                  "H1/conforming ElementRestriction spaces");
+
+      test_restr->FillSparseMatrix(Ae_mem, *A, *trial_restr);
+   }
+
+   template<typename backend_t = LocalQFHOBackend<3>, int T_Q1D = 0>
+   static void derivative_assemble_callback(
+      const IntegratorContext &ctx,
+      const Vector &qp_cache,
+      Vector &Ae_mem,
+      const inputs_t &inputs,
+      const outputs_t &outputs,
+      const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
+      const DofToQuadMap &output_dtq,
+      const Vector &inputs_trial_op_dim,
+      const int test_vdim,
+      const int test_op_dim,
+      const int num_test_dof,
+      const int num_trial_dof,
+      const int num_trial_dof_1d,
+      const int trial_vdim,
+      const int total_trial_op_dim,
+      const int nq,
+      const int ne,
+      const int q1d,
+      const int dim)
+   {
+      static constexpr int DIM = backend_t::DIM;
+      static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr int MNQ = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
+
+      MFEM_VERIFY(dim == DIM,
+                  "DerivativeAssemble: mesh dim does not match backend");
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      MFEM_VERIFY(q1d <= MQ1, "q1d exceeds backend MQ1 limit");
+      MFEM_VERIFY(nq <= MNQ,
+                  "DerivativeAssemble: nq exceeds backend quadrature capacity");
+      MFEM_VERIFY(test_op_dim <= DIM,
+                  "DerivativeAssemble: test_op_dim exceeds spatial DIM");
+      if (ctx.attr.Size() == 0) { return; }
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      const auto qpdc = Reshape(qp_cache.Read(),
+                                nq,
+                                total_trial_op_dim,
+                                trial_vdim,
+                                test_op_dim,
+                                test_vdim,
+                                ne);
+      const auto itod = Reshape(inputs_trial_op_dim.Read(), n_inputs);
+
+      auto Ae = Reshape(Ae_mem.ReadWrite(),
+                        num_test_dof,
+                        test_vdim,
+                        num_trial_dof,
+                        trial_vdim,
+                        ne);
+
+      dfem::forall(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         static constexpr int DIM = backend_t::DIM;
+         static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+
+         MFEM_SHARED typename backend_t::Shared s;
+
+         detail::assemble_element_mat_sumfact<DIM, MQ1>(Ae,
+                                                        qpdc,
+                                                        e,
+                                                        itod,
+                                                        inputs,
+                                                        get<0>(outputs),
+                                                        input_dtq_maps,
+                                                        output_dtq,
+                                                        q1d,
+                                                        num_trial_dof_1d,
+                                                        s);
+      },
+      ne,
+      backend_t::thread_blocks(q1d),
+      0,
+      nullptr);
+   }
+
+   using AssembleKernelType =
+      decltype(&DerivativeAssemble::derivative_assemble_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleHO,
+                                     AssembleKernelType,
+                                     (int, int));
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline
+typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+AssembleKernelType
+DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeAssembleHO::Kernel()
+{
+   static_assert(DIM == 2 || DIM == 3);
+   using assemble_t =
+      DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return assemble_t::template derivative_assemble_callback<
+             LocalQFHOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline
+typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+AssembleKernelType
+DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeAssembleHO::Fallback(int dim, int q1d)
+{
+   using assemble_t =
+      DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeAssembleHO = typename assemble_t::DerivativeAssembleHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleHO, 3, 8>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_assemble_diagonal.hpp

@@ -0,0 +1,540 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+// Assemble diagonal of cached Jacobian (square trial == test, tensor 2D/3D)
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeAssembleDiagonal
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   const IntegratorContext ctx;
+   const Vector &qp_cache;
+   inputs_t inputs;
+   outputs_t outputs;
+   const bool use_sum_factorization;
+   const std::vector<const DofToQuad *> dtqs;
+   const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
+   const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
+   const std::array<bool, n_inputs> input_is_dependent;
+   const size_t trial_field_uf;
+   const size_t test_field_uf;
+   const bool is_square;
+   const int test_vdim;
+   const std::array<int, n_outputs> out_vdim;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int num_test_dof;
+   const int num_test_dof_1d;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int num_trial_dof_1d;
+   const int residual_size_on_qp;
+   const int dim, ne, nq, q1d;
+   const std::array<int, n_inputs> inputs_trial_op_dim;
+   mutable Vector Ye_mem;
+
+public:
+   DerivativeAssembleDiagonal() = delete;
+
+   DerivativeAssembleDiagonal(IntegratorContext ctx_in,
+                              qfunc_t /*qfunc*/,
+                              inputs_t inputs_in,
+                              outputs_t outputs_in,
+                              const Vector &qp_cache_in):
+      ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
+      outputs(outputs_in), use_sum_factorization(
+         [&]
+   {
+      const Element::Type etype =
+         Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
+      return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
+   }()),
+   dtqs(
+      [&]
+   {
+      const DofToQuad::Mode dtq_mode = use_sum_factorization
+      ? DofToQuad::Mode::TENSOR
+      : DofToQuad::Mode::FULL;
+      std::vector<const DofToQuad *> maps;
+      maps.reserve(ctx_in.unionfds.size());
+      for (const auto &field : ctx_in.unionfds)
+      {
+         maps.emplace_back(
+            GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
+      }
+      return maps;
+   }()),
+   input_dtq_maps(create_dtq_maps<Entity::Element>(
+                     inputs,
+                     dtqs,
+                     create_union_field_map_for_dtq(ctx_in, inputs),
+                     ctx_in.unionfds,
+                     ctx_in.ir)),
+   output_dtq_maps(create_dtq_maps<Entity::Element>(
+                      outputs,
+                      dtqs,
+                      create_union_field_map_for_dtq(ctx_in, outputs),
+                      ctx_in.unionfds,
+                      ctx_in.ir)),
+   input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+   trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
+   test_field_uf(
+      find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
+   is_square(
+      [&]
+   {
+      const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[test_field_uf].data);
+      const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[trial_field_uf].data);
+      return test_fes && trial_fes && *test_fes && *trial_fes &&
+      (*test_fes == *trial_fes);
+   }()),
+   test_vdim(get<0>(outputs).vdim),
+   out_vdim(get_vdim(outputs_in)),
+   out_op_dim(compute_out_op_dim(outputs_in)),
+   out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
+   output_size_on_qp(
+      [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs_in).size_on_qp; });
+      return s;
+   }()), num_test_dof(
+      [&]
+   {
+      const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[test_field_uf].data);
+      MFEM_ASSERT(test_fes != nullptr && *test_fes != nullptr,
+                  "LocalQFBackend: test space is not a ParFiniteElementSpace");
+      return (*test_fes)->GetFE(0)->GetDof();
+   }()),
+   num_test_dof_1d(tensor_1d_size(num_test_dof, ctx_in.mesh.Dimension())),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)), total_trial_op_dim(
+      [&]
+   {
+      const auto input_size_on_qp =
+      get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
+      return compute_total_trial_op_dim(
+         inputs, input_is_dependent, input_size_on_qp);
+   }()),
+   num_trial_dof_1d(
+      [&]
+   {
+      const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[trial_field_uf].data);
+      MFEM_ASSERT(trial_fes != nullptr && *trial_fes != nullptr,
+                  "LocalQFBackend: trial space is not a ParFiniteElementSpace");
+      const int num_trial_dof = (*trial_fes)->GetFE(0)->GetDof();
+      return tensor_1d_size(num_trial_dof, ctx_in.mesh.Dimension());
+   }()),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
+   nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
+   inputs_trial_op_dim(
+      [&]
+   {
+      std::array<int, n_inputs> itod{};
+      for_constexpr<n_inputs>([&](auto i)
+      {
+         itod[i] = input_is_dependent[i]
+                   ? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
+                   : 0;
+      });
+      return itod;
+   }()),
+   Ye_mem()
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+      MFEM_ASSERT(
+         trial_field_uf != SIZE_MAX,
+         "DerivativeAssembleDiagonal: trial field not found in unionfds");
+      MFEM_ASSERT(
+         test_field_uf != SIZE_MAX,
+         "DerivativeAssembleDiagonal: test field not found in unionfds");
+      MFEM_ASSERT(trial_vdim > 0,
+                  "LocalQFBackend: could not determine trial vdim");
+      MFEM_ASSERT(total_trial_op_dim > 0,
+                  "LocalQFBackend: no dependent inputs found");
+      for_constexpr<n_outputs>([&](auto o)
+      {
+         MFEM_CONTRACT_VAR(o);
+         MFEM_ASSERT(out_vdim[o] == test_vdim,
+                     "DerivativeAssembleDiagonal: all outputs must share the "
+                     "test field vdim");
+      });
+
+      if (is_square)
+      {
+         Ye_mem.SetSize(num_test_dof * test_vdim * ne);
+         Ye_mem.UseDevice(true);
+      }
+   }
+
+   template<typename Backend>
+   void run_kernels() const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qp_cache,
+                   Ye_mem,
+                   inputs,
+                   outputs,
+                   output_dtq_maps,
+                   input_dtq_maps,
+                   test_vdim,
+                   out_op_dim,
+                   out_offsets,
+                   output_size_on_qp,
+                   num_test_dof,
+                   num_test_dof_1d,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   inputs_trial_op_dim,
+                   nq,
+                   ne,
+                   q1d,
+                   dim);
+   }
+
+   void operator()(Vector &diag_e) const
+   {
+      if (!is_square) { return; }
+      if (ctx.attr.Size() == 0) { return; }
+
+      if (!(use_sum_factorization && (dim == 2 || dim == 3)))
+      {
+         MFEM_ABORT("DerivativeAssembleDiagonal optimized path is implemented "
+                    "for tensor-product 2D/3D elements only");
+      }
+      MFEM_VERIFY(num_test_dof_1d == num_trial_dof_1d,
+                  "DerivativeAssembleDiagonal requires matching tensor dofs");
+      MFEM_VERIFY(num_test_dof_1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+      MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+      Ye_mem = 0.0;
+
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeAssembleDiagonalLO>();
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeAssembleDiagonalHO>();
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+
+      diag_e += Ye_mem;
+   }
+
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void derivative_assemble_diagonal_callback(
+      const IntegratorContext &ctx,
+      const Vector &qp_cache,
+      Vector &Ye_mem,
+      const inputs_t &inputs,
+      const outputs_t &outputs,
+      const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
+      const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
+      const int test_vdim,
+      const std::array<int, n_outputs> &out_op_dim,
+      const std::array<int, n_outputs> &out_offsets,
+      const int output_size_on_qp,
+      const int num_test_dof,
+      const int num_test_dof_1d,
+      const int trial_vdim,
+      const int total_trial_op_dim,
+      const int residual_size_on_qp,
+      const std::array<int, n_inputs> &inputs_trial_op_dim,
+      const int nq,
+      const int ne,
+      const int q1d,
+      const int dim)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr bool B2D = backend_t::DIM == 2;
+      static constexpr int MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      auto cache_tensor = DeviceTensor<3, const real_t>(
+                             qp_cache.Read(), nq, residual_size_on_qp, ne);
+      const int num_dofs_per_elem = num_test_dof * test_vdim;
+      auto Ye = Reshape(Ye_mem.ReadWrite(), num_dofs_per_elem, ne);
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // The cache is written with the quadrature index fastest, then the
+         // trial op index, then the (test vdim, test op) rows of all outputs
+         // stacked via out_offsets.
+         auto qpdc = Reshape(&cache_tensor(0, 0, e),
+                             nq,
+                             total_trial_op_dim,
+                             trial_vdim,
+                             output_size_on_qp);
+
+         // Backend-owned shared scratch for the sum-factorized contraction.
+         MFEM_SHARED typename backend_t::Shared s_diag;
+         const int nz_dof = B2D ? 1 : num_test_dof_1d;
+
+         for (int vd = 0; vd < test_vdim; vd++)
+         {
+            auto Y = Reshape(&Ye(vd * num_test_dof, e),
+                             num_test_dof_1d,
+                             num_test_dof_1d,
+                             nz_dof);
+
+            MFEM_FOREACH_THREAD(dz_t, z, nz_dof)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dy_t, y, num_test_dof_1d)
+               {
+                  MFEM_FOREACH_THREAD_DIRECT(dx_t, x, num_test_dof_1d)
+                  { Y(dx_t, dy_t, dz_t) = 0.0; }
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            // Accumulate every (output o, test op k, dependent input s,
+            // trial op m) block of the cached Jacobian into the diagonal via
+            // the backend driver.
+            for_constexpr<n_outputs>([&](auto o)
+            {
+               using test_fop_t = std::decay_t<decltype(get<o>(outputs))>;
+               const auto &out_dtq = output_dtq_maps[o];
+               const int test_op_dim = out_op_dim[static_cast<int>(o)];
+
+               // Test-basis factor along a spatial axis
+               const auto eval_test =
+                  [&](const int k, const int axis, const int q, const int d)
+               {
+                  const auto &B = out_dtq.B;
+                  const auto &G = out_dtq.G;
+                  if constexpr (is_value_fop<test_fop_t>::value)
+                  {
+                     return (k == 0) ? B(q, 0, d) : 0.0;
+                  }
+                  else if constexpr (is_gradient_fop<test_fop_t>::value)
+                  {
+                     return (k == axis) ? G(q, 0, d) : B(q, 0, d);
+                  }
+                  else
+                  {
+                     return 0.0;
+                  }
+               };
+
+               for (int k = 0; k < test_op_dim; k++)
+               {
+                  const int row =
+                     out_offsets[static_cast<int>(o)] + vd * test_op_dim + k;
+                  int m_offset = 0;
+                  for_constexpr<n_inputs>([&](auto s)
+                  {
+                     using fop_t = std::decay_t<decltype(get<s>(inputs))>;
+                     const int trial_op_dim =
+                        inputs_trial_op_dim[static_cast<int>(s)];
+                     if (trial_op_dim == 0) { return; }
+
+                     const auto &in_dtq = input_dtq_maps[s];
+                     const auto eval_input =
+                        [&](const int m, const int axis, const int q,
+                            const int d)
+                     {
+                        if constexpr (is_value_fop<fop_t>::value)
+                        {
+                           return (m == 0) ? in_dtq.B(q, 0, d) : 0.0;
+                        }
+                        else if constexpr (is_gradient_fop<fop_t>::value)
+                        {
+                           return (m == axis) ? in_dtq.G(q, 0, d)
+                                  : in_dtq.B(q, 0, d);
+                        }
+                        else
+                        {
+                           return 0.0;
+                        }
+                     };
+
+                     for (int m = 0; m < trial_op_dim; m++)
+                     {
+                        const int col = m_offset + m;
+                        backend_t::DiagContract(
+                           s_diag,
+                           num_test_dof_1d,
+                           q1d,
+                           nz_dof,
+                           [&](int axis, int q, int d)
+                        { return eval_test(k, axis, q, d); },
+                        [&](int axis, int q, int d)
+                        { return eval_input(m, axis, q, d); },
+                        [&](int q) { return qpdc(q, col, vd, row); },
+                        [&](int dx, int dy, int dz, real_t u)
+                        { Y(dx, dy, dz) += u; });
+                     }
+                     m_offset += trial_op_dim;
+                  });
+               }
+            });
+         }
+      },
+      ne,
+      backend_t::thread_blocks(std::max(q1d, num_test_dof_1d)),
+      0,
+      nullptr);
+   }
+
+   using DiagonalKernelType =
+      decltype(&DerivativeAssembleDiagonal::
+               derivative_assemble_diagonal_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalLO,
+                                     DiagonalKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalHO,
+                                     DiagonalKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return diag_t::template derivative_assemble_diagonal_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalLO::Fallback(int dim, int q1d)
+{
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeAssembleDiagonalLO =
+      typename diag_t::DerivativeAssembleDiagonalLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalHO::Kernel()
+{
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return diag_t::template derivative_assemble_diagonal_callback<
+             LocalQFHOBackend<DIM>,
+             Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalHO::Fallback(int dim, int q1d)
+{
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeAssembleDiagonalHO =
+      typename diag_t::DerivativeAssembleDiagonalHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_setup.hpp

@@ -0,0 +1,600 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeSetup
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   qfunc_t qfunc;
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   Vector &qp_cache;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // Jacobian cache metadata
+   const std::array<bool, n_inputs> input_is_dependent;
+   const std::array<int, n_inputs> input_size_on_qp;
+   const std::array<int, n_outputs> out_vdim;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int residual_size_on_qp;
+   // other constants
+   const int dim, ne, nq, q1d;
+
+public:
+   //////////////////////////////////////////////////////////////////
+   DerivativeSetup() = delete;
+
+   DerivativeSetup(IntegratorContext ctx,
+                   qfunc_t qfunc,
+                   inputs_t inputs,
+                   outputs_t outputs,
+                   Vector &qp_cache):
+      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
+      qp_cache(qp_cache), dtqs(make_dtqs(ctx)),
+      input_dtq(create_dtq_maps<Entity::Element>(
+                   inputs,
+                   dtqs,
+                   create_union_field_map_for_dtq(ctx, inputs),
+                   ctx.unionfds,
+                   ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+      input_size_on_qp(
+         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
+                           out_vdim(get_vdim(outputs)), out_op_dim(compute_out_op_dim(outputs)),
+                           out_offsets(compute_out_offsets(out_vdim, out_op_dim)), output_size_on_qp(
+                              [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs).size_on_qp; });
+      return s;
+   }()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
+   total_trial_op_dim(compute_total_trial_op_dim(
+                         inputs, input_is_dependent, input_size_on_qp)),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+   q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+
+      qp_cache.SetSize(ne * nq * residual_size_on_qp);
+      qp_cache.UseDevice(true);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> &xe)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      // Quadrature index is fastest-varying so that adjacent threads (one per
+      // quadrature point) touch adjacent addresses.
+      auto cache_tensor = DeviceTensor<3, real_t>(
+                             qp_cache.ReadWrite(), nq, residual_size_on_qp, ne);
+
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeSetupLO>(xe, cache_tensor);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeSetupHO>(xe, cache_tensor);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(const std::vector<Vector *> &xe,
+                    DeviceTensor<3, real_t> &cache_tensor)
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qfunc,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   input_size_on_qp,
+                   input_is_dependent,
+                   // outputs / cache metadata
+                   out_vdim,
+                   out_op_dim,
+                   out_offsets,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   // vectors
+                   xe,
+                   cache_tensor,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   /// Zeroes the q-function *output* slots of an argument tuple.
+   ///
+   /// The argument tuple is built once per quadrature point and reused for
+   /// every trial seed. The q-function writes through its output parameters,
+   /// so those slots have to be restored before each call to give every seed
+   /// the same starting state a freshly value-initialized tuple would.
+   static MFEM_HOST_DEVICE inline void reset_output_args(args_tuple_t &args)
+   {
+      for_constexpr<n_outputs>([&](auto oc)
+      {
+         constexpr size_t ao = n_inputs + oc.value;
+         auto &oarg = get<ao>(args);
+         oarg = std::remove_reference_t<decltype(oarg)> {};
+      });
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   derivative_setup_callback(const IntegratorContext &ctx,
+                             qfunc_t &qfunc,
+                             // inputs: idx, B, G, vdim, d1d, q1d
+                             const std::array<size_t, n_inputs> &in_idx,
+                             const std::array<const real_t *, n_inputs> in_B,
+                             const std::array<const real_t *, n_inputs> in_G,
+                             const std::array<int, n_inputs> &in_vdim,
+                             const std::array<int, n_inputs> &in_d1d,
+                             const std::array<int, n_inputs> &in_q1d,
+                             const std::array<int, n_inputs> &in_size_on_qp,
+                             const std::array<bool, n_inputs> &input_dep,
+                             // outputs / cache metadata
+                             const std::array<int, n_outputs> &out_vdim,
+                             const std::array<int, n_outputs> &out_op_dim,
+                             const std::array<int, n_outputs> &out_offsets,
+                             const int trial_vdim,
+                             const int total_trial_op_dim,
+                             const int residual_size_on_qp,
+                             const std::vector<Vector *> &xe,
+                             DeviceTensor<3, real_t> &cache_tensor,
+                             // fallback arguments
+                             const int dim,
+                             const int q1d)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+      MFEM_CONTRACT_VAR(residual_size_on_qp);
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
+                        "tensor-product IR expected");
+            in_XE[i] = Reshape(
+                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Inputs argument registers + shared memory
+         // -----------------------------------------------
+         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load primal inputs (rargs) once for this element
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            const auto &XE = in_XE[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &rarg = get<i>(rargs);
+            using XE_t = decltype(XE);
+            using rarg_t = decltype(rarg);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::template LoadValue<rarg_t, XE_t>(
+                  smem, e, d, q, Q1D, B, XE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK, rarg_t, XE_t, FieldParamT>(
+                  smem, e, d, q, q1d, B, G, XE, rarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+               // qp values are read directly from in_XE / IR
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Build the primal arguments once per quadrature point, then, for
+         // each trial seed (j, dependent input s, m), differentiate the
+         // q-function with a unit tangent and store the result row in the
+         // cache. Nothing in the primal pull depends on the seed, so the
+         // thread loop is the outermost one here. The seed loops only touch
+         // per-thread state, hence no barrier inside them.
+         // Warning: no 'DIRECT' on the 'Z' direction,
+         // as one backend may need to iterate over it.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+#ifdef MFEM_USE_ENZYME
+                  args_tuple_t primal_args {};
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &parg = get<i>(primal_args);
+                     const auto &XE = in_XE[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        parg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        parg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  for (int j = 0; j < trial_vdim; j++)
+                  {
+                     int m_offset = 0;
+                     for_constexpr<n_inputs>([&](auto sc)
+                     {
+                        constexpr size_t s = sc.value;
+                        if (!input_dep[s]) { return; }
+
+                        const int vdim_s = in_vdim[s];
+                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
+
+                        for (int m = 0; m < op_dim_s; m++)
+                        {
+                           const int col_m = m + m_offset;
+
+                           // Enzyme writes through the output slots of the
+                           // primal tuple, so they are reset per seed.
+                           reset_output_args(primal_args);
+
+                           args_tuple_t shadow_args {};
+                           qf_set_value_at(get<s>(shadow_args), j, m, 1.0);
+
+                           call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
+
+                           for_constexpr<n_outputs>([&](auto oc)
+                           {
+                              constexpr size_t o = oc.value, ao = n_inputs + o;
+                              const auto &tangent = get<ao>(shadow_args);
+                              const int tv = out_vdim[o], to = out_op_dim[o];
+                              for (int i = 0; i < tv; i++)
+                              {
+                                 for (int k = 0; k < to; k++)
+                                 {
+                                    const int row = out_offsets[o] + i * to + k;
+                                    const int cache_idx =
+                                       row * trial_vdim * total_trial_op_dim +
+                                       j * total_trial_op_dim + col_m;
+                                    cache_tensor(q, cache_idx, e) =
+                                       qf_value_at(tangent, i, k);
+                                 }
+                              }
+                           });
+                        }
+                        m_offset += op_dim_s;
+                     });
+                  }
+#else  // MFEM_USE_ENZYME
+                  args_tuple_t qargs {};
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &qarg = get<i>(qargs);
+                     const auto &XE = in_XE[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           qarg = backend_t::template identity_qp_pull_dual<DT>(
+                                     false, XE, XE, qx, qy, qz, e);
+                        }
+                        else
+                        {
+                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        qarg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        qarg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  for (int j = 0; j < trial_vdim; j++)
+                  {
+                     int m_offset = 0;
+                     for_constexpr<n_inputs>([&](auto sc)
+                     {
+                        constexpr size_t s = sc.value;
+                        if (!input_dep[s]) { return; }
+
+                        const int vdim_s = in_vdim[s];
+                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
+
+                        for (int m = 0; m < op_dim_s; m++)
+                        {
+                           const int col_m = m + m_offset;
+
+                           // The q-function writes through the output slots,
+                           // so they are reset per seed.
+                           reset_output_args(qargs);
+
+                           qf_set_gradient_at(get<s>(qargs), j, m, 1.0);
+
+                           call_qfunc_no_move(qfunc, qargs);
+
+                           for_constexpr<n_outputs>([&](auto oc)
+                           {
+                              constexpr size_t o = oc.value, ao = n_inputs + o;
+                              const auto &tangent = get<ao>(qargs);
+                              const int tv = out_vdim[o], to = out_op_dim[o];
+                              for (int i = 0; i < tv; i++)
+                              {
+                                 for (int k = 0; k < to; k++)
+                                 {
+                                    const int row = out_offsets[o] + i * to + k;
+                                    const int cache_idx =
+                                       row * trial_vdim * total_trial_op_dim +
+                                       j * total_trial_op_dim + col_m;
+                                    cache_tensor(q, cache_idx, e) =
+                                       qf_gradient_at(tangent, i, k);
+                                 }
+                              }
+                           });
+
+                           // Clear the seed so the next direction starts from
+                           // the pristine (zero-tangent) primal state.
+                           qf_set_gradient_at(get<s>(qargs), j, m, 0.0);
+                        }
+                        m_offset += op_dim_s;
+                     });
+                  }
+#endif // MFEM_USE_ENZYME
+               }
+            }
+         }
+      },
+      ne,
+      backend_t::thread_blocks(
+         compute_kernel_thread_1d<inputs_t>(q1d, in_d1d)),
+      0,
+      nullptr);
+   }
+
+   using SetupKernelType =
+      decltype(&DerivativeSetup::derivative_setup_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupLO,
+                                     SetupKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupHO,
+                                     SetupKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return setup_t::template derivative_setup_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupLO::Fallback(int dim, int q1d)
+{
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeSetupLO = typename setup_t::DerivativeSetupLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeSetupLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeSetupLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupHO::Kernel()
+{
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return setup_t::template derivative_setup_callback<LocalQFHOBackend<DIM>,
+                                                      Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupHO::Fallback(int dim, int q1d)
+{
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeSetupHO = typename setup_t::DerivativeSetupHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeSetupHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeSetupHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/fwddiff_transformer.hpp

@@ -0,0 +1,274 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../util.hpp"
+
+#ifdef MFEM_USE_ENZYME
+
+namespace mfem::future
+{
+
+template <typename T> struct function_traits;
+
+template <typename R, typename C, typename... Args>
+struct function_traits<R (C::*)(Args...) const>
+{
+   using primal_return_type = R;
+   using args_tuple = std::tuple<Args...>;
+   static constexpr size_t arity = sizeof...(Args);
+};
+
+// Component count and writable counterpart of a per-point argument, which
+// is either a tensor or a plain scalar.
+template <typename Arg> struct qp_traits
+{
+   static_assert(std::is_arithmetic_v<Arg>,
+                 "per-point arguments must be tensors or scalars");
+   using view_type = Arg;
+   static constexpr int components = 1;
+};
+
+template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
+{
+   using view_type = tensor<std::remove_const_t<T>, Sizes...>;
+   static constexpr int components = (Sizes * ... * 1);
+};
+
+// Generic FwdDiff: computes the full gradient of a pointwise qfunction at a
+// single quadrature point.
+//
+// active_input is the index of the argument to differentiate with respect
+// to; active_output is the index of the (scalar) output argument whose
+// derivative is taken.
+//
+// operator()(args...) takes the qfunction's arguments, except that the
+// active output position receives the *gradient*, shaped like the active
+// input (d(output)/d(input component)).
+//
+// Per input component d, one enzyme fwddiff call with the one-hot seed e_d
+// in the input tangent yields gradient entry d, i.e. grad_components enzyme
+// calls per point. The output tangent is not pre-zeroed, so the qfunction
+// must fully write its output (tangent stores overwrite).
+//
+// operator() is MFEM_HOST_DEVICE, stateless and allocation-free, so it is
+// callable inside a GPU kernel: seed, primal scratch and tangent all live
+// on the stack.
+template <typename Func, size_t active_input, size_t active_output>
+struct FwdDiff
+{
+   using traits = function_traits<decltype(&Func::operator())>;
+   using args_tuple = typename traits::args_tuple;
+   static constexpr size_t arity = traits::arity;
+
+   static_assert(std::is_void_v<typename traits::primal_return_type>,
+                 "FwdDiff only supports primal functions with void return type");
+   static_assert(active_input < arity && active_output < arity,
+                 "active argument indices must be within the function arity");
+   static_assert(active_input != active_output,
+                 "active input and output must be different arguments");
+
+   using input_type =
+      std::decay_t<std::tuple_element_t<active_input, args_tuple>>;
+   using output_type =
+      std::decay_t<std::tuple_element_t<active_output, args_tuple>>;
+
+   using grad_type = typename qp_traits<input_type>::view_type;
+   using output_view = typename qp_traits<output_type>::view_type;
+
+   static constexpr int grad_components = qp_traits<input_type>::components;
+
+   static_assert(qp_traits<output_type>::components == 1,
+                 "gradient output requires a scalar output");
+
+   // Signature of the differentiated qfunction: the primal arguments, with
+   // the Active output slot receiving the (writable) gradient instead.
+   // Exposed through create_function_signature below so that
+   // DifferentiableOperator can deduce the parameter types, which it cannot
+   // do from the variadic operator().
+   template <size_t I>
+   using qf_arg_t = std::conditional_t<I == active_output, grad_type &,
+         std::tuple_element_t<I, args_tuple>>;
+
+   template <size_t... Is>
+   static FunctionSignature<void(qf_arg_t<Is>...)>
+   signature_impl(std::index_sequence<Is...>);
+
+   using signature =
+      decltype(signature_impl(std::make_index_sequence<arity> {}));
+
+   // d-th scalar of a per-point argument in flat row-major order, regardless
+   // of rank, built on the native operator[] (tensor has no flat-index
+   // accessor; flatten() returns a copy, so it cannot be written through).
+   MFEM_HOST_DEVICE static double &component(double &t, int) { return t; }
+
+   template <typename T, int n0, int... n>
+   MFEM_HOST_DEVICE static T &component(tensor<T, n0, n...> &t, int d)
+   {
+      if constexpr (sizeof...(n) == 0)
+      {
+         return t[d];
+      }
+      else
+      {
+         constexpr int stride = (n * ... * 1);
+         return component(t[d / stride], d % stride);
+      }
+   }
+
+   // Plain function with the qfunction's exact (reference) signature, so it
+   // can be handed to Enzyme as a function pointer; references are pointers
+   // to Enzyme, so primal arguments and shadows are passed by address below.
+   template <size_t... Is>
+   MFEM_HOST_DEVICE static void
+   static_call(std::tuple_element_t<Is, args_tuple>... args)
+   {
+      Func{}(args...);
+   }
+
+   template <size_t... Is>
+   static constexpr auto fn_ptr(std::index_sequence<Is...>)
+   {
+      return &static_call<Is...>;
+   }
+   static constexpr auto fn = fn_ptr(std::make_index_sequence<arity> {});
+
+   // Writable, zero-initialized scratch with the shape of argument I, used
+   // as its enzyme shadow.
+   template <size_t I>
+   using shadow_t = typename qp_traits<
+                    std::decay_t<std::tuple_element_t<I, args_tuple>>>::view_type;
+
+   template <size_t... Is>
+   MFEM_HOST_DEVICE static auto make_shadows(std::index_sequence<Is...>)
+   {
+      return mfem::future::make_tuple(shadow_t<Is> {}...);
+   }
+
+   template <typename Shadows, size_t... Is>
+   MFEM_HOST_DEVICE static auto make_shadow_ptrs(Shadows &shadows,
+                                                 std::index_sequence<Is...>)
+   {
+      return mfem::future::make_tuple(&mfem::future::get<int(Is)>(shadows)...);
+   }
+
+   // The caller's argument pointers, except the active output slot, which
+   // points to scalar scratch: the caller's slot holds the gradient, while
+   // the primal function writes its scalar output there.
+   template <size_t I, typename Ptrs>
+   MFEM_HOST_DEVICE static auto primal_ptr(Ptrs &ptrs, output_view &primal)
+   {
+      if constexpr (I == active_output) { return &primal; }
+      else { return mfem::future::get<int(I)>(ptrs); }
+   }
+
+   template <typename Ptrs, size_t... Is>
+   MFEM_HOST_DEVICE static auto make_primal_ptrs(Ptrs &ptrs,
+                                                 output_view &primal,
+                                                 std::index_sequence<Is...>)
+   {
+      return mfem::future::make_tuple(primal_ptr<Is>(ptrs, primal)...);
+   }
+
+   // Single flat enzyme call. The activity markers must appear directly in
+   // the __enzyme_fwddiff argument list — Enzyme cannot trace markers that
+   // were forwarded through function parameters (e.g. at -O0, where nothing
+   // is inlined). Every argument is therefore enzyme_dup'd in one sticky
+   // group; Const arguments simply carry a zero tangent, which is equivalent
+   // to marking them enzyme_const.
+   //
+   // always_inline is load-bearing: when FwdDiff is itself differentiated
+   // (second derivatives, forward-over-forward), Enzyme only recognizes this
+   // nested __enzyme_fwddiff call if it sits at most one call level below
+   // the function handed to the outer __enzyme_fwddiff. Without inlining
+   // (-O0) it sits two levels down (wrapper -> operator() -> call_enzyme)
+   // and the outer pass treats it as a regular call: the activity marker
+   // ints then receive undef shadows, which misaligns the argument pairing
+   // (observed as "cannot compute with global variable that doesn't have
+   // marked shadow global" at compile time or null-shadow segfaults at
+   // runtime). The always-inliner runs even at -O0, hoisting this call into
+   // operator() where the nested handling applies.
+   template <typename PrimalPtrs, typename ShadowPtrs, size_t... Is>
+   __attribute__((always_inline))
+   MFEM_HOST_DEVICE static void call_enzyme(PrimalPtrs &primal_ptrs,
+                                            ShadowPtrs &shadow_ptrs,
+                                            std::index_sequence<Is...>)
+   {
+      __enzyme_fwddiff<void>(fn, enzyme_dup,
+                             mfem::future::get<int(Is)>(primal_ptrs)...,
+                             enzyme_interleave,
+                             mfem::future::get<int(Is)>(shadow_ptrs)...,
+                             enzyme_runtime_activity);
+   }
+
+   template <typename... Args>
+   MFEM_HOST_DEVICE void operator()(Args &&...args) const
+   {
+      static_assert(sizeof...(Args) == arity, "Wrong number of arguments");
+      auto ptrs = mfem::future::make_tuple(&args...);
+
+      auto &grad = *mfem::future::get<int(active_output)>(ptrs);
+      static_assert(std::is_same_v<std::decay_t<decltype(grad)>, grad_type>,
+                    "gradient argument must be shaped like the Active input "
+                    "(with writable scalars)");
+
+      constexpr auto seq = std::make_index_sequence<arity> {};
+
+      output_view primal{};
+      auto primal_ptrs = make_primal_ptrs(ptrs, primal, seq);
+
+      auto shadows = make_shadows(seq);
+      auto shadow_ptrs = make_shadow_ptrs(shadows, seq);
+      auto &seed = mfem::future::get<int(active_input)>(shadows);
+      auto &tangent = mfem::future::get<int(active_output)>(shadows);
+
+      // One enzyme call per input component d: seed e_d in the input tangent
+      // and read gradient entry d off the output tangent.
+      for (int d = 0; d < grad_components; d++)
+      {
+         component(seed, d) = 1.0;
+         call_enzyme(primal_ptrs, shadow_ptrs, seq);
+         component(grad, d) = component(tangent, 0);
+         component(seed, d) = 0.0;
+      }
+   }
+
+   static void print() { print_impl(std::make_index_sequence<arity> {}); }
+
+   template <size_t... Is> static void print_impl(std::index_sequence<Is...>)
+   {
+      mfem::out << "for d in [0, " << grad_components
+                << "): __enzyme_fwddiff<void>(fptr, enzyme_dup";
+      ((mfem::out << ", "
+        << get_type_name<std::tuple_element_t<Is, args_tuple>>()),
+       ...);
+      mfem::out << ", enzyme_interleave";
+      (([&]
+      {
+         if constexpr (Is == active_input) { mfem::out << ", e_d seed"; }
+         else if constexpr (Is == active_output) { mfem::out << ", tangent out"; }
+         else { mfem::out << ", zero tangent"; }
+      }()),
+      ...);
+      mfem::out << ")\n";
+   }
+};
+
+template <typename Func, size_t active_input, size_t active_output>
+struct create_function_signature<FwdDiff<Func, active_input, active_output>>
+{
+   using type =
+      typename FwdDiff<Func, active_input, active_output>::signature;
+};
+
+} // namespace mfem::future
+
+#endif // MFEM_USE_ENZYME

fem/dfem/backends/local_qf/kernels.cpp

@@ -0,0 +1,83 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Explicit instantiation of the local Q-function backend device code
+
+#include "../../../../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "kernels.hpp"
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+// Low-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+template struct lo_ker_backend<2, 2>;
+template struct lo_ker_backend<2, 3>;
+template struct lo_ker_backend<2, 4>;
+template struct lo_ker_backend<2, 5>;
+template struct lo_ker_backend<2, 6>;
+template struct lo_ker_backend<2, 7>;
+template struct lo_ker_backend<2, 8>;
+
+template struct lo_ker_backend<3, 2>;
+template struct lo_ker_backend<3, 3>;
+template struct lo_ker_backend<3, 4>;
+template struct lo_ker_backend<3, 5>;
+template struct lo_ker_backend<3, 6>;
+template struct lo_ker_backend<3, 7>;
+template struct lo_ker_backend<3, 8>;
+
+template struct LocalQFLOBackend<2, 2>;
+template struct LocalQFLOBackend<2, 3>;
+template struct LocalQFLOBackend<2, 4>;
+template struct LocalQFLOBackend<2, 5>;
+template struct LocalQFLOBackend<2, 6>;
+template struct LocalQFLOBackend<2, 7>;
+template struct LocalQFLOBackend<2, 8>;
+
+template struct LocalQFLOBackend<3, 2>;
+template struct LocalQFLOBackend<3, 3>;
+template struct LocalQFLOBackend<3, 4>;
+template struct LocalQFLOBackend<3, 5>;
+template struct LocalQFLOBackend<3, 6>;
+template struct LocalQFLOBackend<3, 7>;
+template struct LocalQFLOBackend<3, 8>;
+
+// ────────────────────────────────────────────────────────────────────────────
+// High-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+template struct ho_ker_backend<2, 8>;
+template struct ho_ker_backend<2, 10>;
+template struct ho_ker_backend<2, 12>;
+template struct ho_ker_backend<2, 16>;
+
+template struct ho_ker_backend<3, 8>;
+template struct ho_ker_backend<3, 10>;
+template struct ho_ker_backend<3, 12>;
+template struct ho_ker_backend<3, 16>;
+
+template struct LocalQFHOBackend<2, 8>;
+template struct LocalQFHOBackend<2, 10>;
+template struct LocalQFHOBackend<2, 12>;
+template struct LocalQFHOBackend<2, 16>;
+
+template struct LocalQFHOBackend<3, 8>;
+template struct LocalQFHOBackend<3, 10>;
+template struct LocalQFHOBackend<3, 12>;
+template struct LocalQFHOBackend<3, 16>;
+
+} // namespace mfem::future
+
+#endif // MFEM_USE_MPI

fem/dfem/backends/local_qf/kernels.hpp

@@ -0,0 +1,77 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "kernels_lo.hpp" // IWYU pragma: export
+#include "kernels_ho.hpp" // IWYU pragma: export
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+// Low-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+extern template struct lo_ker_backend<2, 2>;
+extern template struct lo_ker_backend<2, 3>;
+extern template struct lo_ker_backend<2, 4>;
+extern template struct lo_ker_backend<2, 5>;
+extern template struct lo_ker_backend<2, 6>;
+extern template struct lo_ker_backend<2, 7>;
+extern template struct lo_ker_backend<2, 8>;
+
+extern template struct lo_ker_backend<3, 2>;
+extern template struct lo_ker_backend<3, 3>;
+extern template struct lo_ker_backend<3, 4>;
+extern template struct lo_ker_backend<3, 5>;
+extern template struct lo_ker_backend<3, 6>;
+extern template struct lo_ker_backend<3, 7>;
+extern template struct lo_ker_backend<3, 8>;
+
+extern template struct LocalQFLOBackend<2, 2>;
+extern template struct LocalQFLOBackend<2, 3>;
+extern template struct LocalQFLOBackend<2, 4>;
+extern template struct LocalQFLOBackend<2, 5>;
+extern template struct LocalQFLOBackend<2, 6>;
+extern template struct LocalQFLOBackend<2, 7>;
+extern template struct LocalQFLOBackend<2, 8>;
+
+extern template struct LocalQFLOBackend<3, 2>;
+extern template struct LocalQFLOBackend<3, 3>;
+extern template struct LocalQFLOBackend<3, 4>;
+extern template struct LocalQFLOBackend<3, 5>;
+extern template struct LocalQFLOBackend<3, 6>;
+extern template struct LocalQFLOBackend<3, 7>;
+extern template struct LocalQFLOBackend<3, 8>;
+
+// ────────────────────────────────────────────────────────────────────────────
+// High-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+extern template struct ho_ker_backend<2, 8>;
+extern template struct ho_ker_backend<2, 10>;
+extern template struct ho_ker_backend<2, 12>;
+extern template struct ho_ker_backend<2, 16>;
+
+extern template struct ho_ker_backend<3, 8>;
+extern template struct ho_ker_backend<3, 10>;
+extern template struct ho_ker_backend<3, 12>;
+extern template struct ho_ker_backend<3, 16>;
+
+extern template struct LocalQFHOBackend<2, 8>;
+extern template struct LocalQFHOBackend<2, 10>;
+extern template struct LocalQFHOBackend<2, 12>;
+extern template struct LocalQFHOBackend<2, 16>;
+
+extern template struct LocalQFHOBackend<3, 8>;
+extern template struct LocalQFHOBackend<3, 10>;
+extern template struct LocalQFHOBackend<3, 12>;
+extern template struct LocalQFHOBackend<3, 16>;
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/kernels_ho.hpp

@@ -0,0 +1,803 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../../kernels.hpp"
+namespace ker = mfem::kernels::internal;
+
+#include "../../util.hpp" // for ThreadBlocks
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+inline constexpr int LocalQFHOBackendMQ1() { return 16; }
+
+// ────────────────────────────────────────────────────────────────────────────
+/// Register type for one HO q-function parameter
+template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
+struct ho_qreg;
+
+template<typename KerOps, typename T>
+struct ho_qreg<KerOps, T, 0>
+{
+   using type = typename KerOps::template val_reg_t<1>;
+};
+
+template<typename KerOps, typename T>
+struct ho_qreg<KerOps, T, 1>
+{
+   static constexpr int e0 = qf_param_shape<T>::extents[0];
+   using type = typename KerOps::template val_reg_t<e0>;
+};
+
+template<typename KerOps, typename T>
+struct ho_qreg<KerOps, T, 2>
+{
+   static constexpr int VDIM = qf_param_shape<T>::extents[0];
+   static constexpr int SDIM = qf_param_shape<T>::extents[1];
+   using type = typename KerOps::template del_reg_t<VDIM, SDIM>;
+};
+
+template<typename KerOps, typename T>
+using ho_qreg_t = typename ho_qreg<KerOps, T>::type;
+
+// ────────────────────────────────────────────────────────────────────────────
+namespace hok
+{
+
+/// Load one quadrature-point value
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   constexpr int RNK = qf_param_shape<T>::rank;
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz);
+      if constexpr (RNK == 0) { return T{ reg(0, qy, qx) }; }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         T t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qy, qx); }
+         return t;
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         T t;
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qy, qx); }
+         }
+         return t;
+      }
+   }
+   else
+   {
+      if constexpr (RNK == 0) { return T{ reg(0, qz, qy, qx) }; }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         T t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qz, qy, qx); }
+         return t;
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         T t;
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qz, qy, qx); }
+         }
+         return t;
+      }
+   }
+}
+
+template<bool tangent, typename U>
+MFEM_HOST_DEVICE inline auto qp_store(const U &v)
+{
+   if constexpr (tangent) { return qf_store_gradient(v); }
+   else
+   {
+      return qf_store_value(v);
+   }
+}
+
+// Store primal value or dual tangent at one quadrature point
+template<int DIM, typename T, typename Reg, bool tangent>
+MFEM_HOST_DEVICE inline void
+store_at(Reg &reg, int qx, int qy, int qz, const T &out)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   constexpr int RNK = qf_param_shape<T>::rank;
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz);
+      if constexpr (RNK == 0) { reg(0, qy, qx) = qp_store<tangent>(out); }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            reg(dd, qy, qx) = qp_store<tangent>(out(dd));
+         }
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               reg(i, j, qy, qx) = qp_store<tangent>(out(i, j));
+            }
+         }
+      }
+   }
+   else
+   {
+      if constexpr (RNK == 0) { reg(0, qz, qy, qx) = qp_store<tangent>(out); }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            reg(dd, qz, qy, qx) = qp_store<tangent>(out(dd));
+         }
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               reg(i, j, qz, qy, qx) = qp_store<tangent>(out(i, j));
+            }
+         }
+      }
+   }
+}
+
+// Pull primal/tangent pair into a dual q-function argument
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto
+pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
+{
+   if constexpr (!qf_param_uses_dual_v<T>)
+   {
+      return load_at<DIM, T>(preg, qx, qy, qz);
+   }
+   else
+   {
+      if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
+      constexpr int RNK = qf_param_shape<T>::rank;
+      if constexpr (DIM == 2)
+      {
+         MFEM_CONTRACT_VAR(qz);
+         if constexpr (RNK == 0)
+         {
+            return T{ preg(0, qy, qx), sreg(0, qy, qx) };
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            T t{};
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               t(dd) = { preg(dd, qy, qx), sreg(dd, qy, qx) };
+            }
+            return t;
+         }
+         else
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            T t;
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  t(i, j) = { preg(i, j, qy, qx), sreg(i, j, qy, qx) };
+               }
+            }
+            return t;
+         }
+      }
+      else
+      {
+         if constexpr (RNK == 0)
+         {
+            return T{ preg(0, qz, qy, qx), sreg(0, qz, qy, qx) };
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            T t{};
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               t(dd) = { preg(dd, qz, qy, qx), sreg(dd, qz, qy, qx) };
+            }
+            return t;
+         }
+         else
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            T t;
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  t(i, j) = { preg(i, j, qz, qy, qx), sreg(i, j, qz, qy, qx) };
+               }
+            }
+            return t;
+         }
+      }
+   }
+}
+
+} // namespace hok
+
+// ────────────────────────────────────────────────────────────────────────────
+/// HO tensor-product kernels
+template<int T_DIM, int MQ1>
+struct ho_ker_backend
+{
+   static constexpr int DIM = T_DIM;
+   static_assert(DIM == 2 || DIM == 3);
+
+   template<int VDIM>
+   using val_reg_t = std::conditional_t<(DIM == 2),
+         ker::v_regs2d_t<VDIM, MQ1>,
+         ker::v_regs3d_t<VDIM, MQ1>>;
+
+   template<int VDIM, int SDIM>
+   using del_reg_t = std::conditional_t<(DIM == 2),
+         ker::vd_regs2d_t<VDIM, SDIM, MQ1>,
+         ker::vd_regs3d_t<VDIM, SDIM, MQ1>>;
+
+   struct Shared
+   {
+      real_t M[MQ1][MQ1], B[MQ1][MQ1], G[MQ1][MQ1];
+   };
+
+   template<typename XE_t, typename Dofs>
+   static MFEM_HOST_DEVICE void
+   load_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
+   {
+      if constexpr (DIM == 2) { ker::LoadDofs2d(e, d, XE, dofs); }
+      else
+      {
+         ker::LoadDofs3d(e, d, XE, dofs);
+      }
+   }
+
+   template<int VDIM, int SDIM, typename XE_t, typename Dofs>
+   static MFEM_HOST_DEVICE void
+   load_grad_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
+   {
+      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
+      load_dofs(e, d, XE, dofs);
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg>
+   static MFEM_HOST_DEVICE void
+   eval_value(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
+   {
+      if constexpr (DIM == 2) { ker::Eval2d(d, q, s.M, s.B, dofs, rarg); }
+      else
+      {
+         ker::Eval3d(d, q, s.M, s.B, dofs, rarg);
+      }
+   }
+
+   template<int VDIM, int SDIM, typename Smem, typename Dofs, typename ArgReg>
+   static MFEM_HOST_DEVICE void
+   grad(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
+   {
+      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
+      if constexpr (DIM == 2) { ker::Grad2d(d, q, s.M, s.B, s.G, dofs, rarg); }
+      else
+      {
+         ker::Grad3d(d, q, s.M, s.B, s.G, dofs, rarg);
+      }
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
+   static MFEM_HOST_DEVICE void write_value(const int d,
+                                            const int q,
+                                            const int e,
+                                            Smem &s,
+                                            ArgReg &rarg,
+                                            Dofs &dofs,
+                                            YE_t &YE)
+   {
+      if constexpr (DIM == 2)
+      {
+         ker::EvalTranspose2d(d, q, s.M, s.B, rarg, dofs);
+         ker::WriteDofs2d(e, d, dofs, YE);
+      }
+      else
+      {
+         ker::EvalTranspose3d(d, q, s.M, s.B, rarg, dofs);
+         ker::WriteDofs3d(e, d, dofs, YE);
+      }
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
+   static MFEM_HOST_DEVICE void write_gradient_2d(const int d,
+                                                  const int q,
+                                                  const int e,
+                                                  Smem &s,
+                                                  ArgReg &rarg,
+                                                  Dofs &dofs,
+                                                  YE_t &YE)
+   {
+      ker::GradTranspose2d(d, q, s.M, s.B, s.G, rarg, dofs);
+      ker::WriteDofs2d(e, d, dofs, YE);
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
+   static MFEM_HOST_DEVICE void write_gradient_3d(const int d,
+                                                  const int q,
+                                                  const int e,
+                                                  Smem &s,
+                                                  ArgReg &rarg,
+                                                  Dofs &dofs,
+                                                  YE_t &YE)
+   {
+      ker::GradTranspose3d(d, q, s.M, s.B, s.G, rarg, dofs);
+      ker::WriteDofs3d(e, d, dofs, YE);
+   }
+
+   template<int VDIM,
+            int SDIM,
+            typename Smem,
+            typename Dofs,
+            typename ArgReg,
+            typename YE_t>
+   static MFEM_HOST_DEVICE void write_gradient(const int d,
+                                               const int q,
+                                               const int e,
+                                               Smem &s,
+                                               ArgReg &rarg,
+                                               Dofs &dofs,
+                                               YE_t &YE)
+   {
+      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
+      if constexpr (DIM == 2) { write_gradient_2d(d, q, e, s, rarg, dofs, YE); }
+      else
+      {
+         write_gradient_3d(d, q, e, s, rarg, dofs, YE);
+      }
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int T_DIM, int T_Q1D = LocalQFHOBackendMQ1()>
+struct LocalQFHOBackend
+{
+   // ─────────────────────────────────────────────────────
+   static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
+   static_assert(DIM == 2 || DIM == 3);
+
+   // ─────────────────────────────────────────────────────
+   static inline ThreadBlocks thread_blocks(const int q1d)
+   {
+      MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
+      return { q1d, q1d, 1 };
+   }
+
+   // ─────────────────────────────────────────────────────
+   static inline constexpr int MAX_THREADS_PER_BLOCK() { return Q1D * Q1D; }
+
+   // ─────────────────────────────────────────────────────
+   using backend_t = ho_ker_backend<DIM, Q1D>;
+
+   // ─────────────────────────────────────────────────────
+   using Shared = typename backend_t::Shared;
+
+   // ─────────────────────────────────────────────────────
+   template<typename WT, typename WI, typename Cache, typename AddY>
+   static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
+                                                    const int num_dof_1d,
+                                                    const int q1d,
+                                                    const int nz_dof,
+                                                    WT wt,
+                                                    WI wi,
+                                                    Cache cache,
+                                                    AddY add_y)
+   {
+      MFEM_CONTRACT_VAR(nz_dof);
+      const int nqz = (DIM == 3) ? q1d : 1;
+      const int ndz = (DIM == 3) ? num_dof_1d : 1;
+
+      ker::s_regs3d_t<MQ1> rz, ry;
+      auto &smem = s.M;
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            for (int dz = 0; dz < ndz; dz++)
+            {
+               real_t u = 0.0;
+               for (int qz = 0; qz < nqz; qz++)
+               {
+                  const int q = qx + (qy + qz * q1d) * q1d;
+                  const real_t wz =
+                     (DIM == 3) ? (wt(2, qz, dz) * wi(2, qz, dz)) : real_t(1);
+                  u += wz * cache(q);
+               }
+               rz[dz][qy][qx] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int dz = 0; dz < ndz; dz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+            { smem[qy][qx] = rz[dz][qy][qx]; }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < q1d; qy++)
+               {
+                  u += wt(1, qy, dy) * wi(1, qy, dy) * smem[qy][qx];
+               }
+               ry[dz][dy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      for (int dz = 0; dz < ndz; dz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+            { smem[dy][qx] = ry[dz][dy][qx]; }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, num_dof_1d)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < q1d; qx++)
+               {
+                  u += wt(0, qx, dx) * wi(0, qx, dx) * smem[dy][qx];
+               }
+               add_y(dx, dy, dz, u);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   using QReg = ho_qreg_t<backend_t, T>;
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename XE_T>
+   static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
+                                                 const int e,
+                                                 const int d,
+                                                 const int q,
+                                                 const int,
+                                                 const real_t *B,
+                                                 const XE_T &XE,
+                                                 ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      std::remove_reference_t<ArgRegT> dofs;
+      backend_t::load_dofs(e, d, XE, dofs);
+      backend_t::eval_value(d, q, s, dofs, rarg);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename XE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
+                                                    const int e,
+                                                    const int d,
+                                                    const int q,
+                                                    const int,
+                                                    const real_t *B,
+                                                    const real_t *G,
+                                                    const XE_T &XE,
+                                                    ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      static constexpr int VDIM =
+         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
+      static constexpr int SDIM = (RNK == 1)
+                                  ? qf_param_shape<FieldParamT>::extents[0]
+                                  : qf_param_shape<FieldParamT>::extents[1];
+      if constexpr (SDIM == DIM)
+      {
+         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
+         if constexpr (RNK == 1) { backend_t::load_dofs(e, d, XE, dofs); }
+         else
+         {
+            backend_t::template load_grad_dofs<VDIM, SDIM>(e, d, XE, dofs);
+         }
+         backend_t::template grad<VDIM, SDIM>(d, q, s, dofs, rarg);
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto
+   qp_pull(QReg<T> &reg, int qx, int qy, int qz)
+   { return hok::load_at<DIM, T>(reg, qx, qy, qz); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto qp_pull_directional(
+      QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
+   { return hok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename XE_T>
+   static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
+                                                           const XE_T &XP,
+                                                           const XE_T &XD,
+                                                           int qx,
+                                                           int qy,
+                                                           int qz,
+                                                           int e)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (RNK == 0)
+      {
+         DT t{};
+         t.value = XP(0, qx, qy, qz, e);
+         t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
+         return t;
+      }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            t(dd).value = XP(dd, qx, qy, qz, e);
+            t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
+         }
+         return t;
+      }
+      else if constexpr (RNK == 2)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         constexpr int e1 = qf_param_shape<DT>::extents[1];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
+               t(i, j).gradient =
+                  dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
+            }
+         }
+         return t;
+      }
+      else
+      {
+         static_assert(false, "Unsupported");
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   { hok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   {
+      hok::store_at<DIM, T, decltype(reg), qf_param_uses_dual_v<T>>(
+                                                                    reg, qx, qy, qz, out);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_value(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_value(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename YE_T>
+   static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
+                                                  const int e,
+                                                  const int d,
+                                                  const int q,
+                                                  const int,
+                                                  const real_t *B,
+                                                  YE_T &YE,
+                                                  ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      std::remove_reference_t<ArgRegT> dofs;
+      backend_t::write_value(d, q, e, s, rarg, dofs, YE);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename YE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
+                                                     const int e,
+                                                     const int d,
+                                                     const int q,
+                                                     const int,
+                                                     const real_t *B,
+                                                     const real_t *G,
+                                                     YE_T &YE,
+                                                     ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      static constexpr int VDIM =
+         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
+      static constexpr int SDIM = (RNK == 1)
+                                  ? qf_param_shape<FieldParamT>::extents[0]
+                                  : qf_param_shape<FieldParamT>::extents[1];
+      if constexpr (SDIM == DIM)
+      {
+         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
+         backend_t::template write_gradient<VDIM, SDIM>(
+            d, q, e, s, rarg, dofs, YE);
+      }
+   }
+};
+
+/// @brief Dispatch to a compile-time HO kernel with MQ1 >= runtime @a q1d.
+template <typename HOKernelTable, int DIM, int MQ1 = LocalQFHOBackendMQ1()>
+inline typename HOKernelTable::KernelSignature
+DispatchHOKernelByQ1D(int q1d)
+{
+   MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
+               "Unsupported HO quadrature order: " << q1d);
+   return HOKernelTable::template Kernel<DIM, MQ1>();
+}
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/kernels_lo.hpp

@@ -0,0 +1,813 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../../kernels.hpp"
+namespace ker = mfem::kernels::internal;
+
+#include "../../util.hpp" // for ThreadBlocks
+#include "../util.hpp"    // for as_tensor
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+inline constexpr int LocalQFLOBackendMQ1() { return 8; }
+
+// ────────────────────────────────────────────────────────────────────────────
+/// Register type for one LO q-function parameter
+template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
+struct lo_qreg;
+
+template<typename KerOps, typename T>
+struct lo_qreg<KerOps, T, 0>
+{
+   using type = typename KerOps::template qreg_t<1>;
+};
+
+template<typename KerOps, typename T>
+struct lo_qreg<KerOps, T, 1>
+{
+   static constexpr int e0 = qf_param_shape<T>::extents[0];
+   using type = typename KerOps::template qreg_t<e0>;
+};
+
+template<typename KerOps, typename T>
+struct lo_qreg<KerOps, T, 2>
+{
+   static constexpr int e0 = qf_param_shape<T>::extents[0];
+   static constexpr int e1 = qf_param_shape<T>::extents[1];
+   using type = typename KerOps::template qreg_vd_t<e0, e1>;
+};
+
+template<typename KerOps, typename T>
+using lo_qreg_t = typename lo_qreg<KerOps, T>::type;
+
+// ────────────────────────────────────────────────────────────────────────────
+namespace lok
+{
+
+template<int DIM, typename Reg>
+MFEM_HOST_DEVICE inline auto &at(Reg &reg, int qx, int qy, int qz)
+{
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz);
+      return reg[qy][qx];
+   }
+   else
+   {
+      return reg[qz][qy][qx];
+   }
+}
+
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
+{
+   constexpr int RNK = qf_param_shape<T>::rank;
+   auto &qp = at<DIM>(reg, qx, qy, qz);
+   if constexpr (RNK == 0) { return T{ qp[0] }; }
+   else if constexpr (RNK == 1)
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      T t{};
+      MFEM_UNROLL(e0)
+      for (int dd = 0; dd < e0; ++dd) { t(dd) = qp[dd]; }
+      return t;
+   }
+   else
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      constexpr int e1 = qf_param_shape<T>::extents[1];
+      T t;
+      MFEM_UNROLL(e0)
+      for (int i = 0; i < e0; ++i)
+      {
+         MFEM_UNROLL(e1)
+         for (int j = 0; j < e1; ++j) { t(i, j) = qp[i][j]; }
+      }
+      return t;
+   }
+}
+
+template<bool tangent, typename U>
+MFEM_HOST_DEVICE inline auto qp_store(const U &v)
+{
+   if constexpr (tangent) { return qf_store_gradient(v); }
+   else
+   {
+      return qf_store_value(v);
+   }
+}
+
+// Store primal value or dual tangent at one quadrature point
+template<int DIM, typename T, typename Reg, bool tangent>
+MFEM_HOST_DEVICE inline void
+store_at(Reg &reg, int qx, int qy, int qz, const T &out)
+{
+   constexpr int RNK = qf_param_shape<T>::rank;
+   auto &qp = at<DIM>(reg, qx, qy, qz);
+   if constexpr (RNK == 0) { qp[0] = qp_store<tangent>(out); }
+   else if constexpr (RNK == 1)
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      MFEM_UNROLL(e0)
+      for (int dd = 0; dd < e0; ++dd) { qp[dd] = qp_store<tangent>(out(dd)); }
+   }
+   else
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      constexpr int e1 = qf_param_shape<T>::extents[1];
+      MFEM_UNROLL(e0)
+      for (int i = 0; i < e0; ++i)
+      {
+         MFEM_UNROLL(e1)
+         for (int j = 0; j < e1; ++j)
+         {
+            qp[i][j] = qp_store<tangent>(out(i, j));
+         }
+      }
+   }
+}
+
+// Pull primal/tangent pair into a dual q-function argument
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto
+pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
+{
+   if constexpr (!qf_param_uses_dual_v<T>)
+   {
+      return load_at<DIM, T>(preg, qx, qy, qz);
+   }
+   else
+   {
+      if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
+      constexpr int RNK = qf_param_shape<T>::rank;
+      auto &pqp = at<DIM>(preg, qx, qy, qz);
+      auto &sqp = at<DIM>(sreg, qx, qy, qz);
+      if constexpr (RNK == 0) { return T{ pqp[0], sqp[0] }; }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         T t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd) { t(dd) = { pqp[dd], sqp[dd] }; }
+         return t;
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         T t;
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j) { t(i, j) = { pqp[i][j], sqp[i][j] }; }
+         }
+         return t;
+      }
+   }
+}
+
+} // namespace lok
+
+// ────────────────────────────────────────────────────────────────────────────
+/// LO tensor-product kernels
+template<int T_DIM, int MQ1>
+struct lo_ker_backend
+{
+   static constexpr int DIM = T_DIM;
+   static_assert(DIM == 2 || DIM == 3);
+
+   template<int VDIM>
+   using qreg_t = std::conditional_t<(DIM == 2),
+         ker::regs2d_t<VDIM, MQ1>,
+         ker::regs3d_t<VDIM, MQ1>>;
+
+   template<int VDIM, int SDIM>
+   using qreg_vd_t = std::conditional_t<(DIM == 2),
+         ker::regs2d_vd_t<VDIM, SDIM, MQ1>,
+         ker::regs3d_vd_t<VDIM, SDIM, MQ1>>;
+
+   struct Shared2d
+   {
+      real_t M[2][MQ1][MQ1][DIM];
+      real_t B[MQ1][MQ1], G[MQ1][MQ1];
+   };
+
+   struct Shared3d
+   {
+      real_t M[2][MQ1][MQ1][MQ1][DIM];
+      real_t B[MQ1][MQ1], G[MQ1][MQ1];
+   };
+
+   using Shared = std::conditional_t<(DIM == 2), Shared2d, Shared3d>;
+
+   template<typename FieldParamT, typename ArgRegT, typename XE_T>
+   static MFEM_HOST_DEVICE void load_value(Shared &s,
+                                           const int e,
+                                           const int d,
+                                           const int q,
+                                           const real_t *B,
+                                           const XE_T &XE,
+                                           ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      using field_t = std::remove_cv_t<std::remove_reference_t<FieldParamT>>;
+      constexpr int RNK = qf_param_shape<field_t>::rank;
+      constexpr int VDIM = []()
+      {
+         if constexpr (RNK == 0) { return 1; }
+         else { return qf_param_shape<field_t>::extents[RNK - 1]; }
+      }();
+      if constexpr (DIM == 2)
+      {
+         ker::LoadDofs2d<VDIM, DIM, MQ1>(e, d, XE, s.M[0]);
+         ker::Eval2d(d, q, s.B, s.M[0], s.M[1], rarg);
+      }
+      else
+      {
+         ker::LoadDofs3d<VDIM, DIM, MQ1>(e, d, XE, s.M[0]);
+         ker::Eval3d(d, q, s.B, s.M[0], s.M[1], rarg);
+      }
+   }
+
+   template<int RNK,
+            typename ArgRegT,
+            typename XE_T,
+            typename FieldParamT = ArgRegT>
+   static MFEM_HOST_DEVICE void load_gradient(Shared &s,
+                                              const int e,
+                                              const int d,
+                                              const int q,
+                                              const real_t *B,
+                                              const real_t *G,
+                                              const XE_T &XE,
+                                              ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      if constexpr (RNK == 1)
+      {
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[0];
+         if constexpr (SDIM == DIM)
+         {
+            if constexpr (DIM == 2)
+            {
+               ker::LoadDofs2d(e, d, 0, XE, s.M[0]);
+               ker::Grad2d(d, q, s.B, s.G, s.M[0], s.M[1], rarg);
+            }
+            else
+            {
+               ker::LoadDofs3d(e, d, 0, XE, s.M[0]);
+               ker::Grad3d(d, q, s.B, s.G, s.M[0], s.M[1], rarg);
+            }
+         }
+      }
+      if constexpr (RNK == 2)
+      {
+         static constexpr int VDIM = qf_param_shape<FieldParamT>::extents[0];
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[1];
+         if constexpr (SDIM == DIM)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               if constexpr (DIM == 2)
+               {
+                  ker::LoadDofs2d(e, d, c, XE, s.M[0]);
+                  ker::VectorGrad2d(d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+               }
+               else
+               {
+                  ker::LoadDofs3d(e, d, c, XE, s.M[0]);
+                  ker::VectorGrad3d(d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+               }
+            }
+         }
+      }
+   }
+
+   template<typename ArgRegT, typename YE_T>
+   static MFEM_HOST_DEVICE void write_value(Shared &s,
+                                            const int e,
+                                            const int d,
+                                            const int q,
+                                            const real_t *B,
+                                            const YE_T &YE,
+                                            ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      if constexpr (DIM == 2)
+      {
+         ker::EvalTranspose2d(d, q, s.B, rarg, s.M[1], s.M[0]);
+         ker::WriteEvalDofs2d(d, 0, e, rarg, YE);
+      }
+      else
+      {
+         ker::EvalTranspose3d(d, q, s.B, rarg, s.M[1], s.M[0]);
+         ker::WriteEvalDofs3d(d, 0, e, rarg, YE);
+      }
+   }
+
+   template<int RNK,
+            typename ArgRegT,
+            typename YE_T,
+            typename FieldParamT = ArgRegT>
+   static MFEM_HOST_DEVICE void write_gradient(Shared &s,
+                                               const int e,
+                                               const int d,
+                                               const int q,
+                                               const real_t *B,
+                                               const real_t *G,
+                                               YE_T &YE,
+                                               ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      if constexpr (RNK == 1)
+      {
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[0];
+         if constexpr (SDIM == DIM)
+         {
+            if constexpr (DIM == 2)
+            {
+               ker::GradTranspose2d(d, q, s.B, s.G, rarg, s.M[1], s.M[0]);
+               ker::WriteGradDofs2d(d, 0, e, rarg, YE);
+            }
+            else
+            {
+               ker::GradTranspose3d(d, q, s.B, s.G, rarg, s.M[1], s.M[0]);
+               ker::WriteGradDofs3d(d, 0, e, rarg, YE);
+            }
+         }
+      }
+      else if constexpr (RNK == 2)
+      {
+         static constexpr int VDIM = qf_param_shape<FieldParamT>::extents[0];
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[1];
+         if constexpr (SDIM == DIM)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               if constexpr (DIM == 2)
+               {
+                  ker::VectorGradTranspose2d(
+                     d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+                  ker::WriteGradDofs2d(d, c, e, rarg, YE);
+               }
+               else
+               {
+                  ker::VectorGradTranspose3d(
+                     d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+                  ker::WriteGradDofs3d(d, c, e, rarg, YE);
+               }
+            }
+         }
+      }
+      else
+      {
+         static_assert(false, "Unsupported");
+      }
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int T_DIM, int T_Q1D = LocalQFLOBackendMQ1()>
+struct LocalQFLOBackend
+{
+   // ─────────────────────────────────────────────────────
+   static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
+   static_assert(DIM == 2 || DIM == 3);
+
+   // ─────────────────────────────────────────────────────
+   static inline ThreadBlocks thread_blocks(const int q1d)
+   {
+      MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
+      return { q1d, q1d, (DIM == 2) ? 1 : q1d };
+   }
+
+   // ─────────────────────────────────────────────────────
+   static inline constexpr int MAX_THREADS_PER_BLOCK()
+   { return Q1D * Q1D * ((DIM == 2) ? 1 : Q1D); }
+
+   // ─────────────────────────────────────────────────────
+   using backend_t = lo_ker_backend<DIM, Q1D>;
+
+   // ─────────────────────────────────────────────────────
+   using Shared = typename backend_t::Shared;
+
+   // ─────────────────────────────────────────────────────
+   template<typename WT, typename WI, typename Cache, typename AddY>
+   static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
+                                                    const int num_dof_1d,
+                                                    const int q1d,
+                                                    const int nz_dof,
+                                                    WT wt,
+                                                    WI wi,
+                                                    Cache cache,
+                                                    AddY add_y)
+   {
+      MFEM_CONTRACT_VAR(nz_dof);
+      real_t *base = reinterpret_cast<real_t *>(&s.M[0]);
+      auto s0 = reinterpret_cast<real_t(*)[Q1D][Q1D]>(base);
+
+      if constexpr (DIM == 3)
+      {
+         auto s1 =
+            reinterpret_cast<real_t(*)[Q1D][Q1D]>(base + Q1D * Q1D * Q1D);
+
+         // reduce qz → dz : s0[dz][qy][qx]
+         MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t u = 0.0;
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               const int q = qx + (qy + qz * q1d) * q1d;
+               u += wt(2, qz, dz) * wi(2, qz, dz) * cache(q);
+            }
+            s0[dz][qy][qx] = u;
+         }
+         MFEM_SYNC_THREAD;
+
+         // reduce qy → dy : s1[dz][dy][qx]
+         MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t u = 0.0;
+            for (int qy = 0; qy < q1d; qy++)
+            {
+               u += wt(1, qy, dy) * wi(1, qy, dy) * s0[dz][qy][qx];
+            }
+            s1[dz][dy][qx] = u;
+         }
+         MFEM_SYNC_THREAD;
+
+         // reduce qx → dx : Y(dx,dy,dz)
+         MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(dx, x, num_dof_1d)
+         {
+            real_t u = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               u += wt(0, qx, dx) * wi(0, qx, dx) * s1[dz][dy][qx];
+            }
+            add_y(dx, dy, dz, u);
+         }
+         MFEM_SYNC_THREAD;
+      }
+      else
+      {
+         // reduce qy → dy : s0[0][dy][qx]
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t u = 0.0;
+            for (int qy = 0; qy < q1d; qy++)
+            {
+               const int q = qx + qy * q1d;
+               u += wt(1, qy, dy) * wi(1, qy, dy) * cache(q);
+            }
+            s0[0][dy][qx] = u;
+         }
+         MFEM_SYNC_THREAD;
+
+         // reduce qx → dx : Y(dx,dy,0)
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(dx, x, num_dof_1d)
+         {
+            real_t u = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               u += wt(0, qx, dx) * wi(0, qx, dx) * s0[0][dy][qx];
+            }
+            add_y(dx, dy, 0, u);
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   using QReg = lo_qreg_t<backend_t, T>;
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename XE_T>
+   static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
+                                                 const int e,
+                                                 const int d,
+                                                 const int q,
+                                                 const int,
+                                                 const real_t *B,
+                                                 const XE_T &XE,
+                                                 ArgRegT &rarg)
+   {
+      backend_t::template load_value<ArgRegT>(s, e, d, q, B, XE, rarg);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename XE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
+                                                    const int e,
+                                                    const int d,
+                                                    const int q,
+                                                    const int,
+                                                    const real_t *B,
+                                                    const real_t *G,
+                                                    const XE_T &XE,
+                                                    ArgRegT &rarg)
+   {
+      backend_t::template load_gradient<RNK, ArgRegT, XE_T, FieldParamT>(
+         s, e, d, q, B, G, XE, rarg);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto
+   qp_pull(QReg<T> &reg, int qx, int qy, int qz)
+   {
+      if constexpr (qf_param_uses_dual_v<T>)
+      {
+         return lok::load_at<DIM, T>(reg, qx, qy, qz);
+      }
+      else
+      {
+         constexpr int RNK = qf_param_shape<T>::rank;
+         if constexpr (RNK == 0)
+         {
+            return as_tensor<real_t>(&lok::at<DIM>(reg, qx, qy, qz)[0]);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            return as_tensor<real_t, e0>(&lok::at<DIM>(reg, qx, qy, qz)[0]);
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            return as_tensor<real_t, e0, e1>(
+                      &lok::at<DIM>(reg, qx, qy, qz)[0][0]);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto qp_pull_directional(
+      QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
+   { return lok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename XE_T>
+   static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
+                                                           const XE_T &XP,
+                                                           const XE_T &XD,
+                                                           int qx,
+                                                           int qy,
+                                                           int qz,
+                                                           int e)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (RNK == 0)
+      {
+         DT t{};
+         t.value = XP(0, qx, qy, qz, e);
+         t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
+         return t;
+      }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            t(dd).value = XP(dd, qx, qy, qz, e);
+            t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
+         }
+         return t;
+      }
+      else if constexpr (RNK == 2)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         constexpr int e1 = qf_param_shape<DT>::extents[1];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
+               t(i, j).gradient =
+                  dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
+            }
+         }
+         return t;
+      }
+      else
+      {
+         static_assert(false, "Unsupported");
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   {
+      if constexpr (qf_param_uses_dual_v<T>)
+      {
+         lok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out);
+      }
+      else
+      {
+         constexpr int RNK = qf_param_shape<T>::rank;
+         if constexpr (RNK == 0)
+         {
+            as_tensor<real_t>(&lok::at<DIM>(reg, qx, qy, qz)[0]) = out;
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            as_tensor<real_t, e0>(&lok::at<DIM>(reg, qx, qy, qz)[0]) = out;
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            as_tensor<real_t, e0, e1>(&lok::at<DIM>(reg, qx, qy, qz)[0][0]) =
+               out;
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   {
+      if constexpr (!qf_param_uses_dual_v<T>)
+      {
+         qp_push<T>(reg, qx, qy, qz, out);
+      }
+      else
+      {
+         lok::store_at<DIM, T, decltype(reg), true>(reg, qx, qy, qz, out);
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_value(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_value(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename YE_T>
+   static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
+                                                  const int e,
+                                                  const int d,
+                                                  const int q,
+                                                  const int,
+                                                  const real_t *B,
+                                                  const YE_T &YE,
+                                                  ArgRegT &rarg)
+   { backend_t::write_value(s, e, d, q, B, YE, rarg); }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename YE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
+                                                     const int e,
+                                                     const int d,
+                                                     const int q,
+                                                     const int,
+                                                     const real_t *B,
+                                                     const real_t *G,
+                                                     YE_T &YE,
+                                                     ArgRegT &rarg)
+   {
+      backend_t::template write_gradient<RNK, ArgRegT, YE_T, FieldParamT>(
+         s, e, d, q, B, G, YE, rarg);
+   }
+};
+
+/// @brief Dispatch to a compile-time LO kernel matching runtime @a q1d.
+template <typename LOKernelTable, int DIM, int MQ1 = LocalQFLOBackendMQ1()>
+inline typename LOKernelTable::KernelSignature
+DispatchLOKernelByQ1D(int q1d)
+{
+   MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
+               "Unsupported LO quadrature order: " << q1d);
+   return LOKernelTable::template Kernel<DIM, MQ1>();
+}
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/prelude.hpp

@@ -0,0 +1,242 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "action.hpp"
+#include "derivative_action.hpp"
+#include "derivative_setup.hpp"
+#include "derivative_apply.hpp"
+#include "derivative_assemble.hpp"
+#include "derivative_assemble_diagonal.hpp"
+#include "derivative_apply_transpose.hpp"
+
+namespace mfem::future
+{
+
+struct LocalQFBackend
+{
+   /**
+    * @brief Make an action for a local Q-function backend.
+    *
+    * @param ctx The integrator context.
+    * @param args The arguments to the action.
+    * @return The action.
+    */
+   template<typename qfunc_t, typename inputs_t, typename outputs_t>
+   static auto MakeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return LocalQFImpl::Action<qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs);
+   }
+
+   /**
+    * @brief Make a derivative action for a local Q-function backend.
+    *
+    * @tparam derivative_id The id of the derivative.
+    * @param ctx The integrator context.
+    * @param args The arguments to the derivative action.
+    * @return The derivative action.
+    */
+   template<int id, typename qfunc_t, typename inputs_t, typename outputs_t>
+   static auto MakeDerivativeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return LocalQFImpl::DerivativeAction<id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeSetup(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeSetup<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeApply(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeApply<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeApplyTranspose(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeApplyTranspose<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeAssemble(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssemble<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeAssembleDiagonal(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssembleDiagonal<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, typename QT, typename IT, typename OT>
+inline void AddAction()
+{
+   using ker = LocalQFImpl::Action<QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::ActionLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::ActionHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeAction()
+{
+   using ker = LocalQFImpl::DerivativeAction<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeActionLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeActionHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeSetup()
+{
+   using ker = LocalQFImpl::DerivativeSetup<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeSetupLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeSetupHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeApply()
+{
+   using ker = LocalQFImpl::DerivativeApply<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeApplyLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeApplyHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeApplyTranspose()
+{
+   using ker = LocalQFImpl::DerivativeApplyTranspose<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeApplyTransposeLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeApplyTransposeHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, typename QT, typename IT, typename OT,
+         typename derivative_ids_t = std::index_sequence<>>
+inline void AddLocalSpecializations()
+{
+   AddAction<DIM, Q1D, QT, IT, OT>();
+
+   for_constexpr([&](auto i)
+   {
+      using derivative_id = decltype(i);
+      AddDerivativeAction<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+      AddDerivativeSetup<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+      AddDerivativeApply<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+      AddDerivativeApplyTranspose<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+   }, derivative_ids_t{});
+}
+
+}

fem/dfem/backends/local_qf/revdiff_transformer.hpp

@@ -0,0 +1,274 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../util.hpp"
+
+#ifdef MFEM_USE_ENZYME
+
+namespace mfem::future
+{
+
+template <typename T> struct function_traits;
+
+template <typename R, typename C, typename... Args>
+struct function_traits<R (C::*)(Args...) const>
+{
+   using primal_return_type = R;
+   using args_tuple = tuple<Args...>;
+   static constexpr size_t arity = sizeof...(Args);
+};
+
+// Component count and writable counterpart of a per-point argument, which
+// is either a tensor or a plain scalar.
+template <typename Arg> struct qp_traits
+{
+   static_assert(std::is_arithmetic_v<Arg>,
+                 "per-point arguments must be tensors or scalars");
+   using view_type = Arg;
+   static constexpr int components = 1;
+};
+
+template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
+{
+   using view_type = tensor<std::remove_const_t<T>, Sizes...>;
+   static constexpr int components = (Sizes * ... * 1);
+};
+
+template <typename... T1s, typename... T2s>
+constexpr tuple<T1s..., T2s...> concat_tuples(tuple<T1s...>, tuple<T2s...>);
+
+// RevDiff: computes the full gradient of a pointwise qfunction at a single
+// quadrature point using one Enzyme reverse-mode (autodiff) call.
+//
+// operator()(args...) takes all qfunction input primals followed by one
+// writable gradient output per Active input (same shape as that input).
+// The qfunction's own output is not passed; Enzyme writes it to stack
+// scratch (enzyme_dupnoneed).
+//
+// A single __enzyme_autodiff call with the output adjoint seeded to 1
+// yields all gradient blocks simultaneously — O(1) calls per point
+// regardless of input size, vs O(ncomp) for forward mode.
+//
+// operator() is MFEM_HOST_DEVICE, stateless and allocation-free.
+template <typename Func, typename InputActivityTuple,
+          typename OutputActivityTuple>
+struct RevDiff
+{
+   using traits = function_traits<decltype(&Func::operator())>;
+   using args_tuple = typename traits::args_tuple;
+   using activity =
+      decltype(concat_tuples(InputActivityTuple{}, OutputActivityTuple{}));
+   static constexpr size_t arity = traits::arity;
+   static constexpr size_t num_inputs = tuple_size<InputActivityTuple>::value;
+
+   static_assert(std::is_void_v<typename traits::primal_return_type>,
+                 "RevDiff only supports primal functions with void return type");
+   static_assert(tuple_size<activity>::value == arity,
+                 "Number of input and output activity tags must match function "
+                 "arity");
+
+   template <size_t I>
+   static constexpr bool is_active =
+      std::is_same_v<tuple_element_t<I, activity>, Active>;
+
+   // Index of the single Active tag in [Lo, Hi), or arity if not exactly one.
+   template <size_t Lo, size_t Hi, size_t... Is>
+   static constexpr size_t find_single_active(std::index_sequence<Is...>)
+   {
+      size_t idx = arity, count = 0;
+      (((Is >= Lo && Is < Hi && is_active<Is>) ? (idx = Is, ++count) : size_t{0}),
+       ...);
+      return count == 1 ? idx : arity;
+   }
+
+   // Number of Active inputs and their argument indices, in ascending order.
+   // A qfunction may have several Active inputs at once: e.g. a field's value
+   // u and its gradient dudx both feed the output and both must be
+   // differentiated (the chain-rule contraction with the value/gradient shape
+   // functions then happens at the FE-operator level). We produce one gradient
+   // block, d(output)/d(input), per Active input — each computed with the other
+   // Active inputs frozen, so they come out as isolated partials.
+   template <size_t... Is>
+   static constexpr size_t count_active_inputs(std::index_sequence<Is...>)
+   {
+      return ((Is < num_inputs && is_active<Is> ? size_t{1} : size_t{0}) + ...);
+   }
+   static constexpr size_t num_active_inputs =
+      count_active_inputs(std::make_index_sequence<arity> {});
+
+   template <size_t... Is>
+   static constexpr std::array<size_t, num_active_inputs>
+   collect_active_inputs(std::index_sequence<Is...>)
+   {
+      std::array<size_t, num_active_inputs> idx{};
+      size_t j = 0;
+      (((Is < num_inputs && is_active<Is>) ? (idx[j++] = Is) : size_t{0}), ...);
+      return idx;
+   }
+   static constexpr auto active_inputs =
+   collect_active_inputs(std::make_index_sequence<arity> {});
+
+   // Slot index of argument I in the active_inputs array (compile-time).
+   template <size_t I>
+   static constexpr size_t slot_of = []() constexpr -> size_t
+   {
+      for (size_t s = 0; s < num_active_inputs; s++)
+         if (active_inputs[s] == I) { return s; }
+      return num_active_inputs;
+   }();
+
+   static constexpr size_t active_output =
+      find_single_active<num_inputs, arity>(std::make_index_sequence<arity> {});
+   static_assert(active_output < arity,
+                 "gradient mode requires exactly one Active output");
+   static_assert(num_active_inputs >= 1,
+                 "gradient mode requires at least one Active input");
+   static_assert(tuple_size<OutputActivityTuple>::value == 1,
+                 "gradient mode requires exactly one (scalar) output");
+
+   using output_type =
+      std::decay_t<tuple_element_t<active_output, args_tuple>>;
+
+   using output_view = typename qp_traits<output_type>::view_type;
+
+   static_assert(qp_traits<output_type>::components == 1,
+                 "gradient output requires a scalar output");
+
+   template <size_t I>
+   using primal_arg_t = tuple_element_t<I, args_tuple>;
+
+   template <size_t S>
+   using grad_arg_t =
+      typename qp_traits<std::decay_t<tuple_element_t<active_inputs[S], args_tuple>>>::view_type
+      &;
+
+   template <size_t... Is, size_t... Ss>
+   static FunctionSignature<void(primal_arg_t<Is>..., grad_arg_t<Ss>...)>
+   signature_impl(std::index_sequence<Is...>, std::index_sequence<Ss...>);
+
+   using signature = decltype(signature_impl(std::make_index_sequence<num_inputs> {},
+                                             std::make_index_sequence<num_active_inputs> {}));
+
+   // Plain function with the qfunction's exact (reference) signature so it can
+   // be handed to Enzyme as a function pointer.
+   template <size_t... Is>
+   MFEM_HOST_DEVICE static __attribute__((always_inline)) void
+   static_call(tuple_element_t<Is, args_tuple>... args)
+   {
+      Func{}(args...);
+   }
+
+   template <size_t... Is>
+   static constexpr auto fn_ptr(std::index_sequence<Is...>)
+   {
+      return &static_call<Is...>;
+   }
+   static constexpr auto fn = fn_ptr(std::make_index_sequence<arity> {});
+
+   // Recursive builder of the per-argument reverse-mode enzyme call.
+   template <size_t I = 0, typename AllPtrs, typename... Built>
+   MFEM_HOST_DEVICE static __attribute__((always_inline)) void
+   call_enzyme_rev(AllPtrs &ptrs, output_view &scratch, output_view &adjoint,
+                   Built... built)
+   {
+      if constexpr (I == arity)
+      {
+         __enzyme_autodiff<void>(fn, built...);
+      }
+      else if constexpr (I == active_output)
+      {
+         // Output: primal written to scratch (unused), adjoint seeded to 1.
+         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
+                                enzyme_dupnoneed, &scratch, &adjoint);
+      }
+      else if constexpr (is_active<I>)
+      {
+         // Active input: gradient accumulates into its grad-output slot.
+         call_enzyme_rev<I + 1>(
+            ptrs, scratch, adjoint, built..., enzyme_dup,
+            mfem::future::get<int(I)>(ptrs),
+            mfem::future::get<int(num_inputs + slot_of<I>)>(ptrs));
+      }
+      else
+      {
+         // Const input: primal only, no shadow.
+         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
+                                enzyme_const, mfem::future::get<int(I)>(ptrs));
+      }
+   }
+
+   // Zero all gradient outputs before the enzyme call (Enzyme accumulates).
+   template <typename AllPtrs, size_t... Ss>
+   MFEM_HOST_DEVICE static
+   __attribute__((always_inline)) void zero_grads(
+      AllPtrs &ptrs,
+      std::index_sequence<Ss...>)
+   {
+      ((*mfem::future::get<int(num_inputs + Ss)>(ptrs) =
+           std::decay_t<decltype(*mfem::future::get<int(num_inputs + Ss)>(ptrs))> {}),
+       ...);
+   }
+
+   // Called once per quadrature point. Arguments are, in order:
+   //   * the primal value of every qfunction input (active and const), then
+   //   * one gradient output per Active input (ascending index order), each
+   //     shaped like its Active input.
+   // The qfunction's own output slot is not passed; Enzyme writes it to stack
+   // scratch (enzyme_dupnoneed). A single __enzyme_autodiff call yields all
+   // gradient blocks simultaneously.
+   template <typename... Args>
+   MFEM_HOST_DEVICE __attribute__((always_inline)) void operator()(
+      Args &&...args) const
+   {
+      static_assert(sizeof...(Args) == num_inputs + num_active_inputs,
+                    "expected one primal per input plus one gradient output per "
+                    "Active input");
+      auto ptrs = mfem::future::make_tuple(&args...);
+      zero_grads(ptrs, std::make_index_sequence<num_active_inputs> {});
+      output_view out_scratch{};
+      output_view out_adjoint{1.0}; // seed: d(output)/d(output) = 1
+      call_enzyme_rev(ptrs, out_scratch, out_adjoint);
+   }
+
+   static __attribute__((always_inline)) void print() { print_impl(std::make_index_sequence<arity> {}); }
+
+   template <size_t... Is> static __attribute__((always_inline)) void print_impl(
+      std::index_sequence<Is...>)
+   {
+      mfem::out << "__enzyme_autodiff<void>(fptr";
+      (([&]
+      {
+         auto name = get_type_name<tuple_element_t<Is, args_tuple>>();
+         if constexpr (Is == active_output)
+            mfem::out << ", enzyme_dupnoneed, " << name << ", adjoint=1";
+         else if constexpr (is_active<Is>)
+            mfem::out << ", enzyme_dup, " << name << ", grad out";
+         else
+         {
+            mfem::out << ", enzyme_const, " << name;
+         }
+      }()),
+      ...);
+      mfem::out << ")\n";
+   }
+};
+
+template <typename Func, typename InputActivityTuple, typename OutputActivityTuple>
+struct create_function_signature<RevDiff<Func, InputActivityTuple, OutputActivityTuple>>
+{
+   using type = typename
+                RevDiff<Func, InputActivityTuple, OutputActivityTuple>::signature;
+};
+
+} // namespace mfem::future
+
+#endif // MFEM_USE_ENZYME

fem/dfem/backends/local_qf/util.hpp

@@ -0,0 +1,622 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+// Compile-time quadrature argument metadata for local q-functions
+
+#include "../../../../linalg/tensor.hpp"
+
+#include "../../integrator_ctx.hpp"
+#include "../../util.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <type_traits>
+
+namespace mfem::future
+{
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename T>
+MFEM_HOST_DEVICE auto qf_store_value(const T &v)
+{
+   if constexpr (is_dual_number<T>::value) { return v.value; }
+   else { return v; }
+}
+
+template <typename T>
+MFEM_HOST_DEVICE auto qf_store_gradient(const T &v)
+{
+   if constexpr (is_dual_number<T>::value) { return v.gradient; }
+   else { return v; }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// True when quadrature-point values of `T` carry dual-number derivatives
+template <typename T>
+struct qf_param_uses_dual : std::false_type {};
+
+template <typename S, int... Is>
+struct qf_param_uses_dual<tensor<S, Is...>> : is_dual_number<S> {};
+
+template <typename V, typename G>
+struct qf_param_uses_dual<dual<V, G>> : std::true_type {};
+
+template <typename T>
+constexpr bool qf_param_uses_dual_v = qf_param_uses_dual<T>::value;
+
+///////////////////////////////////////////////////////////////////////////////
+/// Static shape for one decayed q-function parameter type
+template <typename T>
+struct qf_param_shape
+{
+   static constexpr int rank = 0;
+   static constexpr std::array<int, 0> extents {};
+};
+
+template <typename scalar_t, int... Is>
+struct qf_param_shape<tensor<scalar_t, Is...>>
+{
+   static constexpr int rank = sizeof...(Is);
+   static constexpr std::array<int, sizeof...(Is)> extents {{Is...}};
+};
+
+template <typename scalar_t>
+struct qf_param_shape<tensor<scalar_t>>
+{
+   static constexpr int rank = 0;
+   static constexpr std::array<int, 0> extents {};
+};
+
+template <>
+struct qf_param_shape<real_t>
+{
+   static constexpr int rank = 0;
+   static constexpr std::array<int, 0> extents {};
+};
+
+///////////////////////////////////////////////////////////////////////////////
+/// Type used in quadrature registers for parameter
+template <typename T>
+struct qf_reg_t { using type = T; };
+
+template <>
+struct qf_reg_t<real_t> { using type = tensor<real_t>; };
+
+///////////////////////////////////////////////////////////////////////////////
+/// Per-parameter tensor info for slot `I` in the decayed q-function parameter tuple
+template <typename qfunc_t, std::size_t I>
+struct qf_param_slot
+{
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using qf_raw_param_t = typename tuple_element<I, qf_param_ts>::type;
+   using qf_decay_param_t =
+      std::remove_cv_t<std::remove_reference_t<qf_raw_param_t>>;
+   using qf_reg_param_t = typename qf_reg_t<qf_decay_param_t>::type;
+
+   static constexpr auto extents = qf_param_shape<qf_decay_param_t>::extents;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+/// Builds a register-bank tuple covering the q-function parameter slots
+/// `[K0, N)`. `K` is the recursion cursor and starts at `K0`; the resulting
+/// tuple is indexed from 0, so a bank starting at `K0 > 0` has its slot indices
+/// rebased by `-K0` relative to the q-function parameter list.
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
+   std::size_t K0, std::size_t K, std::size_t N, typename... Acc>
+struct build_args_reg_tuple_impl;
+
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
+   std::size_t K0, std::size_t N, typename... Acc>
+struct build_args_reg_tuple_impl<backend_t, qfunc_t, inputs_t, outputs_t, MQ1, K0, N, N, Acc...>
+{
+   using type = tuple<Acc...>;
+   static_assert(sizeof...(Acc) == N - K0);
+   static_assert(sizeof...(Acc) <= 9);
+};
+
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
+   std::size_t K0, std::size_t K, std::size_t N, typename... Acc>
+struct build_args_reg_tuple_impl
+{
+   using qf_reg_param_t = typename qf_param_slot<qfunc_t, K>::qf_reg_param_t;
+   using R = typename backend_t::template QReg<qf_reg_param_t>;
+   using type = typename build_args_reg_tuple_impl<backend_t, qfunc_t, inputs_t,
+         outputs_t, MQ1, K0, K + 1, N, Acc..., R>::type;
+};
+
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
+using args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
+      inputs_t, outputs_t, MQ1, 0, 0,
+      tuple_size<inputs_t>::value + tuple_size<outputs_t>::value>::type;
+
+/// Register bank covering q-function inputs only (same types as first
+/// `n_inputs` slots of args_reg_t). Used where shadow / tangent paths never
+/// touch output parameter registers.
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
+using input_args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
+      inputs_t, outputs_t, MQ1, 0, 0,
+      tuple_size<inputs_t>::value>::type;
+
+/// Register bank covering q-function outputs only (same types as the slots
+/// from `n_inputs` onward in args_reg_t). Used where the primal / trial inputs
+/// live in a separate bank and only the test registers are integrated.
+/// Slot `o` of this bank is q-function parameter `n_inputs + o`.
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
+using output_args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
+      inputs_t, outputs_t, MQ1,
+      tuple_size<inputs_t>::value,
+      tuple_size<inputs_t>::value,
+      tuple_size<inputs_t>::value + tuple_size<outputs_t>::value>::type;
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_flat_value(ARG &a, int c, real_t v)
+{
+   if constexpr (std::is_same_v<ARG, real_t>) { MFEM_CONTRACT_VAR(c); a = v; }
+   else if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(c);
+      a.value = v;
+   }
+   else
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      constexpr bool D = qf_param_uses_dual_v<ARG>;
+      if constexpr (RNK == 0)
+      {
+         if constexpr (D) { a(0).value = v; }
+         else { a(0) = v; }
+      }
+      else if constexpr (RNK == 1)
+      {
+         if constexpr (D) { a(c).value = v; }
+         else { a(c) = v; }
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<ARG>::extents[0];
+         if constexpr (D) { a(c % e0, c / e0).value = v; }
+         else { a(c % e0, c / e0) = v; }
+      }
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_flat_gradient(ARG &a, int c, real_t v)
+{
+   if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(c);
+      a.gradient = v;
+   }
+   else if constexpr (qf_param_uses_dual_v<ARG>)
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0) { a(0).gradient = v; }
+      else if constexpr (RNK == 1) { a(c).gradient = v; }
+      else
+      {
+         constexpr int e0 = qf_param_shape<ARG>::extents[0];
+         a(c % e0, c / e0).gradient = v;
+      }
+   }
+   else
+   {
+      // Non-dual argument (e.g. Weight): never an active trial direction.
+      MFEM_CONTRACT_VAR(a);
+      MFEM_CONTRACT_VAR(c);
+      MFEM_CONTRACT_VAR(v);
+   }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// Two-index component access for a q-function argument.
+///
+/// `i` indexes vdim, `k` indexes the operator dimension, matching the
+/// column-major packing `c = i + extents[0]*k` of the flat accessors above.
+/// Prefer these wherever the caller already has both indices: the flat form
+/// would have to undo the packing with an integer division, which is expensive
+/// on device and pointless when `(i, k)` are right there.
+///
+/// The runtime extents of the callers agree with the static extents of `ARG`:
+/// for rank 2, `vdim == extents[0]` and `op_dim == extents[1]`; for rank 1 one
+/// of the two is 1 and the corresponding index is always 0, so `a(i + k)`
+/// selects the right component; for rank 0 both are 1.
+template <typename ARG>
+MFEM_HOST_DEVICE inline real_t qf_value_at(const ARG &a, int i, int k)
+{
+   if constexpr (std::is_same_v<ARG, real_t> || is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      return qf_store_value(a);
+   }
+   else
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         return qf_store_value(a(0));
+      }
+      else if constexpr (RNK == 1) { return qf_store_value(a(i + k)); }
+      else { return qf_store_value(a(i, k)); }
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline real_t qf_gradient_at(const ARG &a, int i, int k)
+{
+   if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      return a.gradient;
+   }
+   else if constexpr (qf_param_uses_dual_v<ARG>)
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         return a(0).gradient;
+      }
+      else if constexpr (RNK == 1) { return a(i + k).gradient; }
+      else { return a(i, k).gradient; }
+   }
+   else
+   {
+      // Non-dual argument carries no tangent: its derivative contribution is 0.
+      MFEM_CONTRACT_VAR(a);
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      return real_t(0);
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_value_at(ARG &a, int i, int k, real_t v)
+{
+   if constexpr (std::is_same_v<ARG, real_t>)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      a = v;
+   }
+   else if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      a.value = v;
+   }
+   else
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      constexpr bool D = qf_param_uses_dual_v<ARG>;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         if constexpr (D) { a(0).value = v; }
+         else { a(0) = v; }
+      }
+      else if constexpr (RNK == 1)
+      {
+         if constexpr (D) { a(i + k).value = v; }
+         else { a(i + k) = v; }
+      }
+      else
+      {
+         if constexpr (D) { a(i, k).value = v; }
+         else { a(i, k) = v; }
+      }
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_gradient_at(ARG &a, int i, int k, real_t v)
+{
+   if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      a.gradient = v;
+   }
+   else if constexpr (qf_param_uses_dual_v<ARG>)
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         a(0).gradient = v;
+      }
+      else if constexpr (RNK == 1) { a(i + k).gradient = v; }
+      else { a(i, k).gradient = v; }
+   }
+   else
+   {
+      // Non-dual argument (e.g. Weight): never an active trial direction.
+      MFEM_CONTRACT_VAR(a);
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      MFEM_CONTRACT_VAR(v);
+   }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// Maps each FOP slot to unionfds indices — used with dtqs / create_dtq_maps
+template<typename C, typename T>
+const auto create_union_field_map_for_dtq(C& ctx, T& io)
+{
+   using FE = Entity::Element;
+   return create_descriptors_to_fields_map<FE>(ctx.unionfds, io);
+}
+
+/// **`xe[i]`** slot per input FOP — indices into **`ctx.infds`** (`SIZE_MAX` for Weight).
+template<typename C, typename T>
+const auto create_input_vector_map(C& ctx, T& io)
+{
+   using FE = Entity::Element;
+   return create_descriptors_to_fields_map<FE>(ctx.infds, io);
+}
+
+/// **`ye[i]`** slot per output FOP — indices into **`ctx.outfds`**.
+template<typename C, typename T>
+const auto create_output_vector_map(C& ctx, T& io)
+{
+   using FE = Entity::Element;
+   return create_descriptors_to_fields_map<FE>(ctx.outfds, io);
+}
+
+template<typename C>
+const auto make_dtqs(C& ctx)
+{
+   std::vector<const DofToQuad*> dtq_vec;
+   dtq_vec.reserve(ctx.unionfds.size());
+   constexpr auto dtq_mode = DofToQuad::Mode::TENSOR;
+   for (const auto &field: ctx.unionfds)
+   {
+      auto dtq = GetDofToQuad<Entity::Element>(field, ctx.ir, dtq_mode);
+      dtq_vec.emplace_back(dtq);
+   }
+   return dtq_vec;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+template<typename Tuple>
+constexpr auto get_vdim(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<int, sizeof...(f)> {f.vdim...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_B(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<const real_t*, sizeof...(f)> {f.B...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_G(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<const real_t*, sizeof...(f)> {f.G...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_D1D(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<int, sizeof...(f)> {f.B.GetShape()[2]...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_Q1D(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<int, sizeof...(f)> {f.B.GetShape()[0]...};
+   }, fields);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// Per-output FOP layout metadata (shared by derivative setup / apply kernels).
+
+template<typename outputs_t>
+constexpr auto compute_out_qp_size(const outputs_t &outs)
+{
+   constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   std::array<int, n_outputs> sizes{};
+   for_constexpr<n_outputs>([&](auto o) { sizes[o] = get<o>(outs).size_on_qp; });
+   return sizes;
+}
+
+template<typename outputs_t>
+constexpr auto compute_out_op_dim(const outputs_t &outs)
+{
+   constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   std::array<int, n_outputs> op{};
+   for_constexpr<n_outputs>([&](auto o)
+   {
+      op[o] = get<o>(outs).size_on_qp / get<o>(outs).vdim;
+   });
+   return op;
+}
+
+template<std::size_t N>
+constexpr std::array<int, N> compute_out_offsets(
+   const std::array<int, N> &vdim,
+   const std::array<int, N> &op_dim)
+{
+   std::array<int, N> offsets{};
+   offsets[0] = 0;
+   for (std::size_t o = 1; o < N; o++)
+   {
+      offsets[o] = offsets[o - 1] + vdim[o - 1] * op_dim[o - 1];
+   }
+   return offsets;
+}
+
+template<std::size_t N>
+constexpr std::array<int, N> compute_out_flat_offsets(
+   const std::array<int, N> &vdim,
+   const std::array<int, N> &op_dim,
+   const int num_qp)
+{
+   std::array<int, N> offsets{};
+   offsets[0] = 0;
+   for (std::size_t o = 1; o < N; o++)
+   {
+      offsets[o] = offsets[o - 1] + vdim[o - 1] * op_dim[o - 1] * num_qp;
+   }
+   return offsets;
+}
+
+template<typename inputs_t>
+const auto compute_input_is_dependent(const inputs_t &ins, int deriv_id)
+{
+   auto dependency_map = make_dependency_map(ins);
+   auto it = dependency_map.find(deriv_id);
+   MFEM_ASSERT(it != dependency_map.end(),
+               "Derivative ID not found in dependency map");
+   return it->second;
+}
+
+template<typename inputs_t>
+constexpr int compute_trial_vdim(const inputs_t &ins, int deriv_id)
+{
+   constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   int v = 1;
+   for_constexpr<n_inputs>([&](auto i)
+   {
+      if (get<i>(ins).GetFieldId() == deriv_id) { v = get<i>(ins).vdim; }
+   });
+   return v;
+}
+
+template<typename inputs_t>
+constexpr int compute_total_trial_op_dim(
+   const inputs_t &ins,
+   const std::array<bool, tuple_size<inputs_t>::value> &dep,
+   const std::array<int, tuple_size<inputs_t>::value> &size_on_qp)
+{
+   constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   int total = 0;
+   for_constexpr<n_inputs>([&](auto i)
+   {
+      if (dep[i]) { total += size_on_qp[i] / get<i>(ins).vdim; }
+   });
+   return total;
+}
+
+inline size_t find_union_field_index(const IntegratorContext &ctx, int field_id)
+{
+   for (size_t uf = 0; uf < ctx.unionfds.size(); uf++)
+   {
+      if (static_cast<int>(ctx.unionfds[uf].id) == field_id) { return uf; }
+   }
+   return SIZE_MAX;
+}
+
+inline size_t find_infd_index(const IntegratorContext &ctx, int field_id)
+{
+   for (size_t i = 0; i < ctx.infds.size(); i++)
+   {
+      if (static_cast<int>(ctx.infds[i].id) == field_id) { return i; }
+   }
+   return SIZE_MAX;
+}
+
+template<typename entity_t = Entity::Element>
+inline int compute_element_dof_sz(
+   const FieldDescriptor &fd,
+   int num_entities,
+   ElementDofOrdering ordering)
+{
+   auto R = get_restriction<entity_t>(fd, ordering);
+   MFEM_ASSERT(R != nullptr, "LocalQF: missing element restriction");
+   return num_entities ? (R->Height() / num_entities) : 0;
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Number of threads per 1D direction to launch the kernel with
+template <typename inputs_t, typename outputs_t,
+          std::size_t N_in, std::size_t N_out>
+inline int compute_kernel_thread_1d(
+   const int q1d,
+   const std::array<int, N_in> &in_d1d,
+   const std::array<int, N_out> &out_d1d)
+{
+   int t1d = q1d;
+   for_constexpr<N_in>([&](auto ic)
+   {
+      using FOP = tuple_element_t<ic.value, inputs_t>;
+      if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+      {
+         t1d = std::max(t1d, in_d1d[ic.value]);
+      }
+   });
+   for_constexpr<N_out>([&](auto ic)
+   {
+      using FOP = tuple_element_t<ic.value, outputs_t>;
+      if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+      {
+         t1d = std::max(t1d, out_d1d[ic.value]);
+      }
+   });
+   return t1d;
+}
+
+// Inputs-only variant: used by kernels whose outputs are written at qp
+template <typename inputs_t, std::size_t N_in>
+inline int compute_kernel_thread_1d(
+   const int q1d,
+   const std::array<int, N_in> &in_d1d)
+{
+   int t1d = q1d;
+   for_constexpr<N_in>([&](auto ic)
+   {
+      using FOP = tuple_element_t<ic.value, inputs_t>;
+      if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+      {
+         t1d = std::max(t1d, in_d1d[ic.value]);
+      }
+   });
+   return t1d;
+}
+
+} // namespace mfem::future

fem/dfem/backends/scratch_bank.hpp

@@ -0,0 +1,277 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+#include "mfem.hpp"
+#include "../tuple.hpp"
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+namespace mfem::future
+{
+// Scratch storage and q-function shadow helpers for dFEM backends. The bank
+// supports two scratch kinds:
+// - quadrature-point scratch: real_t buffers sized as NQ * components_per_qp,
+// - global scratch: one tuple of qfunction-local temporaries, independent of
+//   NQ, used for values such as flags, scalars, or small Vector workspaces.
+template <typename... GlobalScratchTypes>
+struct ScratchBank
+{
+
+   //=================================
+   ///<--- Global scratch utilities.
+   //=================================
+
+   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
+
+   template <typename T>
+   static T MakeGlobalScratchShadow(const T &)
+   {
+      return T {};
+   }
+
+   static Vector MakeGlobalScratchShadow(const Vector &primal)
+   {
+      Vector shadow(primal.Size());
+      shadow.UseDevice(true);
+      shadow = 0.0;
+      return shadow;
+   }
+
+   template <typename Tuple, size_t... Is>
+   static auto MakeGlobalScratchShadowTuple(const Tuple &primal,
+                                            std::index_sequence<Is...>)
+   {
+      return make_tuple(MakeGlobalScratchShadow(get<Is>(primal))...);
+   }
+
+   template <typename Tuple>
+   static auto MakeGlobalScratchShadowTuple(const Tuple &primal)
+   {
+      return MakeGlobalScratchShadowTuple(
+                primal, std::make_index_sequence<tuple_size<Tuple>::value> {});
+   }
+
+
+   //===========================
+   ///<--- Scratch objects
+   //===========================
+
+   mutable GlobalScratchTuple global;
+
+   int nq = 0;
+   std::vector<int> components;
+   std::vector<int> sizes;
+   std::vector<std::shared_ptr<Vector>> owned;
+   std::vector<real_t *> ptrs;
+
+
+   //===========================
+   ///<--- Setter methods
+   //===========================
+
+   void SetScratch(const int nq_,
+                   std::initializer_list<int> components_per_qp = {1})
+   {
+      SetScratch(nq_, std::vector<int>(components_per_qp));
+   }
+
+   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
+   {
+      nq = nq_;
+      components.clear();
+      sizes.clear();
+      owned.clear();
+      ptrs.clear();
+      for (int component_count : components_per_qp)
+      {
+         AddScratch(component_count);
+      }
+   }
+
+   void AddScratch(const int components_per_qp = 1)
+   {
+      MFEM_VERIFY(nq > 0, "SetScratch must be called before AddScratch");
+      MFEM_VERIFY(components_per_qp > 0,
+                  "scratch components per quadrature point must be positive");
+      owned.push_back(std::make_shared<Vector>());
+      Vector &scratch = *owned.back();
+      const int size = components_per_qp * nq;
+      scratch.SetSize(size);
+      scratch.UseDevice(true);
+      scratch = 0.0;
+      components.push_back(components_per_qp);
+      sizes.push_back(scratch.Size());
+      ptrs.push_back(scratch.ReadWrite());
+   }
+
+   void SetGlobalScratch(const GlobalScratchTuple &global_)
+   {
+      global = global_;
+   }
+
+
+   //===========================
+   ///<--- Getter methods
+   //===========================
+
+   real_t *GetScratchPointer(const int i) const { return ptrs[i]; }
+   real_t *operator[](const int i) const { return ptrs[i]; }
+
+   Vector &GetScratchVector(const int i) const { return *owned[i]; }
+
+   template <int I>
+   auto &GetGlobalScratch() const
+   {
+      return get<I>(global);
+   }
+
+
+   //===========================
+   ///<--- Utils methods
+   //===========================
+
+   void CloneScratchLayoutTo(ScratchBank &shadow) const
+   {
+      shadow.SetScratch(nq, components);
+      shadow.SetGlobalScratch(MakeGlobalScratchShadowTuple(global));
+   }
+
+   int Size() const { return static_cast<int>(ptrs.size()); }
+};
+
+// Shared base for Q-functions that use ScratchBank and need a matching scratch
+// shadow for forward differentiation.
+template <typename... GlobalScratchTypes>
+struct QFWithScratch
+{
+   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
+
+   int nq = 0;
+   ScratchBank<GlobalScratchTypes...> scratch;
+
+   void SetScratch(const int nq_,
+                   std::initializer_list<int> components_per_qp = {1})
+   {
+      nq = nq_;
+      scratch.SetScratch(nq, components_per_qp);
+   }
+
+   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
+   {
+      nq = nq_;
+      scratch.SetScratch(nq, components_per_qp);
+   }
+
+   void SetScratch(const int nq_, const int num_scratch_elem,
+                   const int components_per_qp = 1)
+   {
+      nq = nq_;
+      scratch.SetScratch(nq,
+                         std::vector<int>(num_scratch_elem, components_per_qp));
+   }
+
+   void SetGlobalScratch(const GlobalScratchTuple &global_scratch_)
+   {
+      scratch.SetGlobalScratch(global_scratch_);
+   }
+
+   Vector &GetScratchVector(const int i) const
+   {
+      return scratch.GetScratchVector(i);
+   }
+
+   real_t *GetScratchPointer(const int i) const
+   {
+      return scratch.GetScratchPointer(i);
+   }
+
+   template <int I>
+   auto &GetGlobalScratch() const
+   {
+      return scratch.template GetGlobalScratch<I>();
+   }
+
+   void CloneScratchLayoutTo(QFWithScratch &shadow) const
+   {
+      shadow.nq = nq;
+      scratch.CloneScratchLayoutTo(shadow.scratch);
+   }
+
+   QFWithScratch CreateShadow() const
+   {
+      QFWithScratch shadow;
+      CloneScratchLayoutTo(shadow);
+      return shadow;
+   }
+};
+
+using QFWithScratchType = QFWithScratch<>;
+using QFWithGlobalScratchType = QFWithScratch<bool, real_t, Vector>;
+
+namespace detail
+{
+
+template <typename T>
+struct qfunc_uses_scratch
+{
+private:
+   template <typename... GlobalScratchTypes>
+   static std::true_type Test(const QFWithScratch<GlobalScratchTypes...> *);
+
+   static std::false_type Test(...);
+
+public:
+   static constexpr bool value = decltype(Test(
+                                             static_cast<std::remove_cv_t<std::remove_reference_t<T>> *>(nullptr)))::value;
+};
+
+template <typename T>
+inline constexpr bool qfunc_uses_scratch_v =
+   qfunc_uses_scratch<T>::value;
+
+struct unused_qfunc_shadow { };
+
+template <typename qfunc_t, bool uses_scratch>
+struct qfunc_shadow_type
+{
+   using type = unused_qfunc_shadow;
+};
+
+template <typename qfunc_t>
+struct qfunc_shadow_type<qfunc_t, true>
+{
+   using type = decltype(std::declval<const qfunc_t &>().CreateShadow());
+};
+
+template <typename qfunc_t>
+using qfunc_shadow_t = typename qfunc_shadow_type<qfunc_t,
+      qfunc_uses_scratch_v<qfunc_t>>::type;
+
+// Create a persistent q-function shadow if the q-function uses scratch, otherwise return an empty struct.
+template <typename qfunc_t>
+inline qfunc_shadow_t<qfunc_t> MakeQFunctionShadowStorage(
+   const qfunc_t &qfunc)
+{
+   if constexpr (qfunc_uses_scratch_v<qfunc_t>)
+   {
+      return qfunc.CreateShadow();
+   }
+   else
+   {
+      MFEM_CONTRACT_VAR(qfunc);
+      return {};
+   }
+}
+
+} // namespace detail
+
+}

fem/dfem/doperator.cpp

@@ -11,44 +11,387 @@
 
 #include "doperator.hpp"
 
+#include <algorithm>
+
 #ifdef MFEM_USE_MPI
 
 using namespace mfem;
 using namespace mfem::future;
 
-void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
+namespace
 {
-   MFEM_ASSERT(parameters.size() == p.size(),
-               "number of parameters doesn't match descriptors");
-   for (size_t i = 0; i < parameters.size(); i++)
+int GetTotalTrueVSize(const std::vector<FieldDescriptor> &fds)
+{
+   int size = 0;
+   for (const auto &fd : fds)
    {
-      p[i]->Read();
-      parameters_l[i] = *p[i];
+      size += mfem::future::GetTrueVSize(fd);
    }
+   return size;
+}
+
+using DerivativeActionMap =
+   std::map<size_t, std::vector<derivative_action_t>>;
+using DerivativeSetupMap =
+   std::map<size_t, std::vector<derivative_setup_t>>;
+using DerivativeFieldMap =
+   std::map<size_t, std::vector<FieldDescriptor>>;
+using SparseAssemblyMap =
+   std::map<size_t,
+   std::vector<assemble_derivative_sparsematrix_callback_t>>;
+using HypreAssemblyMap =
+    std::map<size_t,
+             std::vector<assemble_derivative_hypreparmatrix_callback_t>>;
+using DiagonalAssemblyMap =
+    std::map<size_t, std::vector<assemble_diagonal_callback_t>>;
+using SecondDerivativeActionMap =
+    std::map<second_derivative_key_t, std::vector<derivative_action_t>>;
+using SecondDerivativeSetupMap =
+    std::map<second_derivative_key_t, std::vector<derivative_setup_t>>;
+using SecondDerivativeFieldMap =
+    std::map<second_derivative_key_t, std::vector<FieldDescriptor>>;
+using SecondSparseAssemblyMap =
+    std::map<second_derivative_key_t,
+             std::vector<assemble_derivative_sparsematrix_callback_t>>;
+using SecondHypreAssemblyMap =
+    std::map<second_derivative_key_t,
+             std::vector<assemble_derivative_hypreparmatrix_callback_t>>;
+using SecondDiagonalAssemblyMap =
+    std::map<second_derivative_key_t, std::vector<assemble_diagonal_callback_t>>;
+
+template <typename map_t>
+const typename map_t::mapped_type &FindOrDefault(
+    const map_t &map, const typename map_t::key_type &id,
+    const typename map_t::mapped_type &fallback)
+{
+   const auto it = map.find(id);
+   return it == map.end() ? fallback : it->second;
+}
+
+template <typename map_t>
+typename map_t::mapped_type FindOrEmpty(
+   const map_t &map, const typename map_t::key_type &id)
+{
+   const auto it = map.find(id);
+   return it == map.end() ? typename map_t::mapped_type{} : it->second;
+}
+
+const std::vector<derivative_action_t> &SelectActionCallbacks(
+   const std::vector<derivative_action_t> &direct_actions,
+   const DerivativeActionMap &cached_actions,
+   size_t derivative_id,
+   bool use_cached_setup)
+{
+   if (use_cached_setup)
+   {
+      const auto it_apply = cached_actions.find(derivative_id);
+      if (it_apply != cached_actions.end() && !it_apply->second.empty())
+      {
+         return it_apply->second;
+      }
+   }
+
+   return direct_actions;
+}
+
+struct DerivativeCallbackSet
+{
+   const DerivativeActionMap &actions;
+   const DerivativeActionMap &cached_actions;
+   const DerivativeActionMap &transpose_actions;
+   const DerivativeFieldMap &outfds;
+   const SparseAssemblyMap &assemble_sparse;
+   const HypreAssemblyMap &assemble_hypre;
+   const DiagonalAssemblyMap &assemble_diagonal;
+   const DerivativeSetupMap &setup;
+   const char *missing_action_message;
+};
+
+struct SecondDerivativeCallbackSet
+{
+   const SecondDerivativeActionMap &actions;
+   const SecondDerivativeActionMap &cached_actions;
+   const SecondDerivativeActionMap &transpose_actions;
+   const SecondDerivativeFieldMap &outfds;
+   const SecondSparseAssemblyMap &assemble_sparse;
+   const SecondHypreAssemblyMap &assemble_hypre;
+   const SecondDiagonalAssemblyMap &assemble_diagonal;
+   const SecondDerivativeSetupMap &setup;
+   const char *missing_action_message;
+};
+
+template <typename vector_t>
+std::shared_ptr<DerivativeOperator> MakeStatefulDerivativeOperator(
+   size_t derivative_id,
+   const vector_t &x,
+   const std::vector<FieldDescriptor> &infds,
+   const std::vector<FieldDescriptor> &default_outfds,
+   const DerivativeCallbackSet &callbacks,
+   bool use_cached_setup,
+   bool lvector_mode,
+   bool functional_gradient = false)
+{
+   const auto it_action = callbacks.actions.find(derivative_id);
+   MFEM_ASSERT(it_action != callbacks.actions.end(),
+               callbacks.missing_action_message << derivative_id);
+
+   const size_t dfidx = FindIdx(derivative_id, infds);
+   const auto &doutfds =
+      FindOrDefault(callbacks.outfds, derivative_id, default_outfds);
+   const auto &mult_callbacks =
+      SelectActionCallbacks(it_action->second, callbacks.cached_actions,
+                            derivative_id, use_cached_setup);
+
+   return std::make_shared<DerivativeOperator>(
+             GetTotalTrueVSize(doutfds),
+             GetTrueVSize(infds[dfidx]),
+             mult_callbacks,
+             FindOrEmpty(callbacks.transpose_actions, derivative_id),
+             infds[dfidx],
+             x,
+             infds,
+             doutfds,
+             FindOrEmpty(callbacks.assemble_sparse, derivative_id),
+             FindOrEmpty(callbacks.assemble_hypre, derivative_id),
+             FindOrEmpty(callbacks.assemble_diagonal, derivative_id),
+             FindOrEmpty(callbacks.setup, derivative_id),
+             lvector_mode,
+             functional_gradient);
+}
+
+const std::vector<derivative_action_t> &SelectSecondDerivativeActionCallbacks(
+    const std::vector<derivative_action_t> &direct_actions,
+    const SecondDerivativeActionMap &cached_actions,
+    second_derivative_key_t derivative_key,
+    bool use_cached_setup)
+{
+   if (use_cached_setup)
+   {
+      const auto it_apply = cached_actions.find(derivative_key);
+      if (it_apply != cached_actions.end() && !it_apply->second.empty())
+      {
+         return it_apply->second;
+      }
+   }
+
+   return direct_actions;
+}
+
+template <typename vector_t>
+std::shared_ptr<DerivativeOperator> MakeStatefulSecondDerivativeOperator(
+    size_t gradient_id,
+    size_t direction_id,
+    const vector_t &x,
+    const std::vector<FieldDescriptor> &infds,
+    const std::vector<FieldDescriptor> &default_outfds,
+    const SecondDerivativeCallbackSet &callbacks,
+    bool use_cached_setup,
+    bool lvector_mode)
+{
+   const second_derivative_key_t derivative_key{gradient_id, direction_id};
+   const auto it_action = callbacks.actions.find(derivative_key);
+   MFEM_ASSERT(it_action != callbacks.actions.end(),
+               callbacks.missing_action_message << "(" << gradient_id << ", "
+                                                << direction_id << ")");
+
+   const size_t dfidx = FindIdx(direction_id, infds);
+   const auto &doutfds =
+       FindOrDefault(callbacks.outfds, derivative_key, default_outfds);
+   const auto &mult_callbacks =
+       SelectSecondDerivativeActionCallbacks(
+           it_action->second, callbacks.cached_actions, derivative_key,
+           use_cached_setup);
+
+   return std::make_shared<DerivativeOperator>(
+       GetTotalTrueVSize(doutfds),
+       GetTrueVSize(infds[dfidx]),
+       mult_callbacks,
+       FindOrEmpty(callbacks.transpose_actions, derivative_key),
+       infds[dfidx],
+       x,
+       infds,
+       doutfds,
+       FindOrEmpty(callbacks.assemble_sparse, derivative_key),
+       FindOrEmpty(callbacks.assemble_hypre, derivative_key),
+       FindOrEmpty(callbacks.assemble_diagonal, derivative_key),
+       FindOrEmpty(callbacks.setup, derivative_key),
+       lvector_mode);
+}
 }
 
 DifferentiableOperator::DifferentiableOperator(
-   const std::vector<FieldDescriptor> &solutions,
-   const std::vector<FieldDescriptor> &parameters,
+   const std::vector<FieldDescriptor> &infds,
+   const std::vector<FieldDescriptor> &outfds,
    const ParMesh &mesh) :
+   Operator(),
    mesh(mesh),
-   solutions(solutions),
-   parameters(parameters)
+   infds(infds),
+   outfds(outfds)
 {
-   fields.resize(solutions.size() + parameters.size());
-   fields_e.resize(fields.size());
-   solutions_l.resize(solutions.size());
-   parameters_l.resize(parameters.size());
+   unionfds.clear();
+   unionfds.insert(unionfds.end(), infds.begin(), infds.end());
+   unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
+   std::sort(unionfds.begin(), unionfds.end());
+   auto last = std::unique(unionfds.begin(), unionfds.end());
+   unionfds.erase(last, unionfds.end());
 
-   for (size_t i = 0; i < solutions.size(); i++)
+   infields_l.resize(infds.size());
+   for (size_t i = 0; i < infds.size(); i++)
    {
-      fields[i] = solutions[i];
+      infields_l[i] = new Vector(GetVSize(infds[i]));
    }
 
-   for (size_t i = 0; i < parameters.size(); i++)
+   infields_e.resize(infds.size());
+}
+
+void DifferentiableOperator::SetMultLevel(MultLevel level)
+{
+   mult_level = level;
+}
+
+void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
+{
+   MFEM_ASSERT(!action_callbacks.empty(),
+               "no integrators have been set");
+
+   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
+               "x needs to be a BlockVector");
+
+   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
+               "y needs to be a BlockVector");
+
+   const auto &bx = static_cast<const BlockVector &>(x);
+   auto &by = static_cast<BlockVector &>(y);
+
+   Mult(bx, by);
+}
+
+void DifferentiableOperator::DisableTensorProductStructure(bool disable)
+{
+   use_tensor_product_structure = !disable;
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id, const Vector &x)
+{
+   return MakeStatefulDerivativeOperator(
+             derivative_id, x, infds, outfds,
    {
-      fields[i + solutions.size()] = parameters[i];
-   }
+      derivative_action_callbacks,
+      derivative_apply_callbacks,
+      daction_transpose_callbacks,
+      derivative_outfds,
+      assemble_derivative_sparsematrix_callbacks,
+      assemble_derivative_hypreparmatrix_callbacks,
+      assemble_diagonal_callbacks,
+      derivative_setup_callbacks,
+      "no derivative action has been found for ID "
+   },
+   true,
+   mult_level == MultLevel::LVECTOR,
+   IsFunctionalDerivative(derivative_id));
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
+{
+   return MakeStatefulDerivativeOperator(
+             derivative_id, x, infds, outfds,
+   {
+      derivative_action_callbacks,
+      derivative_apply_callbacks,
+      daction_transpose_callbacks,
+      derivative_outfds,
+      assemble_derivative_sparsematrix_callbacks,
+      assemble_derivative_hypreparmatrix_callbacks,
+      assemble_diagonal_callbacks,
+      derivative_setup_callbacks,
+      "no derivative action has been found for ID "
+   },
+   use_cached_setup,
+   mult_level == MultLevel::LVECTOR,
+   IsFunctionalDerivative(derivative_id));
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id)
+{
+   MFEM_ASSERT(has_functional_integrator,
+               "stateless GetDerivative is available only for functionals");
+
+   const auto it_action = derivative_action_callbacks.find(derivative_id);
+   MFEM_ASSERT(it_action != derivative_action_callbacks.end(),
+               "no derivative action has been found for ID " << derivative_id);
+
+   const size_t dfidx = FindIdx(derivative_id, infds);
+   const auto &doutfds =
+      FindOrDefault(derivative_outfds, derivative_id, outfds);
+
+   return std::make_shared<DerivativeOperator>(
+             GetTotalTrueVSize(doutfds),
+             GetTrueVSize(infds[dfidx]),
+             it_action->second,
+             infds,
+             doutfds);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t derivative_id, const Vector &x)
+{
+   return GetSecondDerivative(derivative_id, derivative_id, x);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t gradient_id, size_t direction_id, const Vector &x)
+{
+   MFEM_ASSERT(has_functional_integrator,
+               "second derivatives are available only for functionals");
+
+   return MakeStatefulSecondDerivativeOperator(
+             gradient_id, direction_id, x, infds, outfds,
+   {
+      second_derivative_action_callbacks,
+      second_derivative_apply_callbacks,
+      second_daction_transpose_callbacks,
+      second_derivative_outfds,
+      assemble_second_derivative_sparsematrix_callbacks,
+      assemble_second_derivative_hypreparmatrix_callbacks,
+      assemble_second_derivative_diagonal_callbacks,
+      second_derivative_setup_callbacks,
+      "no second derivative action has been found for ID "
+   },
+   false,
+   mult_level == MultLevel::LVECTOR);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
+{
+   return GetSecondDerivative(derivative_id, derivative_id, x, use_cached_setup);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t gradient_id, size_t direction_id, const MultiVector &x,
+   const bool use_cached_setup)
+{
+   MFEM_ASSERT(has_functional_integrator,
+               "second derivatives are available only for functionals");
+
+   return MakeStatefulSecondDerivativeOperator(
+             gradient_id, direction_id, x, infds, outfds,
+   {
+      second_derivative_action_callbacks,
+      second_derivative_apply_callbacks,
+      second_daction_transpose_callbacks,
+      second_derivative_outfds,
+      assemble_second_derivative_sparsematrix_callbacks,
+      assemble_second_derivative_hypreparmatrix_callbacks,
+      assemble_second_derivative_diagonal_callbacks,
+      second_derivative_setup_callbacks,
+      "no second derivative action has been found for ID "
+   },
+   use_cached_setup,
+   mult_level == MultLevel::LVECTOR);
 }
 
 #endif // MFEM_USE_MPI

fem/dfem/fielddescriptor.hpp

@@ -0,0 +1,64 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../fespace.hpp"
+#include "../qspace.hpp"
+#include "parameterspace.hpp"
+
+namespace mfem::future
+{
+
+/// @brief FieldDescriptor struct
+///
+/// This struct is used to store information about a field.
+struct FieldDescriptor
+{
+   using data_variant_t =
+      std::variant<const FiniteElementSpace *,
+      const ParFiniteElementSpace *,
+      const VectorQuadratureSpace *,
+      const ParameterSpace *>;
+
+   /// Field ID
+   std::size_t id;
+
+   /// Field variant
+   data_variant_t data;
+
+   /// Default constructor
+   FieldDescriptor() :
+      id(SIZE_MAX), data(data_variant_t{}) {}
+
+   /// Constructor
+   template <typename T>
+   FieldDescriptor(std::size_t field_id, const T* v) :
+      id(field_id), data(v) {}
+
+   bool operator==(const FieldDescriptor& other) const
+   {
+      return id == other.id;
+   }
+
+   bool operator<(const FieldDescriptor& other) const
+   {
+      return id < other.id;
+   }
+
+   friend void swap(FieldDescriptor& a, FieldDescriptor& b)
+   {
+      using std::swap;
+      swap(a.id, b.id);
+      swap(a.data, b.data);
+   }
+};
+
+}

fem/dfem/fieldoperator.hpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 #pragma once
 
+#include <ostream>
 #include <type_traits>
 
 namespace mfem::future
@@ -74,6 +75,15 @@ struct is_identity_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_identity_fop<Identity<FIELD_ID>> : std::true_type {};
 
+template <typename T>
+constexpr bool is_identity_fop_v = is_identity_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, Identity<FIELD_ID>)
+{
+   return out << "Identity<" << FIELD_ID << ">";
+}
+
 /// @brief Weight FieldOperator.
 ///
 /// This FieldOperator is used to signal that this field contains the quadrature
@@ -90,6 +100,14 @@ struct is_weight_fop : std::false_type {};
 template <>
 struct is_weight_fop<Weight> : std::true_type {};
 
+template <typename T>
+constexpr bool is_weight_fop_v = is_weight_fop<T>::value;
+
+inline std::ostream& operator<<(std::ostream& out, Weight)
+{
+   return out << "Weight";
+}
+
 /// @brief Value FieldOperator.
 ///
 /// This FieldOperator is used to signal that the field contains the
@@ -101,11 +119,20 @@ public:
    constexpr Value() : FieldOperator<FIELD_ID>() {};
 };
 
-template< typename T >
+template <typename T>
 struct is_value_fop : std::false_type {};
 
+template <int T>
+struct is_value_fop<Value<T>> : std::true_type {};
+
+template <typename T>
+constexpr bool is_value_fop_v = is_value_fop<T>::value;
+
 template <int FIELD_ID>
-struct is_value_fop<Value<FIELD_ID>> : std::true_type {};
+inline std::ostream& operator<<(std::ostream& out, Value<FIELD_ID>)
+{
+   return out << "Value<" << FIELD_ID << ">";
+}
 
 /// @brief Gradient FieldOperator.
 ///
@@ -124,6 +151,15 @@ struct is_gradient_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_gradient_fop<Gradient<FIELD_ID>> : std::true_type {};
 
+template <typename T>
+constexpr bool is_gradient_fop_v = is_gradient_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, Gradient<FIELD_ID>)
+{
+   return out << "Gradient<" << FIELD_ID << ">";
+}
+
 /// @brief Sum FieldOperator.
 ///
 /// This FieldOperator is commonly used to signal that an output of a quadrature
@@ -141,4 +177,39 @@ struct is_sum_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_sum_fop<Sum<FIELD_ID>> : std::true_type {};
 
+template <typename T>
+constexpr bool is_sum_fop_v = is_sum_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, Sum<FIELD_ID>)
+{
+   return out << "Sum<" << FIELD_ID << ">";
+}
+
+/// @brief FunctionalValue FieldOperator.
+///
+/// This FieldOperator is commonly used to signal that an output of a quadrature
+/// function should be summed.
+template <int FIELD_ID = -1>
+class FunctionalValue : public FieldOperator<FIELD_ID>
+{
+public:
+   constexpr FunctionalValue() : FieldOperator<FIELD_ID>() {};
+};
+
+template< typename T >
+struct is_functionalvalue_fop : std::false_type {};
+
+template <int FIELD_ID>
+struct is_functionalvalue_fop<FunctionalValue<FIELD_ID>> : std::true_type {};
+
+template <typename T>
+constexpr bool is_functionalvalue_fop_v = is_functionalvalue_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, FunctionalValue<FIELD_ID>)
+{
+   return out << "FunctionalValue<" << FIELD_ID << ">";
+}
+
 } // namespace mfem::future

fem/dfem/integrate.hpp

@@ -1,536 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_impl(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   // assuming the quadrature point residual has to "play nice with
-   // the test function"
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [num_qp, cdim, num_dof] = B.GetShape();
-      const int vdim = output.vdim > 0 ? output.vdim : cdim ;
-      for (int dof = 0; dof < num_dof; dof++)
-      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            real_t acc = 0.0;
-            for (int qp = 0; qp < num_qp; qp++)
-            {
-               acc += B(qp, 0, dof) * f(vd, 0, qp);
-            }
-            y(dof, vd) += acc;
-         }
-      }
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [num_qp, dim, num_dof] = G.GetShape();
-      const int vdim = output.vdim;
-      for (int dof = 0; dof < num_dof; dof++)
-      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            real_t acc = 0.0;
-            for (int d = 0; d < dim; d++)
-            {
-               for (int qp = 0; qp < num_qp; qp++)
-               {
-                  acc += G(qp, d, dof) * f(vd, d, qp);
-               }
-            }
-            y(dof, vd) += acc;
-         }
-      }
-   }
-   else if constexpr (is_sum_fop<std::decay_t<output_t>>::value)
-   {
-      // This is the "integral over all quadrature points type" applying
-      // B = 1 s.t. B^T * C \in R^1.
-      const auto [num_qp, unused, unused1] = B.GetShape();
-      auto cc = Reshape(&f(0, 0, 0), num_qp);
-      for (int i = 0; i < num_qp; i++)
-      {
-         y(0, 0) += cc(i);
-      }
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [num_qp, unused, num_dof] = B.GetShape();
-      const auto vdim = output.vdim;
-      auto cc = Reshape(&f(0, 0, 0), num_qp * vdim);
-      auto yy = Reshape(&y(0, 0), num_qp * vdim);
-      for (int i = 0; i < num_qp * vdim; i++)
-      {
-         yy(i) = cc(i);
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_tensor_impl_1d(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, vdim);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dx, x, d1d)
-         {
-            real_t acc = 0.0;
-            for (int qx = 0; qx < q1d; qx++)
-            {
-               acc += fqp(vd, 0, qx) * B(qx, 0, dx);
-            }
-            yd(dx, vd) = acc;
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, vdim);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dx, x, d1d)
-         {
-            real_t acc = 0.0;
-            for (int qx = 0; qx < q1d; qx++)
-            {
-               acc += fqp(vd, 0, qx) * G(qx, 0, dx);
-            }
-            yd(dx, vd) = acc;
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
-      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
-
-      for (int sq = 0; sq < output.size_on_qp; sq++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            yqp(sq, qx) = fqp(sq, qx);
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        "for this field descriptor with sum factorization on"
-                        " tensor product elements");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_tensor_impl_2d(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t acc = 0.0;
-               for (int qx = 0; qx < q1d; qx++)
-               {
-                  acc += fqp(vd, 0, qx, qy) * B(qx, 0, dx);
-               }
-               s0(qy, dx) = acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t acc = 0.0;
-               for (int qy = 0; qy < q1d; qy++)
-               {
-                  acc += s0(qy, dx) * B(qy, 0, dy);
-               }
-               yd(dx, dy, vd) += acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int qx = 0; qx < q1d; qx++)
-               {
-                  uv[0] += fqp(vd, 0, qx, qy) * G(qx, 0, dx);
-                  uv[1] += fqp(vd, 1, qx, qy) * B(qx, 0, dx);
-               }
-               s0(qy, dx) = uv[0];
-               s1(qy, dx) = uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int qy = 0; qy < q1d; qy++)
-               {
-                  uv[0] += s0(qy, dx) * B(qy, 0, dy);
-                  uv[1] += s1(qy, dx) * G(qy, 0, dy);
-               }
-               yd(dx, dy, vd) += uv[0] + uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-
-      // // TODO: Check if this is the right fix for all cases
-      // auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
-      // auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
-      // for (int sq = 0; sq < output.size_on_qp; sq++)
-      // {
-      //    MFEM_FOREACH_THREAD(qx, x, q1d)
-      //    {
-      //       yqp(sq, qx) = fqp(sq, qx);
-      //    }
-      //    MFEM_SYNC_THREAD;
-      // }
-
-      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d);
-      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
-
-      for (int sq = 0; sq < output.size_on_qp; sq++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               yqp(sq, qx, qy) = fqp(sq, qx, qy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor with sum factorization on"
-                        " tensor product elements");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_tensor_impl_3d(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int qx = 0; qx < q1d; qx++)
-                  {
-                     acc += fqp(vd, 0, qx, qy, qz) * B(qx, 0, dx);
-                  }
-                  s0(qz, qy, dx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int qy = 0; qy < q1d; qy++)
-                  {
-                     acc += s0(qz, qy, dx) * B(qy, 0, dy);
-                  }
-                  s1(qz, dy, dx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(dz, z, d1d)
-               {
-                  real_t acc = 0.0;
-                  for (int qz = 0; qz < q1d; qz++)
-                  {
-                     acc += s1(qz, dy, dx) * B(qz, 0, dz);
-                  }
-                  yd(dx, dy, dz, vd) += acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, q1d, d1d);
-      auto s2 = Reshape(&scratch_mem[2](0), q1d, q1d, d1d);
-      auto s3 = Reshape(&scratch_mem[3](0), q1d, d1d, d1d);
-      auto s4 = Reshape(&scratch_mem[4](0), q1d, d1d, d1d);
-      auto s5 = Reshape(&scratch_mem[5](0), q1d, d1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qx = 0; qx < q1d; qx++)
-                  {
-                     uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
-                     uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
-                     uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
-                  }
-                  s0(qz, qy, dx) = uvw[0];
-                  s1(qz, qy, dx) = uvw[1];
-                  s2(qz, qy, dx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qy = 0; qy < q1d; qy++)
-                  {
-                     uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
-                     uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
-                     uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
-                  }
-                  s3(qz, dy, dx) = uvw[0];
-                  s4(qz, dy, dx) = uvw[1];
-                  s5(qz, dy, dx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qz = 0; qz < q1d; qz++)
-                  {
-                     uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
-                  }
-                  yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d, q1d);
-      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
-
-      for (int sq = 0; sq < output.size_on_qp; sq++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor with sum factorization on"
-                        " tensor product elements");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem);
-      }
-      else if (dimension == 2)
-      {
-         map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem);
-      }
-      else if (dimension == 3)
-      {
-         map_quadrature_data_to_fields_tensor_impl_3d(y, f, output, dtq, scratch_mem);
-      }
-      else { MFEM_ABORT_KERNEL("dimension not supported"); }
-   }
-   else
-   {
-      map_quadrature_data_to_fields_impl(y, f, output, dtq);
-   }
-}
-
-} // namespace mfem::future

fem/dfem/integrator_ctx.hpp

@@ -0,0 +1,37 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include <typeindex>
+#include <unordered_map>
+#include <vector>
+
+#include "../../general/array.hpp"
+#include "fielddescriptor.hpp"
+
+namespace mfem::future
+{
+
+struct IntegratorContext
+{
+   const ParMesh &mesh;
+   const Array<int> *elem_attr;
+   Array<int> attr;
+   const int nentities;
+   const std::vector<FieldDescriptor> &infds;
+   const std::vector<FieldDescriptor> &outfds;
+   const std::vector<FieldDescriptor> &unionfds;
+   const IntegrationRule &ir;
+   std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
+   std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
+};
+
+}

fem/dfem/interpolate.hpp

@@ -1,678 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE inline
-void map_field_to_quadrature_data_tensor_product_3d(
-   DeviceTensor<2> &field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
-   {
-      auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d, q1d);
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int dx = 0; dx < d1d; dx++)
-                  {
-                     acc += B(qx, 0, dx) * field(dx, dy, dz, vd);
-                  }
-                  s0(dz, dy, qx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               MFEM_FOREACH_THREAD(qy, y, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int dy = 0; dy < d1d; dy++)
-                  {
-                     acc += s0(dz, dy, qx) * B(qy, 0, dy);
-                  }
-                  s1(dz, qy, qx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int dz = 0; dz < d1d; dz++)
-                  {
-                     acc += s1(dz, qy, qx) * B(qz, 0, dz);
-                  }
-                  fqp(vd, qx, qy, qz) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
-
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, d1d, q1d);
-      auto s2 = Reshape(&scratch_mem[2](0), d1d, q1d, q1d);
-      auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
-      auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uv[2] = {0.0, 0.0};
-                  for (int dx = 0; dx < d1d; dx++)
-                  {
-                     const real_t f = field(dx, dy, dz, vd);
-                     uv[0] += f * B(qx, 0, dx);
-                     uv[1] += f * G(qx, 0, dx);
-                  }
-                  s0(dz, dy, qx) = uv[0];
-                  s1(dz, dy, qx) = uv[1];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int dy = 0; dy < d1d; dy++)
-                  {
-                     const real_t s0i = s0(dz, dy, qx);
-                     uvw[0] += s1(dz, dy, qx) * B(qy, 0, dy);
-                     uvw[1] += s0i * G(qy, 0, dy);
-                     uvw[2] += s0i * B(qy, 0, dy);
-                  }
-                  s2(dz, qy, qx) = uvw[0];
-                  s3(dz, qy, qx) = uvw[1];
-                  s4(dz, qy, qx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int dz = 0; dz < d1d; dz++)
-                  {
-                     uvw[0] += s2(dz, qy, qx) * B(qz, 0, dz);
-                     uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
-                     uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
-                  }
-                  fqp(vd, 0, qx, qy, qz) = uvw[0];
-                  fqp(vd, 1, qx, qy, qz) = uvw[1];
-                  fqp(vd, 2, qx, qy, qz) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   // TODO: Create separate function for clarity
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      // TODO: eeek
-      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
-      auto w = Reshape(&integration_weights[0], q1d, q1d, q1d);
-      auto f = Reshape(&field_qp[0], q1d, q1d, q1d);
-      MFEM_FOREACH_THREAD(qx, x, q1d)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(qz, z, q1d)
-            {
-               f(qx, qy, qz) = w(qx, qy, qz);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const int q1d = B.GetShape()[0];
-      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d * q1d);
-      field_qp = field;
-   }
-   else
-   {
-      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
-                    "can't map field to quadrature data");
-   }
-}
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE inline
-void map_field_to_quadrature_data_tensor_product_2d(
-   DeviceTensor<2> &field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
-   {
-      auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d);
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               real_t acc = 0.0;
-               for (int dx = 0; dx < d1d; dx++)
-               {
-                  acc += B(qx, 0, dx) * field(dx, dy, vd);
-               }
-               s0(dy, qx) = acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               real_t acc = 0.0;
-               for (int dy = 0; dy < d1d; dy++)
-               {
-                  acc += s0(dy, qx) * B(qy, 0, dy);
-               }
-               fqp(vd, qx, qy) = acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d);
-
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int dx = 0; dx < d1d; dx++)
-               {
-                  const real_t f = field(dx, dy, vd);
-                  uv[0] += f * B(qx, 0, dx);
-                  uv[1] += f * G(qx, 0, dx);
-               }
-               s0(dy, qx) = uv[0];
-               s1(dy, qx) = uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int dy = 0; dy < d1d; dy++)
-               {
-                  const real_t s0i = s0(dy, qx);
-                  uv[0] += s1(dy, qx) * B(qy, 0, dy);
-                  uv[1] += s0i * G(qy, 0, dy);
-               }
-               fqp(vd, 0, qx, qy) = uv[0];
-               fqp(vd, 1, qx, qy) = uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   // TODO: Create separate function for clarity
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      // TODO: eeek
-      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
-      auto w = Reshape(&integration_weights[0], q1d, q1d);
-      auto f = Reshape(&field_qp[0], q1d, q1d);
-      MFEM_FOREACH_THREAD(qx, x, q1d)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            f(qx, qy) = w(qx, qy);
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const int q1d = B.GetShape()[0];
-      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d);
-      field_qp = field;
-   }
-   else
-   {
-      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
-                    "can't map field to quadrature data");
-   }
-}
-
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE inline
-void map_field_to_quadrature_data_tensor_product_1d(
-   DeviceTensor<2> &field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
-   {
-      auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e[0], d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            real_t acc = 0.0;
-            for (int dx = 0; dx < d1d; dx++)
-            {
-               acc += B(qx, 0, dx) * field(dx, vd);
-            }
-            fqp(vd, qx) = acc;
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            real_t acc = 0.0;
-            for (int dx = 0; dx < d1d; dx++)
-            {
-               acc += G(qx, 0, dx) * field(dx, vd);
-            }
-            fqp(vd, 0, qx) = acc;
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   // TODO: Create separate function for clarity
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      // TODO: eeek
-      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
-      auto w = Reshape(&integration_weights[0], q1d);
-      auto f = Reshape(&field_qp[0], q1d);
-      MFEM_FOREACH_THREAD(qx, x, q1d)
-      {
-         f(qx) = w(qx);
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const int q1d = B.GetShape()[0];
-      auto field = Reshape(&field_e[0], input.size_on_qp, q1d);
-      field_qp = field;
-   }
-   else
-   {
-      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
-                    "can't map field to quadrature data");
-   }
-}
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE
-void map_field_to_quadrature_data(
-   DeviceTensor<2> field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-   if constexpr (is_value_fop<field_operator_t>::value)
-   {
-      auto [num_qp, dim, num_dof] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e(0), num_dof, vdim);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         for (int qp = 0; qp < num_qp; qp++)
-         {
-            real_t acc = 0.0;
-            for (int dof = 0; dof < num_dof; dof++)
-            {
-               acc += B(qp, 0, dof) * field(dof, vd);
-            }
-            field_qp(vd, qp) = acc;
-         }
-      }
-   }
-   else if constexpr (is_gradient_fop<field_operator_t>::value)
-   {
-      const auto [num_qp, dim, num_dof] = G.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e(0), num_dof, vdim);
-
-      auto f = Reshape(&field_qp[0], vdim, dim, num_qp);
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         for (int qp = 0; qp < num_qp; qp++)
-         {
-            for (int d = 0; d < dim; d++)
-            {
-               real_t acc = 0.0;
-               for (int dof = 0; dof < num_dof; dof++)
-               {
-                  acc += G(qp, d, dof) * field(dof, vd);
-               }
-               f(vd, d, qp) = acc;
-            }
-         }
-      }
-   }
-   else if constexpr (std::is_same_v<field_operator_t, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      auto f = Reshape(&field_qp[0], num_qp);
-      for (int qp = 0; qp < num_qp; qp++)
-      {
-         f(qp) = integration_weights(qp);
-      }
-   }
-   else if constexpr (is_identity_fop<field_operator_t>::value)
-   {
-      auto [num_qp, unused, num_dof] = B.GetShape();
-      const int size_on_qp = input.size_on_qp;
-      const auto field = Reshape(&field_e[0], size_on_qp * num_qp);
-      auto f = Reshape(&field_qp[0], size_on_qp * num_qp);
-      for (int i = 0; i < size_on_qp * num_qp; i++)
-      {
-         f(i) = field(i);
-      }
-   }
-   else
-   {
-      static_assert(dfem::always_false<field_operator_t>,
-                    "can't map field to quadrature data");
-
-   }
-}
-
-template <typename field_operator_ts, size_t num_inputs, size_t num_fields>
-MFEM_HOST_DEVICE inline
-void map_fields_to_quadrature_data(
-   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
-   const std::array<DeviceTensor<1>, num_fields> &fields_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
-   const std::array<size_t, num_inputs> &input_to_field,
-   const field_operator_ts &fops,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const int &dimension,
-   const bool &use_sum_factorization = false)
-{
-   // When the input_to_field map returns -1, this means the requested input
-   // is the integration weight. Weights don't have a user defined field
-   // attached to them and we create a dummy field which is not accessed
-   // inside the functions it is passed to.
-   const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
-   for_constexpr<num_inputs>([&](auto i)
-   {
-      const DeviceTensor<1> &field_e =
-         (input_to_field[i] == SIZE_MAX) ? dummy_field_weight :
-         fields_e[input_to_field[i]];
-
-      if (use_sum_factorization)
-      {
-         if (dimension == 1)
-         {
-            map_field_to_quadrature_data_tensor_product_1d(
-               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-               integration_weights, scratch_mem);
-         }
-         else if (dimension == 2)
-         {
-            map_field_to_quadrature_data_tensor_product_2d(
-               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-               integration_weights, scratch_mem);
-         }
-         else if (dimension == 3)
-         {
-            map_field_to_quadrature_data_tensor_product_3d(
-               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-               integration_weights, scratch_mem);
-         }
-         else
-         {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-            MFEM_ABORT("unsupported dimension");
-#endif
-         }
-      }
-      else
-      {
-         map_field_to_quadrature_data(
-            fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-            integration_weights);
-      }
-   });
-}
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE
-void map_field_to_quadrature_data_conditional(
-   DeviceTensor<2> &field_qp,
-   const DeviceTensor<1> &field_e,
-   const DofToQuadMap &dtqmap,
-   field_operator_t &fop,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const bool &condition,
-   const int &dimension,
-   const bool &use_sum_factorization = false)
-{
-   if (condition)
-   {
-      if (use_sum_factorization)
-      {
-         if (dimension == 1)
-         {
-            map_field_to_quadrature_data_tensor_product_1d(
-               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
-
-         }
-         else if (dimension == 2)
-         {
-            map_field_to_quadrature_data_tensor_product_2d(
-               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
-         }
-         else if (dimension == 3)
-         {
-            map_field_to_quadrature_data_tensor_product_3d(
-               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
-         }
-      }
-      else
-      {
-         map_field_to_quadrature_data(
-            field_qp, dtqmap, field_e, fop, integration_weights);
-      }
-   }
-}
-
-template <size_t num_fields, size_t num_inputs, typename field_operator_ts>
-MFEM_HOST_DEVICE
-void map_fields_to_quadrature_data_conditional(
-   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
-   const std::array<DeviceTensor<1, const real_t>, num_fields> &fields_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
-   field_operator_ts fops,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const std::array<bool, num_inputs> &conditions,
-   const bool &use_sum_factorization = false)
-{
-   for_constexpr<num_inputs>([&](auto i)
-   {
-      map_field_to_quadrature_data_conditional(
-         fields_qp[i], fields_e[i], dtqmaps[i], get<i>(fops), integration_weights,
-         scratch_mem, conditions[i], use_sum_factorization);
-   });
-}
-
-template <size_t num_inputs, typename field_operator_ts>
-MFEM_HOST_DEVICE
-void map_direction_to_quadrature_data_conditional(
-   std::array<DeviceTensor<2>, num_inputs> &directions_qp,
-   const DeviceTensor<1> &direction_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
-   field_operator_ts fops,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const std::array<bool, num_inputs> &conditions,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   for_constexpr<num_inputs>([&](auto i)
-   {
-      if (conditions[i])
-      {
-         if (use_sum_factorization)
-         {
-            if (dimension == 1)
-            {
-               map_field_to_quadrature_data_tensor_product_1d(
-                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-                  integration_weights, scratch_mem);
-            }
-            else if (dimension == 2)
-            {
-               map_field_to_quadrature_data_tensor_product_2d(
-                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-                  integration_weights, scratch_mem);
-            }
-            else if (dimension == 3)
-            {
-               map_field_to_quadrature_data_tensor_product_3d(
-                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-                  integration_weights, scratch_mem);
-            }
-         }
-         else
-         {
-            map_field_to_quadrature_data(
-               directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-               integration_weights);
-         }
-      }
-   });
-}
-
-}

fem/dfem/parameterspace.hpp

@@ -20,7 +20,7 @@ namespace mfem::future
 class ParameterSpace
 {
 public:
-   ParameterSpace(int vdim = 1) : vdim(vdim) {}
+   ParameterSpace(int vdim = 1) : vdim(vdim) { dtq.FE = nullptr; dtq.IntRule = nullptr; }
 
    /// @brief Get vector dimension at each point
    ///
@@ -43,7 +43,7 @@ public:
    /// Get spatial dimension
    ///
    /// returns always 1.
-   int Dimension() const
+   constexpr int Dimension() const
    {
       return 1;
    }
@@ -65,7 +65,7 @@ public:
    /// It should not be used by a user.
    ///
    /// returns identity by default that is lazy evaluated.
-   virtual const Operator* GetElementRestriction(ElementDofOrdering o) const
+   virtual const Operator* GetElementRestriction(ElementDofOrdering) const
    {
       if (!elem_restr)
       {
@@ -74,11 +74,14 @@ public:
       return elem_restr.get();
    }
 
+   virtual const Operator* GetB() const = 0;
+
+   virtual const Operator* GetBt() const = 0;
+
 protected:
    int vdim;
    DofToQuad dtq;
-   mutable std::unique_ptr<Operator> prolongation;
-   mutable std::unique_ptr<Operator> elem_restr;
+   mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
 };
 
 /// @brief Uniform parameter space

fem/dfem/qfunction_apply.hpp

@@ -1,619 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-#include "qfunction_transform.hpp"
-
-namespace mfem::future
-{
-
-/// @brief Call a qfunction with the given parameters.
-///
-/// @param qfunc the qfunction to call.
-/// @param input_shmem the input shared memory.
-/// @param residual_shmem the residual shared memory.
-/// @param rs_qp the size of the residual.
-/// @param num_qp the number of quadrature points.
-/// @param q1d the number of quadrature points in 1D.
-/// @param dimension the spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-/// @tparam qf_param_ts the tuple type of the qfunction parameters.
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   DeviceTensor<2> &residual_shmem,
-   const int &rs_qp,
-   const int &num_qp,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            auto qf_args = decay_tuple<qf_param_ts> {};
-            auto r = Reshape(&residual_shmem(0, q), rs_qp);
-            apply_kernel(r, qfunc, qf_args, input_shmem, q);
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               auto qf_args = decay_tuple<qf_param_ts> {};
-               auto r = Reshape(&residual_shmem(0, q), rs_qp);
-               apply_kernel(r, qfunc, qf_args, input_shmem, q);
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  auto qf_args = decay_tuple<qf_param_ts> {};
-                  auto r = Reshape(&residual_shmem(0, q), rs_qp);
-                  apply_kernel(r, qfunc, qf_args, input_shmem, q);
-               }
-            }
-         }
-      }
-      else
-      {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-         MFEM_ABORT("unsupported dimension for sum factorization");
-#endif
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else
-   {
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         auto qf_args = decay_tuple<qf_param_ts> {};
-         auto r = Reshape(&residual_shmem(0, q), rs_qp);
-         apply_kernel(r, qfunc, qf_args, input_shmem, q);
-      }
-   }
-}
-
-/// @brief Call a qfunction with the given parameters and
-/// compute it's derivative action.
-///
-/// @param qfunc the qfunction to call.
-/// @param input_shmem the input shared memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param residual_shmem the residual shared memory.
-/// @param das_qp the size of the derivative action.
-/// @param num_qp the number of quadrature points.
-/// @param q1d the number of quadrature points in 1D.
-/// @param dimension the spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-/// @tparam qf_param_ts the tuple type of the qfunction parameters.
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction_derivative_action(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   DeviceTensor<2> &residual_shmem,
-   const int &das_qp,
-   const int &num_qp,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            auto r = Reshape(&residual_shmem(0, q), das_qp);
-            auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                        shadow_shmem, q);
-#else
-            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               auto r = Reshape(&residual_shmem(0, q), das_qp);
-               auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-               auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-               apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                           shadow_shmem, q);
-#else
-               apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  auto r = Reshape(&residual_shmem(0, q), das_qp);
-                  auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-                  auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-                  apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                              shadow_shmem, q);
-#else
-                  apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-               }
-            }
-         }
-      }
-      else
-      {
-         MFEM_ABORT_KERNEL("unsupported dimension");
-      }
-   }
-   else
-   {
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         auto r = Reshape(&residual_shmem(0, q), das_qp);
-         auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-         auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-         apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                     shadow_shmem, q);
-#else
-         apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-namespace detail
-{
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction_derivative(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   DeviceTensor<2> &residual_shmem,
-   DeviceTensor<5> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &das_qp,
-   const int &q)
-{
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-   const int num_qp = qpdc.GetShape()[4];
-   const size_t num_inputs = itod.GetShape()[0];
-
-   for (int j = 0; j < trial_vdim; j++)
-   {
-      int m_offset = 0;
-      for (size_t s = 0; s < num_inputs; s++)
-      {
-         const int trial_op_dim = static_cast<int>(itod(s));
-         if (trial_op_dim == 0)
-         {
-            continue;
-         }
-
-         auto d_qp = Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
-         for (int m = 0; m < trial_op_dim; m++)
-         {
-            d_qp(j, m, q) = 1.0;
-
-            auto r = Reshape(&residual_shmem(0, q), das_qp);
-            auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                        shadow_shmem, q);
-#else
-            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-            d_qp(j, m, q) = 0.0;
-
-            auto f = Reshape(&r(0), test_vdim, test_op_dim);
-            for (int i = 0; i < test_vdim; i++)
-            {
-               for (int k = 0; k < test_op_dim; k++)
-               {
-                  qpdc(i, k, j, m + m_offset, q) = f(i, k);
-               }
-            }
-         }
-         m_offset += trial_op_dim;
-      }
-   }
-}
-}
-
-/// @brief Call a qfunction with the given parameters and
-/// compute it's derivative represented by the Jacobian on
-/// each quadrature point.
-///
-/// @param qfunc the qfunction to call.
-/// @param input_shmem the input shared memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param residual_shmem the residual shared memory.
-/// @param qpdc the quadrature point data cache holding the resulting
-/// Jacobians on each quadrature point.
-/// @param itod inputs trial operator dimension.
-/// If input is dependent the value corresponds to the spatial dimension, otherwise
-/// a zero indicates non-dependence on the variable.
-/// @param das_qp the size of the derivative action.
-/// @param q1d the number of quadrature points in 1D.
-/// @param dimension the spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-/// @tparam qf_param_ts the tuple type of the qfunction parameters.
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction_derivative(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   DeviceTensor<2> &residual_shmem,
-   DeviceTensor<5> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &das_qp,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            detail::call_qfunction_derivative<qf_param_ts>(
-               qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               detail::call_qfunction_derivative<qf_param_ts>(
-                  qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  detail::call_qfunction_derivative<qf_param_ts>(
-                     qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-               }
-            }
-         }
-      }
-      else
-      {
-         MFEM_ABORT_KERNEL("unsupported dimension");
-      }
-   }
-   else
-   {
-      const int num_qp = qpdc.GetShape()[4];
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         detail::call_qfunction_derivative<qf_param_ts>(
-            qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-namespace detail
-{
-
-/// @brief Apply the quadrature point data cache (qpdc) to a vector
-/// (usually a direction) on quadrature point q.
-///
-/// The qpdc consists of compatible data to be used for integration with a test
-/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
-/// function including integration weights and necessesary transformations.
-///
-/// @param fhat the qpdc applied to a vector in shadow_memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param qpdc the quadrature point data cache holding the resulting
-/// Jacobians on each quadrature point.
-/// @param itod inputs trial operator dimension.
-/// If input is dependent the value corresponds to the spatial dimension, otherwise
-/// a zero indicates non-dependence on the variable.
-/// @param q the current quadrature point index.
-template <size_t num_fields>
-MFEM_HOST_DEVICE inline
-void apply_qpdc(
-   DeviceTensor<3> &fhat,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   const DeviceTensor<5, const real_t> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &q)
-{
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-   const int num_qp = qpdc.GetShape()[4];
-   const size_t num_inputs = itod.GetShape()[0];
-
-   for (int i = 0; i < test_vdim; i++)
-   {
-      for (int k = 0; k < test_op_dim; k++)
-      {
-         real_t sum = 0.0;
-         int m_offset = 0;
-         for (size_t s = 0; s < num_inputs; s++)
-         {
-            const int trial_op_dim = static_cast<int>(itod(s));
-            if (trial_op_dim == 0)
-            {
-               continue;
-            }
-            const auto d_qp =
-               Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
-            for (int j = 0; j < trial_vdim; j++)
-            {
-               for (int m = 0; m < trial_op_dim; m++)
-               {
-                  sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
-               }
-            }
-            m_offset += trial_op_dim;
-         }
-         fhat(i, k, q) = sum;
-      }
-   }
-}
-}
-
-/// @brief Apply the quadrature point data cache (qpdc) to a vector
-/// (usually a direction).
-///
-/// The qpdc consists of compatible data to be used for integration with a test
-/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
-/// function including integration weights and necessesary transformations.
-///
-/// @param fhat the qpdc applied to a vector in shadow_memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param qpdc the quadrature point data cache holding the resulting
-/// Jacobians on each quadrature point.
-/// @param itod inputs trial operator dimension.
-/// If input is dependent the value corresponds to the spatial dimension, otherwise
-/// a zero indicates non-dependence on the variable.
-/// @param q1d number of quadrature points in 1D.
-/// @param dimension spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-template <size_t num_fields>
-MFEM_HOST_DEVICE inline
-void apply_qpdc(
-   DeviceTensor<3> &fhat,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   const DeviceTensor<5, const real_t> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-               }
-            }
-         }
-      }
-      else
-      {
-         MFEM_ABORT_KERNEL("unsupported dimension");
-      }
-   }
-   else
-   {
-      const int num_qp = qpdc.GetShape()[4];
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-      }
-   }
-}
-
-template <typename qfunc_t, typename args_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel(
-   DeviceTensor<1, real_t> &f_qp,
-   const qfunc_t &qfunc,
-   args_ts &args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   int qp)
-{
-   process_qf_args(u, args, qp);
-   process_qf_result(f_qp, get<0>(apply(qfunc, args)));
-}
-
-template <typename qfunc_t, typename arg_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel_native_dual(
-   DeviceTensor<1, real_t> &f_qp,
-   const qfunc_t &qfunc,
-   arg_ts &args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   const std::array<DeviceTensor<2>, num_args> &v,
-   const int &qp_idx)
-{
-   process_qf_args(u, v, args, qp_idx);
-   auto r = get<0>(apply(qfunc, args));
-   process_derivative_from_native_dual(f_qp, r);
-}
-
-#ifdef MFEM_USE_ENZYME
-
-template <typename func_t, typename... arg_ts>
-MFEM_HOST_DEVICE inline
-auto qfunction_wrapper(const func_t &f, arg_ts &&...args)
-{
-   return f(args...);
-}
-
-// Version for active function arguments only
-//
-// This is an Enzyme regression and can be removed in later versions.
-template <typename qfunc_t, typename arg_ts, std::size_t... Is,
-          typename inactive_arg_ts>
-MFEM_HOST_DEVICE inline
-auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
-                                  arg_ts &&shadow_args,
-                                  std::index_sequence<Is...>,
-                                  inactive_arg_ts &&inactive_args,
-                                  std::index_sequence<>)
-{
-   using qf_return_t = typename create_function_signature<
-                       decltype(&qfunc_t::operator())>::type::return_t;
-   return __enzyme_fwddiff<qf_return_t>(
-             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...>, enzyme_const,
-             (void *)&qfunc, enzyme_dup, &get<Is>(args)..., enzyme_interleave,
-             &get<Is>(shadow_args)...);
-}
-
-// Interleave function arguments for enzyme
-template <typename qfunc_t, typename arg_ts, std::size_t... Is,
-          typename inactive_arg_ts, std::size_t... Js>
-MFEM_HOST_DEVICE inline
-auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
-                                  arg_ts &&shadow_args,
-                                  std::index_sequence<Is...>,
-                                  inactive_arg_ts &&inactive_args,
-                                  std::index_sequence<Js...>)
-{
-   using qf_return_t = typename create_function_signature<
-                       decltype(&qfunc_t::operator())>::type::return_t;
-   return __enzyme_fwddiff<qf_return_t>(
-             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...,
-             decltype(get<Js>(inactive_args))...>,
-             enzyme_const, (void *)&qfunc, enzyme_dup, &get<Is>(args)...,
-             enzyme_const, &get<Js>(inactive_args)..., enzyme_interleave,
-             &get<Is>(shadow_args)...);
-}
-
-template <typename qfunc_t, typename arg_ts, typename inactive_arg_ts>
-MFEM_HOST_DEVICE inline
-auto fwddiff_apply_enzyme(qfunc_t &qfunc, arg_ts &&args,
-                          arg_ts &&shadow_args,
-                          inactive_arg_ts &&inactive_args)
-{
-   auto arg_indices = std::make_index_sequence<
-                      tuple_size<std::remove_reference_t<arg_ts>>::value> {};
-
-   auto inactive_arg_indices = std::make_index_sequence<
-                               tuple_size<std::remove_reference_t<inactive_arg_ts>>::value> {};
-
-   return fwddiff_apply_enzyme_indexed(qfunc, args, shadow_args, arg_indices,
-                                       inactive_args, inactive_arg_indices);
-}
-
-template <typename qfunc_t, typename arg_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel_fwddiff_enzyme(
-   DeviceTensor<1, real_t> &f_qp,
-   qfunc_t &qfunc,
-   arg_ts &args,
-   arg_ts &shadow_args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   const std::array<DeviceTensor<2>, num_args> &v,
-   int qp_idx)
-{
-   process_qf_args(u, args, qp_idx);
-   process_qf_args(v, shadow_args, qp_idx);
-   process_qf_result(f_qp,
-                     get<0>(fwddiff_apply_enzyme(qfunc, args, shadow_args, tuple<> {})));
-}
-#endif // MFEM_USE_ENZYME
-
-} // namespace mfem::future

fem/dfem/qfunction_transform.hpp

@@ -1,346 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-#include "../../linalg/tensor.hpp"
-
-namespace mfem::future
-{
-
-template <typename T0, typename T1, typename T2>
-MFEM_HOST_DEVICE
-void process_qf_arg(const T0 &, const T1 &, T2 &)
-{
-   static_assert(dfem::always_false<T0, T1, T2>,
-                 "process_qf_arg not implemented for arg type");
-}
-
-template <typename T>
-MFEM_HOST_DEVICE
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   const DeviceTensor<1, T> &v,
-   T &arg)
-{
-   arg = u(0);
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   tensor<dual<T, T>, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i).value = u((i * n) + j);
-      }
-   }
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   dual<T, T> &arg)
-{
-   arg.value = u(0);
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   const DeviceTensor<1> &v,
-   dual<T, T> &arg)
-{
-   arg.value = u(0);
-   arg.gradient = v(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   const DeviceTensor<1> &v,
-   tensor<dual<T, T>, n> &arg)
-{
-   for (int i = 0; i < n; i++)
-   {
-      arg(i).value = u(i);
-      arg(i).gradient = v(i);
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   const DeviceTensor<1> &v,
-   tensor<dual<T, T>, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i).value = u((i * n) + j);
-         arg(j, i).gradient = v((i * n) + j);
-      }
-   }
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      r(i) = x(i).value;
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n, m> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      for (size_t j = 0; j < m; j++)
-      {
-         r(i + n * j) = x(i, j).value;
-      }
-   }
-}
-
-template <typename arg_type>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<2> &u,
-   const DeviceTensor<2> &v,
-   arg_type &arg,
-   const int &qp)
-{
-   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
-   const auto v_qp = Reshape(&v(0, qp), v.GetShape()[0]);
-   process_qf_arg(u_qp, v_qp, arg);
-}
-
-template <size_t num_fields, typename qf_args>
-MFEM_HOST_DEVICE inline
-void process_qf_args(
-   const std::array<DeviceTensor<2>, num_fields> &u,
-   const std::array<DeviceTensor<2>, num_fields> &v,
-   qf_args &args,
-   const int &qp)
-{
-   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
-   {
-      process_qf_arg(u[i], v[i], get<i>(args), qp);
-   });
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_derivative_from_native_dual(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n, m> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      for (size_t j = 0; j < m; j++)
-      {
-         r(i + n * j) = x(i, j).gradient;
-      }
-   }
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_derivative_from_native_dual(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      r(i) = x(i).gradient;
-   }
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_derivative_from_native_dual(
-   DeviceTensor<1, T> &r,
-   const dual<T, T> &x)
-{
-   r(0) = x.gradient;
-}
-
-template <typename T0, typename T1>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(const T0 &, T1 &)
-{
-   static_assert(dfem::always_false<T0, T1>,
-                 "process_qf_arg not implemented for arg type");
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   T &arg)
-{
-   arg = u(0);
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   tensor<T> &arg)
-{
-   arg(0) = u(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   tensor<T, n> &arg)
-{
-   for (int i = 0; i < n; i++)
-   {
-      arg(i) = u(i);
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   tensor<T, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i) = u((i * n) + j);
-      }
-   }
-}
-
-template <typename arg_type>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)
-{
-   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
-   process_qf_arg(u_qp, arg);
-}
-
-template <size_t num_fields, typename qf_args>
-MFEM_HOST_DEVICE inline
-void process_qf_args(
-   const std::array<DeviceTensor<2>, num_fields> &u,
-   qf_args &args,
-   const int &qp)
-{
-   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
-   {
-      process_qf_arg(u[i], get<i>(args), qp);
-   });
-}
-
-template <typename T0, typename T1>
-MFEM_HOST_DEVICE inline
-Vector process_qf_result(T0, T1)
-{
-   static_assert(dfem::always_false<T0, T1>,
-                 "process_qf_result not implemented for result type");
-   return Vector{};
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const T &x)
-{
-   r(0) = x;
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1> &r,
-   const dual<T, T> &x)
-{
-   r(0) = x.value;
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<T> &x)
-{
-   r(0) = x(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<T, n> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      r(i) = x(i);
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<T, n, m> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      for (size_t j = 0; j < m; j++)
-      {
-         r(i + n * j) = x(i, j);
-      }
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   const DeviceTensor<1, T> &v,
-   tensor<T, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i) = u((i * n) + j);
-      }
-   }
-}
-
-} // namespace mfem::future

fem/fem.hpp

@@ -52,7 +52,7 @@
 #include "bounds.hpp"
 #include "particleset.hpp"
 
-#include "dfem/doperator.hpp"
+// #include "dfem/doperator.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/kernel_dispatch.hpp

@@ -48,7 +48,11 @@ namespace mfem
 
 #define MFEM_REGISTER_KERNELS(KernelName, KernelType, ...)                     \
    MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__,2,1,))          \
-      (KernelName,KernelType,__VA_ARGS__))
+      (MFEM_EXPORT, KernelName, KernelType, __VA_ARGS__))
+
+#define MFEM_REGISTER_KERNELS_HEADER_ONLY(KernelName, KernelType, ...)         \
+   MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__,2,1,))          \
+      (, KernelName, KernelType, __VA_ARGS__))
 
 #define MFEM_REGISTER_KERNELS_N(_1, _2, N, ...) MFEM_REGISTER_KERNELS_##N
 
@@ -58,19 +62,19 @@ namespace mfem
 
 // Version of MFEM_REGISTER_KERNELS without any "optional" (non-dispatch)
 // parameters.
-#define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
-   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
+#define MFEM_REGISTER_KERNELS_1(ExportMacro, KernelName, KernelType, Params)   \
+   MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, Params, (), Params)
 
 // Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
 // parameters (e.g. NBZ).
-#define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
-   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \
+#define MFEM_REGISTER_KERNELS_2(ExportMacro, KernelName, KernelType, Params, OptParams) \
+   MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, Params, OptParams,       \
                           (MFEM_PARAM_LIST Params, MFEM_PARAM_LIST OptParams))
 
 // P1 are the parameters, P2 are the optional (non-dispatch parameters), and P3
 // is the concatenation of P1 and P2. We need to pass it as a separate argument
 // to avoid a trailing comma in the case that P2 is empty.
-#define MFEM_REGISTER_KERNELS_(KernelName, KernelType, P1, P2, P3)             \
+#define MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, P1, P2, P3) \
   class KernelName                                                             \
       : public ::mfem::KernelDispatchTable<                                    \
             KernelName, KernelType,                                            \
@@ -80,8 +84,8 @@ namespace mfem
     const char *kernel_name = MFEM_KERNEL_NAME(KernelName);                    \
     using KernelSignature = KernelType;                                        \
     template <MFEM_PARAM_LIST P3> static KernelSignature Kernel();             \
-    static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
-    static MFEM_EXPORT KernelName &Get() {                                     \
+    static ExportMacro KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
+    static ExportMacro KernelName &Get() {                                     \
       static KernelName table;                                                 \
       return table;                                                            \
     }                                                                          \

fem/nonlinearform.hpp

@@ -224,6 +224,9 @@ public:
    /** @see GetGradient(const Vector &) */
    Operator &GetGradient(const Vector &x, bool finalize) const;
 
+   /// Suppress a warning about hiding overloaded virtual function.
+   using Operator::GetGradient;
+
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
        set again. */

fem/qfunction.hpp

@@ -79,6 +79,27 @@ public:
       Vector::operator=(orig);
    }
 
+   /** @brief Construct a QuadratureFunction on the given
+       VectorQuadratureSpace, @a vqspace.
+
+       After construction, the QuadratureFunction does not need the
+       VectorQuadratureSpace object. Instead, it uses directly its underlying
+       QuadratureSpaceBase object. */
+   QuadratureFunction(VectorQuadratureSpace &vqspace)
+      : QuadratureFunction(*vqspace.GetSpace(), vqspace.GetVDim())
+   { }
+
+   /** @brief Construct a QuadratureFunction on the given
+       VectorQuadratureSpace, @a vqspace, with the given MemoryType, @a mt, used
+       for the underlying Vector object.
+
+       After construction, the QuadratureFunction does not need the
+       VectorQuadratureSpace object. Instead, it uses directly its underlying
+       QuadratureSpaceBase object. */
+   QuadratureFunction(VectorQuadratureSpace &vqspace, MemoryType mt)
+      : QuadratureFunction(*vqspace.GetSpace(), mt, vqspace.GetVDim())
+   { }
+
    /// Read a QuadratureFunction from the stream @a in.
    /** The QuadratureFunction assumes ownership of the read QuadratureSpace. */
    QuadratureFunction(Mesh *mesh, std::istream &in);

fem/qinterp/eval_transpose.cpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "eval_transpose.hpp"
+
+namespace mfem
+{
+
+/// @cond Suppress_Doxygen_warnings
+
+QuadratureInterpolator::TensorEvalTransposeKernelType
+QuadratureInterpolator::TensorEvalTransposeKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
+{
+   using namespace internal::quadrature_interpolator;
+
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byNODES>; }
+      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byNODES>; }
+      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byNODES>; }
+   }
+   else
+   {
+      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byVDIM>; }
+      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byVDIM>; }
+      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byVDIM>; }
+   }
+   MFEM_ABORT("Invalid dimension");
+   return nullptr;
+}
+
+/// @endcond
+
+} // namespace mfem

fem/qinterp/eval_transpose.hpp

@@ -0,0 +1,304 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#pragma once
+
+#include "../quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/kernels.hpp"
+#include "../kernels.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+template<QVectorLayout Q_LAYOUT>
+static void ValuesTranspose1D(const int NE,
+                              const real_t *b_,
+                              const real_t *q_,
+                              real_t *e_,
+                              const int vdim,
+                              const int d1d,
+                              const int q1d)
+{
+   const auto b = Reshape(b_, q1d, d1d);
+   const auto qd = Q_LAYOUT == QVectorLayout::byNODES ?
+                   Reshape(q_, q1d, vdim, NE) :
+                   Reshape(q_, vdim, q1d, NE);
+   auto e = Reshape(e_, d1d, vdim, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
+   {
+      for (int c = 0; c < vdim; c++)
+      {
+         for (int d = 0; d < d1d; d++)
+         {
+            real_t u = 0.0;
+            for (int q = 0; q < q1d; q++)
+            {
+               const real_t qval = Q_LAYOUT == QVectorLayout::byVDIM ?
+                                   qd(c, q, el) : qd(q, c, el);
+               u += b(q, d) * qval;
+            }
+            e(d, c, el) += u;
+         }
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void ValuesTranspose2D(const int NE,
+                              const real_t *b_,
+                              const real_t *q_,
+                              real_t *e_,
+                              const int vdim = 0,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, VDIM, NE) :
+                  Reshape(q_, VDIM, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t sB[MQ1*MD1];
+      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
+      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
+
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
+      ConstDeviceMatrix B(sB, D1D, Q1D);
+      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
+      DeviceMatrix DQ(sm1[tidz], MD1, MQ1);
+      DeviceMatrix DD(sm0[tidz], MD1, MD1);
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Load Q data
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               QQ(qx,qy) = Q_LAYOUT == QVectorLayout::byVDIM ?
+                           q(c,qx,qy,el) : q(qx,qy,c,el);
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in y: QQ -> DQ (apply B^T in y-direction)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += B(dy,qy) * QQ(qx,qy);
+               }
+               DQ(dy,qx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in x: DQ -> DD (apply B^T in x-direction)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += B(dx,qx) * DQ(dy,qx);
+               }
+               DD(dx,dy) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Store result
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               e(dx,dy,c,el) += DD(dx,dy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void ValuesTranspose3D(const int NE,
+                              const real_t *b_,
+                              const real_t *q_,
+                              real_t *e_,
+                              const int vdim = 0,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, NE) :
+                  Reshape(q_, VDIM, Q1D, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED real_t sB[MQ1*MD1];
+      MFEM_SHARED real_t sm0[MDQ*MDQ*MDQ];
+      MFEM_SHARED real_t sm1[MDQ*MDQ*MDQ];
+
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
+      ConstDeviceMatrix B(sB, D1D, Q1D);
+      DeviceCube QQQ(sm0, MQ1, MQ1, MQ1);
+      DeviceCube DQQ(sm1, MD1, MQ1, MQ1);
+      DeviceCube DDQ(sm0, MD1, MD1, MQ1);
+      DeviceCube DDD(sm1, MD1, MD1, MD1);
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Load Q data
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  QQQ(qx,qy,qz) = Q_LAYOUT == QVectorLayout::byVDIM ?
+                                  q(c,qx,qy,qz,el) : q(qx,qy,qz,c,el);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in z
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += B(dz,qz) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dz,qx,qy) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in y
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += B(dy,qy) * DQQ(dz,qx,qy);
+                  }
+                  DDQ(dz,dy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in x
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += B(dx,qx) * DDQ(dz,dy,qx);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+
+/// \cond DO_NOT_DOCUMENT
+
+template<int DIM, QVectorLayout Q_LAYOUT,
+         int VDIM, int D1D, int Q1D, int NBZ>
+QuadratureInterpolator::TensorEvalTransposeKernelType
+QuadratureInterpolator::TensorEvalTransposeKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::ValuesTranspose1D<Q_LAYOUT>; }
+   else if (DIM == 2) { return internal::quadrature_interpolator::ValuesTranspose2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::ValuesTranspose3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/qinterp/eval_transpose_by_vdim.cpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "eval_transpose.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+void InitEvalTransposeByVDimKernels()
+{
+   using k = QuadratureInterpolator::TensorEvalTransposeKernels;
+   constexpr auto L = QVectorLayout::byVDIM;
+
+   // 2D
+   k::Specialization<2,L,1,2,4>::Opt<8>::Add();
+   k::Specialization<2,L,1,3,6>::Opt<4>::Add();
+   k::Specialization<2,L,1,4,8>::Opt<2>::Add();
+
+   k::Specialization<2,L,2,2,4>::Opt<8>::Add();
+   k::Specialization<2,L,2,3,4>::Opt<8>::Add();
+   k::Specialization<2,L,2,3,6>::Opt<4>::Add();
+   k::Specialization<2,L,2,4,6>::Opt<2>::Add();
+   k::Specialization<2,L,2,4,8>::Opt<2>::Add();
+
+   // 3D
+   k::Specialization<3,L,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,L,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,L,1,4,8>::Opt<1>::Add();
+   k::Specialization<3,L,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,L,3,3,6>::Opt<1>::Add();
+   k::Specialization<3,L,3,4,8>::Opt<1>::Add();
+
+   k::Specialization<3,L,3,2,2>::Opt<1>::Add();
+   k::Specialization<3,L,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,L,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,L,3,5,5>::Opt<1>::Add();
+   k::Specialization<3,L,3,6,6>::Opt<1>::Add();
+   k::Specialization<3,L,3,7,7>::Opt<1>::Add();
+   k::Specialization<3,L,3,8,8>::Opt<1>::Add();
+   k::Specialization<3,L,3,9,9>::Opt<1>::Add();
+
+   k::Specialization<3,L,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,L,3,3,4>::Opt<1>::Add();
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+} // namespace mfem

fem/qinterp/grad_transpose.cpp

@@ -0,0 +1,62 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "grad_transpose.hpp"
+
+namespace mfem
+{
+
+/// @cond Suppress_Doxygen_warnings
+
+QuadratureInterpolator::GradTransposeKernelType
+QuadratureInterpolator::GradTransposeKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
+{
+   using namespace internal::quadrature_interpolator;
+
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      if (GRAD_PHYS)
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, true>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, true>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, true>; }
+      }
+      else
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, false>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, false>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, false>; }
+      }
+   }
+   else
+   {
+      if (GRAD_PHYS)
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, true>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, true>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, true>; }
+      }
+      else
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, false>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, false>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, false>; }
+      }
+   }
+   MFEM_ABORT("Invalid dimension");
+   return nullptr;
+}
+
+/// @endcond
+
+} // namespace mfem

fem/qinterp/grad_transpose.hpp

@@ -0,0 +1,741 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#pragma once
+
+#include "../quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/kernels.hpp"
+#include "../kernels.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+// Transpose gradient operation: integrate against shape function derivatives
+// This is the adjoint of the Derivatives operation
+
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
+static void DerivativesTranspose1D(const int NE,
+                                   const real_t *b_,
+                                   const real_t *g_,
+                                   const real_t *j_,
+                                   const real_t *q_,
+                                   real_t *e_,
+                                   const int sdim,
+                                   const int vdim,
+                                   const int d1d,
+                                   const int q1d)
+{
+   MFEM_CONTRACT_VAR(b_);
+   const int SDIM = GRAD_PHYS ? sdim : 1;
+   const auto g = Reshape(g_, q1d, d1d);
+   const auto j = Reshape(j_, q1d, SDIM, NE);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, q1d, vdim, SDIM, NE):
+                  Reshape(q_, vdim, SDIM, q1d, NE);
+   auto e = Reshape(e_, d1d, vdim, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
+   {
+      for (int c = 0; c < vdim; c++)
+      {
+         for (int d = 0; d < d1d; d++)
+         {
+            real_t u = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               // Load gradient from q-vector
+               real_t dq[3] = {0.0, 0.0, 0.0};
+               for (int s = 0; s < SDIM; ++s)
+               {
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)  { dq[s] = q(c, s, qx, el); }
+                  if (Q_LAYOUT == QVectorLayout::byNODES) { dq[s] = q(qx, c, s, el); }
+               }
+
+               // Apply inverse Jacobian transpose (adjoint of physical gradient)
+               real_t du = dq[0];
+               if (GRAD_PHYS)
+               {
+                  if (SDIM == 1) { du = dq[0] / j(qx, 0, el); }
+                  else if (SDIM == 2)
+                  {
+                     const real_t Jloc[2] = {j(qx,0,el), j(qx,1,el)};
+                     real_t Jinv[3];
+                     kernels::CalcLeftInverse<2,1>(Jloc, Jinv);
+                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1];
+                  }
+                  else // SDIM == 3
+                  {
+                     const real_t Jloc[3] = {j(qx,0,el), j(qx,1,el), j(qx,2,el)};
+                     real_t Jinv[3];
+                     kernels::CalcLeftInverse<3,1>(Jloc, Jinv);
+                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1] + Jinv[2]*dq[2];
+                  }
+               }
+
+               // Accumulate contribution (transpose of G matrix)
+               u += g(qx, d) * du;
+            }
+            e(d, c, el) += u;
+         }
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void DerivativesTranspose2D(const int NE,
+                                   const real_t *b_,
+                                   const real_t *g_,
+                                   const real_t *j_,
+                                   const real_t *q_,
+                                   real_t *e_,
+                                   const int sdim = 2,
+                                   const int vdim = 0,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   const int SDIM = GRAD_PHYS ? sdim : 2;
+   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto j = Reshape(j_, Q1D, Q1D, SDIM, 2, NE);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, VDIM, SDIM, NE):
+                  Reshape(q_, VDIM, SDIM, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+      DeviceMatrix B(BG[0], D1D, Q1D);
+      DeviceMatrix G(BG[1], D1D, Q1D);
+
+      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
+      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
+
+      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
+      DeviceMatrix DQ0(sm1[tidz], MD1, MQ1);
+      DeviceMatrix DQ1(sm1[tidz], MD1, MQ1);  // Reuse sm1 after DQ0 is done
+      DeviceMatrix DD(sm0[tidz], MD1, MD1);   // Reuse sm0 after QQ is done
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Load Q data and apply inverse Jacobian
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               // Load gradient components
+               real_t dq[3] = {0.0, 0.0, 0.0};
+               for (int d = 0; d < SDIM; ++d)
+               {
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
+                  else { dq[d] = q(qx, qy, c, d, el); }
+               }
+
+               // Apply inverse Jacobian transpose (adjoint of physical gradient)
+               real_t du[2] = {dq[0], dq[1]};
+               if (GRAD_PHYS)
+               {
+                  if (SDIM == 2)
+                  {
+                     real_t Jloc[4], Jinv[4];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,0,1,el);
+                     Jloc[3] = j(qx,qy,1,1,el);
+                     kernels::CalcInverse<2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+                  else // SDIM == 3
+                  {
+                     real_t Jloc[6], Jinv[6];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,2,0,el);
+                     Jloc[3] = j(qx,qy,0,1,el);
+                     Jloc[4] = j(qx,qy,1,1,el);
+                     Jloc[5] = j(qx,qy,2,1,el);
+                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+               }
+               QQ(qx, qy) = du[0];  // Store du/dx component
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in y-direction: QQ -> DQ0
+         // (Transpose of d/dx which uses DQ1(dy,qx)*B(dy,qy))
+         // Must produce DQ0(dy,qx) to match forward's DQ1 indexing
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += B(dy,qy) * QQ(qx,qy);
+               }
+               DQ0(dy,qx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in x-direction: DQ0 -> DD
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += G(dx,qx) * DQ0(dy,qx);
+               }
+               DD(dx,dy) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate to output
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               e(dx,dy,c,el) += DD(dx,dy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Now process du/dy component
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               // Load gradient components
+               real_t dq[3] = {0.0, 0.0, 0.0};
+               for (int d = 0; d < SDIM; ++d)
+               {
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
+                  else { dq[d] = q(qx, qy, c, d, el); }
+               }
+
+               // Apply inverse Jacobian transpose
+               real_t du[2] = {dq[0], dq[1]};
+               if (GRAD_PHYS)
+               {
+                  if (SDIM == 2)
+                  {
+                     real_t Jloc[4], Jinv[4];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,0,1,el);
+                     Jloc[3] = j(qx,qy,1,1,el);
+                     kernels::CalcInverse<2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+                  else // SDIM == 3
+                  {
+                     real_t Jloc[6], Jinv[6];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,2,0,el);
+                     Jloc[3] = j(qx,qy,0,1,el);
+                     Jloc[4] = j(qx,qy,1,1,el);
+                     Jloc[5] = j(qx,qy,2,1,el);
+                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+               }
+               QQ(qx, qy) = du[1];  // Store du/dy component
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in y-direction: QQ -> DQ1
+         // (Transpose of d/dy which uses DQ0(dy,qx)*G(dy,qy))
+         // Must produce DQ1(dy,qx) to match forward's DQ0 indexing
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += G(dy,qy) * QQ(qx,qy);
+               }
+               DQ1(dy,qx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in x-direction: DQ1 -> DD
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += B(dx,qx) * DQ1(dy,qx);
+               }
+               DD(dx,dy) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate to output
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               e(dx,dy,c,el) += DD(dx,dy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void DerivativesTranspose3D(const int NE,
+                                   const real_t *b_,
+                                   const real_t *g_,
+                                   const real_t *j_,
+                                   const real_t *q_,    // q_der
+                                   real_t *e_,          // e_vec
+                                   const int sdim = 3,
+                                   const int vdim = 0,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto j = Reshape(j_, Q1D, Q1D, Q1D, 3, 3, NE);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, 3, NE):
+                  Reshape(q_, VDIM, 3, Q1D, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+      DeviceMatrix B(BG[0], D1D, Q1D);
+      DeviceMatrix G(BG[1], D1D, Q1D);
+
+      MFEM_SHARED real_t sm0[1][MQ1*MQ1*MQ1];
+      MFEM_SHARED real_t sm1[1][MQ1*MQ1*MQ1];
+      DeviceCube QQQ(sm0[0], MQ1, MQ1, MQ1);
+      DeviceCube DQQ(sm1[0], MD1, MQ1, MQ1);
+      DeviceCube DDQ(sm0[0], MD1, MD1, MQ1);
+      DeviceCube DDD(sm1[0], MD1, MD1, MD1);
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Process du/dx component
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t dq[3];
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     dq[0] = q(c,0,qx,qy,qz,el);
+                     dq[1] = q(c,1,qx,qy,qz,el);
+                     dq[2] = q(c,2,qx,qy,qz,el);
+                  }
+                  else
+                  {
+                     dq[0] = q(qx,qy,qz,c,0,el);
+                     dq[1] = q(qx,qy,qz,c,1,el);
+                     dq[2] = q(qx,qy,qz,c,2,el);
+                  }
+
+                  real_t du[3] = {dq[0], dq[1], dq[2]};
+                  if (GRAD_PHYS)
+                  {
+                     real_t Jloc[9], Jinv[9];
+                     for (int col = 0; col < 3; col++)
+                     {
+                        for (int row = 0; row < 3; row++)
+                        {
+                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
+                        }
+                     }
+                     kernels::CalcInverse<3>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
+                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
+                     du[0] = U; du[1] = V; du[2] = W;
+                  }
+                  QQQ(qx,qy,qz) = du[0];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in x: QQQ -> DQQ (transpose of G⊗B⊗B)
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += G(dx,qx) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dx,qy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in y: DQQ -> DDQ
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += B(dy,qy) * DQQ(dx,qy,qz);
+                  }
+                  DDQ(dx,dy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in z: DDQ -> DDD
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += B(dz,qz) * DDQ(dx,dy,qz);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate result
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Process du/dy component
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t dq[3];
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     dq[0] = q(c,0,qx,qy,qz,el);
+                     dq[1] = q(c,1,qx,qy,qz,el);
+                     dq[2] = q(c,2,qx,qy,qz,el);
+                  }
+                  else
+                  {
+                     dq[0] = q(qx,qy,qz,c,0,el);
+                     dq[1] = q(qx,qy,qz,c,1,el);
+                     dq[2] = q(qx,qy,qz,c,2,el);
+                  }
+
+                  real_t du[3] = {dq[0], dq[1], dq[2]};
+                  if (GRAD_PHYS)
+                  {
+                     real_t Jloc[9], Jinv[9];
+                     for (int col = 0; col < 3; col++)
+                     {
+                        for (int row = 0; row < 3; row++)
+                        {
+                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
+                        }
+                     }
+                     kernels::CalcInverse<3>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
+                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
+                     du[0] = U; du[1] = V; du[2] = W;
+                  }
+                  QQQ(qx,qy,qz) = du[1];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in x: QQQ -> DQQ (transpose of B⊗G⊗B)
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += B(dx,qx) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dx,qy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in y: DQQ -> DDQ
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += G(dy,qy) * DQQ(dx,qy,qz);
+                  }
+                  DDQ(dx,dy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in z: DDQ -> DDD
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += B(dz,qz) * DDQ(dx,dy,qz);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate result
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Process du/dz component
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t dq[3];
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     dq[0] = q(c,0,qx,qy,qz,el);
+                     dq[1] = q(c,1,qx,qy,qz,el);
+                     dq[2] = q(c,2,qx,qy,qz,el);
+                  }
+                  else
+                  {
+                     dq[0] = q(qx,qy,qz,c,0,el);
+                     dq[1] = q(qx,qy,qz,c,1,el);
+                     dq[2] = q(qx,qy,qz,c,2,el);
+                  }
+
+                  real_t du[3] = {dq[0], dq[1], dq[2]};
+                  if (GRAD_PHYS)
+                  {
+                     real_t Jloc[9], Jinv[9];
+                     for (int col = 0; col < 3; col++)
+                     {
+                        for (int row = 0; row < 3; row++)
+                        {
+                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
+                        }
+                     }
+                     kernels::CalcInverse<3>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
+                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
+                     du[0] = U; du[1] = V; du[2] = W;
+                  }
+                  QQQ(qx,qy,qz) = du[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in z: QQQ -> DQQ (transpose of B⊗B⊗G)
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += G(dz,qz) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dz,qy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in y: DQQ -> DDQ
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += B(dy,qy) * DQQ(dz,qy,qx);
+                  }
+                  DDQ(dz,dy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in x: DDQ -> DDD
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += B(dx,qx) * DDQ(dz,dy,qx);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate result
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+
+/// \cond DO_NOT_DOCUMENT
+
+template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+         int Q1D, int NBZ>
+QuadratureInterpolator::GradTransposeKernelType
+QuadratureInterpolator::GradTransposeKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::DerivativesTranspose1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return internal::quadrature_interpolator::DerivativesTranspose2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::DerivativesTranspose3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/qinterp/grad_transpose_by_nodes.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "grad_transpose.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+template <bool P>
+void InitGradTransposeByNodesKernels()
+{
+   using k = QuadratureInterpolator::GradTransposeKernels;
+   constexpr auto L = QVectorLayout::byNODES;
+   // 2D
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
+
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
+   // 3D
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
+}
+
+template void InitGradTransposeByNodesKernels<false>();
+template void InitGradTransposeByNodesKernels<true>();
+
+} // namespace quadrature_interpolator
+} // namespace internal
+} // namespace mfem

fem/qinterp/grad_transpose_by_vdim.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "grad_transpose.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+template <bool P>
+void InitGradTransposeByVDimKernels()
+{
+   using k = QuadratureInterpolator::GradTransposeKernels;
+   constexpr auto L = QVectorLayout::byVDIM;
+   // 2D
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
+
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
+   // 3D
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
+}
+
+template void InitGradTransposeByVDimKernels<false>();
+template void InitGradTransposeByVDimKernels<true>();
+
+} // namespace quadrature_interpolator
+} // namespace internal
+} // namespace mfem

fem/qspace.hpp

@@ -27,7 +27,11 @@ enum class QSpaceOffsetStorage
 
 /// Abstract base class for QuadratureSpace and FaceQuadratureSpace.
 /** This class represents the storage layout for QuadratureFunction%s, that may
-    be defined either on mesh elements or mesh faces. */
+    be defined either on mesh elements or mesh faces.
+
+    This class represents the layout for a single scalar value at each
+    quadrature point. Use class VectorQuadratureSpace to represent a space with
+    multiple (vector) values at each quadrature point. */
 class QuadratureSpaceBase
 {
 protected:
@@ -158,8 +162,62 @@ public:
    virtual ~QuadratureSpaceBase() { }
 };
 
+/// Vector version of the scalar class QuadratureSpaceBase.
+class VectorQuadratureSpace
+{
+protected:
+   /// Points to an external object provided during construction. Not owned.
+   QuadratureSpaceBase *qspace;
+   /// Vector dimension.
+   int vdim;
+
+public:
+   /** @brief Construct a VectorQuadratureSpace on the given
+       QuadratureSpaceBase, @a qspace_, with the given vector dimension,
+       @a vdim_.
+
+       The VectorQuadratureSpace does not assume ownership of the
+       QuadratureSpaceBase, @a qspace_. */
+   VectorQuadratureSpace(QuadratureSpaceBase &qspace_, int vdim_)
+      : qspace(&qspace_),
+        vdim(vdim_)
+   { }
+
+   /// Copy constructor: default.
+   VectorQuadratureSpace(const VectorQuadratureSpace &) = default;
+
+   /// Copy assignment: default.
+   VectorQuadratureSpace &operator=(const VectorQuadratureSpace &) = default;
+
+   /// Move construction is not allowed.
+   VectorQuadratureSpace(VectorQuadratureSpace &&) = delete;
+
+   /// Move assignment is not allowed.
+   VectorQuadratureSpace &operator=(VectorQuadratureSpace &&) = delete;
+
+   /// Destructor: default.
+   ~VectorQuadratureSpace() = default;
+
+   /// Get the associated scalar QuadratureSpaceBase object.
+   QuadratureSpaceBase *GetSpace() { return qspace; }
+
+   /// Get the associated scalar QuadratureSpaceBase object (const version).
+   const QuadratureSpaceBase *GetSpace() const { return qspace; }
+
+   /// Get the vector dimension.
+   int GetVDim() const { return vdim; }
+
+   /** @brief Get the total size (on this MPI-rank in parallel) of the
+       VectorQuadratureSpace. */
+   int GetVSize() const { return qspace->GetSize() * vdim; }
+};
+
 /// Class representing the storage layout of a QuadratureFunction.
-/** Multiple QuadratureFunction%s can share the same QuadratureSpace. */
+/** Multiple QuadratureFunction%s can share the same QuadratureSpace.
+
+    This class represents the layout for a single scalar value at each
+    quadrature point. Use class VectorQuadratureSpace to represent a space with
+    multiple (vector) values at each quadrature point. */
 class QuadratureSpace : public QuadratureSpaceBase
 {
 protected:
@@ -209,7 +267,11 @@ public:
 
 /// Class representing the storage layout of a FaceQuadratureFunction.
 /** FaceQuadratureSpace is defined on either the interior or boundary faces
-    of a mesh, depending on the provided FaceType. */
+    of a mesh, depending on the provided FaceType.
+
+    This class represents the layout for a single scalar value at each
+    quadrature point. Use class VectorQuadratureSpace to represent a space with
+    multiple (vector) values at each quadrature point. */
 class FaceQuadratureSpace : public QuadratureSpaceBase
 {
    FaceType face_type; ///< Is the space defined on interior or boundary faces?

fem/quadinterpolator.cpp

@@ -11,6 +11,7 @@
 
 #include "quadinterpolator.hpp"
 #include "qinterp/grad.hpp"
+#include "qinterp/grad_transpose.hpp"
 #include "qinterp/eval.hpp"
 #include "qspace.hpp"
 #include "../general/forall.hpp"
@@ -30,7 +31,10 @@ void InitEvalKernels();
 void InitDetKernels();
 template <bool P> void InitGradByNodesKernels();
 template <bool P> void InitGradByVDimKernels();
+template <bool P> void InitGradTransposeByNodesKernels();
+template <bool P> void InitGradTransposeByVDimKernels();
 void InitTensorEvalHDivKernels();
+void InitEvalTransposeByVDimKernels();
 struct Kernels
 {
    Kernels()
@@ -45,12 +49,19 @@ struct Kernels
       // Phys grad kernels
       InitGradByNodesKernels<true>();
       InitGradByVDimKernels<true>();
+      // Non-phys grad transpose kernels
+      InitGradTransposeByNodesKernels<false>();
+      InitGradTransposeByVDimKernels<false>();
+      // Phys grad transpose kernels
+      InitGradTransposeByNodesKernels<true>();
+      InitGradTransposeByVDimKernels<true>();
       // Determinants
       InitDetKernels();
       // Non-tensor
       InitEvalKernels();
       // Tensor (quad,hex) H(div)
       InitTensorEvalHDivKernels();
+      InitEvalTransposeByVDimKernels();
    }
 };
 }
@@ -646,16 +657,59 @@ void QuadratureInterpolator::MultHDiv(const Vector &e_vec,
    MFEM_CONTRACT_VAR(q_div);
 }
 
-void QuadratureInterpolator::MultTranspose(unsigned eval_flags,
-                                           const Vector &q_val,
-                                           const Vector &q_der,
-                                           Vector &e_vec) const
+void QuadratureInterpolator::AddMultTranspose(unsigned eval_flags,
+                                              const Vector &q_val,
+                                              const Vector &q_der,
+                                              Vector &e_vec) const
 {
-   MFEM_CONTRACT_VAR(eval_flags);
-   MFEM_CONTRACT_VAR(q_val);
-   MFEM_CONTRACT_VAR(q_der);
-   MFEM_CONTRACT_VAR(e_vec);
-   MFEM_ABORT("this method is not implemented yet");
+   const int ne = fespace->GetNE();
+   if (ne == 0) { return; }
+   const FiniteElement *fe = fespace->GetFE(0);
+   const int vdim = fespace->GetVDim();
+   const int sdim = fespace->GetMesh()->SpaceDimension();
+
+   const bool use_tensor_eval =
+      use_tensor_products &&
+      dynamic_cast<const TensorBasisElement*>(fe) != nullptr;
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   const DofToQuad::Mode mode =
+      use_tensor_eval ? DofToQuad::TENSOR : DofToQuad::FULL;
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
+   const int dim = maps.FE->GetDim();
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+
+   const GeometricFactors *geom = nullptr;
+   if (eval_flags & PHYSICAL_DERIVATIVES)
+   {
+      const int jacobians = GeometricFactors::JACOBIANS;
+      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
+   }
+
+   if (use_tensor_eval)
+   {
+      if (eval_flags & (VALUES | PHYSICAL_VALUES))
+      {
+         TensorEvalTransposeKernels::Run(dim, q_layout, vdim, nd, nq, ne,
+                                         maps.B.Read(), q_val.Read(),
+                                         e_vec.ReadWrite(), vdim, nd, nq);
+      }
+      if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
+      {
+         const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
+         const real_t *J = phys ? geom->J.Read() : nullptr;
+         const int s_dim = phys ? sdim : dim;
+         GradTransposeKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
+                                   maps.B.Read(), maps.G.Read(), J,
+                                   q_der.Read(), e_vec.ReadWrite(),
+                                   s_dim, vdim, nd, nq);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Non-tensor MultTranspose not yet implemented");
+   }
 }
 
 void QuadratureInterpolator::Values(const Vector &e_vec,

fem/quadinterpolator.hpp

@@ -152,8 +152,8 @@ public:
    void Determinants(const Vector &e_vec, Vector &q_det) const;
 
    /// Perform the transpose operation of Mult(). (TODO)
-   void MultTranspose(unsigned eval_flags, const Vector &q_val,
-                      const Vector &q_der, Vector &e_vec) const;
+   void AddMultTranspose(unsigned eval_flags, const Vector &q_val,
+                         const Vector &q_der, Vector &e_vec) const;
 
    /// @brief Returns true if the given finite element space is supported by
    /// QuadratureInterpolator.
@@ -178,6 +178,13 @@ public:
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
+   using TensorEvalTransposeKernelType = void(*)(const int, const real_t *,
+                                                 const real_t *,
+                                                 real_t *, const int, const int, const int);
+   using GradTransposeKernelType = void(*)(const int, const real_t *,
+                                           const real_t *, const real_t *,
+                                           const real_t *, real_t *,
+                                           const int, const int, const int, const int);
 
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
@@ -189,6 +196,10 @@ public:
                          (int, QVectorLayout, bool, int, int), (int));
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
+   MFEM_REGISTER_KERNELS(TensorEvalTransposeKernels, TensorEvalTransposeKernelType,
+                         (int, QVectorLayout, int, int, int), (int));
+   MFEM_REGISTER_KERNELS(GradTransposeKernels, GradTransposeKernelType,
+                         (int, QVectorLayout, bool, int, int, int), (int));
 };
 
 }

fem/restriction.cpp

@@ -299,6 +299,17 @@ void ElementRestriction::FillSparseMatrix(const Vector &mat_ea,
    FillJAndData(mat_ea, mat);
 }
 
+void ElementRestriction::FillSparseMatrix(
+   const Vector &mat_ea, SparseMatrix &mat,
+   const ElementRestriction &trial_restr) const
+{
+   mat.GetMemoryI().New(mat.Height()+1, mat.GetMemoryI().GetMemoryType());
+   const int nnz = FillI(mat, trial_restr);
+   mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
+   mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
+   FillJAndData(mat_ea, mat, trial_restr);
+}
+
 static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
                                       const int *nbr_elts, const int nbrNbElts)
 {
@@ -328,6 +339,23 @@ static MFEM_HOST_DEVICE int GetAndIncrementNnzIndex(const int i_L, int* I)
    return ind;
 }
 
+static MFEM_HOST_DEVICE int DofToVDof(const int dof, const int c,
+                                      const int ndofs, const int vdim,
+                                      const bool byvdim)
+{
+   return byvdim ? dof*vdim + c : c*ndofs + dof;
+}
+
+static MFEM_HOST_DEVICE int SignedIndexAbs(const int i)
+{
+   return (i >= 0) ? i : -1 - i;
+}
+
+static MFEM_HOST_DEVICE int SignedIndexSign(const int i)
+{
+   return (i >= 0) ? 1 : -1;
+}
+
 int ElementRestriction::FillI(SparseMatrix &mat) const
 {
    const int all_dofs = ndofs;
@@ -405,6 +433,100 @@ int ElementRestriction::FillI(SparseMatrix &mat) const
    return h_I[nTdofs];
 }
 
+int ElementRestriction::FillI(SparseMatrix &mat,
+                              const ElementRestriction &trial_restr) const
+{
+   MFEM_VERIFY(ne == trial_restr.ne,
+               "ElementRestriction::FillI: test/trial NE mismatch");
+   const int test_all_dofs = ndofs;
+   const int trial_all_dofs = trial_restr.ndofs;
+   const int test_vd = vdim;
+   const int trial_vd = trial_restr.vdim;
+   const int test_elt_dofs = dof;
+   const int trial_elt_dofs = trial_restr.dof;
+   const bool test_byvdim = byvdim;
+   const bool trial_byvdim = trial_restr.byvdim;
+   auto I = mat.ReadWriteI();
+   auto test_offsets = offsets.Read();
+   auto test_indices = indices.Read();
+   auto test_gather_map = gather_map.Read();
+   auto trial_offsets = trial_restr.offsets.Read();
+   auto trial_indices = trial_restr.indices.Read();
+   auto trial_gather_map = trial_restr.gather_map.Read();
+
+   Array<int> test_elts(indices.Size());
+   Array<int> trial_elts(trial_restr.indices.Size());
+   auto d_test_elts = test_elts.Write();
+   auto d_trial_elts = trial_elts.Write();
+
+   mfem::forall(test_vd*test_all_dofs+1, [=] MFEM_HOST_DEVICE (int i_L)
+   {
+      I[i_L] = 0;
+   });
+   mfem::forall(ne*test_elt_dofs*test_vd, [=] MFEM_HOST_DEVICE (int iE)
+   {
+      const int e = iE/(test_elt_dofs*test_vd);
+      const int it = iE%(test_elt_dofs*test_vd);
+      const int i = it%test_elt_dofs;
+      const int test_c = it/test_elt_dofs;
+
+      const int i_gm = e*test_elt_dofs + i;
+      const int i_dof = SignedIndexAbs(test_gather_map[i_gm]);
+      const int i_L = DofToVDof(i_dof, test_c, test_all_dofs, test_vd,
+                                test_byvdim);
+      const int i_offset = test_offsets[i_dof];
+      const int i_next_offset = test_offsets[i_dof+1];
+      const int i_nbElts = i_next_offset - i_offset;
+
+      int *i_elts = &d_test_elts[i_offset];
+      for (int e_i = 0; e_i < i_nbElts; ++e_i)
+      {
+         const int i_loc = SignedIndexAbs(test_indices[i_offset+e_i]);
+         i_elts[e_i] = i_loc/test_elt_dofs;
+      }
+      for (int trial_c = 0; trial_c < trial_vd; ++trial_c)
+      {
+         MFEM_CONTRACT_VAR(trial_c);
+         for (int j = 0; j < trial_elt_dofs; j++)
+         {
+            const int j_gm = e*trial_elt_dofs + j;
+            const int j_dof = SignedIndexAbs(trial_gather_map[j_gm]);
+            const int j_offset = trial_offsets[j_dof];
+            const int j_next_offset = trial_offsets[j_dof+1];
+            const int j_nbElts = j_next_offset - j_offset;
+            if (i_nbElts == 1 || j_nbElts == 1)
+            {
+               GetAndIncrementNnzIndex(i_L, I);
+            }
+            else
+            {
+               int *j_elts = &d_trial_elts[j_offset];
+               for (int e_j = 0; e_j < j_nbElts; ++e_j)
+               {
+                  const int j_loc = SignedIndexAbs(trial_indices[j_offset+e_j]);
+                  j_elts[e_j] = j_loc/trial_elt_dofs;
+               }
+               const int min_e = GetMinElt(i_elts, i_nbElts,
+                                           j_elts, j_nbElts);
+               if (e == min_e) { GetAndIncrementNnzIndex(i_L, I); }
+            }
+         }
+      }
+   });
+
+   auto h_I = mat.HostReadWriteI();
+   const int nTdofs = test_vd*test_all_dofs;
+   int sum = 0;
+   for (int i = 0; i < nTdofs; i++)
+   {
+      const int nnz = h_I[i];
+      h_I[i] = sum;
+      sum += nnz;
+   }
+   h_I[nTdofs] = sum;
+   return h_I[nTdofs];
+}
+
 void ElementRestriction::FillJAndData(const Vector &ea_data,
                                       SparseMatrix &mat) const
 {
@@ -501,6 +623,135 @@ void ElementRestriction::FillJAndData(const Vector &ea_data,
    h_I[0] = 0;
 }
 
+void ElementRestriction::FillJAndData(
+   const Vector &ea_data, SparseMatrix &mat,
+   const ElementRestriction &trial_restr) const
+{
+   MFEM_VERIFY(ne == trial_restr.ne,
+               "ElementRestriction::FillJAndData: test/trial NE mismatch");
+   const int test_all_dofs = ndofs;
+   const int trial_all_dofs = trial_restr.ndofs;
+   const int test_vd = vdim;
+   const int trial_vd = trial_restr.vdim;
+   const int test_elt_dofs = dof;
+   const int trial_elt_dofs = trial_restr.dof;
+   const bool test_byvdim = byvdim;
+   const bool trial_byvdim = trial_restr.byvdim;
+   auto I = mat.ReadWriteI();
+   auto J = mat.WriteJ();
+   auto Data = mat.WriteData();
+   auto test_offsets = offsets.Read();
+   auto test_indices = indices.Read();
+   auto test_gather_map = gather_map.Read();
+   auto trial_offsets = trial_restr.offsets.Read();
+   auto trial_indices = trial_restr.indices.Read();
+   auto trial_gather_map = trial_restr.gather_map.Read();
+   auto mat_ea = Reshape(ea_data.Read(), test_elt_dofs, test_vd,
+                         trial_elt_dofs, trial_vd, ne);
+
+   Array<int> test_el(indices.Size() * 3);
+   Array<int> trial_el(trial_restr.indices.Size() * 3);
+   auto d_test_el = Reshape(test_el.Write(), indices.Size(), 3);
+   auto d_trial_el = Reshape(trial_el.Write(), trial_restr.indices.Size(), 3);
+
+   mfem::forall(ne*test_elt_dofs*test_vd, [=] MFEM_HOST_DEVICE (int iE)
+   {
+      const int e = iE/(test_elt_dofs*test_vd);
+      const int it = iE%(test_elt_dofs*test_vd);
+      const int i = it%test_elt_dofs;
+      const int test_c = it/test_elt_dofs;
+
+      const int i_gm = e*test_elt_dofs + i;
+      const int i_gm_s = test_gather_map[i_gm];
+      const int i_dof = SignedIndexAbs(i_gm_s);
+      const int i_sgn = SignedIndexSign(i_gm_s);
+      const int i_L = DofToVDof(i_dof, test_c, test_all_dofs, test_vd,
+                                test_byvdim);
+      const int i_offset = test_offsets[i_dof];
+      const int i_next_offset = test_offsets[i_dof+1];
+      const int i_nbElts = i_next_offset - i_offset;
+
+      int *i_elts = &d_test_el(i_offset, 0);
+      int *i_B = &d_test_el(i_offset, 1);
+      int *i_sgns = &d_test_el(i_offset, 2);
+      for (int e_i = 0; e_i < i_nbElts; ++e_i)
+      {
+         const int i_idx_s = test_indices[i_offset+e_i];
+         const int i_idx = SignedIndexAbs(i_idx_s);
+         i_elts[e_i] = i_idx/test_elt_dofs;
+         i_B[e_i] = i_idx%test_elt_dofs;
+         i_sgns[e_i] = SignedIndexSign(i_idx_s);
+      }
+      for (int trial_c = 0; trial_c < trial_vd; ++trial_c)
+      {
+         for (int j = 0; j < trial_elt_dofs; j++)
+         {
+            const int j_gm = e*trial_elt_dofs + j;
+            const int j_gm_s = trial_gather_map[j_gm];
+            const int j_dof = SignedIndexAbs(j_gm_s);
+            const int j_sgn = SignedIndexSign(j_gm_s);
+            const int j_L = DofToVDof(j_dof, trial_c, trial_all_dofs,
+                                      trial_vd, trial_byvdim);
+            const int j_offset = trial_offsets[j_dof];
+            const int j_next_offset = trial_offsets[j_dof+1];
+            const int j_nbElts = j_next_offset - j_offset;
+            if (i_nbElts == 1 || j_nbElts == 1)
+            {
+               const int nnz = GetAndIncrementNnzIndex(i_L, I);
+               J[nnz] = j_L;
+               Data[nnz] = i_sgn*j_sgn*mat_ea(i, test_c, j, trial_c, e);
+            }
+            else
+            {
+               int *j_elts = &d_trial_el(j_offset, 0);
+               int *j_B = &d_trial_el(j_offset, 1);
+               int *j_sgns = &d_trial_el(j_offset, 2);
+               for (int e_j = 0; e_j < j_nbElts; ++e_j)
+               {
+                  const int j_idx_s = trial_indices[j_offset+e_j];
+                  const int j_idx = SignedIndexAbs(j_idx_s);
+                  j_elts[e_j] = j_idx/trial_elt_dofs;
+                  j_B[e_j] = j_idx%trial_elt_dofs;
+                  j_sgns[e_j] = SignedIndexSign(j_idx_s);
+               }
+               const int min_e = GetMinElt(i_elts, i_nbElts,
+                                           j_elts, j_nbElts);
+               if (e == min_e)
+               {
+                  real_t val = 0.0;
+                  for (int k = 0; k < i_nbElts; k++)
+                  {
+                     const int e_i = i_elts[k];
+                     const int i_Bloc = i_B[k];
+                     for (int l = 0; l < j_nbElts; l++)
+                     {
+                        const int e_j = j_elts[l];
+                        const int j_Bloc = j_B[l];
+                        if (e_i == e_j)
+                        {
+                           val += i_sgns[k]*j_sgns[l]*
+                                  mat_ea(i_Bloc, test_c, j_Bloc, trial_c, e_i);
+                        }
+                     }
+                  }
+                  const int nnz = GetAndIncrementNnzIndex(i_L, I);
+                  J[nnz] = j_L;
+                  Data[nnz] = val;
+               }
+            }
+         }
+      }
+   });
+
+   auto h_I = mat.HostReadWriteI();
+   const int size = test_vd*test_all_dofs;
+   for (int i = 0; i < size; i++)
+   {
+      h_I[size-i] = h_I[size-(i+1)];
+   }
+   h_I[0] = 0;
+}
+
 L2ElementRestriction::L2ElementRestriction(const FiniteElementSpace &fes)
    : ne(fes.GetNE()),
      vdim(fes.GetVDim()),

fem/restriction.hpp

@@ -87,12 +87,25 @@ public:
    /// Fill a Sparse Matrix with Element Matrices.
    void FillSparseMatrix(const Vector &mat_ea, SparseMatrix &mat) const;
 
+   /** Fill a SparseMatrix with element matrices for this (test) restriction and
+       the given trial restriction. The element matrix layout is
+       test_dof x test_vdim x trial_dof x trial_vdim x ne. */
+   void FillSparseMatrix(const Vector &mat_ea, SparseMatrix &mat,
+                         const ElementRestriction &trial_restr) const;
+
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
        given by this ElementRestriction. */
    int FillI(SparseMatrix &mat) const;
+   /** Fill the I array for this (test) and the given trial restriction. */
+   int FillI(SparseMatrix &mat, const ElementRestriction &trial_restr) const;
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
        pattern given by this ElementRestriction, and the values of ea_data. */
    void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
+   /** Fill the J and Data arrays for this (test) and the given trial
+       restriction, using element matrix layout
+       test_dof x test_vdim x trial_dof x trial_vdim x ne. */
+   void FillJAndData(const Vector &ea_data, SparseMatrix &mat,
+                     const ElementRestriction &trial_restr) const;
    /// @private Not part of the public interface (device kernel limitation).
    ///
    /// Performs either MultTranspose or AddMultTranspose depending on the

general/enzyme.hpp

@@ -8,12 +8,22 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-
-#include "../config/config.hpp"
-
 #ifndef MFEM_ENZYME_HPP
 #define MFEM_ENZYME_HPP
 
+#include "../config/config.hpp" // IWYU pragma: keep
+
+namespace mfem
+{
+inline constexpr bool UseEnzyme =
+#ifdef MFEM_USE_ENZYME
+   true
+#else
+   false
+#endif
+   ;
+} // namespace mfem
+
 #ifdef MFEM_USE_ENZYME
 /*
  * Variables prefixed with enzyme_* or function types prefixed with __enzyme_*,
@@ -27,6 +37,7 @@ extern int enzyme_dupnoneed;
 extern int enzyme_out;
 extern int enzyme_const;
 extern int enzyme_interleave;
+extern int enzyme_runtime_activity;
 
 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
 #define MFEM_DEVICE_EXTERN_STMT(name) extern __device__ int name;
@@ -39,6 +50,7 @@ MFEM_DEVICE_EXTERN_STMT(enzyme_dupnoneed)
 MFEM_DEVICE_EXTERN_STMT(enzyme_out)
 MFEM_DEVICE_EXTERN_STMT(enzyme_const)
 MFEM_DEVICE_EXTERN_STMT(enzyme_interleave)
+MFEM_DEVICE_EXTERN_STMT(enzyme_runtime_activity)
 
 // warning: if inlined, triggers function '__enzyme_autodiff' is not defined
 template <typename return_type, typename... Args>
@@ -54,13 +66,15 @@ return_type __enzyme_fwddiff(Args...);
 #define MFEM_ENZYME_INACTIVE   __attribute__((enzyme_inactive))
 #define MFEM_ENZYME_FN_LIKE(x)   __attribute__((enzyme_function_like(#x)))
 
-#else
+#else // MFEM_USE_ENZYME
+
 #define MFEM_ENZYME_INACTIVENOFREE
 #define MFEM_ENZYME_INACTIVE
 #define MFEM_ENZYME_FN_LIKE(x)
-#endif
+
+#endif // MFEM_USE_ENZYME
 
 #define MFEM_ENZYME_FN_LIKE_FREE MFEM_ENZYME_FN_LIKE(free)
 #define MFEM_ENZYME_FN_LIKE_DYNCAST MFEM_ENZYME_FN_LIKE(__dynamic_cast)
 
-#endif
+#endif // MFEM_ENZYME_HPP

general/forall.hpp

@@ -753,7 +753,7 @@ static void CuKernel3DLaunchBounds(const int N, BODY body)
    for (int k = blockIdx.x; k < N; k += gridDim.x) { body(k); }
 }
 
-template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+template <const int BLCK, typename DBODY>
 void CuWrap1D(const int N, DBODY &&d_body)
 {
    if (N==0) { return; }
@@ -812,6 +812,11 @@ void CuWrap3DLaunchBounds(const int N, DBODY &&d_body,
    MFEM_GPU_CHECK(cudaGetLastError());
 }
 
+#ifdef MFEM_USE_ENZYME
+template <const int BLCK, typename DBODY>
+void CuWrap1DWithEnzyme(const int N, DBODY &&d_body);
+#endif
+
 template <int Dim, int MAX_THREADS_PER_BLOCK> struct CuWrap;
 
 template <int MAX_THREADS_PER_BLOCK>
@@ -870,6 +875,87 @@ struct CuWrap<3, MAX_THREADS_PER_BLOCK>
    }
 };
 
+#ifdef MFEM_USE_ENZYME
+template <typename BODY> struct DerivativeKernelWrapperStruct
+{
+   MFEM_DEVICE static void CuWrap1DEnzymeBody(BODY *body, const int k)
+   {
+      (*body)(k);
+   }
+
+   __global__ static void FwdLaunch(const int N, BODY body, BODY d_body)
+   {
+      const int k = blockDim.x * blockIdx.x + threadIdx.x;
+      if (k >= N)
+      {
+         return;
+      }
+
+      __enzyme_fwddiff<void>(
+         (void*)CuWrap1DEnzymeBody, enzyme_dup,
+         (void*)&body, (void*)&d_body, enzyme_const, k, enzyme_runtime_activity);
+   }
+
+   __global__ static void Launch(const int N, BODY body)
+   {
+      const int k = blockDim.x * blockIdx.x + threadIdx.x;
+      if (k >= N)
+      {
+         return;
+      }
+      body(k);
+   }
+};
+
+template <const int BLCK, typename DBODY> struct CuWrap1DStruct
+{
+   static constexpr int ACTUAL_BLCK = BLCK == 0 ? MFEM_CUDA_BLOCKS : BLCK;
+
+   static void Call(const int N, DBODY *body)
+   {
+      if (N == 0)
+      {
+         return;
+      }
+      const int GRID = (N + ACTUAL_BLCK - 1) / ACTUAL_BLCK;
+      DerivativeKernelWrapperStruct<DBODY>::Launch<<<GRID, ACTUAL_BLCK>>>(N, *body);
+      MFEM_GPU_CHECK(cudaGetLastError());
+   }
+
+   static void FwdCall(const int N, int dN, DBODY *body, DBODY *d_body)
+   {
+      MFEM_CONTRACT_VAR(dN);
+      if (N == 0)
+      {
+         return;
+      }
+      const int GRID = (N + ACTUAL_BLCK - 1) / ACTUAL_BLCK;
+      DerivativeKernelWrapperStruct<DBODY>::FwdLaunch<<<GRID, ACTUAL_BLCK>>>(N, *body,
+                                                                             *d_body);
+      MFEM_GPU_CHECK(cudaGetLastError());
+   }
+
+   inline static void *__enzyme_register_derivative_CuWrap1D[2] =
+   {
+      (void *)&Call, (void *)&FwdCall
+   };
+};
+
+template <const int BLCK, typename DBODY>
+void CuWrap1DWithEnzyme(const int N, DBODY &&d_body)
+{
+   using DBODY_BASE = std::remove_reference_t<DBODY>;
+   // Taking the address forces instantiation/emission of the registration
+   // global for this lambda type so Enzyme can find the custom derivative for
+   // CuWrap1DStruct<..., DBODY_BASE>::Call before trying to differentiate the
+   // CUDA runtime launch inside it.
+   [[maybe_unused]] auto *enzyme_registration =
+      &CuWrap1DStruct<BLCK, DBODY_BASE>::__enzyme_register_derivative_CuWrap1D;
+   MFEM_CONTRACT_VAR(enzyme_registration);
+   CuWrap1DStruct<BLCK, DBODY_BASE>::Call(N, &d_body);
+}
+#endif
+
 #endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 
 
@@ -1037,7 +1123,7 @@ struct HipWrap<3, MAX_THREADS_PER_BLOCK>
 
 ///////////////////////////////////////////////////////////////////////////////
 /// Forall host & device kernel dispatch
-template <int DIM, int MAX_THREADS_PER_BLOCK = 0,
+template <int DIM, int MAX_THREADS_PER_BLOCK = 0, bool use_enzyme = false,
           typename d_lambda, typename h_lambda>
 inline void ForallWrap(const bool use_dev, const int N,
                        d_lambda &&d_body, h_lambda &&h_body,
@@ -1071,6 +1157,20 @@ inline void ForallWrap(const bool use_dev, const int N,
    // If Backend::CUDA is allowed, use it
    if (Device::Allows(Backend::CUDA))
    {
+      if constexpr (use_enzyme)
+      {
+#ifdef MFEM_USE_ENZYME
+         static_assert(DIM == 1,
+                       "Enzyme CUDA Wrappers are only implemented "
+                       "for one dimensional thread blocks");
+
+         constexpr int BLCK = MAX_THREADS_PER_BLOCK == 0 ? MFEM_CUDA_BLOCKS :
+                              MAX_THREADS_PER_BLOCK;
+         return CuWrap1DWithEnzyme<BLCK>(N, d_body);
+#else
+         MFEM_ABORT("Enzyme not available");
+#endif
+      }
       return CuWrap<DIM, MAX_THREADS_PER_BLOCK>::run(N, d_body, X, Y, Z, G);
    }
 #endif
@@ -1110,26 +1210,25 @@ backend_cpu:
 
 ///////////////////////////////////////////////////////////////////////////////
 /// Forall host & device kernel wrappers
-template <int DIM, typename lambda>
+template <int DIM, int MAX_THREADS_PER_BLOCK = 0, bool use_enzyme = false,
+          typename lambda>
 inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
                        const int X=0, const int Y=0, const int Z=0,
                        const int G=0)
 {
-   ForallWrap<DIM>(use_dev, N, body, body, X, Y, Z, G);
-}
-
-template <int DIM, int MAX_THREADS_PER_BLOCK, typename lambda>
-inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
-                       const int X=0, const int Y=0, const int Z=0,
-                       const int G=0)
-{
-   ForallWrap<DIM, MAX_THREADS_PER_BLOCK>(use_dev, N, body, body, X, Y, Z, G);
+   ForallWrap<DIM, MAX_THREADS_PER_BLOCK, use_enzyme>(use_dev, N, body, body, X, Y,
+                                                      Z, G);
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 // forall interfaces
-template<typename lambda>
-inline void forall(int N, lambda &&body) { ForallWrap<1>(true, N, body); }
+
+template<bool use_enzyme = false, typename lambda>
+inline void forall(int N, lambda &&body)
+{
+   constexpr int MAX_THREADS_PER_BLOCK = 0;
+   ForallWrap<1, MAX_THREADS_PER_BLOCK, use_enzyme>(true, N, body);
+}
 
 template<typename lambda>
 inline void forall(int Nx, int Ny, lambda &&body)
@@ -1277,9 +1376,9 @@ template<typename lambda>
 inline void hypre_forall_gpu(int N, lambda &&body)
 {
 #if defined(HYPRE_USING_CUDA)
-   CuWrap1D(N, body);
+   CuWrap1D<MFEM_CUDA_BLOCKS>(N, body);
 #elif defined(HYPRE_USING_HIP)
-   HipWrap1D(N, body);
+   HipWrap1D<MFEM_HIP_BLOCKS>(N, body);
 #else
 #error Unknown HYPRE GPU backend!
 #endif

linalg/CMakeLists.txt

@@ -27,6 +27,7 @@ list(APPEND SRCS
   handle.cpp
   matrix.cpp
   mma.cpp
+  multivector.cpp
   ode.cpp
   operator.cpp
   ordering.cpp
@@ -63,6 +64,7 @@ list(APPEND HDRS
   linalg.hpp
   matrix.hpp
   mma.hpp
+  multivector.hpp
   ode.hpp
   operator.hpp
   ordering.hpp

linalg/linalg.hpp

@@ -15,6 +15,7 @@
 // Linear algebra header file
 
 #include "vector.hpp"
+#include "multivector.hpp"
 #include "operator.hpp"
 #include "matrix.hpp"
 #include "sparsemat.hpp"
@@ -37,6 +38,7 @@
 #include "batched/gpu_blas.hpp"
 #include "batched/solver.hpp"
 #include "tensor.hpp"
+#include "tensor_arrays.hpp"
 #include "filteredsolver.hpp"
 #include "ordering.hpp"
 #include "particlevector.hpp"

linalg/multivector.cpp

@@ -0,0 +1,60 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "multivector.hpp"
+
+namespace mfem
+{
+
+MultiVector::MultiVector(const Array<int> &vector_sizes)
+{
+   SetSizes(vector_sizes);
+}
+
+MultiVector::MultiVector(const Array<int> &vector_sizes, MemoryType mt)
+{
+   SetSizes(vector_sizes, mt);
+}
+
+MultiVector::MultiVector(Vector &base, const Array<int> &vector_sizes)
+{
+   MakeRef(base, vector_sizes);
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i]);
+   }
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes, MemoryType mt)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i], mt);
+   }
+}
+
+void MultiVector::MakeRef(Vector &base, const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int offset = 0, i = 0; i < vector_sizes.Size(); i++)
+   {
+      blocks[i].emplace<0>(base, offset, vector_sizes[i]);
+      offset += vector_sizes[i];
+   }
+}
+
+} // namespace mfem

linalg/multivector.hpp

@@ -0,0 +1,240 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MULTIVECTOR_HPP
+#define MFEM_MULTIVECTOR_HPP
+
+#include "../general/array.hpp"
+#include "vector.hpp"
+#include <vector>
+#include <array>
+#include <variant>
+
+namespace mfem
+{
+
+/// Class representing an array of Vectors with generally different sizes.
+/** This class is similar to BlockVector with the following two main
+    differences:
+    - the data for the individual Vector blocks does not need to be part of one
+      big contiguous memory allocation;
+    - this class does not inherit from class Vector (as a consequence of the
+      first bullet).
+
+    Internally, each Vector block is represented as either:
+    - (default) a Vector object constructed and owned by this class; this
+      object, in turn, as any Vector object, can own its Memory allocation or
+      refer to a sub-Memory of another Memory object; or
+    - a pointer to an externally allocated Vector or classes derived from
+      Vector. */
+class MultiVector
+{
+private:
+   std::vector<std::variant<Vector,Vector*,const Vector*>> blocks;
+
+public:
+   /// Create an empty MultiVector with zero blocks.
+   MultiVector() = default;
+
+   /** @brief Create a MultiVector with @a num_blocks blocks. The individual
+       Vector blocks are default initialized, i.e. they all have size zero. */
+   MultiVector(int num_blocks)
+      : blocks(num_blocks) { }
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes. All Vector blocks use the
+       MemoryType @a mt.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Construct a MultiVector referencing data within a given monolithic
+       Vector @a base.
+
+       With this constructor, the Memory flags of @a base and of the individual
+       Vector blocks may need to be explicitly synchronized when data is moved
+       between host and device. */
+   MultiVector(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector referencing multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       With this constructor, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   MultiVector(VectorTypes &...vs) { MakeRef(vs...); }
+
+   /// Return the number of Vectors in the MultiVector.
+   int NumBlocks() const { return blocks.size(); }
+
+   /** @brief Set the number of Vectors in the MultiVector. Existing Vector
+       blocks will remain unmodified. New Vector blocks will be default
+       initialized, i.e. they all have size zero. */
+   void SetNumBlocks(int num_blocks) { blocks.resize(num_blocks); }
+
+   /// Read-write access to the i-th Vector.
+   inline Vector &operator[](int i);
+
+   /// Read-only access to the i-th Vector.
+   inline const Vector &operator[](int i) const;
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes are updated using the method Vector::SetSize(int). */
+   void SetSizes(const Array<int> &vector_sizes);
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes and
+       MemoryType @a mt.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes and MemoryType are updated using the method
+       Vector::SetSize(int, MemoryType). */
+   void SetSizes(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Update the MultiVector to reference data within a given monolithic
+       Vector @a base.
+
+       After calling this method, the Memory flags of @a base and of the
+       individual Vector blocks may need to be explicitly synchronized when data
+       is moved between host and device.*/
+   void MakeRef(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Update the @a i-th MultiVector block to reference data within the
+       given monolithic Vector @a base at the given @a offset and with the given
+       @a size.
+
+       After calling this method, the Memory flags of @a base and of the @a i-th
+       Vector block may need to be explicitly synchronized when data is moved
+       between host and device.*/
+   inline void MakeRef(int i, Vector &base, int offset, int size)
+   {
+      blocks[i].emplace<0>(base, offset, size);
+   }
+
+   /** @brief Update the MultiVector to reference multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       After calling this method, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   inline void MakeRef(VectorTypes &...vs);
+
+   /** @brief Update the @a i-th MultiVector block to reference the given
+       Vector @a v.
+
+       After calling this method, operations on the @a i-th Vector block are
+       performed directly on the Vector @a v. In particular, there is no need
+       to synchronize the Memory flags of @a v and the ones of the @a i-th
+       Vector blocks when data is moved between host and device. */
+   inline void MakeRef(int i, Vector &v) { blocks[i] = &v; }
+
+   /** @brief Construct a MultiVector referencing multiple const Vectors given
+       as arguments. Individual blocks are read-only; non-const operator[]
+       will assert. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<const VectorTypes&,const Vector&>...>, bool> = true>
+   MultiVector(const VectorTypes &...vs) { MakeRef(vs...); }
+
+   /** @brief Update the @a i-th MultiVector block to reference the given
+       const Vector @a v. The block becomes read-only. */
+   inline void MakeRef(int i, const Vector &v) { blocks[i] = &v; }
+
+   /** @brief Update the MultiVector to reference multiple const Vectors given
+       as arguments. Individual blocks are read-only; non-const operator[]
+       will assert. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<const VectorTypes&,const Vector&>...>, bool> = true>
+   inline void MakeRef(const VectorTypes &...vs);
+};
+
+// Inline and template methods
+
+inline Vector &MultiVector::operator[](int i)
+{
+   auto &bi = blocks[i];
+   MFEM_ASSERT(bi.index() != 2, "Non-const access to a const Vector block");
+   return (bi.index() == 0) ? std::get<0>(bi) : *std::get<1>(bi);
+}
+
+inline const Vector &MultiVector::operator[](int i) const
+{
+   auto &bi = blocks[i];
+   return (bi.index() == 0) ? std::get<0>(bi) :
+          (bi.index() == 1) ? *std::get<1>(bi) :
+          /**/                *std::get<2>(bi);
+}
+
+template <typename... VectorTypes,
+          std::enable_if_t<
+             std::conjunction_v<
+                std::is_convertible<VectorTypes&,Vector&>...>, bool>>
+inline void MultiVector::MakeRef(VectorTypes &...vs)
+{
+   blocks.resize(sizeof...(vs));
+   if constexpr (sizeof...(vs) > 0)
+   {
+      const std::array vs_p{&static_cast<Vector&>(vs)...};
+      for (std::size_t i = 0; i < sizeof...(vs); i++)
+      {
+         blocks[i] = vs_p[i];
+      }
+   }
+}
+
+template <typename... VectorTypes,
+          std::enable_if_t<
+             std::conjunction_v<
+                std::is_convertible<const VectorTypes&,const Vector&>...>, bool>>
+inline void MultiVector::MakeRef(const VectorTypes &...vs)
+{
+   blocks.resize(sizeof...(vs));
+   if constexpr (sizeof...(vs) > 0)
+   {
+      const std::array vs_p{&static_cast<const Vector&>(vs)...};
+      for (std::size_t i = 0; i < sizeof...(vs); i++)
+      {
+         blocks[i] = vs_p[i];
+      }
+   }
+}
+
+} // namespace mfem
+
+#endif // MFEM_MULTIVECTOR_HPP

linalg/operator.hpp

@@ -13,6 +13,7 @@
 #define MFEM_OPERATOR
 
 #include "vector.hpp"
+#include "multivector.hpp"
 
 namespace mfem
 {

linalg/tensor.hpp

@@ -17,150 +17,145 @@
 
 #pragma once
 
-#include "../general/backends.hpp"
 #include "dual.hpp"
 #include <limits>
+#include <utility>
 #include <type_traits> // for std::false_type
 
+// Force-inline every tensor operation under clang
+#if defined(__clang__)
+#pragma clang attribute push (__attribute__((always_inline)), apply_to = function)
+#endif
+
 namespace mfem
 {
 namespace future
 {
 
-template <typename T, int... n>
+template <typename T, int... Dims>
 struct tensor;
 
-/// The implementation can be drastically generalized by using concepts of the
-/// c++17 standard.
-
-template < typename T >
+template <typename T>
 struct tensor<T>
 {
    using type = T;
-   static constexpr int ndim      = 1;
+   static constexpr int ndim = 1;
    static constexpr int first_dim = 0;
-   MFEM_HOST_DEVICE T& operator[](int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const T& operator[](int /*unused*/) const { return values; }
-   MFEM_HOST_DEVICE T& operator()(int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const T& operator()(int /*unused*/) const { return values; }
+
+   MFEM_HOST_DEVICE tensor() = default;
+   MFEM_HOST_DEVICE tensor(T val) : values(val) {}
+
+   MFEM_HOST_DEVICE T& operator[](int) { return values; }
+   MFEM_HOST_DEVICE const T& operator[](int) const { return values; }
+   MFEM_HOST_DEVICE T& operator()() { return values; }
+   MFEM_HOST_DEVICE const T& operator()() const { return values; }
+   MFEM_HOST_DEVICE T& operator()(int) { return values; }
+   MFEM_HOST_DEVICE const T& operator()(int) const { return values; }
    MFEM_HOST_DEVICE operator T() const { return values; }
+   MFEM_HOST_DEVICE tensor& operator=(T val) { values = val; return *this; }
+
    T values;
 };
 
-template < typename T, int n0 >
-struct tensor<T, n0>
+template <typename T, int N0>
+struct tensor<T, N0>
 {
    using type = T;
-   static constexpr int ndim      = 1;
-   static constexpr int first_dim = n0;
+   static constexpr int ndim = 1;
+   static constexpr int first_dim = N0;
+
    MFEM_HOST_DEVICE T& operator[](int i) { return values[i]; }
    MFEM_HOST_DEVICE const T& operator[](int i) const { return values[i]; }
    MFEM_HOST_DEVICE T& operator()(int i) { return values[i]; }
    MFEM_HOST_DEVICE const T& operator()(int i) const { return values[i]; }
-   T values[n0];
+
+   T values[N0];
 };
 
-template < typename T >
+template <typename T>
 struct tensor<T, 0>
 {
    using type = T;
-   static constexpr int ndim      = 1;
+   static constexpr int ndim = 1;
    static constexpr int first_dim = 0;
-   MFEM_HOST_DEVICE T& operator[](int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const T& operator[](int /*unused*/) const { return values; }
-   MFEM_HOST_DEVICE T& operator()(int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const T& operator()(int /*unused*/) const { return values; }
+
+   MFEM_HOST_DEVICE T& operator[](int) { return values; }
+   MFEM_HOST_DEVICE const T& operator[](int) const { return values; }
+   MFEM_HOST_DEVICE T& operator()(int) { return values; }
+   MFEM_HOST_DEVICE const T& operator()(int) const { return values; }
+
    T values;
 };
 
-template < typename T, int n0, int n1 >
-struct tensor<T, n0, n1>
+template <typename T, int N0, int N1, int... Rest>
+struct tensor<T, N0, N1, Rest...>
 {
    using type = T;
-   static constexpr int ndim      = 2;
-   static constexpr int first_dim = n0;
-   MFEM_HOST_DEVICE tensor< T, n1 >& operator[](int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1 >& operator[](int i) const { return values[i]; }
-   MFEM_HOST_DEVICE tensor< T, n1 >& operator()(int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1 >& operator()(int i) const { return values[i]; }
-   MFEM_HOST_DEVICE T& operator()(int i, int j) { return values[i][j]; }
-   MFEM_HOST_DEVICE const T& operator()(int i, int j) const { return values[i][j]; }
-   tensor < T, n1 > values[n0];
-};
+   using sub_tensor = tensor<T, N1, Rest...>;
+   static constexpr int ndim = 2 + sizeof...(Rest);
+   static constexpr int first_dim = N0;
 
-template < typename T, int n1 >
-struct tensor<T, 0, n1>
-{
-   using type = T;
-   static constexpr int ndim      = 2;
-   static constexpr int first_dim = 0;
-   MFEM_HOST_DEVICE tensor< T, n1 >& operator[](int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const tensor< T, n1 >& operator[](int /*unused*/) const { return values; }
-   MFEM_HOST_DEVICE tensor< T, n1 >& operator()(int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const tensor< T, n1 >& operator()(int /*unused*/) const { return values; }
-   MFEM_HOST_DEVICE T& operator()(int /*unused*/, int j) { return values[j]; }
-   MFEM_HOST_DEVICE const T& operator()(int /*unused*/, int j) const { return values[j]; }
-   tensor < T, n1 > values;
-};
+   static constexpr bool is_zero_dim = (N0 == 0);
+   static constexpr int storage_size = is_zero_dim ? 1 : N0;
+   using storage_type =
+      std::conditional_t<is_zero_dim, sub_tensor, sub_tensor[storage_size]>;
+   storage_type values;
 
-template < typename T, int n0, int n1, int n2 >
-struct tensor<T, n0, n1, n2>
-{
-   using type = T;
-   static constexpr int ndim      = 3;
-   static constexpr int first_dim = n0;
-   MFEM_HOST_DEVICE tensor< T, n1, n2 >& operator[](int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2 >& operator[](int i) const { return values[i]; }
-   MFEM_HOST_DEVICE tensor< T, n1, n2 >& operator()(int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2 >& operator()(int i) const { return values[i]; }
-   MFEM_HOST_DEVICE tensor< T, n2 >& operator()(int i, int j) { return values[i][j]; }
-   MFEM_HOST_DEVICE const tensor< T, n2 >& operator()(int i, int j) const { return values[i][j]; }
-   MFEM_HOST_DEVICE T& operator()(int i, int j, int k) { return values[i][j][k]; }
-   MFEM_HOST_DEVICE const T& operator()(int i, int j, int k) const { return values[i][j][k]; }
-   tensor < T, n1, n2 > values[n0];
-};
+   MFEM_HOST_DEVICE sub_tensor& operator[](int i)
+   {
+      if constexpr (is_zero_dim)
+      {
+         static_cast<void>(i);
+         return values;
+      }
+      else { return values[i]; }
+   }
 
-template < typename T, int n0, int n1, int n2, int n3 >
-struct tensor<T, n0, n1, n2, n3>
-{
-   using type = T;
-   static constexpr int ndim      = 4;
-   static constexpr int first_dim = n0;
-   MFEM_HOST_DEVICE tensor< T, n1, n2, n3 >& operator[](int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2, n3 >& operator[](int i) const { return values[i]; }
-   MFEM_HOST_DEVICE tensor< T, n1, n2, n3 >& operator()(int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2, n3 >& operator()(int i) const { return values[i]; }
-   MFEM_HOST_DEVICE tensor< T, n2, n3 >& operator()(int i, int j) { return values[i][j]; }
-   MFEM_HOST_DEVICE const tensor< T, n2, n3 >& operator()(int i, int j) const { return values[i][j]; }
-   MFEM_HOST_DEVICE tensor< T, n3 >& operator()(int i, int j, int k) { return values[i][j][k]; }
-   MFEM_HOST_DEVICE const tensor< T, n3 >& operator()(int i, int j, int k) const { return values[i][j][k]; }
-   MFEM_HOST_DEVICE T& operator()(int i, int j, int k, int l) { return values[i][j][k][l]; }
-   MFEM_HOST_DEVICE const T&  operator()(int i, int j, int k, int l) const { return values[i][j][k][l]; }
-   tensor < T, n1, n2, n3 > values[n0];
-};
+   MFEM_HOST_DEVICE const sub_tensor& operator[](int i) const
+   {
+      if constexpr (is_zero_dim)
+      {
+         static_cast<void>(i);
+         return values;
+      }
+      else { return values[i]; }
+   }
 
-template < typename T, int n0, int n1, int n2, int n3, int n4 >
-struct tensor<T, n0, n1, n2, n3, n4>
-{
-   using type = T;
-   static constexpr int ndim      = 5;
-   static constexpr int first_dim = n0;
-   MFEM_HOST_DEVICE tensor< T, n1, n2, n3, n4 >& operator[](int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2, n3, n4 >& operator[](int i) const { return values[i]; }
-   MFEM_HOST_DEVICE tensor< T, n1, n2, n3, n4 >& operator()(int i) { return values[i]; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2, n3, n4 >& operator()(int i) const { return values[i]; }
-   MFEM_HOST_DEVICE tensor< T, n2, n3, n4 >& operator()(int i, int j) { return values[i][j]; }
-   MFEM_HOST_DEVICE const tensor< T, n2, n3, n4 >& operator()(int i,
-                                                              int j) const { return values[i][j]; }
-   MFEM_HOST_DEVICE tensor< T, n3, n4>& operator()(int i, int j, int k) { return values[i][j][k]; }
-   MFEM_HOST_DEVICE const tensor< T, n3, n4>& operator()(int i, int j,
-                                                         int k) const { return values[i][j][k]; }
-   MFEM_HOST_DEVICE tensor< T, n4 >& operator()(int i, int j, int k, int l) { return values[i][j][k][l]; }
-   MFEM_HOST_DEVICE const tensor< T, n4 >& operator()(int i, int j, int k,
-                                                      int l) const { return values[i][j][k][l]; }
-   MFEM_HOST_DEVICE T& operator()(int i, int j, int k, int l, int m) { return values[i][j][k][l][m]; }
-   MFEM_HOST_DEVICE const T& operator()(int i, int j, int k, int l, int m) const { return values[i][j][k][l][m]; }
-   tensor < T, n1, n2, n3, n4 > values[n0];
+   MFEM_HOST_DEVICE sub_tensor& operator()(int i)
+   {
+      return (*this)[i];
+   }
+
+   MFEM_HOST_DEVICE const sub_tensor& operator()(int i) const
+   {
+      return (*this)[i];
+   }
+
+   template <typename... Is>
+   MFEM_HOST_DEVICE auto& operator()(int i, int j, Is... rest)
+   {
+      if constexpr (sizeof...(rest) == 0)
+      {
+         return (*this)[i][j];
+      }
+      else
+      {
+         return (*this)[i](j, rest...);
+      }
+   }
+
+   template <typename... Is>
+   MFEM_HOST_DEVICE const auto& operator()(int i, int j, Is... rest) const
+   {
+      if constexpr (sizeof...(rest) == 0)
+      {
+         return (*this)[i][j];
+      }
+      else
+      {
+         return (*this)[i](j, rest...);
+      }
+   }
 };
 
 /**
@@ -270,10 +265,10 @@ MFEM_HOST_DEVICE constexpr zero operator/(zero, T /*other*/)
    return zero{};
 }
 
-/** @brief `zero` plus `zero` is `zero */
+/** @brief `zero` plus `zero` is `zero` */
 MFEM_HOST_DEVICE constexpr zero operator+=(zero, zero) { return zero{}; }
 
-/** @brief `zero` minus `zero` is `zero */
+/** @brief `zero` minus `zero` is `zero` */
 MFEM_HOST_DEVICE constexpr zero operator-=(zero, zero) { return zero{}; }
 
 /** @brief let `zero` be accessed like a tuple */
@@ -305,13 +300,10 @@ MFEM_HOST_DEVICE zero dot(zero, const T&)
  * @tparam n2 The second dimension
  */
 template <typename T, int n1, int n2 = 1>
-using reduced_tensor = typename std::conditional<
-                       (n1 == 1 && n2 == 1), T,
-                       typename std::conditional<n1 == 1, tensor<T, n2>,
-                       typename std::conditional<n2 == 1, tensor<T, n1>, tensor<T, n1, n2>
-                       >::type
-                       >::type
-                       >::type;
+using reduced_tensor =
+   std::conditional_t<(n1 == 1 && n2 == 1), T,
+   std::conditional_t<(n1 == 1), tensor<T, n2>,
+   std::conditional_t<(n2 == 1), tensor<T, n1>, tensor<T, n1, n2>>>>;
 
 /**
  * @brief Creates a tensor of requested dimension by subsequent calls to a functor
@@ -462,7 +454,7 @@ tensor<T, n> get_col(tensor<T, m, n> A, int j)
 
 /// @overload
 template <typename T> MFEM_HOST_DEVICE
-tensor<T, 1> get_col(tensor<T, 1, 1> A, int j)
+tensor<T, 1> get_col(tensor<T, 1, 1> A, [[maybe_unused]] int j)
 {
    return tensor<T, 1> {A[0][0]};
 }
@@ -541,9 +533,16 @@ MFEM_HOST_DEVICE auto operator*(S scale, const tensor<T, n...>& A) ->
 tensor<decltype(S {} * T{}), n...>
 {
    tensor<decltype(S{} * T{}), n...> C{};
-   for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+   if constexpr (sizeof...(n) == 0)
    {
-      C[i] = scale * A[i];
+      C.values = scale * A.values;
+   }
+   else
+   {
+      for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+      {
+         C[i] = scale * A[i];
+      }
    }
    return C;
 }
@@ -563,9 +562,16 @@ MFEM_HOST_DEVICE auto operator*(const tensor<T, n...>& A, S scale) ->
 tensor<decltype(T {} * S{}), n...>
 {
    tensor<decltype(T{} * S{}), n...> C{};
-   for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+   if constexpr (sizeof...(n) == 0)
    {
-      C[i] = A[i] * scale;
+      C.values = A.values * scale;
+   }
+   else
+   {
+      for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+      {
+         C[i] = A[i] * scale;
+      }
    }
    return C;
 }
@@ -729,9 +735,9 @@ auto outer(S A, T B) -> decltype(A * B)
 }
 
 template <typename T, int n, int m> MFEM_HOST_DEVICE
-tensor<T, n + m> flatten(tensor<T, n, m> A)
+tensor<T, n * m> flatten(tensor<T, n, m> A)
 {
-   tensor<T, n + m> B{};
+   tensor<T, n * m> B{};
    for (int i = 0; i < n; i++)
    {
       for (int j = 0; j < m; j++)
@@ -1218,6 +1224,32 @@ decltype(S {} * T{})
    return AB;
 }
 
+// tensor<T> (rank-0 scalar wrapper) acts as a scalar in multiplication
+template <typename S, typename T, int... m,
+          typename = typename std::enable_if<(sizeof...(m) > 0)>::type>
+MFEM_HOST_DEVICE
+auto operator*(const tensor<S, m...>& A, const tensor<T>& scale) ->
+tensor<decltype(S {} * T{}), m...>
+{
+   return A * static_cast<T>(scale);
+}
+
+template <typename S, typename T, int... m,
+          typename = typename std::enable_if<(sizeof...(m) > 0)>::type>
+MFEM_HOST_DEVICE
+auto operator*(const tensor<T>& scale, const tensor<S, m...>& A) ->
+tensor<decltype(T {} * S{}), m...>
+{
+   return static_cast<T>(scale) * A;
+}
+
+template <typename S, typename T> MFEM_HOST_DEVICE
+auto operator*(const tensor<S>& A, const tensor<T>& B) ->
+tensor<decltype(S {} * T{})>
+{
+   return tensor<decltype(S{} * T{})>{A.values * B.values};
+}
+
 /**
  * @brief this is a shorthand for dot(A, B)
  */
@@ -1507,7 +1539,7 @@ void GetScalingFactor(const T &d_max, T &mult)
 }
 
 template <typename T> MFEM_HOST_DEVICE
-T calcsv(const tensor<T, 1, 1> A, const int i)
+T calcsv(const tensor<T, 1, 1> A, [[maybe_unused]] const int i)
 {
    return A[0][0];
 }
@@ -2288,3 +2320,7 @@ auto ddot(const isotropic_tensor<S, m, m, m, m>& I,
 
 } // namespace future
 } // namespace mfem
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#endif

linalg/tensor_arrays.hpp

@@ -0,0 +1,390 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_TENSOR_ARRAYS_HPP
+#define MFEM_TENSOR_ARRAYS_HPP
+
+#include "tensor.hpp"
+#include <array>       // std::array, std::size_t (indirectly)
+#include <type_traits> /* std::remove_cv_t, std::remove_reference_t,
+                          std::is_const_v */
+#include <utility>     /* std::forward, std::index_sequence,
+                          std::make_index_sequence */
+#include <algorithm>   // std::min
+#include <tuple>       // std::apply, std::tuple_size_v
+#include <numeric>     // std::iota
+
+namespace mfem::future
+{
+
+template <std::size_t... Is, typename Fn>
+constexpr inline void for_unrolled_simple(std::index_sequence<Is...>, Fn &&fn)
+{
+   (fn(Is), ...);
+}
+
+
+template <int... loop_sizes>
+#ifdef MFEM_USE_ENZYME
+__attribute__((annotate("enzyme_inactive")))
+#endif
+constexpr inline auto to_multiindex(std::size_t i)
+{
+   constexpr auto dims = sizeof...(loop_sizes);
+   constexpr std::array<std::size_t,dims> sizes{loop_sizes...};
+   std::array<std::size_t,dims> is{};  // value initialization with zeros
+   for (std::size_t d = 0; d < dims; d++)
+   {
+      is[d] = i%sizes[d];
+      i /= sizes[d];
+   }
+   return is;
+}
+
+
+/// lambda_t:
+/// - input: const std::array<std::size_t,sizeof...(loop_sizes)> &
+/// - output: void
+/// Note: 0D loop executes the lambda one time with an array of dim 0.
+template <int... loop_sizes, typename lambda_t>
+constexpr inline void for_multiindex(lambda_t f)
+{
+   constexpr auto dims = sizeof...(loop_sizes);
+   if constexpr (dims == 0)
+   {
+      f(std::array<std::size_t,0> {});
+   }
+   else
+   {
+      if constexpr (std::min({loop_sizes...}) <= 0) { return; }
+      constexpr auto total_loop_size = (loop_sizes * ...);
+      for_unrolled_simple(std::make_index_sequence<total_loop_size> {},
+                          [&f](std::size_t i)
+      {
+         f(to_multiindex<loop_sizes...>(i));
+      });
+   }
+}
+
+
+namespace detail
+{
+
+template <typename Fn, typename Tuple, std::size_t... Is>
+inline constexpr decltype(auto) apply_impl(Fn&& f, Tuple&& t,
+                                           std::index_sequence<Is...>)
+{
+   using std::get;
+   return std::forward<Fn>(f)(get<Is>(std::forward<Tuple>(t))...);
+}
+
+}  // namespace detail
+
+/// Extend std::apply to work with 0-size arrays and mfem::future::tuple.
+template <typename Fn, typename Tuple>
+inline constexpr decltype(auto) apply(Fn&& f, Tuple&& t)
+{
+   constexpr auto size = std::tuple_size_v<std::remove_reference_t<Tuple>>;
+   if constexpr (size == 0)
+   { return std::forward<Fn>(f)(); }
+   return detail::apply_impl(std::forward<Fn>(f), std::forward<Tuple>(t),
+                             std::make_index_sequence<size> {});
+}
+
+
+/// Multi-dimensional array of tensors of the same size.
+/** The array sizes are dynamic while the tensor sizes are static, i.e. template
+    parameters.
+
+    This class provides flexible global data layout where the dynamic (array)
+    dimnsions and the tensor dimnsions are stored in memory using a runtime
+    defined strided layout. */
+template <typename scalar_t, int ndims, int... tensor_sizes>
+class tensor_ndarray
+{
+public:
+   using scalar_type = scalar_t;
+   using tensor_type = tensor<std::remove_cv_t<scalar_t>,tensor_sizes...>;
+
+   static constexpr auto tensor_dims = sizeof...(tensor_sizes);
+   static constexpr auto total_dims = ndims + tensor_dims;
+   static constexpr auto tensor_sizes_array() noexcept
+   {
+      return std::array<std::size_t, tensor_dims> {tensor_sizes...};
+   }
+
+private:
+   scalar_t *data;  /// Not owned
+public:
+   std::array<std::size_t,ndims> dyn_sizes;
+   mutable std::array<std::size_t,total_dims> strides;
+
+public:
+   /** @brief Constructor with the default, column-major or left, layout where
+       the dynamic dimensions are first, on the left, and the tensor dimensions
+       are second. */
+   tensor_ndarray(scalar_t *ptr, std::array<std::size_t,ndims> dynamic_sizes)
+      : data(ptr), dyn_sizes(dynamic_sizes)
+   {
+      std::array<std::size_t,total_dims> default_perm;
+      std::iota(default_perm.begin(), default_perm.end(), 0); // 0, 1, 2, ...
+      set_layout(default_perm);
+   }
+
+   /// Number of dynamic array dimensions.
+   static constexpr std::size_t rank() { return ndims; }
+
+   /// Array size in the @a k-th dynamic dimension.
+   MFEM_HOST_DEVICE std::size_t size(int k = 0) const { return dyn_sizes[k]; }
+
+   /// Returns the product of all sizes of the dynamic dimensions.
+   MFEM_HOST_DEVICE std::size_t total_size() const
+   {
+      std::size_t t = 1;
+      for (int d = 0; d < ndims; d++)
+      {
+         t *= dyn_sizes[d];
+      }
+      return t;
+   }
+
+   /// Number of tensor (static) dimensions.
+   static constexpr std::size_t tensor_rank()
+   { return sizeof...(tensor_sizes); }
+
+   /// Tensor size in the @a k-th tensor (static) dimension.
+   static constexpr std::size_t tensor_size(int k = 0)
+   {
+      return tensor_sizes_array()[k];
+   }
+
+   /// Returns the product of all sizes of the static (tensor) dimensions.
+   static constexpr std::size_t total_tensor_size()
+   { return (tensor_sizes * ...); }
+
+   /// Set the global data layout based on the given permutation @a perm.
+   /** The entries of @a perm are numbers identifying either a dynamic or a
+       tensor (static) dimension. Values in the range [0,rank()) identify the
+       dynamic dimensions and values in the range [rank(),rank()+tensor_rank())
+       identify the tensor dymensions. The first entry in @a perm determines
+       which dynamic or tensor dimension will have stride 1. The k-th entry of
+       @a perm determines which dimension will use the next stride which is
+       defined as the product of the sizes of all k-1 previous dimensions from
+       @a perm.
+
+       @note The default layout corresponds to the identity permutation:
+       { 0, 1, ..., rank()+tensor_rank()-1 }.
+
+       @note This method does not permute the global 1D data array. */
+   MFEM_HOST_DEVICE
+#ifdef MFEM_USE_ENZYME
+   __attribute__((annotate("enzyme_inactive")))
+#endif
+   void set_layout(std::array<std::size_t,rank()+tensor_rank()> perm) const
+   {
+      std::size_t stride = 1;
+      for (std::size_t d_g = 0; d_g < total_dims; d_g++)
+      {
+         const auto d_l = perm[d_g];
+         strides[d_l] = stride;
+         stride *= (d_l < ndims) ? dyn_sizes[d_l] :
+                   tensor_sizes_array()[d_l-ndims];
+      }
+   }
+
+   /** @brief Comute the dynamic offset for a given dynamic multi-index @a is.
+       The total offset in the global data array is the sum of the dynamic and
+       static (tensor) offsets. */
+   MFEM_HOST_DEVICE
+   std::size_t get_dynamic_offset(
+      const std::array<std::size_t,rank()> &is) const
+   {
+      std::size_t dynamic_offset = 0;
+      for (std::size_t d = 0; d < ndims; d++)
+      {
+         dynamic_offset += is[d]*strides[d];
+      }
+      return dynamic_offset;
+   }
+
+   /** @brief Comute the static (tensor) offset for a given tensor multi-index
+       @a js. The total offset in the global data array is the sum of the
+       dynamic and static (tensor) offsets. */
+   MFEM_HOST_DEVICE
+   std::size_t get_static_offset(
+      const std::array<std::size_t,tensor_rank()> &js) const
+   {
+      std::size_t static_offset = 0;
+      if constexpr (tensor_dims > 0)
+      {
+         for (std::size_t d = 0; d < tensor_dims; d++)
+         {
+            static_offset += js[d]*strides[ndims+d];
+         }
+      }
+      return static_offset;
+   }
+
+   /** @brief Return a local tensor extracted from the global data array
+       corresponding to the given dynamic multi-index @a is. */
+   /** @note Return a const tensor to prevent attempts to assign to the
+       temporary object which is considered a mistake. */
+   MFEM_HOST_DEVICE
+   const tensor_type get_tensor(std::array<std::size_t,rank()> is) const
+   {
+      tensor_type result;
+      const std::size_t dynamic_offset = get_dynamic_offset(is);
+      for_multiindex<tensor_sizes...>(
+         [&result, this, dynamic_offset](
+            const std::array<std::size_t,tensor_rank()> &js)
+      {
+         ::mfem::future::apply(result, js) =
+            data[dynamic_offset + get_static_offset(js)];
+      });
+      return result;
+   }
+
+   /** @brief Return a local tensor extracted from the global data array
+       corresponding to the given dynamic indices @a is. */
+   /** @note Return a const tensor to prevent attempts to assign to the
+       temporary object which is considered a mistake. */
+   template <typename... index_types> MFEM_HOST_DEVICE
+   const tensor_type get_tensor(index_types... is) const
+   {
+      static_assert(sizeof...(is) == rank(), "invalid number of indices!");
+      return get_tensor(std::array<std::size_t,rank()> {std::size_t(is)...});
+   }
+
+   /** @brief Returns one of the following depending on the type scalar_t:
+       - get_tensor(std::array<std::size_t,rank()>) iff scalar_t is const,
+       - get_accessor(std::array<std::size_t,rank()>) iff scalar_t is not
+         const. */
+   MFEM_HOST_DEVICE
+   decltype(auto) operator()(std::array<std::size_t,rank()> is) const
+   {
+      if constexpr (std::is_const_v<scalar_t>) { return get_tensor(is); }
+      else { return get_accessor(is); }
+   }
+
+   /** @brief Returns one of the following depending on the type scalar_t:
+       - get_tensor(index_types...) iff scalar_t is const,
+       - get_accessor(index_types...) iff scalar_t is not const. */
+   template <typename... index_types> MFEM_HOST_DEVICE
+   decltype(auto) operator()(index_types... is) const
+   {
+      if constexpr (std::is_const_v<scalar_t>) { return get_tensor(is...); }
+      else { return get_accessor(is...); }
+   }
+
+   /** @brief Helper class facilitating the reading/writing of local tensor
+       objects to the global data array of the tensor_ndarray. */
+   class tensor_accessor
+   {
+   private:
+      const tensor_ndarray &base_array;
+      scalar_t *offset_data;  /// Not owned
+
+   public:
+      /** @brief Construct a tensor_accessor to @a base for the given dynamic
+          multi-index @a is.
+
+          During its life time, this object assumes that the @a base object
+          remains unmodified. */
+      MFEM_HOST_DEVICE
+      tensor_accessor(const tensor_ndarray &base,
+                      const std::array<std::size_t,rank()> &is)
+         : base_array(base)
+      {
+         offset_data = base_array.data + base_array.get_dynamic_offset(is);
+      }
+
+      /// Read-write access to a particular entry of the referenced tensor.
+      /** The returned reference points to the corresponding entry in the global
+          data array of the base tensor_ndarray. */
+      MFEM_HOST_DEVICE
+      scalar_t &operator()(const std::array<std::size_t,tensor_rank()> &js)
+      {
+         return offset_data[base_array.get_static_offset(js)];
+      }
+
+      /** @brief Write a tensor to the referenced tensor in the global data
+          array of the base tensor_ndarray. */
+      MFEM_HOST_DEVICE
+      tensor_accessor &operator=(const tensor_type &rhs)
+      {
+         for_multiindex<tensor_sizes...>(
+            [&](const std::array<std::size_t,tensor_rank()> &js)
+         {
+            operator()(js) = ::mfem::future::apply(rhs, js);
+         });
+         return *this;
+      }
+
+      /** @brief For 0D (scalar) tensor arrays, allow direct binding as a
+          scalar reference so that @c array(i) can be passed to functions
+          expecting a @c scalar_t& in/out parameter. */
+      template <bool Enable = (sizeof...(tensor_sizes) == 0),
+                std::enable_if_t<Enable, int> = 0>
+      MFEM_HOST_DEVICE
+      operator scalar_t &()
+      {
+         return *offset_data;
+      }
+   };
+
+   /** @brief Get a tensor_accessor object referencing the tensor stored at the
+       dynamic multi-index @a is. This object can be used to write tensor
+       objects into the global data array of the tensor_ndarray. */
+   MFEM_HOST_DEVICE
+   tensor_accessor get_accessor(std::array<std::size_t,rank()> is) const
+   {
+      return tensor_accessor(*this, is);
+   }
+
+   /** @brief Get a tensor_accessor object referencing the tensor stored at the
+       dynamic indices @a is. This object can be used to write tensor objects
+       into the global data array of the tensor_ndarray. */
+   template <typename... index_types> MFEM_HOST_DEVICE
+   tensor_accessor get_accessor(index_types... is) const
+   {
+      static_assert(sizeof...(is) == rank(), "invalid number of indices!");
+      return get_accessor(std::array<std::size_t,rank()> {std::size_t(is)...});
+   }
+};
+
+
+/** @brief Construct a tensor_ndarray where only the tensor sizes have to be
+    explicitly given as template parameters, the rest can be deduced from the
+    function call arguments. */
+template <int... tensor_sizes, typename scalar_t, typename... dyn_sizes_t>
+decltype(auto) make_tensor_ndarray(scalar_t *ptr, dyn_sizes_t... dynamic_sizes)
+{
+   return tensor_ndarray<scalar_t,sizeof...(dynamic_sizes),tensor_sizes...>(
+             ptr, {std::size_t(dynamic_sizes)...});
+}
+
+
+/// Alias for make_tensor_ndarray = make_tensor_array.
+template <int... tensor_sizes, typename scalar_t, typename... dyn_sizes_t>
+decltype(auto) make_tensor_array(scalar_t *ptr, dyn_sizes_t... dynamic_sizes)
+{
+   return tensor_ndarray<scalar_t,sizeof...(dynamic_sizes),tensor_sizes...>(
+             ptr, {std::size_t(dynamic_sizes)...});
+}
+
+
+/// Short name for one-dimensional tensor_ndarray: tensor_array.
+template <typename scalar_t, int... tensor_sizes>
+using tensor_array = tensor_ndarray<scalar_t, 1, tensor_sizes...>;
+
+} // namespace mfem::future
+
+#endif // MFEM_TENSOR_ARRAYS_HPP

makefile

@@ -451,7 +451,8 @@ DIRS = general linalg linalg/batched linalg/simd mesh mesh/submesh fem \
        fem/ceed/integrators/diffusion fem/ceed/integrators/nlconvection \
        fem/ceed/interface fem/ceed/solvers fem/eltrans fem/fe fem/gslib \
        fem/integ fem/lor fem/moonolith fem/qinterp fem/tmop fem/dfem \
-	   fem/tmop/assemble fem/tmop/metrics fem/tmop/mult fem/tmop/tools
+       fem/dfem/backends fem/dfem/backends/local_qf fem/dfem/backends/global_qf\
+       fem/tmop/assemble fem/tmop/metrics fem/tmop/mult fem/tmop/tools
 
 ifeq ($(MFEM_USE_MOONOLITH),YES)
    MFEM_CXXFLAGS += $(MOONOLITH_CXX_FLAGS)

mesh/pmesh.hpp

@@ -318,7 +318,7 @@ protected:
    void LoadSharedEntities(std::istream &input);
 
    /// If the mesh is curved, make sure 'Nodes' is ParGridFunction.
-   /** Note that this method is not related to the public 'Mesh::EnsureNodes`.*/
+   /** Note that this method is not related to the public 'Mesh::EnsureNodes'.*/
    void EnsureParNodes();
 
    /// Internal function used in ParMesh::MakeRefined (and related constructor)

miniapps/dfem/dfem-minimal-surface.cpp

@@ -23,10 +23,8 @@
 //               mpirun -np 4 dfem-minimal-surface -der 2
 //
 // Device sample runs:
-//               mpirun -np 4 dfem-minimal-surface -der 0 -r 1 -o 2 -d cuda
-//               mpirun -np 4 dfem-minimal-surface -der 1 -r 1 -o 2 -d cuda
-//             * mpirun -np 4 dfem-minimal-surface -der 0 -r 1 -o 2 -d hip
-//             * mpirun -np 4 dfem-minimal-surface -der 1 -r 1 -o 2 -d hip
+//               mpirun -np 4 dfem-minimal-surface -der 0 -r 1 -o 2 -d gpu
+//               mpirun -np 4 dfem-minimal-surface -der 1 -r 1 -o 2 -d gpu
 //
 // Description:  This example code demonstrates the use of MFEM to solve the
 //               minimal surface problem in 2D:
@@ -48,6 +46,8 @@
 //               visualization.
 
 #include "mfem.hpp"
+#include "../../fem/dfem/doperator.hpp"
+#include "../../fem/dfem/backends/local_qf/prelude.hpp"
 
 using namespace mfem;
 
@@ -116,11 +116,12 @@ public:
       auto operator()(
          const tensor<dscalar_t, dim> &dudxi,
          const tensor<real_t, dim, dim> &J,
-         const real_t &w) const
+         const real_t &w,
+         tensor<dscalar_t, dim> &dvdx) const
       {
          const auto invJ = inv(J);
          const auto dudx = dudxi * invJ;
-         return tuple{coeff(dudx) * dudx * transpose(invJ) * det(J) * w};
+         dvdx = coeff(dudx) * dudx * transpose(invJ) * det(J) * w;
       }
    };
 
@@ -135,7 +136,8 @@ public:
          const tensor<real_t, dim> &ddelta_udxi,
          const tensor<real_t, dim> &dudxi,
          const tensor<real_t, dim, dim> &J,
-         const real_t &w) const
+         const real_t &w,
+         tensor<real_t, dim> &dvdx) const
       {
          const auto invJ = inv(J);
          const auto dudx = dudxi * invJ;
@@ -145,7 +147,7 @@ public:
          const auto term1 = c * ddelta_udx;
          const auto term2 = c * c * c * dot(dudx, ddelta_udx) * dudx;
 
-         return tuple{(term1 - term2) * transpose(invJ) * det(J) * w};
+         dvdx = (term1 - term2) * transpose(invJ) * det(J) * w;
       }
    };
 
@@ -162,14 +164,9 @@ public:
          z(minsurface->Height())
       {
          minsurface->u.SetFromTrueDofs(x);
-         auto mesh_nodes = static_cast<ParGridFunction*>
-                           (minsurface->H1.GetParMesh()->GetNodes());
 
-         // One can retrieve the derivative of a DifferentiableOperator wrt a
-         // field variable if the derivative has been requested during the
-         // DifferentiableOperator::AddDomainIntegrator call.
-         dres_du = minsurface->res->GetDerivative(
-                      SOLUTION_U, {&minsurface->u}, {mesh_nodes});
+         MultiVector X{x, minsurface->mesh_nodes_tdofs};
+         dres_du = minsurface->res->GetDerivative(SOLUTION_U, X);
       }
 
       void Mult(const Vector &x, Vector &y) const override
@@ -177,7 +174,8 @@ public:
          z = x;
          z.SetSubVector(minsurface->ess_tdofs, 0.0);
 
-         dres_du->Mult(z, y);
+         MultiVector Y{y};
+         dres_du->Mult(z, Y);
 
          auto d_y = y.ReadWrite();
          const auto d_x = x.Read();
@@ -216,22 +214,20 @@ public:
          Array<int> all_domain_attr(minsurface->H1.GetMesh()->attributes.Max());
          all_domain_attr = 1;
 
-         auto &mesh_nodes = *static_cast<ParGridFunction *>
-                            (minsurface->H1.GetParMesh()->GetNodes());
-         auto &mesh_nodes_fes = *mesh_nodes.ParFESpace();
-
-         std::vector<FieldDescriptor> solutions =
-         {
-            {DIRECTION_U, &minsurface->H1}
-         };
-         std::vector<FieldDescriptor> parameters =
+         std::vector<FieldDescriptor> inputs =
          {
+            {DIRECTION_U, &minsurface->H1},
             {SOLUTION_U, &minsurface->H1},
-            {MESH_NODES, &mesh_nodes_fes}
+            {MESH_NODES, minsurface->mesh_nodes_fes}
+         };
+
+         std::vector<FieldDescriptor> outputs =
+         {
+            {SOLUTION_U, &minsurface->H1}
          };
 
          dres_du = std::make_shared<DifferentiableOperator>(
-                      solutions, parameters, *minsurface->H1.GetParMesh());
+                      inputs, outputs, *minsurface->H1.GetParMesh());
 
          auto input_operators = tuple
          {
@@ -247,12 +243,13 @@ public:
          };
 
          ManualDerivativeApply manual_derivative_apply;
-         dres_du->AddDomainIntegrator(manual_derivative_apply, input_operators,
-                                      output_operators, minsurface->ir,
-                                      all_domain_attr);
+         dres_du->AddDomainIntegrator<LocalQFBackend>(manual_derivative_apply,
+                                                      input_operators,
+                                                      output_operators, minsurface->ir,
+                                                      all_domain_attr);
 
-         minsurface->u.SetFromTrueDofs(x);
-         dres_du->SetParameters({&minsurface->u, &mesh_nodes});
+         x0.SetSize(x.Size());
+         x0 = x;
       }
 
       void Mult(const Vector &x, Vector &y) const override
@@ -260,7 +257,9 @@ public:
          z = x;
          z.SetSubVector(minsurface->ess_tdofs, 0.0);
 
-         dres_du->Mult(z, y);
+         MultiVector X{z, x0, minsurface->mesh_nodes_tdofs};
+         MultiVector Y{y};
+         dres_du->Mult(X, Y);
 
          auto d_y = y.HostReadWrite();
          const auto d_x = x.HostRead();
@@ -272,10 +271,10 @@ public:
 
       const MinimalSurface *minsurface = nullptr;
       std::shared_ptr<DifferentiableOperator> dres_du;
+      Vector x0;
       mutable Vector z;
    };
 
-
 public:
    MinimalSurface(ParFiniteElementSpace &H1,
                   const IntegrationRule &ir,
@@ -291,7 +290,8 @@ public:
 
       auto &mesh_nodes =
          *static_cast<ParGridFunction *>(H1.GetParMesh()->GetNodes());
-      auto &mesh_nodes_fes = *mesh_nodes.ParFESpace();
+      mesh_nodes_fes = mesh_nodes.ParFESpace();
+      mesh_nodes.GetTrueDofs(mesh_nodes_tdofs);
 
       // The following section is the heart of this example. It shows how to
       // create and interact with the DifferentialOperator class.
@@ -299,14 +299,16 @@ public:
       // The constructor of DifferentiableOperator takes two vectors of
       // FieldDescriptors. A FieldDescriptor can be viewed as a a pair of an
       // identifier (the field ID) and it's accompanying space.
-      std::vector<FieldDescriptor> solutions;
-      solutions.push_back(FieldDescriptor(SOLUTION_U, &H1));
-      std::vector<FieldDescriptor> parameters;
-      parameters.push_back(FieldDescriptor(MESH_NODES, &mesh_nodes_fes));
+      std::vector<FieldDescriptor> inputs;
+      inputs.emplace_back(SOLUTION_U, &H1);
+      inputs.emplace_back(MESH_NODES, mesh_nodes_fes);
+
+      std::vector<FieldDescriptor> outputs;
+      outputs.emplace_back(SOLUTION_U, &H1);
 
       // Create the DifferentiableOperator on the desired mesh.
       res = std::make_shared<DifferentiableOperator>(
-               solutions, parameters, *H1.GetParMesh());
+               inputs, outputs, *H1.GetParMesh());
 
       // DifferentiableOperator::AddIntegrator consists mainly of multiple
       // components. The input and output operators and the pointwise
@@ -353,16 +355,9 @@ public:
       // formed integrator should be formed. This is necessary to specify at
       // compile time in order to instantiate the correct functions.
       auto derivatives = std::integer_sequence<size_t, SOLUTION_U> {};
-      res->AddDomainIntegrator(mf_apply_qf, input_operators, output_operators,
-                               ir, all_domain_attr, derivatives);
-
-      // Before we are able to use DifferentiableOperator::Mult, we need to call
-      // DifferentiableOperator::SetParameters to set the parameters of the
-      // operator. Here, only the mesh node function is required. We do this
-      // here once, because we know that the nodes won't change. If they do,
-      // we'd have to call SetParameters before each call to Mult. This is done
-      // to be mathematically consistent with fixing paramaters.
-      res->SetParameters({&mesh_nodes});
+      res->AddDomainIntegrator<LocalQFBackend>(
+         mf_apply_qf, input_operators, output_operators,
+         ir, all_domain_attr, derivatives);
 
       Array<int> ess_bdr(H1.GetParMesh()->bdr_attributes.Max());
       ess_bdr = 1;
@@ -371,7 +366,9 @@ public:
 
    void Mult(const Vector &x, Vector &y) const override
    {
-      res->Mult(x, y);
+      MultiVector X{x, mesh_nodes_tdofs};
+      MultiVector Y{y};
+      res->Mult(X, Y);
       y.SetSubVector(ess_tdofs, 0.0);
    }
 
@@ -408,10 +405,11 @@ public:
    }
 
 private:
-   ParFiniteElementSpace &H1;
+   ParFiniteElementSpace &H1, *mesh_nodes_fes = nullptr;
    const IntegrationRule &ir;
 
    mutable ParGridFunction u;
+   Vector mesh_nodes_tdofs;
 
    Array<int> ess_tdofs;
 

miniapps/hdiv-linear-solver/change_basis.hpp

@@ -80,6 +80,8 @@ public:
    // limitations
    void MultRT_2D(const Vector &x, Vector &y, Mode mode) const;
    void MultRT_3D(const Vector &x, Vector &y, Mode mode) const;
+   // suppress warning about hiding overloaded virtual function:
+   using Operator::Mult;
 };
 
 } // namespace mfem

miniapps/mtop/CMakeLists.txt

@@ -12,8 +12,9 @@
 # Skip dFEM qfunction with floats
 if(MFEM_USE_MPI AND MFEM_USE_DOUBLE)
 
-	list(APPEND PAR_MTOP_SOURCES mtop_solvers.cpp)
-	list(APPEND PAR_MTOP_HEADERS mtop_solvers.hpp)
+	list(APPEND PAR_MTOP_SOURCES mtop_solvers.cpp tmop_ad_err.cpp)
+	list(APPEND PAR_MTOP_HEADERS mtop_solvers.hpp tmop_ad_err.hpp)
+
 
 	convert_filenames_to_full_paths(PAR_MTOP_SOURCES)
 	convert_filenames_to_full_paths(PAR_MTOP_HEADERS)
@@ -27,4 +28,10 @@ if(MFEM_USE_MPI AND MFEM_USE_DOUBLE)
 					 ${PAR_MTOP_COMMON_FILES}
 					 LIBRARIES mfem)
 
+	add_mfem_miniapp(mtop_Rref_TO
+					 MAIN mtop_Rref_TO.cpp
+					 ${PAR_MTOP_COMMON_FILES}
+					 LIBRARIES mfem)
+
+
 endif (MFEM_USE_MPI AND MFEM_USE_DOUBLE)

miniapps/mtop/mtop_solvers.cpp

@@ -35,12 +35,13 @@ template <int DIM> struct QFunction
       MFEM_HOST_DEVICE inline auto operator()(const matd_t &dudxi,
                                               const real_t &L, const real_t &M,
                                               const matd_t &J,
-                                              const real_t &w) const
+                                              const real_t &w,
+                                              matd_t &rtn) const
       {
          const matd_t JxW = transpose(inv(J)) * det(J) * w;
          constexpr auto I = mfem::future::IsotropicIdentity<DIM>();
          const auto eps = mfem::future::sym(dudxi * mfem::future::inv(J));
-         return tuple{(L * tr(eps) * I + 2.0 * M * eps) * JxW};
+         rtn = (L * tr(eps) * I + 2.0 * M * eps) * JxW;
       }
    };
 };
@@ -78,12 +79,12 @@ IsoLinElasticSolver::IsoLinElasticSolver(ParMesh *mesh, int vorder,
    ir(IntRules.Get(fe->GetGeomType(),
                    fe->GetOrder() + fe->GetOrder() + fe->GetDim() - 1)),
    qs(*pmesh, ir),
-   Lambda_ps(*pmesh, ir, 1),
-   Mu_ps(*pmesh, ir, 1),
+   Lambda_qs(qs, 1),
+   Mu_qs(qs, 1),
    lf(nullptr)
 {
-   MFEM_VERIFY(qs.GetSize() == Lambda_ps.GetTrueVSize(),
-               "QuadratureSpace and ParameterSpace size mismatch");
+   MFEM_VERIFY(qs.GetSize() == Lambda_qs.GetVSize(),
+               "QuadratureSpace and VectorQuadratureSpace size mismatch");
 
    sol = 0.0;
    rhs = 0.0;
@@ -120,13 +121,13 @@ IsoLinElasticSolver::~IsoLinElasticSolver()
 
    delete lvforce;
 
-   for (auto it = load_coeff.begin(); it != load_coeff.end(); it++)
-   {
-      delete it->second;
-   }
+   // for (auto it = load_coeff.begin(); it != load_coeff.end(); it++)
+   // {
+   //    delete it->second;
+   // }
 
-   delete lambda;
-   delete mu;
+   // delete lambda;
+   // delete mu;
 }
 
 void IsoLinElasticSolver::SetLinearSolver(real_t rtol,
@@ -374,12 +375,16 @@ void IsoLinElasticSolver::Assemble()
 #ifdef MFEM_USE_DOUBLE
       // define the differentiable operator
       dop = std::make_unique<mfem::future::DifferentiableOperator>(
-      std::vector<mfem::future::FieldDescriptor> {{ U, vfes }},
+               std::vector<mfem::future::FieldDescriptor>
+      {
+         { U, vfes },
+         { LCoeff, &Lambda_qs},
+         { MuCoeff, &Mu_qs},
+         { Coords, mfes }
+      },
       std::vector<mfem::future::FieldDescriptor>
       {
-         { LCoeff, &Lambda_ps},
-         { MuCoeff, &Mu_ps},
-         { Coords, mfes }
+         { U, vfes }
       },
       *pmesh);
 
@@ -389,25 +394,23 @@ void IsoLinElasticSolver::Assemble()
       // sample mu on the integration points
       Mu_cv = std::make_unique<CoefficientVector>(*mu, qs);
 
-      // set the parameters of the differentiable operator
-      dop->SetParameters({ Lambda_cv.get(), Mu_cv.get(), nodes });
-
       // define the q-function for dimensions 2 and 3
-      const auto inputs =
-         mfem::future::tuple{ Gradient<U>{},
-                              Identity<LCoeff>{}, Identity<MuCoeff>{},
-                              Gradient<Coords>{},
-                              Weight{} };
-      const auto output = mfem::future::tuple{ Gradient<U>{} };
+      auto inputs =
+         mfem::future::Inputs< Gradient<U>,
+         Identity<LCoeff>, Identity<MuCoeff>,
+         Gradient<Coords>,
+         Weight> {};
+      auto output = mfem::future::Outputs< Gradient<U>> {};
+      using Backend = mfem::future::LocalQFBackend;
       if (2 == spaceDim)
       {
          typename QFunction<2>::Elasticity e2qf;
-         dop->AddDomainIntegrator(e2qf, inputs, output, ir, domain_attributes);
+         dop->AddDomainIntegrator<Backend>(e2qf, inputs, output, ir, domain_attributes);
       }
       else if (3 == spaceDim)
       {
          typename QFunction<3>::Elasticity e3qf;
-         dop->AddDomainIntegrator(e3qf, inputs, output, ir, domain_attributes);
+         dop->AddDomainIntegrator<Backend>(e3qf, inputs, output, ir, domain_attributes);
       }
       else { MFEM_ABORT("Space dimension not supported"); }
 #else
@@ -500,12 +503,16 @@ void IsoLinElasticSolver::Assemble()
       else
       {
          prec = new HypreBoomerAMG();
+         //precILU->SetLevelOfFill (5);
+         //precILU->SetOperator(*K);
          // set the rigid body modes
          prec->SetElasticityOptions(vfes);
+         prec->SetPrintLevel(0);
          ls->SetPreconditioner(*prec);
       }
+   
       ls->SetOperator(((pa||dfem) ? *Kh->Ptr() : *K));
-      ls->SetPrintLevel(1);
+      ls->SetPrintLevel(0);
    }
    else
    {
@@ -547,3 +554,14 @@ void IsoLinElasticSolver::FSolve()
    delete lf;
    lf = nullptr;
 }
+
+void IsoLinElasticSolver::ASolve( mfem::Vector dQdu )
+{
+   if (pa || dfem) { Kc->EliminateRHS(adj, dQdu); }
+   else
+   {
+      K->EliminateBC(*Ke, ess_tdofv, adj, dQdu);
+   }
+
+   ls->Mult(dQdu, adj);
+}

miniapps/mtop/mtop_solvers.hpp

@@ -14,6 +14,9 @@
 
 #include "mfem.hpp"
 
+#include "../../fem/dfem/doperator.hpp"
+#include "../../fem/dfem/parameterspace.hpp"
+
 using real_t = mfem::real_t;
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -97,6 +100,10 @@ public:
    /// Solves the forward problem.
    void FSolve();
 
+   /// Solves the adjoint problem.
+   void ASolve( mfem::Vector dQdu );
+
+
    /// Adds displacement BC in direction 0(x), 1(y), 2(z), or -1(all).
    void AddDispBC(int id, int dir, real_t val);
 
@@ -191,6 +198,13 @@ public:
       mu = new IsoElasticySchearCoeff(E, nu);
    }
 
+   void SetMaterialLame(mfem::Coefficient &lambda_, mfem::Coefficient &mu_)
+   {
+      lambda = &lambda_;
+      mu = &mu_;
+   }
+
+
    class NqptUniformParameterSpace : public
       mfem::future::UniformParameterSpace
    {
@@ -202,6 +216,10 @@ public:
       {
          dtq.nqpt = ir.GetNPoints();
       }
+
+      const Operator* GetB() const override { return nullptr; }
+
+      const Operator* GetBt() const override { return nullptr; }
    };
 
    // creates a list with essential dofs
@@ -240,6 +258,7 @@ private:
    int linear_iter;
 
    mfem::HypreBoomerAMG *prec; // preconditioner
+   mfem::HypreILU *precILU; // preconditioner
    mfem::CGSolver *ls;         // linear solver
 
    // PA LOR preconditioner
@@ -298,7 +317,7 @@ private:
    mfem::Array<int> domain_attributes;
    const mfem::IntegrationRule &ir;
    mfem::QuadratureSpace qs;
-   NqptUniformParameterSpace Lambda_ps, Mu_ps;
+   mfem::VectorQuadratureSpace Lambda_qs, Mu_qs;
    std::unique_ptr<mfem::CoefficientVector> Lambda_cv, Mu_cv;
    std::unique_ptr<mfem::future::DifferentiableOperator> dop;
    // end of dFEM definitions

miniapps/mtop/tmop_ad_err.hpp

@@ -0,0 +1,954 @@
+#ifndef TMOP_AD_ERR_HPP
+#define TMOP_AD_ERR_HPP
+
+#include "mfem.hpp"
+
+namespace mfem{
+void IdentityMatrix(int dim, DenseMatrix &I);
+
+void Vectorize(const DenseMatrix &A, Vector &a);
+
+double MatrixInnerProduct(const DenseMatrix &A, const DenseMatrix &B);
+
+void ConjugationProduct(const DenseMatrix &A, const DenseMatrix &B, const DenseMatrix &C, DenseMatrix &D);
+
+void KroneckerProduct(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &C);
+
+void IsotropicStiffnessMatrix(int dim, double mu, double lambda, DenseMatrix &C);
+
+void IsotropicStiffnessMatrix3D(double E, double v, DenseMatrix &C);
+
+void FourthOrderSymmetrizer(int dim, DenseMatrix &S);
+
+void FourthOrderIdentity(int dim, DenseMatrix &I4);
+
+void FourthOrderTranspose(int dim, DenseMatrix &T);
+
+void VectorOuterProduct(const Vector &a, const Vector &b, DenseMatrix &C);
+void   UnitStrain(int dim, int i, int j, DenseMatrix &E);
+void   UnitStrain(int dim, int i, int j, Vector &E);
+void   MatrixConjugationProduct(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &C);
+
+class NodeAwareTMOPQuality
+{
+public:
+    NodeAwareTMOPQuality(mfem::ParMesh* mesh_, int order_, TMOP_QualityMetric *metric, TargetConstructor *target_c)
+    {
+        pmesh=mesh_;
+        int dim=pmesh->Dimension();
+
+        fec = new H1_FECollection(order_,dim);
+        coord_fes_ = new ParFiniteElementSpace(pmesh,fec,dim);
+
+        X0_.SetSpace(coord_fes_);
+        designVar.SetSpace(coord_fes_);
+        mfem::Vector tempX0_;
+        pmesh->GetNodes(tempX0_);
+
+        X0_ = tempX0_;
+
+        dQdx_ = new mfem::ParLinearForm(coord_fes_);
+        metric_in = metric;
+        target_in = target_c;
+    }
+
+    ~NodeAwareTMOPQuality()
+    {
+    }
+
+    void UpdateMesh(mfem::Vector const &U);
+    double EvalQoI();
+    void EvalQoIGrad();
+
+    mfem::ParLinearForm * GetDQDx(){ return dQdx_; };
+
+    void SetDesign( mfem::ParGridFunction & design){ designVar = design; };
+
+    private:
+
+    mfem::ParMesh* pmesh;
+    mfem::ParGridFunction X0_;
+    mfem::ParGridFunction designVar;
+
+    mfem::FiniteElementCollection *fec;
+    mfem::ParFiniteElementSpace	  *coord_fes_;
+
+    mfem::ParLinearForm * dQdx_;
+    TMOP_QualityMetric *metric_in = nullptr;
+    TargetConstructor *target_in = nullptr;
+};
+
+
+enum QoIType
+{
+  L2_ERROR,
+  H1S_ERROR,
+  ZZ_ERROR,
+  AVG_ERROR,
+  ENERGY,
+  GZZ_ERROR,
+  H1_ERROR,
+  STRUC_COMPLIANCE
+};
+
+class QoIBaseCoefficient : public Coefficient {
+public:
+  QoIBaseCoefficient() {};
+
+  virtual ~QoIBaseCoefficient() {};
+
+  virtual const DenseMatrix &explicitSolutionDerivative
+      (ElementTransformation &T, const IntegrationPoint &ip) = 0;
+
+  virtual const DenseMatrix &explicitSolutionGradientDerivative(ElementTransformation &T,
+      const IntegrationPoint &ip)
+    = 0;
+
+  virtual const DenseMatrix &gradTimesexplicitSolutionGradientDerivative(ElementTransformation &T,
+      const IntegrationPoint &ip)
+    = 0;
+
+  virtual const DenseMatrix &explicitShapeDerivative(ElementTransformation &T, const IntegrationPoint &ip) = 0;
+
+    virtual const Vector DerivativeExactWRTX(ElementTransformation &T, const IntegrationPoint &ip) {
+    Vector vec(T.GetSpaceDim());
+    vec = 0.0;
+    return vec;
+  }
+private:
+};
+
+
+
+
+class Energy_QoI : public QoIBaseCoefficient {
+public:
+  Energy_QoI(mfem::ParGridFunction * solutionField, mfem::Coefficient * force, VectorCoefficient * forceGrad, int Dim)
+    : solutionField_(solutionField), force_(force), forceGrad_(forceGrad), Dim_(Dim)
+  {};
+
+  ~Energy_QoI() {};
+
+  double Eval( mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip) override
+  {
+    double fieldVal = solutionField_->GetValue( T, ip );
+    double Val = force_->Eval( T, ip );
+
+    double energy = fieldVal*Val;
+    return energy;
+  };
+
+  const mfem::DenseMatrix &explicitSolutionDerivative( mfem::ElementTransformation & T, const mfem::IntegrationPoint & ip) override
+  {
+    dtheta_dU.SetSize(1);
+
+    double Val = force_->Eval( T, ip );
+    double & matVal = dtheta_dU.Elem(0,0);
+    matVal = Val;
+    return dtheta_dU;
+  };
+
+  const mfem::DenseMatrix &explicitSolutionGradientDerivative( mfem::ElementTransformation &T,
+      const mfem::IntegrationPoint &ip) override
+  {
+    dtheta_dGradU.SetSize(1, Dim_);
+    dtheta_dGradU = 0.0;
+
+    return dtheta_dGradU;
+  };
+
+  const mfem::DenseMatrix &explicitShapeDerivative( mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip) override
+  {
+    dtheta_dX.SetSize(1, Dim_);
+    dtheta_dX = 0.0;
+
+    return dtheta_dX;
+  };
+
+  virtual const mfem::DenseMatrix &gradTimesexplicitSolutionGradientDerivative( mfem::ElementTransformation &T,
+      const mfem::IntegrationPoint &ip) override
+  {
+    dtheta_dX.SetSize(Dim_, Dim_);
+    dtheta_dX = 0.0;
+
+    return dtheta_dX;
+  };
+
+  const Vector DerivativeExactWRTX(ElementTransformation &T, const IntegrationPoint &ip) override
+  {
+    Vector trueGrad;
+    forceGrad_->Eval(trueGrad, T, ip);
+
+    double fieldVal = solutionField_->GetValue( T, ip );
+    trueGrad *= fieldVal;
+    return trueGrad;
+  }
+
+private:
+
+  mfem::ParGridFunction * solutionField_;
+  mfem::Coefficient * force_;
+  VectorCoefficient * forceGrad_ = nullptr;
+
+  int Dim_;
+
+  double theta = 0.0;
+  mfem::DenseMatrix dtheta_dX;
+  mfem::DenseMatrix dtheta_dU;
+  mfem::DenseMatrix dtheta_dGradU;
+};
+
+class LFNodeCoordinateSensitivityIntegrator : public LinearFormIntegrator {
+public:
+  LFNodeCoordinateSensitivityIntegrator( int IntegrationOrder = INT_MAX);
+  ~LFNodeCoordinateSensitivityIntegrator() {};
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+  void SetQoI(std::shared_ptr<QoIBaseCoefficient> QoI) { QoI_ = QoI; };
+  void SetGLLVec(Array<double> &gllvec) { gllvec_ = gllvec;}
+  void SetNqptsPerEl(int nqp) { nqptsperel = nqp; }
+private:
+  std::shared_ptr<QoIBaseCoefficient> QoIFactoryFunction(const int dim);
+
+  const int IntegrationOrder_;
+
+  std::shared_ptr<QoIBaseCoefficient> QoI_ = nullptr;
+
+  Array<double> gllvec_;
+  int nqptsperel;
+};
+
+class LFAvgErrorNodeCoordinateSensitivityIntegrator : public LinearFormIntegrator {
+public:
+  LFAvgErrorNodeCoordinateSensitivityIntegrator(
+     ParGridFunction * solutionField, GridFunctionCoefficient * elementVol,
+     int IntegrationOrder = INT_MAX);
+  ~LFAvgErrorNodeCoordinateSensitivityIntegrator() {};
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+  void SetQoI(std::shared_ptr<QoIBaseCoefficient> QoI) { QoI_ = QoI; };
+private:
+  std::shared_ptr<QoIBaseCoefficient> QoIFactoryFunction(const int dim);
+
+  ParGridFunction * solutionField_ = nullptr;
+  GridFunctionCoefficient * elementVol_ = nullptr;
+  const int IntegrationOrder_;
+
+  std::shared_ptr<QoIBaseCoefficient> QoI_ = nullptr;
+};
+
+class LFErrorIntegrator : public LinearFormIntegrator {
+public:
+  LFErrorIntegrator( int IntegrationOrder = INT_MAX);
+  ~LFErrorIntegrator() {};
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+  void SetQoI(std::shared_ptr<QoIBaseCoefficient> QoI) { QoI_ = QoI; };
+    void SetGLLVec(Array<double> &gllvec) { gllvec_ = gllvec;}
+    void SetNqptsPerEl(int nqp) { nqptsperel = nqp; }
+private:
+  std::shared_ptr<QoIBaseCoefficient> QoIFactoryFunction(const int dim);
+
+  const int IntegrationOrder_;
+
+  std::shared_ptr<QoIBaseCoefficient> QoI_ = nullptr;
+    Array<double> gllvec_;
+    int nqptsperel;
+};
+
+class LFErrorDerivativeIntegrator : public LinearFormIntegrator {
+public:
+  LFErrorDerivativeIntegrator( );
+  ~LFErrorDerivativeIntegrator() {};
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+  void SetQoI(std::shared_ptr<QoIBaseCoefficient> QoI) { QoI_ = QoI; };
+private:
+  std::shared_ptr<QoIBaseCoefficient> QoIFactoryFunction(const int dim);
+
+  std::shared_ptr<QoIBaseCoefficient> QoI_ = nullptr;
+};
+
+class LFErrorDerivativeIntegrator_2 : public LinearFormIntegrator {
+public:
+  LFErrorDerivativeIntegrator_2( ParFiniteElementSpace * fespace, Array<int> count, int IntegrationOrder = INT_MAX);
+  ~LFErrorDerivativeIntegrator_2() {};
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+  void SetQoI(std::shared_ptr<QoIBaseCoefficient> QoI) { QoI_ = QoI; };
+private:
+  std::shared_ptr<QoIBaseCoefficient> QoIFactoryFunction(const int dim);
+
+  ParFiniteElementSpace * fespace_ = nullptr;
+  Array<int> count_;
+  const int IntegrationOrder_;
+
+  std::shared_ptr<QoIBaseCoefficient> QoI_ = nullptr;
+};
+
+class LFFilteredFieldErrorDerivativeIntegrator : public LinearFormIntegrator {
+public:
+  LFFilteredFieldErrorDerivativeIntegrator( );
+  ~LFFilteredFieldErrorDerivativeIntegrator() {};
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+  void SetQoI(std::shared_ptr<QoIBaseCoefficient> QoI) { QoI_ = QoI; };
+private:
+  std::shared_ptr<QoIBaseCoefficient> QoIFactoryFunction(const int dim);
+
+  std::shared_ptr<QoIBaseCoefficient> QoI_ = nullptr;
+};
+
+class LFAverageErrorDerivativeIntegrator : public LinearFormIntegrator {
+public:
+  LFAverageErrorDerivativeIntegrator( ParFiniteElementSpace * fespace, GridFunctionCoefficient * elementVol, int IntegrationOrder = INT_MAX);
+  ~LFAverageErrorDerivativeIntegrator() {};
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+  void SetQoI(std::shared_ptr<QoIBaseCoefficient> QoI) { QoI_ = QoI; };
+private:
+  std::shared_ptr<QoIBaseCoefficient> QoIFactoryFunction(const int dim);
+
+  ParFiniteElementSpace * fespace_ = nullptr;
+  Array<int> count_;
+  const int IntegrationOrder_;
+
+  GridFunctionCoefficient * elementVol_ = nullptr;
+  std::shared_ptr<QoIBaseCoefficient> QoI_ = nullptr;
+};
+
+class PenaltyMassShapeSensitivityIntegrator : public LinearFormIntegrator {
+public:
+  PenaltyMassShapeSensitivityIntegrator(Coefficient &penalty, const ParGridFunction &t_primal,
+      const ParGridFunction &t_adjoint);
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+private:
+  Coefficient *penalty_;
+  const ParGridFunction *t_primal_;
+  const ParGridFunction *t_adjoint_;
+};
+
+class PenaltyShapeSensitivityIntegrator : public LinearFormIntegrator {
+public:
+  PenaltyShapeSensitivityIntegrator(Coefficient &t_primal, const ParGridFunction &t_adjoint, Coefficient &t_penalty, VectorCoefficient *SolGrad_= nullptr, int oa = 2, int ob = 2);
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+private:
+  Coefficient *t_primal_ = nullptr;
+  Coefficient *t_penalty_ = nullptr;
+  VectorCoefficient *SolGradCoeff_= nullptr;
+  const ParGridFunction *t_adjoint_;
+  int oa_, ob_;
+};
+
+
+class GradProjectionShapeSensitivityIntegrator : public LinearFormIntegrator {
+public:
+  GradProjectionShapeSensitivityIntegrator(const ParGridFunction &t_primal, const ParGridFunction &t_adjoin, VectorCoefficient & tempCoeff);
+  void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+private:
+  const ParGridFunction *t_primal_;
+  const ParGridFunction *t_adjoint_;
+  VectorCoefficient *tempCoeff_;
+};
+
+class ElasticityStiffnessShapeSensitivityIntegrator : public LinearFormIntegrator
+{
+public:
+    ElasticityStiffnessShapeSensitivityIntegrator(Coefficient &lambda, Coefficient &mu,
+            const ParGridFunction &u_primal, const ParGridFunction &u_adjoint);
+    void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+private:
+    Coefficient       *lambda_;
+    Coefficient       *mu_;
+    const ParGridFunction *u_primal_;
+    const ParGridFunction *u_adjoint_;
+};
+
+class ElasticityTractionIntegrator : public LinearFormIntegrator
+{
+public:
+    ElasticityTractionIntegrator(VectorCoefficient &f, int oa=2, int ob=2);
+    void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+private:
+    VectorCoefficient *f_;
+    int oa_, ob_;
+};
+
+class ElasticityTractionShapeSensitivityIntegrator : public LinearFormIntegrator
+{
+public:
+    ElasticityTractionShapeSensitivityIntegrator(VectorCoefficient &f,
+            const ParGridFunction &u_adjoint, int oa=2, int ob=2);
+    void AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &T, Vector &elvect);
+private:
+    VectorCoefficient *f_;
+    const ParGridFunction *u_adjoint_;
+    int oa_, ob_;
+};
+
+
+class QuantityOfInterest
+{
+public:
+    QuantityOfInterest(ParMesh* mesh_, enum QoIType qoiType, int order_, int physics_order_, Array<int> NeumannBdr = {} ,int pdim = 1)
+    : pmesh(mesh_), qoiType_(qoiType), bdr(NeumannBdr)
+    {
+        int dim=pmesh->Dimension();
+
+        pmesh->GetNodes(X0_);
+
+        fec = new H1_FECollection(order_,dim);
+        pfec = new H1_FECollection(physics_order_,dim);
+        temp_fes_ = new ParFiniteElementSpace(pmesh,pfec,pdim);
+        coord_fes_ = new ParFiniteElementSpace(pmesh,fec,dim);
+        temp_fes_grad_ = new ParFiniteElementSpace(pmesh,pfec,dim*pdim);
+
+        solgf_.SetSpace(temp_fes_);
+
+        dQdu_ = new ParLinearForm(temp_fes_);
+        dQdx_ = new ParLinearForm(coord_fes_);
+    }
+
+    ~QuantityOfInterest()
+    {
+        delete temp_fes_;
+        delete coord_fes_;
+        delete fec;
+
+        delete dQdu_;
+        delete dQdx_;
+    }
+
+    void setTrueSolCoeff( Coefficient * trueSolution ){ trueSolution_ = trueSolution; };
+    void setTrueSolGradCoeff( VectorCoefficient * trueSolutionGrad ){ trueSolutionGrad_ = trueSolutionGrad; };
+    void setTrueSolHessCoeff( MatrixCoefficient * trueSolutionHess ){ trueSolutionHess_ = trueSolutionHess; };
+    void setTrueSolHessCoeff( VectorCoefficient * trueSolutionHessV ){ trueSolutionHessV_ = trueSolutionHessV; };
+    void setTractionCoeff( VectorCoefficient * tractionLoad ){ tractionLoad_ = tractionLoad; }
+    void SetManufacturedSolution( Coefficient * QCoef ){ QCoef_ = QCoef; }
+    void SetManufacturedSolutionGrad( VectorCoefficient * QCoefGrad ){ QCoefGrad_ = QCoefGrad; }
+    void SetDesign( Vector & design){ designVar = design; };
+    void SetNodes( Vector & coords){ X0_ = coords; };
+    void SetDesignVarFromUpdatedLocations( Vector & design)
+    {
+        designVar = design;
+        designVar -= X0_;
+    };
+    void SetDiscreteSol( ParGridFunction & sol){ solgf_ = sol; };
+    void UpdateMesh(Vector const &U);
+    double EvalQoI();
+    void EvalQoIGrad();
+    ParLinearForm * GetDQDu(){ return dQdu_; };
+    ParLinearForm * GetDQDx(){ return dQdx_; };
+    void SetGLLVec(Array<double> &gllvec) { gllvec_ = gllvec;}
+    void SetNqptsPerEl(int nqp) { nqptsperel = nqp; }
+    void SetIntegrationRules(IntegrationRules *irule_, int quad_order_) { irules = irule_; quad_order = quad_order_; }
+    Coefficient * GetTrueSolCoeff() { return trueSolution_; }
+    VectorCoefficient *GetTrueSolGradCoeff() { return trueSolutionGrad_; }
+private:
+    Coefficient * trueSolution_ = nullptr;
+    VectorCoefficient * trueSolutionGrad_ = nullptr;
+    MatrixCoefficient * trueSolutionHess_ = nullptr;
+    VectorCoefficient * trueSolutionHessV_ = nullptr;
+
+    VectorCoefficient * tractionLoad_ = nullptr;
+    Coefficient       * QCoef_ = nullptr;
+    VectorCoefficient * QCoefGrad_ = nullptr;
+
+    ParMesh* pmesh;
+    enum QoIType qoiType_;
+
+    Vector X0_;
+    Vector designVar;
+
+    FiniteElementCollection *fec;
+    FiniteElementCollection *pfec;
+    ParFiniteElementSpace	  *temp_fes_;
+    ParFiniteElementSpace	  *coord_fes_;
+    ParFiniteElementSpace	  *temp_fes_grad_;
+
+    ParLinearForm * dQdu_;
+    ParLinearForm * dQdx_;
+
+    ParGridFunction solgf_;
+
+    ParaViewDataCollection *debug_pdc;
+    int pdc_cycle = 0;
+
+    std::shared_ptr<QoIBaseCoefficient> ErrorCoefficient_ = nullptr;
+    Array<double> gllvec_;
+    int nqptsperel;
+
+    IntegrationRules *irules;
+    int quad_order;
+
+    Array<int> bdr;
+};
+
+class PhysicsSolverBase
+{
+  public:
+    PhysicsSolverBase( ParMesh* mesh_, int order_, int physics_order_)
+    {
+        pmesh=mesh_;
+        int dim=pmesh->Dimension();
+
+        pmesh->GetNodes(X0_);
+
+        fec = new H1_FECollection(order_,dim);
+        pfec = new H1_FECollection(physics_order_,dim);
+        coord_fes_ = new ParFiniteElementSpace(pmesh,fec,dim);
+
+        dQdx_ = new ParLinearForm(coord_fes_);
+
+        SetLinearSolver();
+    };
+
+    virtual ~PhysicsSolverBase()
+    {
+        delete physics_fes_;
+        delete coord_fes_;
+        delete fec;
+
+        delete dQdu_;
+        delete dQdx_;
+    };
+
+    void UpdateMesh(Vector const &U);
+
+    void SetLinearSolver(double rtol=1e-8, double atol=1e-12, int miter=2000)
+    {
+        linear_rtol=rtol;
+        linear_atol=atol;
+        linear_iter=miter;
+    }
+
+    virtual void FSolve() = 0;
+
+    virtual void ASolve( Vector & rhs ) = 0;
+
+    void SetDesign( Vector & design)
+    {
+        designVar = design;
+    };
+
+    void SetDesignVarFromUpdatedLocations( Vector & design)
+    {
+        designVar = design;
+        designVar -= X0_;
+    };
+
+    /// Returns the solution
+    ParGridFunction& GetSolution(){return solgf;}
+
+    /// Returns the solution vector.
+    Vector& GetSol(){return sol;}
+
+    /// Returns the adjoint solution vector.
+    Vector& GetAdj(){return adj;}
+
+    ParLinearForm * GetImplicitDqDx(){ return dQdx_; };
+
+  protected:
+    ParMesh* pmesh;
+
+    Vector X0_;
+    Vector designVar;
+
+    FiniteElementCollection *fec;
+    FiniteElementCollection *pfec;
+    ParFiniteElementSpace	  *physics_fes_;
+    ParFiniteElementSpace	  *coord_fes_;
+
+    //solution true vector
+    Vector sol;
+    Vector adj;
+    Vector rhs;
+    ParGridFunction solgf, projsolgf;
+    ParGridFunction adjgf;
+    ParGridFunction bcGridFunc_;
+
+    ParLinearForm * dQdu_;
+    ParLinearForm * dQdx_;
+
+        //Linear solver parameters
+    double linear_rtol;
+    double linear_atol;
+    int linear_iter;
+
+    int print_level = 1;
+};
+
+class Elasticity_Solver : public PhysicsSolverBase
+{
+public:
+    Elasticity_Solver(ParMesh* mesh_, std::vector<std::pair<int, double>> ess_bdr, const Array<int> & neumannBdr, int order_)
+    : PhysicsSolverBase( mesh_, order_ , order_), bdr(neumannBdr)
+    {
+        int dim=pmesh->Dimension();
+        physics_fes_ = new ParFiniteElementSpace(pmesh,fec,dim);
+
+        sol.SetSize(physics_fes_->GetTrueVSize()); sol=0.0;
+        rhs.SetSize(physics_fes_->GetTrueVSize()); rhs=0.0;
+        adj.SetSize(physics_fes_->GetTrueVSize()); adj=0.0;
+
+        solgf.SetSpace(physics_fes_);
+        adjgf.SetSpace(physics_fes_);
+
+        dQdu_ = new ParLinearForm(physics_fes_);
+
+        // store list of essential dofs
+        int maxAttribute = pmesh->bdr_attributes.Max();
+        Array<int> bdr_attr_is_ess(maxAttribute);
+        ess_tdof_list_.DeleteAll();
+        Vector ess_bc(physics_fes_->GetTrueVSize());
+        ess_bc = 0.0;
+
+        // loop over input attribute, value pairs
+        for (const auto &bc: ess_bdr)
+        {
+            int attribute = bc.first;
+
+            // get dofs associated with this attribute, component pair
+            bdr_attr_is_ess = 0;
+            bdr_attr_is_ess[attribute - 1] = 1; // mfem attributes 1-indexed, arrays 0-indexed
+            Array<int> u_tdofs;
+            physics_fes_->GetEssentialTrueDofs(bdr_attr_is_ess, u_tdofs);
+
+            // append to global dof list
+            ess_tdof_list_.Append(u_tdofs);
+
+            // set value in grid function
+            double value = bc.second;
+            ess_bc.SetSubVector(u_tdofs, value);
+        }
+        bcGridFunc_.SetSpace(physics_fes_);
+        bcGridFunc_.SetFromTrueDofs(ess_bc);
+
+        firstLameCoef = new ConstantCoefficient(0.5769230769);
+        secondLameCoef = new ConstantCoefficient(1.0/2.6);
+    }
+
+    ~Elasticity_Solver(){
+    }
+
+    /// Solves the forward problem.
+    void FSolve() override ;
+
+    void ASolve( Vector & rhs ) override ;
+
+    void SetLoad( VectorCoefficient * QCoef )
+    {
+      QCoef_ = QCoef;
+    }
+
+    void setMaterial( Coefficient * firstLameCoef_, Coefficient * secondLameCoef_)
+    {
+      // delete(firstLameCoef);
+      // delete(secondLameCoef);
+      firstLameCoef = firstLameCoef_;
+      secondLameCoef = secondLameCoef_;
+    }
+
+private:
+
+    // holds NBC in coefficient form
+    std::map<int, Coefficient*> ncc;
+
+    Array<int> ess_tdof_list_;
+
+    VectorCoefficient * QCoef_ = nullptr;
+
+    Array<int> bdr;
+
+    Coefficient * firstLameCoef = nullptr;
+    Coefficient * secondLameCoef = nullptr;
+};
+
+class VectorHelmholtz
+{
+public:
+    VectorHelmholtz(ParMesh* mesh_, std::vector<std::pair<int, int>> ess_bdr, real_t radius, int order_, int physics_order_)
+    {
+
+        radius_ = new ConstantCoefficient(radius);
+        pmesh=mesh_;
+        int dim=pmesh->Dimension();
+
+        pmesh->GetNodes(X0_);
+
+        fec = new H1_FECollection(order_,dim);
+        pfec = new H1_FECollection(physics_order_,dim);
+        temp_fes_ = new ParFiniteElementSpace(pmesh,pfec,dim);
+        temp_fes_scalar_ = new ParFiniteElementSpace(pmesh,pfec);
+        coord_fes_ = new ParFiniteElementSpace(pmesh,fec,dim);
+
+        // sol.SetSize(coord_fes_->GetTrueVSize()); sol=0.0;
+        rhs.SetSize(coord_fes_->GetTrueVSize()); rhs=0.0;
+        // adj.SetSize(coord_fes_->GetTrueVSize()); adj=0.0;
+
+        solgf.SetSpace(temp_fes_);
+        // adjgf.SetSpace(coord_fes_);
+
+        dQdx_ = new ParLinearForm(coord_fes_);
+        dQdu_ = new ParLinearForm(temp_fes_scalar_);
+        dQdxshape_ = new ParLinearForm(coord_fes_);
+
+        SetLinearSolver();
+
+        // store list of essential dofs
+        int maxAttribute = pmesh->bdr_attributes.Max();
+        Array<int> bdr_attr_is_ess(maxAttribute);
+        ess_tdof_list_.DeleteAll();
+        Vector ess_bc(coord_fes_->GetTrueVSize());
+        ess_bc = 0.0;
+
+        // loop over input attribute, value pairs
+        for (const auto &bc: ess_bdr)
+        {
+            int attribute = bc.first;
+            int component = bc.second;
+
+            // get dofs associated with this attribute, component pair
+            bdr_attr_is_ess = 0;
+            bdr_attr_is_ess[attribute - 1] = 1; // mfem attributes 1-indexed, arrays 0-indexed
+            Array<int> u_tdofs;
+            coord_fes_->GetEssentialTrueDofs(bdr_attr_is_ess, u_tdofs, component);
+
+            // append to global dof list
+            ess_tdof_list_.Append(u_tdofs);
+        }
+    }
+
+    VectorHelmholtz(ParMesh* mesh_, std::vector<std::pair<int, int>> ess_bdr, ProductCoefficient *radius, int order_, int physics_order_)
+    {
+        pradius_ = radius;
+        pmesh=mesh_;
+        int dim=pmesh->Dimension();
+
+        pmesh->GetNodes(X0_);
+
+        fec = new H1_FECollection(order_,dim);
+        pfec = new H1_FECollection(physics_order_,dim);
+        temp_fes_ = new ParFiniteElementSpace(pmesh,pfec, dim);
+        temp_fes_scalar_ = new ParFiniteElementSpace(pmesh,pfec);
+        coord_fes_ = new ParFiniteElementSpace(pmesh,fec,dim);
+
+        // sol.SetSize(coord_fes_->GetTrueVSize()); sol=0.0;
+        rhs.SetSize(coord_fes_->GetTrueVSize()); rhs=0.0;
+        // adj.SetSize(coord_fes_->GetTrueVSize()); adj=0.0;
+
+        solgf.SetSpace(temp_fes_);
+        // adjgf.SetSpace(coord_fes_);
+
+        dQdx_ = new ParLinearForm(coord_fes_);
+        dQdu_ = new ParLinearForm(temp_fes_scalar_);
+        dQdxshape_ = new ParLinearForm(coord_fes_);
+
+        SetLinearSolver();
+
+        // store list of essential dofs
+        int maxAttribute = pmesh->bdr_attributes.Max();
+        Array<int> bdr_attr_is_ess(maxAttribute);
+        ess_tdof_list_.DeleteAll();
+        Vector ess_bc(coord_fes_->GetTrueVSize());
+        ess_bc = 0.0;
+
+        // loop over input attribute, value pairs
+        for (const auto &bc: ess_bdr)
+        {
+            int attribute = bc.first;
+            int component = bc.second;
+
+            // get dofs associated with this attribute, component pair
+            bdr_attr_is_ess = 0;
+            bdr_attr_is_ess[attribute - 1] = 1; // mfem attributes 1-indexed, arrays 0-indexed
+            Array<int> u_tdofs;
+            coord_fes_->GetEssentialTrueDofs(bdr_attr_is_ess, u_tdofs, component);
+
+            // append to global dof list
+            ess_tdof_list_.Append(u_tdofs);
+        }
+    }
+
+    ~VectorHelmholtz(){
+        delete coord_fes_;
+        delete temp_fes_;
+        delete fec;
+
+        delete dQdx_;
+        delete radius_;
+        delete dQdu_;
+        delete dQdxshape_;
+
+        delete QGF_;
+        delete QCoef_;
+    }
+
+    /// Set the Linear Solver
+    void SetLinearSolver(double rtol=1e-8, double atol=1e-12, int miter=2000)
+    {
+        linear_rtol=rtol;
+        linear_atol=atol;
+        linear_iter=miter;
+    }
+
+    /// Solves the forward problem.
+    void FSolve( );
+
+    void ASolve( Vector & rhs, bool isGradX = true );
+
+    void setLoadGridFunction( Vector & loadGF)
+    {
+        if(coeffSet) { mfem_error("coeff already set"); }
+        GFSet = true;
+        delete QGF_;
+        delete QCoef_;
+        QGF_ = new ParGridFunction(coord_fes_);
+        *QGF_ = loadGF;
+        // QGF_->SetFromTrueDofs(loadGF);
+        QCoef_ = new VectorGridFunctionCoefficient(QGF_);
+    };
+
+    void setLoadCoeff(VectorCoefficient * loadCoeff)
+    {
+      if(coeffSet) { mfem_error("coeff already set"); }
+      coeffSet = true;
+      QCoef_ =loadCoeff; };
+
+    /// Returns the solution
+    ParGridFunction& GetSolution(){return solgf;}
+
+    /// Returns the solution vector.
+    Vector& GetSolutionVec(){return solgf;}
+    Vector GetSolutionTVec(){
+      solgf.SetTrueVector();
+      return solgf.GetTrueVector();}
+
+    /// Returns the adjoint solution vector.
+    // Vector& GetAdj(){return adj;}
+
+    ParLinearForm * GetImplicitDqDx(){ return dQdx_; };
+    Vector GetImplicitDqDxVec(){ return *dQdx_; };
+
+    ParLinearForm * GetImplicitDqDxshape(){ return dQdxshape_; };
+
+    ParLinearForm * GetImplicitDqDu(){ return dQdu_; };
+
+private:
+    ParMesh* pmesh;
+
+    Vector X0_;
+
+    //solution true vector
+    // Vector sol;
+    // Vector adj;
+    Vector rhs;
+    ParGridFunction solgf;
+    // ParGridFunction adjgf;
+    ParGridFunction bcGridFunc_;
+
+    ParLinearForm * dQdx_;
+    ParLinearForm * dQdxshape_;
+    ParLinearForm * dQdu_;
+
+    FiniteElementCollection *fec;
+    FiniteElementCollection *pfec;
+    ParFiniteElementSpace	  *temp_fes_;
+    ParFiniteElementSpace	  *coord_fes_;
+    ParFiniteElementSpace	  *temp_fes_scalar_;
+
+    //Linear solver parameters
+    double linear_rtol;
+    double linear_atol;
+    int linear_iter;
+
+    int print_level = 1;
+
+    // holds NBC in coefficient form
+    std::map<int, Coefficient*> ncc;
+
+    Array<int> ess_tdof_list_;
+
+    ParGridFunction* QGF_ = nullptr;
+    VectorCoefficient * QCoef_ = nullptr;
+
+    Coefficient * radius_;
+    ProductCoefficient *pradius_ = nullptr;
+
+    bool GFSet = false;
+    bool coeffSet = false;
+};
+
+class DiffusionSolver
+{
+private:
+   mfem::Mesh * mesh = nullptr;
+   int order = 1;
+   // diffusion coefficient
+   mfem::Coefficient * diffcf = nullptr;
+   // mass coefficient
+   mfem::Coefficient * masscf = nullptr;
+   mfem::Coefficient * rhscf = nullptr;
+   mfem::Coefficient * essbdr_cf = nullptr;
+   mfem::Coefficient * neumann_cf = nullptr;
+   mfem::VectorCoefficient * gradient_cf = nullptr;
+
+   // FEM solver
+   int dim;
+   mfem::FiniteElementCollection * fec = nullptr;
+   mfem::FiniteElementSpace * fes = nullptr;
+   mfem::Array<int> ess_bdr;
+   mfem::Array<int> neumann_bdr;
+   mfem::GridFunction * u = nullptr;
+   mfem::LinearForm * b = nullptr;
+   bool parallel;
+#ifdef MFEM_USE_MPI
+   mfem::ParMesh * pmesh = nullptr;
+   mfem::ParFiniteElementSpace * pfes = nullptr;
+#endif
+
+public:
+   DiffusionSolver() { }
+   DiffusionSolver(mfem::Mesh * mesh_, int order_, mfem::Coefficient * diffcf_,
+                   mfem::Coefficient * cf_);
+
+   void SetMesh(mfem::Mesh * mesh_)
+   {
+      mesh = mesh_;
+      parallel = false;
+#ifdef MFEM_USE_MPI
+      pmesh = dynamic_cast<mfem::ParMesh *>(mesh);
+      if (pmesh) { parallel = true; }
+#endif
+   }
+   void SetOrder(int order_) { order = order_ ; }
+   void SetDiffusionCoefficient(mfem::Coefficient * diffcf_) { diffcf = diffcf_; }
+   void SetMassCoefficient(mfem::Coefficient * masscf_) { masscf = masscf_; }
+   void SetRHSCoefficient(mfem::Coefficient * rhscf_) { rhscf = rhscf_; }
+   void SetEssentialBoundary(const mfem::Array<int> & ess_bdr_) { ess_bdr = ess_bdr_;};
+   void SetNeumannBoundary(const mfem::Array<int> & neumann_bdr_) { neumann_bdr = neumann_bdr_;};
+   void SetNeumannData(mfem::Coefficient * neumann_cf_) {neumann_cf = neumann_cf_;}
+   void SetEssBdrData(mfem::Coefficient * essbdr_cf_) {essbdr_cf = essbdr_cf_;}
+   void SetGradientData(mfem::VectorCoefficient * gradient_cf_) {gradient_cf = gradient_cf_;}
+
+   void ResetFEM();
+   void SetupFEM();
+
+   void Solve();
+   mfem::GridFunction * GetFEMSolution();
+   mfem::LinearForm * GetLinearForm() {return b;}
+#ifdef MFEM_USE_MPI
+   mfem::ParGridFunction * GetParFEMSolution();
+   mfem::ParLinearForm * GetParLinearForm()
+   {
+      if (parallel)
+      {
+         return dynamic_cast<mfem::ParLinearForm *>(b);
+      }
+      else
+      {
+         MFEM_ABORT("Wrong code path. Call GetLinearForm");
+         return nullptr;
+      }
+   }
+#endif
+
+   ~DiffusionSolver();
+
+};
+
+}
+#endif

tests/benchmarks/CMakeLists.txt

@@ -17,7 +17,23 @@ include_directories(BEFORE ${PROJECT_SOURCE_DIR})
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 
 # Include the Google Benchmark include directory
-include_directories(BEFORE ${BENCHMARK_INCLUDE_DIRS})
+if (MFEM_USE_BENCHMARK)
+    include_directories(BEFORE ${BENCHMARK_INCLUDE_DIRS})
+endif(MFEM_USE_BENCHMARK)
+
+#-------------------------------------------------------------------------------
+# Simple timer-based benchmark that intentionally does not use Google Benchmark.
+#-------------------------------------------------------------------------------
+if (MFEM_USE_MPI)
+    set(DFEM_LINEARIZED_SIMPLE_BENCH_SRCS bench_dfem_linearized_simple.cpp)
+    if (MFEM_USE_CUDA)
+       set_property(SOURCE ${DFEM_LINEARIZED_SIMPLE_BENCH_SRCS} PROPERTY LANGUAGE CUDA)
+    endif(MFEM_USE_CUDA)
+
+    mfem_add_executable(bench_dfem_linearized_simple ${DFEM_LINEARIZED_SIMPLE_BENCH_SRCS})
+    target_link_libraries(bench_dfem_linearized_simple PRIVATE mfem)
+    add_dependencies(${MFEM_ALL_BENCHMARKS_TARGET_NAME} bench_dfem_linearized_simple)
+endif(MFEM_USE_MPI)
 
 #-------------------------------------------------------------------------------
 # Function to add one benchmark from the tests/benchmarks directory.
@@ -32,7 +48,7 @@ function(add_benchmark name)
     endif(MFEM_USE_CUDA)
 
     add_executable(bench_${name} ${${NAME}_BENCH_SRCS})
-    target_link_libraries(bench_${name} mfem pthread)
+    target_link_libraries(bench_${name} PRIVATE mfem pthread)
     add_dependencies(${MFEM_ALL_BENCHMARKS_TARGET_NAME} bench_${name})
 
     add_test(NAME bench_${name}_cpu
@@ -49,10 +65,13 @@ function(add_benchmark name)
 endfunction(add_benchmark)
 
 #-------------------------------------------------------------------------------
-add_benchmark(assembly_levels)
-add_benchmark(ceed)
-add_benchmark(dg_amr)
-add_benchmark(elasticity)
-add_benchmark(tmop)
-add_benchmark(vector)
-add_benchmark(virtuals)
+if (MFEM_USE_BENCHMARK)
+    add_benchmark(assembly_levels)
+    add_benchmark(ceed)
+    add_benchmark(dfem)
+    add_benchmark(dg_amr)
+    add_benchmark(elasticity)
+    add_benchmark(tmop)
+    add_benchmark(vector)
+    add_benchmark(virtuals)
+endif(MFEM_USE_BENCHMARK)

tests/benchmarks/bench_dfem_linearized_simple.cpp

@@ -0,0 +1,342 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "fem/dfem/doperator.hpp"
+#include "fem/dfem/backends/global_qf/prelude.hpp"
+#include "fem/dfem/backends/local_qf/prelude.hpp"
+
+#include <cmath>
+#include <iomanip>
+#include <iostream>
+#include <string>
+
+using namespace mfem;
+using namespace mfem::future;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+template <int DIM>
+struct DiffusionLocalMode
+{
+   using dvecd_t = tensor<dscalar_t, DIM>;
+   using matd_t = tensor<real_t, DIM, DIM>;
+
+   MFEM_HOST_DEVICE inline auto operator()(const dvecd_t &dudxi,
+                                          const matd_t &J,
+                                          const real_t &w,
+                                          dvecd_t &dvdxi) const
+   {
+      const auto invJ = inv(J);
+      const auto invJt = transpose(invJ);
+      dvdxi = (dudxi * invJ) * invJt * det(J) * w;
+   }
+};
+
+template <int DIM>
+struct DiffusionGlobalMode
+{
+   void operator()(tensor_array<const real_t, DIM> &dudxi,
+                   tensor_array<const real_t, DIM, DIM> &J,
+                   tensor_array<const real_t> &w,
+                   tensor_array<real_t, DIM> &dvdxi) const
+   {
+      mfem::forall<UseEnzyme>(J.size(), [=] MFEM_HOST_DEVICE (int q)
+      {
+         const auto invJ = inv(J(q));
+         const auto invJt = transpose(invJ);
+         dvdxi(q) = (dudxi(q) * invJ) * invJt * det(J(q)) * w(q);
+      });
+   }
+};
+
+template <int DIM, typename QFBackend> struct DiffusionQF;
+template <int DIM> struct DiffusionQF<DIM, LocalQFBackend>
+{
+   using type = DiffusionLocalMode<DIM>;
+};
+template <int DIM> struct DiffusionQF<DIM, GlobalQFBackend>
+{
+   using type = DiffusionGlobalMode<DIM>;
+};
+
+struct Timings
+{
+   double forward = 0.0;
+   double derivative = 0.0;
+   real_t error = 0.0;
+};
+
+template <typename F>
+double TimeIt(const int iterations, MPI_Comm comm, F &&f)
+{
+   MPI_Barrier(comm);
+   MFEM_DEVICE_SYNC;
+
+   StopWatch timer;
+   timer.Start();
+   for (int i = 0; i < iterations; i++)
+   {
+      f();
+   }
+   MFEM_DEVICE_SYNC;
+   timer.Stop();
+
+   const double local_time = timer.RealTime();
+   double global_time = 0.0;
+   MPI_Allreduce(&local_time, &global_time, 1, MPI_DOUBLE, MPI_MAX, comm);
+   return global_time / iterations;
+}
+
+template <int DIM>
+Mesh MakeSimpleTensorMesh(int n)
+{
+   if constexpr (DIM == 2)
+   {
+      return Mesh::MakeCartesian2D(n, n, Element::QUADRILATERAL,
+                                   true, 1.0, 1.0);
+   }
+   else
+   {
+      return Mesh::MakeCartesian3D(n, n, n, Element::HEXAHEDRON,
+                                   1.0, 1.0, 1.0);
+   }
+}
+
+template <int DIM>
+int MeshSizeFromTargetDofs(const int order, const int target_dofs)
+{
+   const real_t dofs_1d = std::pow(static_cast<real_t>(target_dofs),
+                                   1.0 / DIM);
+   return std::max(1, static_cast<int>(std::ceil((dofs_1d - 1.0) / order)));
+}
+
+template <int DIM>
+auto GetGlobalTrueVSize(const int order, const int mesh_n)
+{
+   Mesh smesh = MakeSimpleTensorMesh<DIM>(mesh_n);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   smesh.Clear();
+
+   pmesh.EnsureNodes();
+   const int p = std::max(order, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace pfes(&pmesh, &fec);
+   return pfes.GlobalTrueVSize();
+}
+
+template <int DIM, typename QFBackend>
+Timings RunBackendCase(const int order, const int mesh_n, const int warmup,
+                       const int iterations)
+{
+   Mesh smesh = MakeSimpleTensorMesh<DIM>(mesh_n);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   smesh.Clear();
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+
+   const int p = std::max(order, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace pfes(&pmesh, &fec);
+   const IntegrationRule *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(),
+                                             2 * p);
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   static constexpr int U = 0, Coords = 1;
+   const auto in_fds = std::vector
+   {
+      FieldDescriptor{ U, &pfes },
+      FieldDescriptor{ Coords, mfes }
+   };
+   const auto out_fds = std::vector{ FieldDescriptor{ U, &pfes } };
+
+   DifferentiableOperator dop_mf(in_fds, out_fds, pmesh);
+   typename DiffusionQF<DIM, QFBackend>::type mf_apply_qf;
+   dop_mf.AddDomainIntegrator<QFBackend>(
+      mf_apply_qf,
+      Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+      Outputs<Gradient<U>> {},
+      *ir, all_domain_attr,
+      Derivatives<U> {});
+
+   Vector xtvec(pfes.GetTrueVSize()), ztvec(pfes.GetTrueVSize());
+   xtvec.Randomize(567);
+
+   Vector nodestv;
+   nodes->GetTrueDofs(nodestv);
+
+   // Move the fixed input state to device before warmup/timing so repeated
+   // applies do not pay host-to-device copies through aliased L-vectors.
+   xtvec.Read();
+   nodestv.Read();
+
+   MultiVector X{xtvec, nodestv};
+   MultiVector Z{ztvec};
+   auto ddop = dop_mf.GetDerivative(U, X, false);
+
+   Vector dztvec(ztvec.Size());
+   MultiVector DZ{dztvec};
+
+   for (int i = 0; i < warmup; i++)
+   {
+      dop_mf.Mult(X, Z);
+      ddop->Mult(X[0], DZ);
+   }
+   MFEM_DEVICE_SYNC;
+
+   // One correctness check: this diffusion action is linear in U, so the
+   // forward action and derivative action should agree for the same direction.
+   dop_mf.Mult(X, Z);
+   ddop->Mult(X[0], DZ);
+   Vector diff(ztvec);
+   diff -= dztvec;
+   const real_t local_error = diff.Normlinf();
+   real_t global_error = 0.0;
+   MPI_Allreduce(&local_error, &global_error, 1, MPI_DOUBLE, MPI_MAX,
+                 pmesh.GetComm());
+
+   Timings timings;
+   timings.error = global_error;
+   timings.forward = TimeIt(iterations, pmesh.GetComm(), [&]()
+   {
+      dop_mf.Mult(X, Z);
+   });
+
+   timings.derivative = TimeIt(iterations, pmesh.GetComm(), [&]()
+   {
+      ddop->Mult(X[0], DZ);
+   });
+
+   return timings;
+}
+
+template <int DIM>
+void RunCase(const int order, const int mesh_n, const int warmup,
+             const int iterations)
+{
+   const char *mesh_name = DIM == 2 ? "quad" : "hex";
+   const auto tdofs = GetGlobalTrueVSize<DIM>(order, mesh_n);
+   if (Mpi::WorldRank() == 0)
+   {
+      mfem::out << mesh_name << " #dofs=" << tdofs << std::endl;
+   }
+
+   const Timings global = RunBackendCase<DIM, GlobalQFBackend>(order, mesh_n,
+                                                               warmup,
+                                                               iterations);
+   const Timings local = RunBackendCase<DIM, LocalQFBackend>(order, mesh_n,
+                                                             warmup,
+                                                             iterations);
+
+   const real_t error = std::max(global.error, local.error);
+   if (Mpi::WorldRank() == 0)
+   {
+      mfem::out << mesh_name << " Scalar Action Linearized timings (seconds):\n"
+                << "  forward global=" << global.forward
+                << ", derivative global=" << global.derivative
+                << ", overhead=" << global.derivative / global.forward << "x\n"
+                << "  forward local =" << local.forward
+                << ", derivative local =" << local.derivative
+                << ", overhead=" << local.derivative / local.forward << "x\n"
+                << "  forward local/global=" << local.forward / global.forward
+                << "x\n"
+                << "  derivative local/global="
+                << local.derivative / global.derivative << "x\n"
+                << "  linf_error=" << error << "\n";
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init(argc, argv);
+   Hypre::Init();
+
+   int order = 3;
+   int quad_mesh_n = 32;
+   int hex_mesh_n = 8;
+   int target_dofs = 0;
+   int warmup = 3;
+   int iterations = 25;
+   const char *device_config = "cpu";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order", "Finite element order.");
+   args.AddOption(&quad_mesh_n, "-nq", "--quad-mesh-size",
+                  "Number of quad elements per direction.");
+   args.AddOption(&hex_mesh_n, "-nh", "--hex-mesh-size",
+                  "Number of hex elements per direction.");
+   args.AddOption(&target_dofs, "-nd", "--num-dofs",
+                  "Target number of dofs. If positive, overrides -nq/-nh and "
+                  "computes element counts from the order.");
+   args.AddOption(&warmup, "-w", "--warmup", "Number of warmup iterations.");
+   args.AddOption(&iterations, "-i", "--iterations",
+                  "Number of timed iterations.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "MFEM device configuration string.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::WorldRank() == 0) { args.PrintUsage(mfem::out); }
+      return 1;
+   }
+
+   Device device(device_config);
+   if (Mpi::WorldRank() == 0)
+   {
+      args.PrintOptions(mfem::out);
+      device.Print(mfem::out);
+      mfem::out << std::scientific << std::setprecision(6)
+                << "\nForward DifferentiableOperator::Mult vs "
+                << "GetDerivative(...)->Mult\n";
+   }
+
+   if (target_dofs > 0)
+   {
+      quad_mesh_n = MeshSizeFromTargetDofs<2>(order, target_dofs);
+      hex_mesh_n = MeshSizeFromTargetDofs<3>(order, target_dofs);
+      if (Mpi::WorldRank() == 0)
+      {
+         mfem::out << "target #dofs=" << target_dofs
+                   << " -> quad n=" << quad_mesh_n
+                   << ", hex n=" << hex_mesh_n << std::endl;
+      }
+   }
+
+   RunCase<2>(order, quad_mesh_n, warmup, iterations);
+   RunCase<3>(order, hex_mesh_n, warmup, iterations);
+
+   return 0;
+}
+
+#else
+
+int main(int, char *[])
+{
+   mfem::out << "This benchmark requires MFEM_USE_MPI=YES.\n";
+   return MFEM_SKIP_RETURN_VALUE;
+}
+
+#endif // MFEM_USE_MPI

tests/benchmarks/makefile

@@ -20,8 +20,8 @@ CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_TESTS = bench_assembly_levels bench_ceed bench_dg_amr bench_elasticity \
-            bench_tmop bench_vector bench_virtuals
+SEQ_TESTS = bench_assembly_levels bench_ceed bench_dfem bench_dg_amr \
+			bench_elasticity bench_tmop bench_vector bench_virtuals
 PAR_TESTS = 
 ifeq ($(MFEM_USE_MPI),NO)
    TESTS = $(SEQ_TESTS)

tests/unit/CMakeLists.txt

@@ -37,8 +37,21 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR})
 set(UNIT_TESTS_SRCS
   dfem/test_diffusion.cpp
   dfem/test_divergence.cpp
+  dfem/test_functional.cpp  
+  dfem/test_dfem_scratch.cpp
+  enzyme/test_enzyme_scratch.cpp
+  enzyme/test_enzyme_scratch_multikernel.cpp
+  enzyme/test_enzyme_reverse_tape.cpp
+  dfem/test_hyperelasticity.cpp
+  dfem/test_functional.cpp
+  dfem/test_functional_gradient.cpp
+  dfem/test_jvp_vjp.cpp
   dfem/test_lvector_interface.cpp
   dfem/test_mass.cpp
+  dfem/test_multiple_inputs.cpp
+  dfem/test_multiple_outputs.cpp
+  dfem/test_second_derivative.cpp
+  dfem/test_tmop.cpp
   general/test_array.cpp
   general/test_scan.cpp
   general/test_arrays_by_name.cpp
@@ -180,11 +193,11 @@ set(UNIT_TESTS_SRCS
 # SERIAL CPU TESTS: unit_tests
 #-----------------------------------------------------------
 if (MFEM_USE_CUDA)
-    set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
+  set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
                 PROPERTY LANGUAGE CUDA)
 endif()
 if (MFEM_USE_HIP)
-    set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
+  set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
                PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 
@@ -208,7 +221,7 @@ add_dependencies(unit_tests copy_miniapps_nurbs_data)
 #   make unit_tests
 #   ctest -R unit_tests [-V]
 if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-    add_test(NAME unit_tests COMMAND unit_tests)
+  add_test(NAME unit_tests COMMAND unit_tests)
 endif()
 
 #-----------------------------------------------------------
@@ -216,15 +229,15 @@ endif()
 #-----------------------------------------------------------
 # Create CUDA executable and test
 if (MFEM_USE_CUDA)
-    # gpu_unit_tests
-    set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
-    set_property(SOURCE ${GPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-    mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS})
-    target_link_libraries(gpu_unit_tests unit_tests_srcs)
-    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
-    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-        add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
-    endif()
+  # gpu_unit_tests
+  set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
+  set_property(SOURCE ${GPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+  mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS})
+  target_link_libraries(gpu_unit_tests unit_tests_srcs)
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
+  if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+    add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
+  endif()
 endif()
 
 #-----------------------------------------------------------
@@ -232,14 +245,14 @@ endif()
 #-----------------------------------------------------------
 # Create HIP 'gpu_unit_tests' executable and test
 if (MFEM_USE_HIP)
-    # gpu_unit_tests
-    set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
-    mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS})
-    target_link_libraries(gpu_unit_tests unit_tests_srcs)
-    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
-    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-        add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
-    endif()
+  # gpu_unit_tests
+  set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
+  mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS})
+  target_link_libraries(gpu_unit_tests unit_tests_srcs)
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
+  if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+    add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
+  endif()
 endif()
 
 #-----------------------------------------------------------
@@ -251,61 +264,61 @@ endif()
 # All device unit tests are built into a separate executable, in order to be
 # able to change the device.
 function(add_serial_miniapp_test name test_uvm)
-    string(TOUPPER ${name} NAME)
+  string(TOUPPER ${name} NAME)
 
-    set(${NAME}_TESTS_SRCS miniapps/test_${name}.cpp)
-    if (MFEM_USE_CUDA)
-        set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-    endif(MFEM_USE_CUDA)
-    if (MFEM_USE_HIP)
-       set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
-    endif(MFEM_USE_HIP)
+  set(${NAME}_TESTS_SRCS miniapps/test_${name}.cpp)
+  if (MFEM_USE_CUDA)
+    set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+  endif(MFEM_USE_CUDA)
+  if (MFEM_USE_HIP)
+    set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
+  endif(MFEM_USE_HIP)
 
-    mfem_add_executable(${name}_tests_cpu ${${NAME}_TESTS_SRCS})
-    add_dependencies(${name}_tests_cpu copy_miniapps_meshing_data)
-    add_dependencies(${name}_tests_cpu ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
-    target_compile_definitions(${name}_tests_cpu PUBLIC MFEM_${NAME}_DEVICE="cpu")
-    target_link_libraries(${name}_tests_cpu mfem)
-    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_cpu)
+  mfem_add_executable(${name}_tests_cpu ${${NAME}_TESTS_SRCS})
+  add_dependencies(${name}_tests_cpu copy_miniapps_meshing_data)
+  add_dependencies(${name}_tests_cpu ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
+  target_compile_definitions(${name}_tests_cpu PUBLIC MFEM_${NAME}_DEVICE="cpu")
+  target_link_libraries(${name}_tests_cpu mfem)
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_cpu)
+  if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+    add_test(NAME ${name}_tests_cpu COMMAND ${name}_tests_cpu)
+  endif()
+
+  mfem_add_executable(${name}_tests_debug ${${NAME}_TESTS_SRCS})
+  add_dependencies(${name}_tests_debug copy_miniapps_meshing_data)
+  add_dependencies(${name}_tests_debug ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
+  target_compile_definitions(${name}_tests_debug PUBLIC MFEM_${NAME}_DEVICE="debug")
+  target_link_libraries(${name}_tests_debug mfem)
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_debug)
+  if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+    add_test(NAME ${name}_tests_debug COMMAND ${name}_tests_debug)
+  endif()
+
+  if (MFEM_USE_CUDA OR MFEM_USE_HIP)
+    mfem_add_executable(${name}_tests_gpu ${${NAME}_TESTS_SRCS})
+    add_dependencies(${name}_tests_gpu copy_miniapps_meshing_data)
+    add_dependencies(${name}_tests_gpu ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
+    target_compile_definitions(${name}_tests_gpu PUBLIC MFEM_${NAME}_DEVICE="gpu")
+    target_link_libraries(${name}_tests_gpu mfem)
+    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu)
     if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-        add_test(NAME ${name}_tests_cpu COMMAND ${name}_tests_cpu)
+      add_test(NAME ${name}_tests_gpu COMMAND ${name}_tests_gpu)
     endif()
 
-    mfem_add_executable(${name}_tests_debug ${${NAME}_TESTS_SRCS})
-    add_dependencies(${name}_tests_debug copy_miniapps_meshing_data)
-    add_dependencies(${name}_tests_debug ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
-    target_compile_definitions(${name}_tests_debug PUBLIC MFEM_${NAME}_DEVICE="debug")
-    target_link_libraries(${name}_tests_debug mfem)
-    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_debug)
-    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-        add_test(NAME ${name}_tests_debug COMMAND ${name}_tests_debug)
-    endif()
-
-    if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-        mfem_add_executable(${name}_tests_gpu ${${NAME}_TESTS_SRCS})
-        add_dependencies(${name}_tests_gpu copy_miniapps_meshing_data)
-        add_dependencies(${name}_tests_gpu ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
-        target_compile_definitions(${name}_tests_gpu PUBLIC MFEM_${NAME}_DEVICE="gpu")
-        target_link_libraries(${name}_tests_gpu mfem)
-        add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu)
-        if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-            add_test(NAME ${name}_tests_gpu COMMAND ${name}_tests_gpu)
-        endif()
-
-        if (test_uvm)
-            mfem_add_executable(${name}_tests_gpu_uvm ${${NAME}_TESTS_SRCS})
-            add_dependencies(${name}_tests_gpu_uvm copy_miniapps_meshing_data)
-            add_dependencies(${name}_tests_gpu_uvm ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
-            target_compile_definitions(${name}_tests_gpu_uvm PUBLIC
+    if (test_uvm)
+      mfem_add_executable(${name}_tests_gpu_uvm ${${NAME}_TESTS_SRCS})
+      add_dependencies(${name}_tests_gpu_uvm copy_miniapps_meshing_data)
+      add_dependencies(${name}_tests_gpu_uvm ${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME})
+      target_compile_definitions(${name}_tests_gpu_uvm PUBLIC
                MFEM_${NAME}_DEVICE="gpu:uvm")
-            target_link_libraries(${name}_tests_gpu_uvm mfem)
-            add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME}
+      target_link_libraries(${name}_tests_gpu_uvm mfem)
+      add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME}
                ${name}_tests_gpu_uvm)
-            if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-                add_test(NAME ${name}_tests_gpu_uvm COMMAND ${name}_tests_gpu_uvm)
-            endif()
-        endif()
+      if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+        add_test(NAME ${name}_tests_gpu_uvm COMMAND ${name}_tests_gpu_uvm)
+      endif()
     endif()
+  endif()
 endfunction(add_serial_miniapp_test)
 
 add_serial_miniapp_test(sedov ON)  # UVM ON
@@ -318,25 +331,25 @@ add_serial_miniapp_test(tmop_pa OFF)  # UVM OFF
 #-----------------------------------------------------------
 # Add 'ceed_tests' executable and test; add extra tests 'ceed_test_*'
 if (MFEM_USE_CEED)
-    set(CEED_TESTS_SRCS
+  set(CEED_TESTS_SRCS
       ceed/test_ceed.cpp
       ceed/test_ceed_main.cpp)
-    if (MFEM_USE_CUDA)
-        set_property(SOURCE ${CEED_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-    endif(MFEM_USE_CUDA)
-    mfem_add_executable(ceed_tests ${CEED_TESTS_SRCS})
-    target_link_libraries(ceed_tests mfem)
-    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ceed_tests)
-    # Add CEED tests
-    add_test(NAME ceed_tests COMMAND ceed_tests)
-    if (MFEM_USE_CUDA)
-        add_test(NAME ceed_tests_cuda_ref
+  if (MFEM_USE_CUDA)
+    set_property(SOURCE ${CEED_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+  endif(MFEM_USE_CUDA)
+  mfem_add_executable(ceed_tests ${CEED_TESTS_SRCS})
+  target_link_libraries(ceed_tests mfem)
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ceed_tests)
+  # Add CEED tests
+  add_test(NAME ceed_tests COMMAND ceed_tests)
+  if (MFEM_USE_CUDA)
+    add_test(NAME ceed_tests_cuda_ref
                COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/ref)
-        add_test(NAME ceed_tests_cuda_shared
+    add_test(NAME ceed_tests_cuda_shared
                COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/shared)
-        add_test(NAME ceed_tests_cuda_gen
+    add_test(NAME ceed_tests_cuda_gen
                COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/gen)
-    endif()
+  endif()
 endif()
 
 #-----------------------------------------------------------
@@ -344,52 +357,52 @@ endif()
 #-----------------------------------------------------------
 # Define executables and tests
 if (MFEM_USE_MPI)
-    # punit_tests
-    if (MFEM_USE_CUDA)
-        set_property(SOURCE punit_test_main.cpp PROPERTY LANGUAGE CUDA)
-    endif()
-    mfem_add_executable(punit_tests punit_test_main.cpp)
-    target_link_libraries(punit_tests unit_tests_srcs)
-    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} punit_tests)
-    foreach(np 1 ${MFEM_MPI_NP})
-        if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-            add_test(NAME punit_tests_np=${np}
+  # punit_tests
+  if (MFEM_USE_CUDA)
+    set_property(SOURCE punit_test_main.cpp PROPERTY LANGUAGE CUDA)
+  endif()
+  mfem_add_executable(punit_tests punit_test_main.cpp)
+  target_link_libraries(punit_tests unit_tests_srcs)
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} punit_tests)
+  foreach(np 1 ${MFEM_MPI_NP})
+    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+      add_test(NAME punit_tests_np=${np}
                   COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
                   ${MPIEXEC_PREFLAGS} $<TARGET_FILE:punit_tests>
                   ${MPIEXEC_POSTFLAGS})
-        endif()
+    endif()
+  endforeach()
+  if (MFEM_USE_CUDA)
+    # pgpu_unit_tests
+    set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
+    set_property(SOURCE ${PGPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+    mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS})
+    target_link_libraries(pgpu_unit_tests unit_tests_srcs)
+    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
+    foreach(np 1 ${MFEM_MPI_NP})
+      if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+        add_test(NAME pgpu_unit_tests_np=${np}
+                      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
+                      ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pgpu_unit_tests>
+                      ${MPIEXEC_POSTFLAGS})
+      endif()
     endforeach()
-    if (MFEM_USE_CUDA)
-        # pgpu_unit_tests
-        set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
-        set_property(SOURCE ${PGPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-        mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS})
-        target_link_libraries(pgpu_unit_tests unit_tests_srcs)
-        add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
-        foreach(np 1 ${MFEM_MPI_NP})
-            if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-                add_test(NAME pgpu_unit_tests_np=${np}
+  endif()
+  if (MFEM_USE_HIP)
+    # pgpu_unit_tests
+    set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
+    mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS})
+    target_link_libraries(pgpu_unit_tests unit_tests_srcs)
+    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
+    foreach(np 1 ${MFEM_MPI_NP})
+      if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+        add_test(NAME pgpu_unit_tests_np=${np}
                       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
                       ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pgpu_unit_tests>
                       ${MPIEXEC_POSTFLAGS})
-            endif()
-        endforeach()
-    endif()
-    if (MFEM_USE_HIP)
-        # pgpu_unit_tests
-        set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
-        mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS})
-        target_link_libraries(pgpu_unit_tests unit_tests_srcs)
-        add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
-        foreach(np 1 ${MFEM_MPI_NP})
-            if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-                add_test(NAME pgpu_unit_tests_np=${np}
-                      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
-                      ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pgpu_unit_tests>
-                      ${MPIEXEC_POSTFLAGS})
-            endif()
-        endforeach()
-    endif()
+      endif()
+    endforeach()
+  endif()
 endif(MFEM_USE_MPI)
 
 #-----------------------------------------------------------
@@ -399,72 +412,72 @@ endif(MFEM_USE_MPI)
 #-----------------------------------------------------------
 # Function to add one MPI executable for a test.
 function(add_mpi_executable_test name dev)
-    string(TOUPPER ${name} NAME)
-    string(REPLACE "_" ":" DEV ${dev})
-    mfem_add_executable(p${name}_tests_${dev} ${PAR_${NAME}_TESTS_SRCS})
-    target_compile_definitions(p${name}_tests_${dev} PUBLIC MFEM_${NAME}_MPI=1)
-    target_compile_definitions(p${name}_tests_${dev} PUBLIC MFEM_${NAME}_DEVICE="${DEV}")
-    target_link_libraries(p${name}_tests_${dev} mfem)
-    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} p${name}_tests_${dev})
+  string(TOUPPER ${name} NAME)
+  string(REPLACE "_" ":" DEV ${dev})
+  mfem_add_executable(p${name}_tests_${dev} ${PAR_${NAME}_TESTS_SRCS})
+  target_compile_definitions(p${name}_tests_${dev} PUBLIC MFEM_${NAME}_MPI=1)
+  target_compile_definitions(p${name}_tests_${dev} PUBLIC MFEM_${NAME}_DEVICE="${DEV}")
+  target_link_libraries(p${name}_tests_${dev} mfem)
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} p${name}_tests_${dev})
 endfunction(add_mpi_executable_test)
 
 # Function to add one test from the tests/unit/miniapp directory.
 function(add_parallel_miniapp_test name HYPRE_MM)
-    string(TOUPPER ${name} NAME)
+  string(TOUPPER ${name} NAME)
 
-    function(add_mpi_unit_test DEV NP)
-        set(test_name p${name}_tests_${DEV})
-        if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-            add_test(NAME ${test_name}_np=${NP}
+  function(add_mpi_unit_test DEV NP)
+    set(test_name p${name}_tests_${DEV})
+    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+      add_test(NAME ${test_name}_np=${NP}
                 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${NP}
                 ${MPIEXEC_PREFLAGS} $<TARGET_FILE:${test_name}>
                 ${MPIEXEC_POSTFLAGS})
-        endif()
-    endfunction()
-
-    set(PAR_${NAME}_TESTS_SRCS miniapps/test_${name}.cpp)
-    if (MFEM_USE_CUDA)
-        set_property(SOURCE ${PAR_${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
     endif()
+  endfunction()
 
-    set(backends cpu)
-    if (HYPRE_MM)
-        # psedov_tests_debug_* will return MFEM_SKIP_RETURN_VALUE when all of
-        # the following conditions are met:
-        # * MFEM_SEDOV_MPI is defined (true here)
-        # * MFEM_DEBUG is defined
-        # * MFEM_SEDOV_DEVICE is "debug" (the case added here)
-        # * HypreUsingGPU() is true; this is the same as: HYPRE_USING_GPU is
-        #   defined and MFEM_HYPRE_VERSION < 23100 (if the version is >= 23100,
-        #   the code will switch to HYPRE running on CPU).
-        # We check these conditions here to skip the "debug" backend and avoid
-        # the ctest failure.
-        if (NOT ((${name} STREQUAL "sedov") AND MFEM_DEBUG AND
+  set(PAR_${NAME}_TESTS_SRCS miniapps/test_${name}.cpp)
+  if (MFEM_USE_CUDA)
+    set_property(SOURCE ${PAR_${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+  endif()
+
+  set(backends cpu)
+  if (HYPRE_MM)
+    # psedov_tests_debug_* will return MFEM_SKIP_RETURN_VALUE when all of
+    # the following conditions are met:
+    # * MFEM_SEDOV_MPI is defined (true here)
+    # * MFEM_DEBUG is defined
+    # * MFEM_SEDOV_DEVICE is "debug" (the case added here)
+    # * HypreUsingGPU() is true; this is the same as: HYPRE_USING_GPU is
+    #   defined and MFEM_HYPRE_VERSION < 23100 (if the version is >= 23100,
+    #   the code will switch to HYPRE running on CPU).
+    # We check these conditions here to skip the "debug" backend and avoid
+    # the ctest failure.
+    if (NOT ((${name} STREQUAL "sedov") AND MFEM_DEBUG AND
                  (HYPRE_USING_CUDA OR HYPRE_USING_HIP) AND
                  (${MFEM_HYPRE_VERSION} LESS "23100")))
-            list(APPEND backends debug)
-        endif()
+      list(APPEND backends debug)
     endif()
-    if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-        list(APPEND backends gpu)
-        if (HYPRE_MM)
-            list(APPEND backends gpu_uvm)
-        endif()
+  endif()
+  if (MFEM_USE_CUDA OR MFEM_USE_HIP)
+    list(APPEND backends gpu)
+    if (HYPRE_MM)
+      list(APPEND backends gpu_uvm)
     endif()
+  endif()
 
-    set(MPI_NPS 1 ${MFEM_MPI_NP})
-    foreach(dev ${backends})
-        add_mpi_executable_test(${name} ${dev})
-        foreach(np ${MPI_NPS})
-            add_mpi_unit_test(${dev} ${np})
-        endforeach()
+  set(MPI_NPS 1 ${MFEM_MPI_NP})
+  foreach(dev ${backends})
+    add_mpi_executable_test(${name} ${dev})
+    foreach(np ${MPI_NPS})
+      add_mpi_unit_test(${dev} ${np})
     endforeach()
+  endforeach()
 endfunction(add_parallel_miniapp_test)
 
 # Additional MPI unit tests
 if (MFEM_USE_MPI)
-    add_parallel_miniapp_test(sedov TRUE)
-    add_parallel_miniapp_test(tmop_pa FALSE)
+  add_parallel_miniapp_test(sedov TRUE)
+  add_parallel_miniapp_test(tmop_pa FALSE)
 endif(MFEM_USE_MPI)
 
 #-----------------------------------------------------------
@@ -473,10 +486,10 @@ endif(MFEM_USE_MPI)
 #-----------------------------------------------------------
 set(DEBUG_DEVICE_SRCS miniapps/test_debug_device.cpp)
 if (MFEM_USE_CUDA)
-    set_property(SOURCE ${DEBUG_DEVICE_SRCS} PROPERTY LANGUAGE CUDA)
+  set_property(SOURCE ${DEBUG_DEVICE_SRCS} PROPERTY LANGUAGE CUDA)
 endif()
 if (MFEM_USE_HIP)
-    set_property(SOURCE ${DEBUG_DEVICE_SRCS}
+  set_property(SOURCE ${DEBUG_DEVICE_SRCS}
                 PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 mfem_add_executable(debug_device_tests ${DEBUG_DEVICE_SRCS})

tests/unit/dfem/test_dfem_scratch.cpp

@@ -0,0 +1,660 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+
+// Multi-kernel scratch regression for the weak residual
+//
+//   F(u)_i = int_Omega phi_i c u^3 dx,
+//
+// evaluated as the scratch chain s = u, s = s*u, y = c*s*u.  The directional
+// derivative is
+//
+//   DF(u)[du]_i = int_Omega phi_i 3 c u^2 du dx.
+//
+// If the tangent stored in the qfunction scratch shadow is lost between the
+// split scratch updates, the final product only sees the direct derivative of
+// the last factor and produces int_Omega phi_i c u^2 du dx instead.  This test
+// checks both the direct DerivativeAction path and the cached
+// DerivativeSetup+DerivativeApply path against the factor-of-3 result.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Test for the dFEM global qfunction with split computation and scratch space.
+
+#include "../unit_tests.hpp"
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/global_qf/prelude.hpp"
+
+#include "../../../linalg/tensor_arrays.hpp"
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::future;
+
+using dscalar_t = real_t;
+
+///<--- Q-functions
+constexpr int U = 1;
+constexpr int Y = 2;
+constexpr int COEF = 3;
+constexpr int COORDINATES = 4;
+
+// Global qf with splitting and scratch space.
+// The user only writes operator(); the shared base handles scratch setup.
+struct CubicQFWithScratch : QFWithScratchType
+{
+   void operator()(tensor_array<const dscalar_t> &x,
+                   tensor_array<const dscalar_t> &coef,
+                   tensor_array<const real_t, 2, 2> &J,
+                   tensor_array<const real_t> &w,
+                   tensor_array<dscalar_t> &y) const
+   {
+      const int NQ = nq;
+      MFEM_ASSERT(NQ == static_cast<int>(x.size()),
+                  "unexpected number of quadrature points");
+
+      auto scratch_q = make_tensor_array<>(scratch[0], NQ);
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         scratch_q(q) = x(q);
+      }
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         scratch_q(q) = scratch_q(q) * x(q);
+      }
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         y(q) = coef(q) * scratch_q(q) * x(q) * det(J(q)) * w(q);
+      }
+   }
+};
+
+template <int DIM>
+struct CubicQFWithScratchMultipleSizes : QFWithScratchType
+{
+   void operator()(tensor_array<const dscalar_t> &x,
+                   tensor_array<const dscalar_t> &coef,
+                   tensor_array<const real_t, 2, 2> &J,
+                   tensor_array<const real_t> &w,
+                   tensor_array<dscalar_t> &y) const
+   {
+      const int NQ = nq;
+      MFEM_ASSERT(NQ == static_cast<int>(x.size()),
+                  "unexpected number of quadrature points");
+
+      auto scratch_scalar = make_tensor_array<>(scratch[0], NQ);
+      auto scratch_vector = make_tensor_array<DIM>(scratch[1], NQ);
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         scratch_vector(q)(0) = x(q);
+         scratch_vector(q)(1) = x(q) * x(q);
+      }
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         scratch_scalar(q) = scratch_vector(q)(0) * scratch_vector(q)(1);
+      }
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         y(q) = coef(q) * scratch_scalar(q) * det(J(q)) * w(q);
+      }
+   }
+};
+
+struct CubicQFWithGlobalScratch : QFWithGlobalScratchType
+{
+   void operator()(tensor_array<const dscalar_t> &x,
+                   tensor_array<const dscalar_t> &coef,
+                   tensor_array<const real_t, 2, 2> &J,
+                   tensor_array<const real_t> &w,
+                   tensor_array<dscalar_t> &y) const
+   {
+      const int NQ = nq;
+      MFEM_ASSERT(NQ == static_cast<int>(x.size()),
+                  "unexpected number of quadrature points");
+
+      // Unpack the scratch vectors from the scratch bank
+      auto scratch_q = make_tensor_array<>(GetScratchPointer(0), NQ);
+
+      // Unpack global scratch
+      auto &has_scale = GetGlobalScratch<0>();
+      const auto scale = GetGlobalScratch<1>();
+      auto &global_vector = GetGlobalScratch<2>();
+
+      has_scale = global_vector.Size() >
+                  0;  // If the global vector is non-empty, we will use it to scale the output
+      if (has_scale)
+      {
+         global_vector(0) = 1.0;
+      }
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         scratch_q(q) = x(q) * x(q);
+      }
+
+      for (int q = 0; q < NQ; ++q)
+      {
+         const real_t global_scale = has_scale ? scale * global_vector(0) : 0.0;
+         y(q) = global_scale * coef(q) * scratch_q(q) * x(q) * det(J(q)) * w(q);
+      }
+   }
+};
+
+///<--- Utils
+
+/// @param fes
+/// @param x
+void FillInput(ParFiniteElementSpace &fes, Coefficient &input_coeff, Vector &x)
+{
+   GridFunction x_gf(&fes);
+   x_gf.ProjectCoefficient(input_coeff);
+   x_gf.GetTrueDofs(x);
+}
+
+void FillQData(FiniteElementSpace &fes, const IntegrationRule &ir,
+               Coefficient &coeff_fc, QuadratureFunction &coef)
+{
+   QuadratureSpace qspace(*fes.GetMesh(), ir);
+   QuadratureFunction coef_qf(qspace);
+   coeff_fc.Project(coef_qf);
+   coef = coef_qf;
+}
+
+void CheckResults(ParFiniteElementSpace &fes, const IntegrationRule &ir,
+                  Vector &y, Vector &dy)
+{
+   FunctionCoefficient expected_coeff([](const Vector &p)
+   {
+      const real_t input = 1.0 + p(0) + 0.25 * (p.Size() > 1 ? p(1) : 0.0);
+      const real_t coeff = 0.5 + p(0) + 0.125 * (p.Size() > 1 ? p(1) : 0.0);
+      return coeff * input * input * input;
+   });
+
+   ParLinearForm expected_lf(&fes);
+   expected_lf.AddDomainIntegrator(new DomainLFIntegrator(expected_coeff, &ir));
+   expected_lf.Assemble();
+
+   Vector expected_y(fes.GetTrueVSize());
+   fes.GetProlongationMatrix()->MultTranspose(expected_lf, expected_y);
+
+   y -= expected_y;
+   const real_t local_err = y.Normlinf();
+   real_t global_err = 0.0;
+   MPI_Allreduce(&local_err, &global_err, 1, MPITypeMap<real_t>::mpi_type,
+                 MPI_MAX, MPI_COMM_WORLD);
+
+   FunctionCoefficient expected_deriv_coeff([](const Vector &p)
+   {
+      const real_t input = 1.0 + p(0) + 0.25 * (p.Size() > 1 ? p(1) : 0.0);
+      const real_t coeff = 0.5 + p(0) + 0.125 * (p.Size() > 1 ? p(1) : 0.0);
+      return 3.0 * coeff * input * input;
+   });
+
+   ParLinearForm expected_deriv_lf(&fes);
+   expected_deriv_lf.AddDomainIntegrator(new DomainLFIntegrator(
+                                            expected_deriv_coeff, &ir));
+   expected_deriv_lf.Assemble();
+
+   Vector expected_dy(fes.GetTrueVSize());
+   fes.GetProlongationMatrix()->MultTranspose(expected_deriv_lf, expected_dy);
+
+   dy -= expected_dy;
+   const real_t local_deriv_err = dy.Normlinf();
+   real_t global_deriv_err = 0.0;
+   MPI_Allreduce(&local_deriv_err, &global_deriv_err, 1,
+                 MPITypeMap<real_t>::mpi_type,
+                 MPI_MAX, MPI_COMM_WORLD);
+
+   if (verbose_tests && Mpi::Root())
+   {
+      mfem::out << "Primal output max error: " << global_err << endl;
+      mfem::out << "Derivative output max error: "
+                << global_deriv_err << endl;
+   }
+
+   REQUIRE(global_err == MFEM_Approx(0.0));
+   REQUIRE(global_deriv_err == MFEM_Approx(0.0));
+}
+
+void CheckScratchResults(ParMesh &pmesh, const IntegrationRule &ir,
+                         const Vector &scratch, const Vector &scratch_d)
+{
+   REQUIRE(scratch.Size() == scratch_d.Size());
+   REQUIRE(scratch.Size() == pmesh.GetNE() * ir.GetNPoints());
+
+   const real_t *scratch_h = scratch.HostRead();
+   const real_t *scratch_d_h = scratch_d.HostRead();
+
+   real_t local_scratch_err = 0.0;
+   real_t local_scratch_d_err = 0.0;
+   for (int e = 0; e < pmesh.GetNE(); e++)
+   {
+      ElementTransformation *T = pmesh.GetElementTransformation(e);
+      for (int q = 0; q < ir.GetNPoints(); q++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(q);
+         T->SetIntPoint(&ip);
+         Vector p;
+         T->Transform(ip, p);
+         const real_t u_q = 1.0 + p(0) + 0.25 * (p.Size() > 1 ? p(1) : 0.0);
+         const int idx = q + ir.GetNPoints() * e;
+         local_scratch_err = std::max(local_scratch_err,
+                                      std::abs(scratch_h[idx] - u_q * u_q));
+         local_scratch_d_err = std::max(local_scratch_d_err,
+                                        std::abs(scratch_d_h[idx] - 2.0 * u_q));
+      }
+   }
+
+   real_t global_scratch_err = 0.0;
+   real_t global_scratch_d_err = 0.0;
+   MPI_Allreduce(&local_scratch_err, &global_scratch_err, 1,
+                 MPITypeMap<real_t>::mpi_type, MPI_MAX, MPI_COMM_WORLD);
+   MPI_Allreduce(&local_scratch_d_err, &global_scratch_d_err, 1,
+                 MPITypeMap<real_t>::mpi_type, MPI_MAX, MPI_COMM_WORLD);
+
+   if (verbose_tests && Mpi::Root())
+   {
+      mfem::out << "Scratch max error: " << global_scratch_err << endl;
+      mfem::out << "Scratch derivative max error: "
+                << global_scratch_d_err << endl;
+   }
+
+   REQUIRE(global_scratch_err == MFEM_Approx(0.0));
+   REQUIRE(global_scratch_d_err == MFEM_Approx(0.0));
+}
+
+///<--- Test
+TEST_CASE("dFEM Scratch scalar", "[Parallel][dFEM][Scratch-Scalar]")
+{
+   int order = 2;
+   int ref_levels = 1;
+
+   ///<--- Mesh and finite element space setup
+   Mesh mesh = Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL, true, 1.0,
+                                     1.0);
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   pmesh.EnsureNodes();
+   H1_FECollection fec(order, pmesh.Dimension());
+   ParFiniteElementSpace fes(&pmesh, &fec);
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *nodes_fes = nodes->ParFESpace();
+   Vector nodes_tvec;
+   nodes->GetTrueDofs(nodes_tvec);
+
+   ///<--- dFEM setup
+   const IntegrationRule &ir = IntRules.Get(Geometry::SQUARE, 2 * order + 1);
+   QuadratureSpace qspace(pmesh, ir);
+   VectorQuadratureSpace coef_qspace(qspace, 1);
+   QuadratureFunction coef(coef_qspace);
+   coef.UseDevice(true);
+   FunctionCoefficient coeff_fc([](const Vector &p)
+   { return 0.5 + p(0) + 0.125 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillQData(fes, ir, coeff_fc, coef);
+
+   Array<int> all_domain_attr(pmesh.attributes.Max());
+   all_domain_attr = 1;
+
+   const std::vector<FieldDescriptor> inputs
+   {
+      {U, &fes},
+      {COEF, &coef_qspace},
+      {COORDINATES, nodes_fes}};
+   const std::vector<FieldDescriptor> outputs
+   {
+      {Y, &fes}};
+   DifferentiableOperator dop(inputs, outputs, pmesh);
+
+   // Define the cubic qfunction with scratch space
+   // Requesting one scalar scratch vector
+   // Equivalent to
+   // cubic_qf.SetScratch(pmesh.GetNE() * ir.GetNPoints(), 1, 1);
+
+   CubicQFWithScratch cubic_qf;
+   cubic_qf.SetScratch(pmesh.GetNE() * ir.GetNPoints(), {1});
+   dop.AddDomainIntegrator<GlobalQFBackend>(
+      cubic_qf,
+      Inputs<Value<U>, Identity<COEF>, Gradient<COORDINATES>, Weight> {},
+      Outputs<Value<Y>> {},
+      ir, all_domain_attr,
+      Derivatives<U> {});
+
+
+   Vector x(fes.GetTrueVSize()), y(fes.GetTrueVSize()), dx(fes.GetTrueVSize()),
+          dy(fes.GetTrueVSize());
+   x.UseDevice(true);
+   y.UseDevice(true);
+   dx.UseDevice(true);
+   dy.UseDevice(true);
+   FunctionCoefficient input_coeff([](const Vector &p)
+   { return 1.0 + p(0) + 0.25 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillInput(fes, input_coeff, x);
+   ConstantCoefficient direction_coeff(1.0);
+   FillInput(fes, direction_coeff, dx);
+   y = 0.0;
+   dy = 0.0;
+
+   ///<--- Apply the operator
+   MultiVector X{x, coef, nodes_tvec};
+   MultiVector Y{y};
+   dop.Mult(X, Y);
+
+   //<--- Apply derivative operator
+   auto dop_deriv = dop.GetDerivative(U, X);
+   MultiVector dY{dy};
+   dop_deriv->Mult(dx, dY);
+
+   ///<--- Check the result against the expected output
+   CheckResults(fes, ir, y, dy);
+}
+
+
+TEST_CASE("dFEM Scratch multiple sizes",
+          "[Parallel][dFEM][Scratch-Multiple-Sizes]")
+{
+   int order = 2;
+   int ref_levels = 1;
+   int DIM = 2;
+
+   ///<--- Mesh and finite element space setup
+   Mesh mesh = Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL, true, 1.0,
+                                     1.0);
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   pmesh.EnsureNodes();
+   H1_FECollection fec(order, pmesh.Dimension());
+   ParFiniteElementSpace fes(&pmesh, &fec);
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *nodes_fes = nodes->ParFESpace();
+   Vector nodes_tvec;
+   nodes->GetTrueDofs(nodes_tvec);
+
+   ///<--- dFEM setup
+   const IntegrationRule &ir = IntRules.Get(Geometry::SQUARE, 2 * order + 1);
+   QuadratureSpace qspace(pmesh, ir);
+   VectorQuadratureSpace coef_qspace(qspace, 1);
+   QuadratureFunction coef(coef_qspace);
+   coef.UseDevice(true);
+   FunctionCoefficient coeff_fc([](const Vector &p)
+   { return 0.5 + p(0) + 0.125 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillQData(fes, ir, coeff_fc, coef);
+
+   Array<int> all_domain_attr(pmesh.attributes.Max());
+   all_domain_attr = 1;
+
+   const std::vector<FieldDescriptor> inputs
+   {
+      {U, &fes},
+      {COEF, &coef_qspace},
+      {COORDINATES, nodes_fes}};
+   const std::vector<FieldDescriptor> outputs
+   {
+      {Y, &fes}};
+   DifferentiableOperator dop(inputs, outputs, pmesh);
+
+   // Define the cubic qfunction with scratch space
+   // Requesting one scalar scratch vector per dimension
+   CubicQFWithScratch cubic_qf;
+   cubic_qf.SetScratch(pmesh.GetNE() * ir.GetNPoints(), {1, DIM});
+   dop.AddDomainIntegrator<GlobalQFBackend>(
+      cubic_qf,
+      Inputs<Value<U>, Identity<COEF>, Gradient<COORDINATES>, Weight> {},
+      Outputs<Value<Y>> {},
+      ir, all_domain_attr,
+      Derivatives<U> {});
+
+
+   Vector x(fes.GetTrueVSize()), y(fes.GetTrueVSize()), dx(fes.GetTrueVSize()),
+          dy(fes.GetTrueVSize());
+   x.UseDevice(true);
+   y.UseDevice(true);
+   dx.UseDevice(true);
+   dy.UseDevice(true);
+   FunctionCoefficient input_coeff([](const Vector &p)
+   { return 1.0 + p(0) + 0.25 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillInput(fes, input_coeff, x);
+   ConstantCoefficient direction_coeff(1.0);
+   FillInput(fes, direction_coeff, dx);
+   y = 0.0;
+   dy = 0.0;
+
+   ///<--- Apply the operator
+   MultiVector X{x, coef, nodes_tvec};
+   MultiVector Y{y};
+   dop.Mult(X, Y);
+
+   //<--- Apply derivative operator
+   auto dop_deriv = dop.GetDerivative(U, X);
+   MultiVector dY{dy};
+   dop_deriv->Mult(dx, dY);
+
+   ///<--- Check the result against the expected output
+   CheckResults(fes, ir, y, dy);
+}
+
+TEST_CASE("dFEM Global Scratch with tuple objects",
+          "[Parallel][dFEM][GlobalScratch]")
+{
+   int order = 2;
+   int ref_levels = 1;
+
+   ///<--- Mesh and finite element space setup
+   Mesh mesh = Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL, true, 1.0,
+                                     1.0);
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   pmesh.EnsureNodes();
+   H1_FECollection fec(order, pmesh.Dimension());
+   ParFiniteElementSpace fes(&pmesh, &fec);
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *nodes_fes = nodes->ParFESpace();
+   Vector nodes_tvec;
+   nodes->GetTrueDofs(nodes_tvec);
+
+   ///<--- dFEM setup
+   const IntegrationRule &ir = IntRules.Get(Geometry::SQUARE, 2 * order + 1);
+   QuadratureSpace qspace(pmesh, ir);
+   VectorQuadratureSpace coef_qspace(qspace, 1);
+   QuadratureFunction coef(coef_qspace);
+   coef.UseDevice(true);
+   FunctionCoefficient coeff_fc([](const Vector &p)
+   { return 0.5 + p(0) + 0.125 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillQData(fes, ir, coeff_fc, coef);
+
+   Array<int> all_domain_attr(pmesh.attributes.Max());
+   all_domain_attr = 1;
+
+   const std::vector<FieldDescriptor> inputs
+   {
+      {U, &fes},
+      {COEF, &coef_qspace},
+      {COORDINATES, nodes_fes}};
+   const std::vector<FieldDescriptor> outputs
+   {
+      {Y, &fes}};
+   DifferentiableOperator dop(inputs, outputs, pmesh);
+
+   Vector global_vec(1);
+   global_vec.UseDevice(true);
+   global_vec = 0.0;
+   real_t global_scalar = 1.0;
+   bool global_flag;
+
+   CubicQFWithGlobalScratch cubic_qf;
+   cubic_qf.SetScratch(pmesh.GetNE() * ir.GetNPoints(), {1});
+   cubic_qf.SetGlobalScratch(
+      mfem::future::make_tuple(global_flag, global_scalar, global_vec));
+   dop.AddDomainIntegrator<GlobalQFBackend>(
+      cubic_qf,
+      Inputs<Value<U>, Identity<COEF>, Gradient<COORDINATES>, Weight> {},
+      Outputs<Value<Y>> {},
+      ir, all_domain_attr,
+      Derivatives<U> {});
+
+   Vector x(fes.GetTrueVSize()), y(fes.GetTrueVSize()), dx(fes.GetTrueVSize()),
+          dy(fes.GetTrueVSize());
+   x.UseDevice(true);
+   y.UseDevice(true);
+   dx.UseDevice(true);
+   dy.UseDevice(true);
+   FunctionCoefficient input_coeff([](const Vector &p)
+   { return 1.0 + p(0) + 0.25 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillInput(fes, input_coeff, x);
+   ConstantCoefficient direction_coeff(1.0);
+   FillInput(fes, direction_coeff, dx);
+   y = 0.0;
+   dy = 0.0;
+
+   ///<--- Apply the operator
+   MultiVector X{x, coef, nodes_tvec};
+   MultiVector Y{y};
+   dop.Mult(X, Y);
+
+   //<--- Apply derivative operator
+   auto dop_deriv = dop.GetDerivative(U, X);
+   MultiVector dY{dy};
+   dop_deriv->Mult(dx, dY);
+
+   ///<--- Check the result against the expected output
+   CheckResults(fes, ir, y, dy);
+}
+
+
+TEST_CASE("dFEM Scratch multi-kernel persists tangents",
+          "[Parallel][dFEM][Scratch-MultiKernel]")
+{
+   // This test checks that the tangent stored in the qfunction scratch shadow is preserved between split scratch updates, and persists to the final product.
+   // With the old implementation, the tangent of the temporary scratch vector was used internally but lost after the derivative action.
+
+   int order = 2;
+   int ref_levels = 1;
+
+   Mesh mesh = Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL, true, 1.0,
+                                     1.0);
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   pmesh.EnsureNodes();
+   H1_FECollection fec(order, pmesh.Dimension());
+   ParFiniteElementSpace fes(&pmesh, &fec);
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *nodes_fes = nodes->ParFESpace();
+   Vector nodes_tvec;
+   nodes->GetTrueDofs(nodes_tvec);
+
+   const IntegrationRule &ir = IntRules.Get(Geometry::SQUARE, 2 * order + 1);
+   QuadratureSpace qspace(pmesh, ir);
+   VectorQuadratureSpace coef_qspace(qspace, 1);
+   QuadratureFunction coef(coef_qspace);
+   coef.UseDevice(true);
+   FunctionCoefficient coeff_fc([](const Vector &p)
+   { return 0.5 + p(0) + 0.125 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillQData(fes, ir, coeff_fc, coef);
+
+   Array<int> all_domain_attr(pmesh.attributes.Max());
+   all_domain_attr = 1;
+
+   const std::vector<FieldDescriptor> inputs
+   {
+      {U, &fes},
+      {COEF, &coef_qspace},
+      {COORDINATES, nodes_fes}};
+   const std::vector<FieldDescriptor> outputs
+   {
+      {Y, &fes}};
+   DifferentiableOperator dop(inputs, outputs, pmesh);
+
+   CubicQFWithScratch cubic_qf;
+   cubic_qf.SetScratch(pmesh.GetNE() * ir.GetNPoints(), {1});
+   dop.AddDomainIntegrator<GlobalQFBackend>(
+      cubic_qf,
+      Inputs<Value<U>, Identity<COEF>, Gradient<COORDINATES>, Weight> {},
+      Outputs<Value<Y>> {},
+      ir, all_domain_attr,
+      Derivatives<U> {});
+
+   Vector x(fes.GetTrueVSize()), y(fes.GetTrueVSize()), dx(fes.GetTrueVSize());
+   Vector dy_action(fes.GetTrueVSize()), dy_cached(fes.GetTrueVSize());
+   x.UseDevice(true);
+   y.UseDevice(true);
+   dx.UseDevice(true);
+   dy_action.UseDevice(true);
+   dy_cached.UseDevice(true);
+   FunctionCoefficient input_coeff([](const Vector &p)
+   { return 1.0 + p(0) + 0.25 * (p.Size() > 1 ? p(1) : 0.0); });
+   FillInput(fes, input_coeff, x);
+   ConstantCoefficient direction_coeff(1.0);
+   FillInput(fes, direction_coeff, dx);
+   y = 0.0;
+   dy_action = 0.0;
+   dy_cached = 0.0;
+
+   // DifferentiableOperator action
+   MultiVector X{x, coef, nodes_tvec};
+   MultiVector Y{y};
+   dop.Mult(X, Y);
+
+   // Derivative action (non-cached)
+   auto dop_deriv_action = dop.GetDerivative(U, X, false);
+   MultiVector dY_action{dy_action};
+   dop_deriv_action->Mult(dx, dY_action);
+   Vector y_action_check(y);
+   CheckResults(fes, ir, y_action_check, dy_action);
+
+   auto *stored_qf = dop.GetDerivativeActionQFunction<CubicQFWithScratch>(U);
+   auto *stored_qf_shadow =
+      dop.GetDerivativeActionShadowQFunction<CubicQFWithScratch>(U);
+   REQUIRE(stored_qf != nullptr);
+   REQUIRE(stored_qf_shadow != nullptr);
+
+   CheckScratchResults(pmesh, ir, stored_qf->GetScratchVector(0),
+                       stored_qf_shadow->GetScratchVector(0));
+
+   // Derivative action (cached)
+   auto dop_deriv_cached = dop.GetDerivative(U, X, true);
+   MultiVector dY_cached{dy_cached};
+   dop_deriv_cached->Mult(dx, dY_cached);
+
+   Vector y_cached_check(y);
+   CheckResults(fes, ir, y_cached_check, dy_cached);
+
+   auto *stored_setup_qf = dop.GetDerivativeSetupQFunction<CubicQFWithScratch>(U);
+   auto *stored_setup_qf_shadow =
+      dop.GetDerivativeSetupShadowQFunction<CubicQFWithScratch>(U);
+   REQUIRE(stored_setup_qf != nullptr);
+   REQUIRE(stored_setup_qf_shadow != nullptr);
+   CheckScratchResults(pmesh, ir, stored_setup_qf->GetScratchVector(0),
+                       stored_setup_qf_shadow->GetScratchVector(0));
+}
+
+#endif // MFEM_USE_MPI

tests/unit/dfem/test_diffusion.cpp

@@ -10,65 +10,117 @@
 // CONTRIBUTING.md for details.
 
 #include "../unit_tests.hpp"
-#include "../linalg/test_same_matrices.hpp"
-#include "mfem.hpp"
 
-#include <utility>
+#include "mfem.hpp"
 
 #ifdef MFEM_USE_MPI
 
+#include "../linalg/test_same_matrices.hpp"
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+#include "../../../linalg/tensor_arrays.hpp"
+
 using namespace mfem;
 using namespace mfem::future;
-using mfem::future::tensor;
 
 #ifdef MFEM_USE_ENZYME
 using dscalar_t = real_t;
 #else
-using mfem::future::dual;
 using dscalar_t = dual<real_t, real_t>;
 #endif
 
-using DOperator = DifferentiableOperator;
-
-template <int DIM> struct Diffusion
+// ────────────────────────────────────────────────────────────────────────────
+template <int DIM>
+struct Diffusion
 {
    using dvecd_t = tensor<dscalar_t, DIM>;
    using matd_t = tensor<real_t, DIM, DIM>;
 
    struct MFApply
    {
-      MFEM_HOST_DEVICE inline auto operator()(const dvecd_t &dudxi,
-                                              const real_t &rho,
-                                              const matd_t &J,
-                                              const real_t &w) const
+      MFEM_HOST_DEVICE inline auto operator()(
+         const dvecd_t &dudxi,
+         const matd_t &J,
+         const real_t &w,
+         dvecd_t &dvdxi) const
       {
-         const auto invJ = inv(J), TinJ = transpose(invJ);
-         return tuple{ (dudxi * invJ) * TinJ * det(J) * w * rho };
+         const auto invJ = inv(J);
+         const auto invJt = transpose(invJ);
+         dvdxi = (dudxi * invJ) * invJt * det(J) * w;
       }
    };
 
    struct PASetup
    {
-      MFEM_HOST_DEVICE inline auto operator()(const real_t u,
-                                              const real_t &rho,
-                                              const matd_t &J,
-                                              const real_t &w) const
+      MFEM_HOST_DEVICE inline auto operator()(
+         const matd_t &J,
+         const real_t &w,
+         matd_t &qdata) const
       {
-         return tuple{ inv(J) * transpose(inv(J)) * det(J) * w * rho };
+         qdata = inv(J) * transpose(inv(J)) * det(J) * w;
       }
    };
 
    struct PAApply
    {
-      MFEM_HOST_DEVICE inline auto operator()(const dvecd_t &dudxi,
-                                              const matd_t &q) const
+      MFEM_HOST_DEVICE inline auto operator()(
+         const dvecd_t &dudxi,
+         const matd_t &qdata,
+         dvecd_t &dvdxi) const
       {
-         return tuple{ q * dudxi };
+         dvdxi = qdata * dudxi;
       };
    };
 };
 
+// ────────────────────────────────────────────────────────────────────────────
+// Global-QF diffusion. Used to exercise the GlobalQFBackend derivative cache
+// with residual_size_on_qp = DIM*DIM > 1; the other GlobalQFBackend tests are
+// all scalar Value-in/Value-out, where residual_size_on_qp == 1 and the cache
+// index layout is degenerate.
 template <int DIM>
+struct GlobalDiffusion
+{
+   struct MFApply
+   {
+      void operator()(tensor_array<const dscalar_t, DIM> &dudxi,
+                      tensor_array<const real_t, DIM, DIM> &J,
+                      tensor_array<const real_t> &w,
+                      tensor_array<dscalar_t, DIM> &dvdxi) const
+      {
+         mfem::forall(w.size(), [=] MFEM_HOST_DEVICE(int q)
+         {
+            const auto invJ = inv(J(q));
+            dvdxi(q) = (dudxi(q) * invJ) * transpose(invJ) * det(J(q)) * w(q);
+         });
+      }
+   };
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template <int DIM> struct VectorDiffusion
+{
+   using dmatd_t = tensor<dscalar_t, DIM, DIM>;
+   using matd_t = tensor<real_t, DIM, DIM>;
+
+   struct MFApply
+   {
+      MFEM_HOST_DEVICE inline auto operator()(
+         const dmatd_t &dudxi,
+         const matd_t &J,
+         const real_t &w,
+         dmatd_t &dvdxi) const
+      {
+         const auto invJ = inv(J);
+         const auto invJt = transpose(invJ);
+         dvdxi = (dudxi * invJ) * invJt * det(J) * w;
+      }
+   };
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template <int DIM, typename QFBackend = LocalQFBackend>
 void diffusion(const char *filename, int p)
 {
    CAPTURE(filename, DIM, p);
@@ -79,9 +131,10 @@ void diffusion(const char *filename, int p)
 
    pmesh.EnsureNodes();
    auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
-   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
    smesh.Clear();
 
+   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+
    Array<int> all_domain_attr;
    if (pmesh.attributes.Size() > 0)
    {
@@ -89,248 +142,551 @@ void diffusion(const char *filename, int p)
       all_domain_attr = 1;
    }
 
-   H1_FECollection fec(p, DIM);
-   ParFiniteElementSpace pfes(&pmesh, &fec);
    ParFiniteElementSpace *mfes = nodes->ParFESpace();
+   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
 
-   const int NE = pfes.GetNE(), d1d(p + 1), q = 2 * p;
-   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), q);
-   const int q1d(IntRules.Get(Geometry::SEGMENT, ir->GetOrder()).GetNPoints());
-   MFEM_VERIFY(d1d <= q1d, "q1d should be >= d1d");
+   H1_FECollection fec(p, DIM);
 
-   ParGridFunction x(&pfes), y(&pfes), z(&pfes);
-   Vector X(pfes.GetTrueVSize()), Y(pfes.GetTrueVSize()), Z(pfes.GetTrueVSize());
+   static constexpr int U = 0, Coords = 1;
 
-   X.Randomize(1);
-   x.SetFromTrueDofs(X);
-
-   auto rho = [](const Vector &xyz)
+   SECTION("Scalar")
    {
-      const real_t x = xyz(0), y = xyz(1), z = DIM == 3 ? xyz(2) : 0.0;
-      real_t r = M_PI * pow(x, 2);
-      if (DIM >= 2) { r += pow(y, 3); }
-      if (DIM >= 3) { r += pow(z, 4); }
-      return r;
-   };
-   FunctionCoefficient rho_coeff(rho);
+      ParFiniteElementSpace pfes(&pmesh, &fec);
 
-   ParBilinearForm blf_fa(&pfes);
-   blf_fa.AddDomainIntegrator(new DiffusionIntegrator(rho_coeff, ir));
-   blf_fa.SetAssemblyLevel(AssemblyLevel::FULL);
-   blf_fa.Assemble();
-   blf_fa.Finalize();
+      ParGridFunction x(&pfes), y(&pfes), z(&pfes);
+      Vector xtvec(pfes.GetTrueVSize()), ytvec(pfes.GetTrueVSize()),
+             ztvec(pfes.GetTrueVSize());
 
-   QuadratureSpace qs(pmesh, *ir);
-   CoefficientVector rho_coeff_cv(rho_coeff, qs);
-   MFEM_VERIFY(rho_coeff_cv.GetVDim() == 1, "Coefficient should be scalar");
-   MFEM_VERIFY(rho_coeff_cv.Size() == q1d * q1d * (DIM == 3 ? q1d : 1) * NE, "");
+      xtvec.Randomize(1);
+      x.SetFromTrueDofs(xtvec);
 
-   UniformParameterSpace rho_ps(pmesh, *ir, 1);
+      ParBilinearForm blf_fa(&pfes);
+      blf_fa.AddDomainIntegrator(new DiffusionIntegrator(ir));
+      blf_fa.SetAssemblyLevel(AssemblyLevel::FULL);
+      blf_fa.Assemble();
+      blf_fa.Finalize();
 
-   static constexpr int U = 0, Coords = 1, Rho = 3;
-   const auto sol = std::vector{ FieldDescriptor{ U, &pfes } };
+      const auto in_fds = std::vector
+      {
+         FieldDescriptor{ U, &pfes },
+         FieldDescriptor{ Coords, mfes }
+      };
+      const auto out_fds = std::vector{ FieldDescriptor{ U, &pfes } };
 
-   SECTION("action")
-   {
-      DOperator dop_mf(sol, {{Rho, &rho_ps}, {Coords, mfes}}, pmesh);
-      typename Diffusion<DIM>::MFApply mf_apply_qf;
-      dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                 tuple{ Gradient<U>{}, Identity<Rho>{},
-                                        Gradient<Coords>{}, Weight{} },
-                                 tuple{ Gradient<U>{} }, *ir,
-                                 all_domain_attr);
-      dop_mf.SetParameters({ &rho_coeff_cv, nodes });
+      SECTION("Scalar Action")
+      {
+         DifferentiableOperator dop_mf(in_fds, out_fds, pmesh);
+         typename Diffusion<DIM>::MFApply mf_apply_qf;
+         dop_mf.AddDomainIntegrator<QFBackend>(
+            mf_apply_qf,
+            Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr);
 
-      pfes.GetRestrictionMatrix()->Mult(x, X);
-      dop_mf.Mult(X, Z);
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
 
-      blf_fa.Mult(x, y);
-      pfes.GetProlongationMatrix()->MultTranspose(y, Y);
-      Y -= Z;
+         pfes.GetRestrictionMatrix()->Mult(x, xtvec);
 
-      real_t norm_global = 0.0;
-      real_t norm_local = Y.Normlinf();
-      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
-                    pmesh.GetComm());
+         MultiVector X{xtvec, nodestv};
+         MultiVector Z{ztvec};
 
-      REQUIRE(norm_global == MFEM_Approx(0.0));
-      MPI_Barrier(MPI_COMM_WORLD);
+         dop_mf.Mult(X, Z);
+
+         blf_fa.Mult(x, y);
+         pfes.GetProlongationMatrix()->MultTranspose(y, ytvec);
+         ytvec -= ztvec;
+
+         real_t norm_global = 0.0;
+         real_t norm_local = ytvec.Normlinf();
+         MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                       pmesh.GetComm());
+
+         REQUIRE(norm_global == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+
+      SECTION("Scalar Action Partial Assembly")
+      {
+         static constexpr int QData = 2;
+         QuadratureSpace qspace(pmesh, *ir);
+         VectorQuadratureSpace qspace_vec(qspace, DIM * DIM);
+         QuadratureFunction qd(qspace_vec);
+
+         DifferentiableOperator setupPAData(
+         {
+            {Coords, mfes}
+         },
+         {
+            {QData, &qspace_vec}
+         }, pmesh);
+
+         typename Diffusion<DIM>::PASetup pa_setup_qf;
+         setupPAData.AddDomainIntegrator<QFBackend>(
+            pa_setup_qf,
+            Inputs<Gradient<Coords>, Weight> {},
+            Outputs<Identity<QData>> {},
+            *ir, all_domain_attr);
+
+         {
+            Vector nodestv;
+            nodes->GetTrueDofs(nodestv);
+            MultiVector X{nodestv};
+            MultiVector Y{qd};
+            setupPAData.Mult(X, Y);
+         }
+
+         DifferentiableOperator applyPAData(
+         {
+            {U, &pfes}, {QData, &qspace_vec}
+         },
+         {
+            {U, &pfes}
+         }, pmesh);
+         typename Diffusion<DIM>::PAApply pa_apply_qf;
+         applyPAData.AddDomainIntegrator<QFBackend>(
+            pa_apply_qf,
+            Inputs<Gradient<U>, Identity<QData>> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr);
+
+         {
+            pfes.GetRestrictionMatrix()->Mult(x, xtvec);
+            MultiVector X{xtvec, qd};
+            MultiVector Z{ztvec};
+            applyPAData.Mult(X, Z);
+         }
+
+         blf_fa.Mult(x, y);
+         pfes.GetProlongationMatrix()->MultTranspose(y, ytvec);
+         ytvec -= ztvec;
+
+         real_t norm_global = 0.0;
+         real_t norm_local = ytvec.Normlinf();
+         MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                       pmesh.GetComm());
+
+         REQUIRE(norm_global == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+
+      SECTION("Scalar Action Linearized")
+      {
+         DifferentiableOperator dop_mf(in_fds, out_fds, pmesh);
+         typename Diffusion<DIM>::MFApply mf_apply_qf;
+         dop_mf.AddDomainIntegrator<QFBackend>(
+            mf_apply_qf,
+            Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr,
+            Derivatives<U> {});
+
+         pfes.GetRestrictionMatrix()->Mult(x, xtvec);
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         MultiVector X{xtvec, nodestv};
+         MultiVector Z{ztvec};
+         auto ddop = dop_mf.GetDerivative(U, X);
+
+         // Randomize again s.t. the PA setup like cache can't
+         // trivially succeed by caching one direction only.
+         xtvec.Randomize(567);
+         x.SetFromTrueDofs(xtvec);
+
+         Vector dztvec(ztvec.Size());
+         MultiVector DZ{dztvec};
+         ddop->Mult(X[0], DZ);
+
+         blf_fa.Mult(x, y);
+         pfes.GetProlongationMatrix()->MultTranspose(y, ytvec);
+
+         ytvec -= dztvec;
+
+         real_t norm_global = 0.0;
+         real_t norm_local = ytvec.Normlinf();
+         MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                       pmesh.GetComm());
+
+         REQUIRE(norm_global == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+
+      SECTION("Scalar SparseMatrix")
+      {
+         DifferentiableOperator dop_mf(in_fds, out_fds, pmesh);
+         typename Diffusion<DIM>::MFApply mf_apply_qf;
+         dop_mf.AddDomainIntegrator<QFBackend>(
+            mf_apply_qf,
+            Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr,
+            Derivatives<U> {});
+
+         pfes.GetRestrictionMatrix()->Mult(x, xtvec);
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         MultiVector X{xtvec, nodestv};
+         auto dRdU = dop_mf.GetDerivative(U, X);
+
+         SparseMatrix *A = nullptr;
+         dRdU->Assemble(A);
+
+         TestSameMatrices(*A, blf_fa.SpMat());
+         delete A;
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+
+      SECTION("Scalar Assemble Diagonal")
+      {
+         DifferentiableOperator dop_mf(in_fds, out_fds, pmesh);
+         typename Diffusion<DIM>::MFApply mf_apply_qf;
+         dop_mf.AddDomainIntegrator<QFBackend>(
+            mf_apply_qf,
+            Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr,
+            Derivatives<U> {});
+
+         pfes.GetRestrictionMatrix()->Mult(x, xtvec);
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         MultiVector X{xtvec, nodestv};
+         auto dRdU = dop_mf.GetDerivative(U, X);
+
+         Vector dfem_diagonal(pfes.GetTrueVSize());
+         dRdU->AssembleDiagonal(dfem_diagonal);
+
+         Vector mfem_diagonal(pfes.GetTrueVSize());
+         blf_fa.AssembleDiagonal(mfem_diagonal);
+
+         dfem_diagonal -= mfem_diagonal;
+
+         real_t norm_global = 0.0;
+         real_t norm_local = dfem_diagonal.Normlinf();
+         MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                       pmesh.GetComm());
+
+         REQUIRE(norm_global == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
    }
 
-   SECTION("action partial assembly")
-   {
-      static constexpr int QData = 2;
-      UniformParameterSpace qd_ps(pmesh, *ir, DIM * DIM);
-      ParameterFunction qdata(qd_ps);
-      qdata.UseDevice(true);
-
-      DOperator dSetup(sol, {{Rho, &rho_ps}, {Coords, mfes}, {QData, &qd_ps}}, pmesh);
-      typename Diffusion<DIM>::PASetup pa_setup_qf;
-      dSetup.AddDomainIntegrator(
-         pa_setup_qf,
-         tuple{ Value<U>{}, Identity<Rho>{}, Gradient<Coords>{}, Weight{} },
-         tuple{ Identity<QData>{} }, *ir, all_domain_attr);
-      dSetup.SetParameters({ &rho_coeff_cv, nodes, &qdata });
-      pfes.GetRestrictionMatrix()->Mult(x, X);
-      dSetup.Mult(X, qdata);
-
-      DOperator dop_pa(sol, { { QData, &qd_ps } }, pmesh);
-      typename Diffusion<DIM>::PAApply pa_apply_qf;
-      dop_pa.AddDomainIntegrator(pa_apply_qf,
-                                 tuple{ Gradient<U>{}, Identity<QData>{} },
-                                 tuple{ Gradient<U>{} },
-                                 *ir, all_domain_attr);
-      dop_pa.SetParameters({ &qdata });
-
-      pfes.GetRestrictionMatrix()->Mult(x, X);
-      dop_pa.Mult(X, Z);
-
-      blf_fa.Mult(x, y);
-      pfes.GetProlongationMatrix()->MultTranspose(y, Y);
-      Y -= Z;
-
-      real_t norm_global = 0.0;
-      real_t norm_local = Y.Normlinf();
-      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
-                    pmesh.GetComm());
-
-      REQUIRE(norm_global == MFEM_Approx(0.0));
-      MPI_Barrier(MPI_COMM_WORLD);
-   }
-
-   SECTION("action linearized")
-   {
-      DOperator dop_mf(sol, {{Rho, &rho_ps}, {Coords, mfes}}, pmesh);
-      typename Diffusion<DIM>::MFApply mf_apply_qf;
-      auto derivatives = std::integer_sequence<size_t, U> {};
-      dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                 tuple{ Gradient<U>{}, Identity<Rho>{},
-                                        Gradient<Coords>{}, Weight{} },
-                                 tuple{ Gradient<U>{} }, *ir,
-                                 all_domain_attr, derivatives);
-      dop_mf.SetParameters({ &rho_coeff_cv, nodes });
-      auto dRdU = dop_mf.GetDerivative(U, {&x}, {&rho_coeff_cv, nodes});
-
-      pfes.GetRestrictionMatrix()->Mult(x, X);
-      dRdU->Mult(X, Z);
-
-      blf_fa.Mult(x, y);
-      pfes.GetProlongationMatrix()->MultTranspose(y, Y);
-      Y -= Z;
-
-      real_t norm_global = 0.0;
-      real_t norm_local = Y.Normlinf();
-      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
-                    pmesh.GetComm());
-
-      REQUIRE(norm_global == MFEM_Approx(0.0));
-      MPI_Barrier(MPI_COMM_WORLD);
-   }
-
-   SECTION("action vector")
+   SECTION("Vector")
    {
       ParFiniteElementSpace vpfes(&pmesh, &fec, DIM);
       ParGridFunction vx(&vpfes), vy(&vpfes);
       Vector vX(vpfes.GetTrueVSize()), vY(vpfes.GetTrueVSize()),
              vZ(vpfes.GetTrueVSize());
 
-      vX.Randomize(1);
-      vx.SetFromTrueDofs(vX);
+      SECTION("Vector Diffusion Action")
       {
-         const auto vsol = std::vector{ FieldDescriptor{ U, &vpfes } };
-         DOperator dop_mf(vsol, {{Coords, mfes}}, pmesh);
-         const auto mf_vector_diffusion_qf =
-            [] MFEM_HOST_DEVICE (const tensor<dscalar_t, DIM, DIM> &dudxi,
-                                 const tensor<real_t, DIM, DIM> &J,
-                                 const real_t &w)
+         vX.Randomize(1);
+         vx.SetFromTrueDofs(vX);
+
+         DifferentiableOperator dop_mf(
          {
-            const auto invJ = inv(J), TinJ = transpose(invJ);
-            return tuple{ (dudxi * invJ) * TinJ * det(J) * w };
-         };
-         dop_mf.AddDomainIntegrator(mf_vector_diffusion_qf,
-                                    tuple{ Gradient<U>{}, Gradient<Coords>{}, Weight{} },
-                                    tuple{ Gradient<U>{} },
-                                    *ir, all_domain_attr);
-         dop_mf.SetParameters({ nodes });
-         vpfes.GetRestrictionMatrix()->Mult(vx, vX), dop_mf.Mult(vX, vZ);
-      }
-      {
-         ConstantCoefficient one(1.0);
+            {U, &vpfes},
+            {Coords, mfes},
+         },
+         {
+            {U, &vpfes}
+         }, pmesh);
+
+         typename VectorDiffusion<DIM>::MFApply mf_vector_diffusion_qf;
+         dop_mf.AddDomainIntegrator<QFBackend>(
+            mf_vector_diffusion_qf,
+            Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr);
+
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         MultiVector X{vX, nodestv};
+         MultiVector Z{vZ};
+         dop_mf.Mult(X, Z);
+
          ParBilinearForm vblf_fa(&vpfes);
-         vblf_fa.AddDomainIntegrator(new VectorDiffusionIntegrator(one, ir));
+         vblf_fa.AddDomainIntegrator(new VectorDiffusionIntegrator(ir));
          vblf_fa.SetAssemblyLevel(AssemblyLevel::LEGACYFULL);
          vblf_fa.Assemble();
          vblf_fa.Finalize();
          vblf_fa.Mult(vx, vy);
          vpfes.GetProlongationMatrix()->MultTranspose(vy, vY);
-      }
-      vY -= vZ;
-      real_t norm_global = 0.0, norm_local = vY.Normlinf();
-      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
-                    pmesh.GetComm());
-      // Account for ill conditioning of the RT mesh
-      if (std::string(filename).compare("../../data/rt-2d-q3.mesh") == 0)
-      {
-         REQUIRE(norm_global == MFEM_Approx(0.0, 5e-12, 5e-12));
-      }
-      else
-      {
+
+         vY -= vZ;
+         real_t norm_global = 0.0;
+         real_t norm_local = vY.Normlinf();
+         MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                       pmesh.GetComm());
+
          REQUIRE(norm_global == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
       }
-      MPI_Barrier(MPI_COMM_WORLD);
+
+      SECTION("Vector Diffusion Action Linearized")
+      {
+         vX.Randomize(1);
+         vx.SetFromTrueDofs(vX);
+
+         DifferentiableOperator dop_mf(
+         {
+            {U, &vpfes},
+            {Coords, mfes},
+         },
+         {
+            {U, &vpfes}
+         }, pmesh);
+
+         typename VectorDiffusion<DIM>::MFApply mf_vector_diffusion_qf;
+         dop_mf.AddDomainIntegrator<QFBackend>(
+            mf_vector_diffusion_qf,
+            Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr,
+            Derivatives<U> {});
+
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         MultiVector X{vX, nodestv};
+         const auto ddop = dop_mf.GetDerivative(U, X);
+
+         MultiVector Z{vZ};
+         ddop->Mult(vX, Z);
+
+         ParBilinearForm vblf_fa(&vpfes);
+         vblf_fa.AddDomainIntegrator(new VectorDiffusionIntegrator(ir));
+         vblf_fa.SetAssemblyLevel(AssemblyLevel::LEGACYFULL);
+         vblf_fa.Assemble();
+         vblf_fa.Finalize();
+         vblf_fa.Mult(vx, vy);
+         vpfes.GetProlongationMatrix()->MultTranspose(vy, vY);
+
+         vY -= vZ;
+         real_t norm_global = 0.0;
+         real_t norm_local = vY.Normlinf();
+         MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                       pmesh.GetComm());
+
+         REQUIRE(norm_global == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+
+      SECTION("Vector SparseMatrix")
+      {
+         vX.Randomize(1);
+         vx.SetFromTrueDofs(vX);
+
+         DifferentiableOperator dop_mf(
+         {
+            {U, &vpfes},
+            {Coords, mfes},
+         },
+         {
+            {U, &vpfes}
+         }, pmesh);
+
+         typename VectorDiffusion<DIM>::MFApply mf_vector_diffusion_qf;
+         dop_mf.AddDomainIntegrator<QFBackend>(
+            mf_vector_diffusion_qf,
+            Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<U>> {},
+            *ir, all_domain_attr,
+            Derivatives<U> {});
+
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         MultiVector X{vX, nodestv};
+         const auto ddop = dop_mf.GetDerivative(U, X);
+
+         MultiVector Z{vZ};
+         ddop->Mult(vX, Z);
+
+         ParBilinearForm vblf_fa(&vpfes);
+         vblf_fa.AddDomainIntegrator(new VectorDiffusionIntegrator(ir));
+         vblf_fa.SetAssemblyLevel(AssemblyLevel::LEGACYFULL);
+         vblf_fa.Assemble();
+         vblf_fa.Finalize();
+
+         SparseMatrix *A = nullptr;
+         ddop->Assemble(A);
+
+         TestSameMatrices(*A, vblf_fa.SpMat());
+         delete A;
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM Diffusion 2D", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3});
+   const auto meshs = { "../../data/inline-quad.mesh" };
+   const auto extra = { "../../data/star.mesh",
+                        "../../data/star-q3.mesh",
+                        "../../data/rt-2d-q3.mesh",
+                        "../../data/periodic-square.mesh"
+                      };
+   diffusion<2>(GenAll(meshs, extra), p);
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// The GlobalQFBackend writes the derivative qp_cache, but Assemble and
+// AssembleDiagonal are served by the LocalQF implementations (see
+// GlobalQFBackend::MakeDerivativeAssemble*), so writer and readers live in
+// different backends and must agree on the cache layout.
+template <int DIM>
+void diffusion_globalqf(const char *filename, int p)
+{
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   smesh.Clear();
+
+   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
    }
 
-   SECTION("spmat")
-   {
-      DOperator dop_mf(sol, {{Rho, &rho_ps}, {Coords, mfes}}, pmesh);
-      typename Diffusion<DIM>::MFApply mf_apply_qf;
-      auto derivatives = std::integer_sequence<size_t, U> {};
-      dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                 tuple{ Gradient<U>{}, Identity<Rho>{},
-                                        Gradient<Coords>{}, Weight{} },
-                                 tuple{ Gradient<U>{} }, *ir,
-                                 all_domain_attr, derivatives);
-      dop_mf.SetParameters({ &rho_coeff_cv, nodes });
-      auto dRdU = dop_mf.GetDerivative(U, {&x}, {&rho_coeff_cv, nodes});
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
 
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace pfes(&pmesh, &fec);
+
+   static constexpr int U = 0, Coords = 1;
+
+   ParGridFunction x(&pfes), y(&pfes);
+   Vector xtvec(pfes.GetTrueVSize()), ytvec(pfes.GetTrueVSize()),
+          ztvec(pfes.GetTrueVSize());
+   xtvec.Randomize(1);
+   x.SetFromTrueDofs(xtvec);
+
+   ParBilinearForm blf_fa(&pfes);
+   blf_fa.AddDomainIntegrator(new DiffusionIntegrator(ir));
+   blf_fa.SetAssemblyLevel(AssemblyLevel::FULL);
+   blf_fa.Assemble();
+   blf_fa.Finalize();
+
+   const auto in_fds = std::vector
+   {
+      FieldDescriptor{ U, &pfes },
+      FieldDescriptor{ Coords, mfes }
+   };
+   const auto out_fds = std::vector{ FieldDescriptor{ U, &pfes } };
+
+   DifferentiableOperator dop(in_fds, out_fds, pmesh);
+   typename GlobalDiffusion<DIM>::MFApply global_qfn;
+   dop.AddDomainIntegrator<GlobalQFBackend>(
+      global_qfn,
+      Inputs<Gradient<U>, Gradient<Coords>, Weight> {},
+      Outputs<Gradient<U>> {},
+      *ir, all_domain_attr,
+      Derivatives<U> {});
+
+   Vector nodestv;
+   nodes->GetTrueDofs(nodestv);
+   pfes.GetRestrictionMatrix()->Mult(x, xtvec);
+   MultiVector X{ xtvec, nodestv };
+   auto dRdU = dop.GetDerivative(U, X);
+
+   const auto max_error = [&](const Vector &v)
+   {
+      real_t norm_global = 0.0, norm_local = v.Normlinf();
+      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                    pmesh.GetComm());
+      return norm_global;
+   };
+
+   // GlobalQF setup writer -> GlobalQF apply reader
+   {
+      MultiVector Z{ ztvec };
+      dRdU->Mult(xtvec, Z);
+      blf_fa.Mult(x, y);
+      pfes.GetProlongationMatrix()->MultTranspose(y, ytvec);
+      ytvec -= ztvec;
+      REQUIRE(max_error(ytvec) == MFEM_Approx(0.0));
+   }
+
+   // GlobalQF setup writer -> GlobalQF apply-transpose reader
+   {
+      Vector wtvec(pfes.GetTrueVSize());
+      wtvec.Randomize(0x9e3779b9);
+      MultiVector W{ wtvec }, Z{ ztvec };
+      dRdU->MultTranspose(W, Z);
+
+      ParGridFunction w(&pfes);
+      w.SetFromTrueDofs(wtvec);
+      blf_fa.MultTranspose(w, y);
+      pfes.GetProlongationMatrix()->MultTranspose(y, ytvec);
+      ytvec -= ztvec;
+      REQUIRE(max_error(ytvec) == MFEM_Approx(0.0));
+   }
+
+   // GlobalQF setup writer -> LocalQF assemble reader
+   {
       SparseMatrix *A = nullptr;
       dRdU->Assemble(A);
       TestSameMatrices(*A, blf_fa.SpMat());
       delete A;
    }
+
+   // GlobalQF setup writer -> LocalQF assemble-diagonal reader
+   {
+      Vector dfem_diag(pfes.GetTrueVSize()), mfem_diag(pfes.GetTrueVSize());
+      dRdU->AssembleDiagonal(dfem_diag);
+      blf_fa.AssembleDiagonal(mfem_diag);
+      dfem_diag -= mfem_diag;
+      REQUIRE(max_error(dfem_diag) == MFEM_Approx(0.0));
+   }
+
+   MPI_Barrier(MPI_COMM_WORLD);
 }
 
-TEST_CASE("dFEM Diffusion", "[Parallel][dFEM][GPU]")
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM Diffusion GlobalQF cache 2D", "[Parallel][dFEM]")
 {
-   const bool all_tests = launch_all_non_regression_tests;
-
-   const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
-
-   SECTION("2d")
+   if constexpr (!mfem_use_gpu)
    {
-      const auto filename2d =
-         GENERATE(
-            "../../data/star.mesh",
-            "../../data/star-q3.mesh",
-            "../../data/rt-2d-q3.mesh",
-            "../../data/inline-quad.mesh",
-            "../../data/periodic-square.mesh"
-         );
-      diffusion<2>(filename2d, p);
+      const auto p = GenAll({1}, {2, 3});
+      const auto meshs = { "../../data/inline-quad.mesh" };
+      const auto extra = { "../../data/star.mesh" };
+      diffusion_globalqf<2>(GenAll(meshs, extra), p);
    }
+}
 
-   SECTION("3d")
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM Diffusion GlobalQF cache 3D", "[Parallel][dFEM]")
+{
+   if constexpr (!mfem_use_gpu)
    {
-      const auto filename3d =
-         GENERATE(
-            "../../data/fichera.mesh",
-            "../../data/fichera-q3.mesh",
-            "../../data/inline-hex.mesh",
-            "../../data/toroid-hex.mesh",
-            "../../data/periodic-cube.mesh"
-         );
-      diffusion<3>(filename3d, p);
+      const auto p = GenAll({1}, {2, 3});
+      const auto meshs = { "../../data/inline-hex.mesh" };
+      const auto extra = { "../../data/fichera.mesh" };
+      diffusion_globalqf<3>(GenAll(meshs, extra), p);
    }
 }
 
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM Diffusion 3D", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3});
+   const auto meshs = { "../../data/inline-hex.mesh" };
+   const auto extra = { "../../data/fichera.mesh",
+                        "../../data/fichera-q3.mesh",
+                        "../../data/toroid-hex.mesh",
+                        "../../data/periodic-cube.mesh"
+                      };
+   diffusion<3>(GenAll(meshs, extra), p);
+}
+
 #endif // MFEM_USE_MPI

tests/unit/dfem/test_divergence.cpp

@@ -10,18 +10,20 @@
 // CONTRIBUTING.md for details.
 
 #include "../unit_tests.hpp"
+
 #include "mfem.hpp"
 
 #ifdef MFEM_USE_MPI
 
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
 using namespace mfem;
 using namespace mfem::future;
-using mfem::future::tensor;
 
 #ifdef MFEM_USE_ENZYME
 using dscalar_t = real_t;
 #else
-using mfem::future::dual;
 using dscalar_t = dual<real_t, real_t>;
 #endif
 
@@ -50,69 +52,123 @@ void vectordivergence(const char *filename, int p)
    ParFiniteElementSpace psfes(&pmesh, &fec);
    ParFiniteElementSpace pvfes(&pmesh, &fec, DIM);
 
-   const int d1d(p + 1), q = 3 * p + 1;
+   const int q = 3 * p + 1;
    const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), q);
-   const int q1d(IntRules.Get(Geometry::SEGMENT, ir->GetOrder()).GetNPoints());
-   MFEM_VERIFY(d1d <= q1d, "q1d should be >= d1d");
 
-   ParGridFunction vx(&pvfes);
-   ParGridFunction sy(&psfes), sz(&psfes);
-   Vector vX(pvfes.GetTrueVSize());
-   Vector sY(psfes.GetTrueVSize()), sZ(psfes.GetTrueVSize());
+   ParGridFunction xv(&pvfes);
+   ParGridFunction ys(&psfes), sz(&psfes);
+   Vector Xv(pvfes.GetTrueVSize());
+   Vector Ys(psfes.GetTrueVSize()), Zs(psfes.GetTrueVSize());
 
-   vX.Randomize(1), vx.SetFromTrueDofs(vX);
+   Xv.Randomize(1), xv.SetFromTrueDofs(Xv);
 
    MixedBilinearForm mblf_fa(&pvfes, &psfes);
    mblf_fa.AddDomainIntegrator(new VectorDivergenceIntegrator);
-   mblf_fa.Assemble(), mblf_fa.Finalize();
-   mblf_fa.Mult(vx, sy);
+   mblf_fa.Assemble();
+   mblf_fa.Finalize();
+   mblf_fa.Mult(xv, ys);
 
-   MixedBilinearForm mblf_pa(&pvfes, &psfes);
-   mblf_pa.AddDomainIntegrator(new VectorDivergenceIntegrator);
-   mblf_pa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   mblf_pa.Assemble();
-   mblf_pa.Mult(vx, sz);
-   sy -= sz;
-   REQUIRE(sy.Normlinf() == MFEM_Approx(0.0));
-   MPI_Barrier(MPI_COMM_WORLD);
+   static constexpr int P = 0, V = 1, Coords = 2;
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
 
+   const auto inputs = std::vector
    {
-      static constexpr int P = 0, V = 1, Coords = 2;
-      ParFiniteElementSpace *mfes = nodes->ParFESpace();
+      FieldDescriptor{V, &pvfes},
+      FieldDescriptor{Coords, mfes}
+   };
+   const auto outputs = std::vector
+   {
+      FieldDescriptor{P, &psfes}
+   };
 
-      const auto solutions = std::vector{ FieldDescriptor{ P, &psfes } };
-      const auto parameters = std::vector
-      {
-         FieldDescriptor{ V, &pvfes },
-         FieldDescriptor{ Coords, mfes }
-      };
+   DifferentiableOperator dop_mf(inputs, outputs, pmesh);
 
-      DifferentiableOperator dop_mf(solutions, parameters, pmesh);
+   const auto mf_vector_divergence_qf =
+      [] MFEM_HOST_DEVICE(const tensor<dscalar_t, DIM, DIM> &dudxi,
+                          const tensor<real_t, DIM, DIM> &J,
+                          const real_t &w,
+                          dscalar_t &v)
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      v = tr(dudx) * det(J) * w;
+   };
 
-      const auto mf_vector_divergence_qf =
-         [] MFEM_HOST_DEVICE(const tensor<dscalar_t, DIM, DIM> &dudxi,
-                             const tensor<mfem::real_t, DIM, DIM> &J,
-                             const real_t &w)
-      {
-         const auto invJ = inv(J);
-         const auto dudx = dudxi * invJ;
-         return tuple{ tr(dudx) * det(J) * w };
-      };
+   const auto derivatives = std::integer_sequence<size_t, V> {};
+   dop_mf.AddDomainIntegrator<LocalQFBackend>(
+      mf_vector_divergence_qf,
+      tuple{Gradient<V>{}, Gradient<Coords>{}, Weight{}},
+      tuple{Value<P>{}},
+      *ir, all_domain_attr, derivatives);
 
-      dop_mf.AddDomainIntegrator(mf_vector_divergence_qf,
-                                 tuple{ Gradient<V>{}, Gradient<Coords>{}, Weight{} },
-                                 tuple{ Value<P>{} },
-                                 *ir, all_domain_attr);
+   SECTION("Action")
+   {
+      Vector nodestv;
+      nodes->GetTrueDofs(nodestv);
+      MultiVector X{Xv, nodestv};
+      MultiVector Z{Zs};
+      dop_mf.Mult(X, Z);
 
-      dop_mf.SetParameters({ &vx, nodes });
-      Vector unused(pvfes.GetTrueVSize());
-      dop_mf.Mult(unused, sZ);
+      mblf_fa.Mult(xv, ys);
+      psfes.GetProlongationMatrix()->MultTranspose(ys, Ys);
 
-      mblf_fa.Mult(vx, sy);
-      psfes.GetProlongationMatrix()->MultTranspose(sy, sY);
+      Ys -= Zs;
+      real_t norm_global = 0.0;
+      real_t norm_local = Ys.Normlinf();
+      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                    pmesh.GetComm());
+      REQUIRE(norm_global == MFEM_Approx(0.0));
+      MPI_Barrier(MPI_COMM_WORLD);
+   }
 
-      sY -= sZ;
-      real_t norm_global = M_PI, norm_local = sY.Normlinf();
+   SECTION("Derivative Action")
+   {
+      Vector nodestv;
+      nodes->GetTrueDofs(nodestv);
+      MultiVector X{Xv, nodestv};
+      MultiVector Z{Zs};
+      auto dRdV = dop_mf.GetDerivative(V, X);
+      dRdV->Mult(X[0], Z);
+
+      mblf_fa.Mult(xv, ys);
+      psfes.GetProlongationMatrix()->MultTranspose(ys, Ys);
+
+      Ys -= Zs;
+      real_t norm_global = 0.0;
+      real_t norm_local = Ys.Normlinf();
+      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                    pmesh.GetComm());
+      REQUIRE(norm_global == MFEM_Approx(0.0));
+      MPI_Barrier(MPI_COMM_WORLD);
+   }
+
+   SECTION("Derivative Transpose Action")
+   {
+      Vector nodestv;
+      nodes->GetTrueDofs(nodestv);
+
+      // Build cache with full primal state
+      MultiVector state{Xv, nodestv};
+      auto dRdV = dop_mf.GetDerivative(V, state);
+
+      // Direction in output (test) T-space: use Ys computed from mblf_fa.
+      psfes.GetProlongationMatrix()->MultTranspose(ys, Ys);
+      MultiVector direction{Ys};
+
+      // Result in derivative (trial) T-space.
+      Vector result_v(pvfes.GetTrueVSize());
+      result_v = 0.0;
+      MultiVector result{result_v};
+      dRdV->MultTranspose(direction, result);
+
+      psfes.GetProlongationMatrix()->Mult(Ys, ys);
+      mblf_fa.MultTranspose(ys, xv);
+      Vector ref_v(pvfes.GetTrueVSize());
+      pvfes.GetProlongationMatrix()->MultTranspose(xv, ref_v);
+
+      result_v -= ref_v;
+      real_t norm_global = 0.0;
+      real_t norm_local = result_v.Normlinf();
       MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
                     pmesh.GetComm());
       REQUIRE(norm_global == MFEM_Approx(0.0));
@@ -120,32 +176,31 @@ void vectordivergence(const char *filename, int p)
    }
 }
 
-TEST_CASE("dFEM VectorDivergence", "[Parallel][dFEM]")
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM VectorDivergence", "[Parallel][dFEM][GPU]")
 {
-   const bool all_tests = launch_all_non_regression_tests;
-
-   const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
+   const auto p = GenAll({1}, {2, 3});
 
    SECTION("2D p=" + std::to_string(p))
    {
-      const auto filename =
-         GENERATE("../../data/star.mesh",
-                  "../../data/star-q3.mesh",
-                  "../../data/rt-2d-q3.mesh",
-                  "../../data/inline-quad.mesh",
-                  "../../data/periodic-square.mesh");
-      vectordivergence<2>(filename, p);
+      const auto meshs = { "../../data/inline-quad.mesh" };
+      const auto extra = { "../../data/star.mesh",
+                           "../../data/star-q3.mesh",
+                           "../../data/rt-2d-q3.mesh",
+                           "../../data/periodic-square.mesh"
+                         };
+      vectordivergence<2>(GenAll(meshs, extra), p);
    }
 
    SECTION("3D p=" + std::to_string(p))
    {
-      const auto filename =
-         GENERATE("../../data/fichera.mesh",
-                  "../../data/fichera-q3.mesh",
-                  "../../data/inline-hex.mesh",
-                  "../../data/toroid-hex.mesh",
-                  "../../data/periodic-cube.mesh");
-      vectordivergence<3>(filename, p);
+      const auto meshs = { "../../data/inline-hex.mesh" };
+      const auto extra = { "../../data/fichera.mesh",
+                           "../../data/fichera-q3.mesh",
+                           "../../data/toroid-hex.mesh",
+                           "../../data/periodic-cube.mesh"
+                         };
+      vectordivergence<3>(GenAll(meshs, extra), p);
    }
 }
 

tests/unit/dfem/test_functional.cpp

@@ -0,0 +1,279 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+template <typename dscalar_t, int dim>
+struct CubicH1Functional
+{
+   static constexpr real_t alpha = 0.3;
+   static constexpr real_t beta  = 0.7;
+
+   MFEM_HOST_DEVICE inline
+   auto operator()(const dscalar_t &u,
+                   const tensor<dscalar_t, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   dscalar_t &f) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      const auto dx = det(J) * w;
+
+      const auto z =
+         + 0.5_r * u * u
+         + 0.25_r * alpha * u * u * u * u
+         + 0.5_r * beta * dot(dudx, dudx);
+
+      f = z * dx;
+   }
+};
+
+template <int dim>
+class MyFunctional
+{
+   static constexpr int U = 0, Coords = 1, Q = 2;
+
+public:
+   MyFunctional(const ParFiniteElementSpace &fes,
+                const ParFiniteElementSpace &mfes,
+                const IntegrationRule &ir) :
+      comm(fes.GetComm()),
+      qspace(*fes.GetParMesh(), ir),
+      qspace_vec(qspace, 1),
+      q(qspace_vec)
+   {
+      const auto in_fds = std::vector
+      {
+         FieldDescriptor{U, &fes},
+         FieldDescriptor{Coords, &mfes}
+      };
+      const auto out_fds = std::vector
+      {
+         FieldDescriptor{Q, &qspace_vec}
+      };
+
+      const auto &mesh = *fes.GetParMesh();
+      Array<int> all_domain_attr;
+      if (mesh.attributes.Size() > 0)
+      {
+         all_domain_attr.SetSize(mesh.attributes.Max());
+         all_domain_attr = 1;
+      }
+
+      dop = std::make_unique<DifferentiableOperator>(in_fds, out_fds, mesh);
+      CubicH1Functional<dscalar_t, dim> apply;
+      auto derivatives = std::integer_sequence<size_t, U> {};
+      dop->AddDomainIntegrator<LocalQFBackend>(
+         apply,
+         tuple{Value<U>{}, Gradient<U>{}, Gradient<Coords>{}, Weight{}},
+         tuple{Identity<Q>{}},
+         ir, all_domain_attr, derivatives);
+
+      mesh.GetNodes()->GetTrueDofs(coords);
+   }
+
+   real_t Eval(const Vector &u) const
+   {
+      real_t local = EvalLocal(u), global;
+      MPI_Allreduce(&local, &global, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM, comm);
+      return global;
+   }
+
+   // Returns the directional derivative dJ/du · du.
+   real_t dJdu_dir(const Vector &u, const Vector &du) const
+   {
+      real_t local = dJdu_dir_local(u, du), global;
+      MPI_Allreduce(&local, &global, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM, comm);
+      return global;
+   }
+
+   // Computes the full gradient \nabla J(u) in the trial space via J^T.
+   // Since Eval sums raw QF values (J = \sum f_q), the adjoint of the
+   // summation is a QF of ones.
+   void grad(const Vector &u, Vector &g) const
+   {
+      MultiVector X{u, coords};
+      q = 1.0;
+      MultiVector ones{q};
+      dop->GetDerivative(U, X)->MultTranspose(ones, g);
+   }
+
+   // Computes the full gradient via element-wise central differences.
+   void grad_fd(const Vector &u, Vector &g, real_t eps = 1e-5) const
+   {
+      const int local_size = u.Size();
+
+      // Global offset for this rank's DOFs and total DOF count.
+      int offset = 0, global_size = local_size;
+      MPI_Exscan(&local_size, &offset, 1, MPITypeMap<int>::mpi_type, MPI_SUM, comm);
+      MPI_Allreduce(MPI_IN_PLACE, &global_size, 1, MPITypeMap<int>::mpi_type, MPI_SUM,
+                    comm);
+
+      g.SetSize(local_size);
+      Vector up(u), um(u);
+
+      // Loop over global DOF indices. Each rank perturbs only when gi falls in
+      // its local range [offset, offset+local_size); all ranks call Eval together.
+      for (int gi = 0; gi < global_size; ++gi)
+      {
+         const int li = gi - offset;
+         if (li >= 0 && li < local_size) { up(li) += eps; um(li) -= eps; }
+         const real_t Jp = Eval(up);
+         const real_t Jm = Eval(um);
+         if (li >= 0 && li < local_size)
+         {
+            g(li) = (Jp - Jm) / (2.0 * eps);
+            up(li) = u(li);
+            um(li) = u(li);
+         }
+      }
+   }
+
+private:
+   real_t EvalLocal(const Vector &u) const
+   {
+      MultiVector X{u, coords};
+      MultiVector Y{q};
+      dop->Mult(X, Y);
+      return q.Sum();
+   }
+
+   real_t dJdu_dir_local(const Vector &u, const Vector &du) const
+   {
+      MultiVector X{u, coords};
+      MultiVector dY{q};
+      dop->GetDerivative(U, X)->Mult(du, dY);
+      return q.Sum();
+   }
+
+   MPI_Comm comm;
+   std::unique_ptr<DifferentiableOperator> dop;
+   QuadratureSpace qspace;
+   VectorQuadratureSpace qspace_vec;
+   mutable QuadratureFunction q;
+   Vector coords;
+};
+
+template <int DIM>
+void functional(const char *filename, int p)
+{
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+   smesh.Clear();
+
+   H1_FECollection fec(p, DIM);
+
+   ParFiniteElementSpace fes(&pmesh, &fec);
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+
+   const auto ir = IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   Vector u(fes.GetTrueVSize());
+   Vector du(fes.GetTrueVSize());
+
+   u.Randomize(5532);
+   du.Randomize(3251);
+
+   MyFunctional<DIM> functional(fes, *mfes, ir);
+
+   const real_t dJ_ad = functional.dJdu_dir(u, du);
+
+   Vector g(fes.GetTrueVSize());
+   functional.grad(u, g);
+   const real_t dJ_ad_grad = InnerProduct(pmesh.GetComm(), g, du);
+
+   real_t best_error_dir  = infinity();
+   real_t best_error_grad = infinity();
+
+   for (real_t eps : {1e-3, 3e-4, 1e-4, 3e-5})
+   {
+      Vector up(u), um(u);
+      up.Add(eps, du);
+      um.Add(-eps, du);
+
+      const real_t Jp = functional.Eval(up);
+      const real_t Jm = functional.Eval(um);
+
+      const real_t dJ_fd  = (Jp - Jm) / (2.0 * eps);
+      const real_t scale  = std::max(real_t(1.0), std::abs(dJ_fd));
+
+      best_error_dir  = std::min(best_error_dir,
+                                 std::abs(dJ_ad - dJ_fd) / scale);
+      best_error_grad = std::min(best_error_grad,
+                                 std::abs(dJ_ad_grad - dJ_fd) / scale);
+   }
+
+   REQUIRE(best_error_dir  < 1e-7);
+   REQUIRE(best_error_grad < 1e-7);
+
+   // Must match entry-wise FD gradient
+   Vector g_fd;
+   functional.grad_fd(u, g_fd);
+
+   Vector diff(g);
+   diff -= g_fd;
+   const real_t scale = std::max(real_t(1.0), g_fd.Normlinf());
+   real_t local_norm = diff.Normlinf();
+   real_t global_norm;
+   MPI_Allreduce(&local_norm, &global_norm, 1, MPITypeMap<real_t>::mpi_type,
+                 MPI_SUM, pmesh.GetComm());
+   REQUIRE(diff.Normlinf() / scale < 1e-5);
+}
+
+TEST_CASE("dFEM functional derivative action matches finite differences",
+          "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3});
+   SECTION("2d")
+   {
+      const auto meshs = { "../../data/inline-quad.mesh" };
+      const auto extra = { "../../data/star.mesh",
+                           "../../data/star-q3.mesh",
+                           "../../data/rt-2d-q3.mesh",
+                           "../../data/periodic-square.mesh"
+                         };
+      functional<2>(GenAll(meshs, extra), p);
+   }
+
+   SECTION("3d")
+   {
+      const auto meshs = { "../../data/inline-hex.mesh" };
+      const auto extra = { "../../data/fichera.mesh",
+                           "../../data/fichera-q3.mesh",
+                           "../../data/toroid-hex.mesh",
+                           "../../data/periodic-cube.mesh"
+                         };
+      functional<3>(GenAll(meshs, extra), p);
+   }
+}
+
+#endif // MFEM_USE_MPI

tests/unit/dfem/test_functional_gradient.cpp

@@ -0,0 +1,237 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_ENZYME
+#ifdef MFEM_USE_MPI
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+using mfem::future::tensor;
+
+namespace functional_gradient_test
+{
+
+// Dirichlet-type energy 1/2 \int |grad u|^2 dx. It depends on the solution and
+// on the mesh coordinates, so both derivatives are nontrivial. The coordinate
+// field enters only through Gradient<Coords>, while the solution enters
+// through both Value<U> and Gradient<U>, so the two derivatives have a
+// different number of outputs.
+template <int dim>
+struct EnergyFunctional
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const real_t &u,
+                   const tensor<real_t, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &f) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      f = (0.5_r * sqnorm(dudx) + 0.25_r * u * u) * det(J) * w;
+   }
+};
+
+/// Energy functional with derivatives w.r.t. both the solution and the mesh
+/// coordinates, plus a finite-difference reference for either one.
+template <int dim>
+class EnergyWithShapeDerivative
+{
+   static constexpr int U = 0, Coords = 1, Q = 2;
+
+public:
+   EnergyWithShapeDerivative(const ParFiniteElementSpace &fes,
+                             const ParFiniteElementSpace &mfes,
+                             const IntegrationRule &ir) :
+      comm(fes.GetComm()),
+      qspace(*fes.GetParMesh(), ir),
+      qspace_vec(qspace, 1),
+      q(qspace_vec)
+   {
+      const auto &pmesh = *fes.GetParMesh();
+      Array<int> all_domain_attr;
+      if (pmesh.attributes.Size() > 0)
+      {
+         all_domain_attr.SetSize(pmesh.attributes.Max());
+         all_domain_attr = 1;
+      }
+
+      const auto in = std::vector
+      {
+         FieldDescriptor{U, &fes},
+         FieldDescriptor{Coords, &mfes}
+      };
+      const auto out = std::vector
+      {
+         FieldDescriptor{Q, &qspace_vec}
+      };
+
+      dop = std::make_unique<DifferentiableOperator>(in, out, pmesh);
+      EnergyFunctional<dim> energy;
+      dop->AddDomainIntegrator<LocalQFBackend>(
+         energy,
+         Inputs<Value<U>, Gradient<U>, Gradient<Coords>, Weight> {},
+         Outputs<FunctionalValue<Q>> {},
+         ir, all_domain_attr, Derivatives<U, Coords> {});
+   }
+
+   real_t Eval(const Vector &u, const Vector &x) const
+   {
+      MultiVector X{u, x};
+      MultiVector Y{q};
+      dop->Mult(X, Y);
+      const real_t local = q.Sum();
+      real_t global;
+      MPI_Allreduce(&local, &global, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                    comm);
+      return global;
+   }
+
+   /// Gradient w.r.t. @a derivative_id, assembled into a Vector.
+   void AssembleGradient(size_t derivative_id, const Vector &u,
+                         const Vector &x, Vector &g) const
+   {
+      MultiVector X{u, x};
+      dop->GetDerivative(derivative_id, X)->Assemble(g);
+   }
+
+   /// Central-difference directional derivative of the energy.
+   real_t DirectionalFD(const Vector &u, const Vector &x,
+                        const Vector &du, const Vector &dx,
+                        real_t eps) const
+   {
+      Vector up(u), um(u), xp(x), xm(x);
+      up.Add(eps, du);
+      um.Add(-eps, du);
+      xp.Add(eps, dx);
+      xm.Add(-eps, dx);
+      return (Eval(up, xp) - Eval(um, xm)) / (2.0_r * eps);
+   }
+
+   /// Global inner product of two T-vectors (true dofs are uniquely owned).
+   real_t Dot(const Vector &a, const Vector &b) const
+   {
+      const real_t local = a * b;
+      real_t global;
+      MPI_Allreduce(&local, &global, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                    comm);
+      return global;
+   }
+
+private:
+   MPI_Comm comm;
+   std::unique_ptr<DifferentiableOperator> dop;
+   QuadratureSpace qspace;
+   VectorQuadratureSpace qspace_vec;
+   mutable QuadratureFunction q;
+};
+
+template <int DIM>
+void functional_gradient(const char *filename, int p)
+{
+   CAPTURE(filename, DIM, p);
+
+   static constexpr int U = 0, Coords = 1;
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace fes(&pmesh, &fec);
+
+   const IntegrationRule &ir =
+      IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p + 2);
+
+   EnergyWithShapeDerivative<DIM> energy(fes, *mfes, ir);
+
+   ParGridFunction u_gf(&fes), du_gf(&fes);
+   FunctionCoefficient u_coeff(
+      [](const auto &x) { return 1.0_r + x[0] + 0.25_r * x[1] * x[1]; });
+   FunctionCoefficient du_coeff(
+      [](const auto &x) { return sin(M_PI * x[0]) + 0.5_r * x[0] * x[1]; });
+   u_gf.ProjectCoefficient(u_coeff);
+   du_gf.ProjectCoefficient(du_coeff);
+
+   ParGridFunction dx_gf(mfes);
+   VectorFunctionCoefficient dx_coeff(
+      DIM, [](const Vector &x, Vector &v)
+   {
+      v = 0.0;
+      v(0) = 0.1_r * sin(M_PI * x[0]) * x[1];
+      v(1) = 0.1_r * cos(M_PI * x[1]) * x[0];
+   });
+   dx_gf.ProjectCoefficient(dx_coeff);
+
+   Vector u(fes.GetTrueVSize()), du(fes.GetTrueVSize());
+   Vector x(mfes->GetTrueVSize()), dx(mfes->GetTrueVSize());
+   u_gf.GetTrueDofs(u);
+   du_gf.GetTrueDofs(du);
+   nodes->GetTrueDofs(x);
+   dx_gf.GetTrueDofs(dx);
+
+   Vector zero_u(fes.GetTrueVSize()), zero_x(mfes->GetTrueVSize());
+   zero_u = 0.0;
+   zero_x = 0.0;
+
+   const real_t eps = 1e-6;
+
+   // Derivative w.r.t. the solution.
+   Vector g_u;
+   energy.AssembleGradient(U, u, x, g_u);
+   REQUIRE(g_u.Size() == fes.GetTrueVSize());
+
+   const real_t dJdu_fd = energy.DirectionalFD(u, x, du, zero_x, eps);
+   REQUIRE(energy.Dot(g_u, du) == MFEM_Approx(dJdu_fd, 1e-6, 1e-8));
+
+   // Derivative w.r.t. the mesh coordinates. This is the case that used to
+   // assemble into the solution space instead of the coordinate space.
+   Vector g_x;
+   energy.AssembleGradient(Coords, u, x, g_x);
+   REQUIRE(g_x.Size() == mfes->GetTrueVSize());
+
+   const real_t dJdx_fd = energy.DirectionalFD(u, x, zero_u, dx, eps);
+   REQUIRE(energy.Dot(g_x, dx) == MFEM_Approx(dJdx_fd, 1e-6, 1e-8));
+}
+
+} // namespace functional_gradient_test
+
+TEST_CASE("dFEM functional gradient assembles into a Vector",
+          "[Parallel][dFEM][functional-gradient]")
+{
+   const bool all_tests = launch_all_non_regression_tests;
+   const auto p = !all_tests ? 1 : GENERATE(1, 2, 3);
+
+   SECTION("2d")
+   {
+      const auto f = GENERATE("../../data/inline-quad.mesh",
+                              "../../data/star.mesh");
+      functional_gradient_test::functional_gradient<2>(f, p);
+   }
+
+   SECTION("3d")
+   {
+      const auto f = GENERATE("../../data/inline-hex.mesh");
+      functional_gradient_test::functional_gradient<3>(f, p);
+   }
+}
+
+#endif // MFEM_USE_MPI
+#endif // MFEM_USE_ENZYME

tests/unit/dfem/test_hyperelasticity.cpp

@@ -0,0 +1,312 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+#include "mfem.hpp"
+#ifdef MFEM_USE_MPI
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+using mfem::future::tensor;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using mfem::future::dual;
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+
+constexpr int dim = 3;
+constexpr int Displacement = 0;
+constexpr int Coords = 1;
+constexpr int Energy = 2;
+
+template <typename dscalar_t>
+struct NeoHookeanEnergy
+{
+   real_t D1 = 100.0;
+   real_t C1 = 50.0;
+
+   MFEM_HOST_DEVICE inline
+   void operator()(const tensor<dscalar_t, dim, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   dscalar_t &energy) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      const auto F = IdentityMatrix<dim>() + dudx;
+      const auto C = transpose(F) * F;
+      const auto JF = det(F);
+      const auto I1_bar = pow(JF, -2.0_r / 3.0_r) * tr(C);
+      const auto psi = D1 * (JF - 1.0_r) * (JF - 1.0_r)
+                       + C1 * (I1_bar - real_t(dim));
+      energy = psi * det(J) * w;
+   }
+};
+
+template <typename dscalar_t>
+struct NeoHookeanStress
+{
+   real_t D1 = 100.0;
+   real_t C1 = 50.0;
+
+   MFEM_HOST_DEVICE inline
+   void operator()(const tensor<dscalar_t, dim, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   tensor<dscalar_t, dim, dim> &dvdxi) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      const auto F = IdentityMatrix<dim>() + dudx;
+      const auto JF = det(F);
+      const auto FinvT = inv(transpose(F));
+      const auto I1 = tr(transpose(F) * F);
+      const auto P = 2.0_r * D1 * JF * (JF - 1.0_r) * FinvT
+                     + 2.0_r * C1 * pow(JF, -2.0_r / 3.0_r)
+                     * (F - (I1 / 3.0_r) * FinvT);
+      dvdxi = P * transpose(invJ) * det(J) * w;
+   }
+};
+
+class HyperelasticityProblem
+{
+public:
+   HyperelasticityProblem(ParFiniteElementSpace &fes,
+                          const IntegrationRule &ir,
+                          bool use_energy) :
+      fes(fes),
+      use_energy(use_energy),
+      qspace(*fes.GetParMesh(), ir),
+      qspace_vec(qspace, 1),
+      q(qspace_vec)
+   {
+      auto &mesh_nodes = *static_cast<ParGridFunction *>
+                         (fes.GetParMesh()->GetNodes());
+      mesh_nodes_fes = mesh_nodes.ParFESpace();
+      mesh_nodes.GetTrueDofs(mesh_nodes_tdofs);
+
+      const std::vector<FieldDescriptor> inputs =
+      {
+         {Displacement, &fes},
+         {Coords, mesh_nodes_fes}
+      };
+      std::vector<FieldDescriptor> outputs;
+      if (use_energy)
+      {
+         outputs = std::vector<FieldDescriptor>
+         {
+            {Energy, &qspace_vec}
+         };
+      }
+      else
+      {
+         outputs = std::vector<FieldDescriptor>
+         {
+            {Displacement, &fes}
+         };
+      }
+
+      dop = std::make_shared<DifferentiableOperator>(inputs, outputs,
+                                                     *fes.GetParMesh());
+
+      Array<int> all_domain_attr;
+      if (fes.GetMesh()->attributes.Size() > 0)
+      {
+         all_domain_attr.SetSize(fes.GetMesh()->attributes.Max());
+         all_domain_attr = 1;
+      }
+
+      auto derivatives = std::integer_sequence<size_t, Displacement> {};
+      if (use_energy)
+      {
+         NeoHookeanEnergy<dscalar_t> energy;
+         dop->AddDomainIntegrator<LocalQFBackend>(
+            energy,
+            Inputs<Gradient<Displacement>, Gradient<Coords>, Weight> {},
+            Outputs<FunctionalValue<Energy>> {},
+            ir, all_domain_attr, derivatives);
+      }
+      else
+      {
+         NeoHookeanStress<dscalar_t> stress;
+         dop->AddDomainIntegrator<LocalQFBackend>(
+            stress,
+            Inputs<Gradient<Displacement>, Gradient<Coords>, Weight> {},
+            Outputs<Gradient<Displacement>> {},
+            ir, all_domain_attr, derivatives);
+      }
+   }
+
+   void SetEssentialAttributes(const Array<int> &ess_bdr)
+   {
+      fes.GetEssentialTrueDofs(ess_bdr, ess_tdofs);
+   }
+
+   void SetPrescribedDisplacementAttributes(const Array<int> &disp_bdr)
+   {
+      fes.GetEssentialTrueDofs(disp_bdr, prescribed_tdofs);
+   }
+
+   const Array<int>& GetPrescribedDisplacementTDofs() const
+   {
+      return prescribed_tdofs;
+   }
+
+   const Array<int>& GetEssentialTDofs() const
+   {
+      return ess_tdofs;
+   }
+
+   void Residual(const Vector &x, Vector &r) const
+   {
+      MultiVector X{x, mesh_nodes_tdofs};
+      MultiVector R{r};
+      if (use_energy)
+      {
+         dop->GetDerivative(Displacement)->Mult(X, R);
+      }
+      else
+      {
+         dop->Mult(X, R);
+      }
+      r.SetSubVector(ess_tdofs, 0.0);
+   }
+
+   void GradientAction(const Vector &x, const Vector &dx, Vector &y) const
+   {
+      MultiVector X{x, mesh_nodes_tdofs};
+      std::shared_ptr<DerivativeOperator> derivative;
+      if (use_energy)
+      {
+         derivative = dop->GetSecondDerivative(Displacement, X);
+      }
+      else
+      {
+         derivative = dop->GetDerivative(Displacement, X);
+      }
+
+      Vector local_dx(dx);
+      local_dx.SetSubVector(ess_tdofs, 0.0);
+
+      MultiVector Y{y};
+      derivative->Mult(local_dx, Y);
+      y.SetSubVector(ess_tdofs, 0.0);
+   }
+
+private:
+   ParFiniteElementSpace &fes;
+   bool use_energy = false;
+   ParFiniteElementSpace *mesh_nodes_fes = nullptr;
+   QuadratureSpace qspace;
+   VectorQuadratureSpace qspace_vec;
+   QuadratureFunction q;
+   Vector mesh_nodes_tdofs;
+   Array<int> ess_tdofs;
+   Array<int> prescribed_tdofs;
+   std::shared_ptr<DifferentiableOperator> dop;
+};
+
+struct HyperelasticityTestContext
+{
+   HyperelasticityTestContext(bool use_energy)
+   {
+      Mesh mesh = Mesh::MakeCartesian3D(8, 2, 2, Element::HEXAHEDRON, 8.0, 1.0, 1.0);
+      mesh.EnsureNodes();
+      pmesh = std::make_unique<ParMesh>(MPI_COMM_WORLD, mesh);
+      mesh.Clear();
+      pmesh->EnsureNodes();
+
+      fec = std::make_unique<H1_FECollection>(1, dim);
+      fes = std::make_unique<ParFiniteElementSpace>(pmesh.get(), fec.get(), dim,
+                                                    Ordering::byNODES);
+      const IntegrationRule &ir = IntRules.Get(pmesh->GetTypicalElementGeometry(), 3);
+      problem = std::make_unique<HyperelasticityProblem>(*fes, ir, use_energy);
+
+      Array<int> ess_attr(pmesh->bdr_attributes.Max());
+      ess_attr = 0;
+      ess_attr[4] = 1;
+      ess_attr[2] = 1;
+      problem->SetEssentialAttributes(ess_attr);
+
+      Array<int> disp_attr(pmesh->bdr_attributes.Max());
+      disp_attr = 0;
+      disp_attr[2] = 1;
+      problem->SetPrescribedDisplacementAttributes(disp_attr);
+
+      state.SetSize(fes->GetTrueVSize());
+      state.Randomize(11);
+      state -= 0.5;
+      state *= 1.0e-3;
+      state.SetSubVector(problem->GetEssentialTDofs(), 0.0);
+      state.SetSubVector(problem->GetPrescribedDisplacementTDofs(), 1.0e-2);
+
+      direction.SetSize(fes->GetTrueVSize());
+      direction.Randomize(17);
+      direction -= 0.5;
+   }
+
+   std::unique_ptr<ParMesh> pmesh;
+   std::unique_ptr<H1_FECollection> fec;
+   std::unique_ptr<ParFiniteElementSpace> fes;
+   std::unique_ptr<HyperelasticityProblem> problem;
+   Vector state;
+   Vector direction;
+};
+
+
+TEST_CASE("dfem neo-hookean energy and stress agree",
+          "[Parallel][dFEM][GPU][Hyperelasticity]")
+{
+   HyperelasticityTestContext energy(true);
+   HyperelasticityTestContext stress(false);
+
+   REQUIRE(energy.state.Size() == stress.state.Size());
+
+   // Check residuals from energy and stress formulations should match.
+   Vector energy_residual(energy.state.Size());
+   Vector stress_residual(stress.state.Size());
+   energy.problem->Residual(energy.state, energy_residual);
+   stress.problem->Residual(stress.state, stress_residual);
+
+   Vector residual_diff(energy_residual);
+   residual_diff -= stress_residual;
+   REQUIRE(residual_diff.Norml2() < 1e-12);
+
+   // Check the energy Hessian action should match the stress Jacobian action.
+   Vector energy_action(energy.state.Size());
+   Vector stress_action(stress.state.Size());
+   energy.problem->GradientAction(energy.state, energy.direction, energy_action);
+   stress.problem->GradientAction(stress.state, stress.direction, stress_action);
+
+   Vector action_diff(energy_action);
+   action_diff -= stress_action;
+
+   if (verbose_tests)
+   {
+      mfem::out << "Energy Hessian action norm: " << energy_action.Norml2() <<
+                std::endl;
+      mfem::out << "Stress Jacobian action norm: " << stress_action.Norml2() <<
+                std::endl;
+   }
+
+   REQUIRE(action_diff.Norml2() < 1e-10);
+
+   REQUIRE(energy_action.Norml2() > 0.0);
+   REQUIRE(stress_action.Norml2() > 0.0);
+}
+
+#endif

tests/unit/dfem/test_jvp_vjp.cpp

@@ -0,0 +1,198 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+#ifndef _WIN32
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../linalg/tensor_arrays.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+
+#ifdef MFEM_USE_ENZYME
+using dreal_t = real_t;
+#else
+using dreal_t = dual<real_t, real_t>;
+#endif
+
+static constexpr int U = 0, V = 1, X = 2;
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM>
+struct global_qf
+{
+   void operator()(tensor_array<const dreal_t> &x,
+                   tensor_array<const dreal_t> &y,
+                   tensor_array<const real_t, DIM, DIM> &J,
+                   tensor_array<const real_t> &w,
+                   tensor_array<dreal_t> &z) const
+   {
+      mfem::forall(x.size(),
+                   [=] MFEM_HOST_DEVICE(int q)
+      {
+         const dreal_t xq = x(q), yq = y(q);
+         z(q) = sin(xq) * cos(yq) * (xq + yq) * w(q) * det(J(q));
+      });
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM>
+struct local_qf
+{
+   inline MFEM_HOST_DEVICE void operator()(const dreal_t &x,
+                                           const dreal_t &y,
+                                           const tensor<real_t, DIM, DIM> &J,
+                                           const real_t &w,
+                                           dreal_t &z) const
+   { z = sin(x) * cos(y) * (x + y) * w * det(J); }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM>
+static void VerifyJvpVjp(DifferentiableOperator &F,
+                         ParFiniteElementSpace &fes,
+                         ParGridFunction &nodes)
+{
+   const auto nfes = nodes.ParFESpace();
+   const auto tvsize = fes.GetTrueVSize();
+   const auto ntvsize = nfes->GetTrueVSize();
+   const auto comm = fes.GetParMesh()->GetComm();
+
+   Vector X_bar(tvsize), Y_bar(tvsize), N_bar(ntvsize);
+
+   X_bar.Randomize(0x9e3779b9);
+   Y_bar.Randomize(0x9e3779b1);
+   nodes.GetTrueDofs(N_bar);
+
+   MultiVector state{ X_bar, Y_bar, N_bar };
+
+   Vector dX(tvsize), dY(tvsize), dZ(tvsize);
+   dX.Randomize(0x01000193);
+   dY.Randomize(0x1b873593);
+
+   Vector dU(tvsize), dV(tvsize);
+   MultiVector mdU{ dU }, mdV{ dV };
+
+   const auto dFu = F.GetDerivative(U, state);
+   const auto dFv = F.GetDerivative(V, state);
+   dFu->Mult(dX, mdU); // dU = (∂F/∂u) dX
+   dFv->Mult(dY, mdV); // dV = (∂F/∂v) dY
+   add(dU, dV, dZ);
+
+   Vector dX_star(tvsize), dY_star(tvsize), dZ_star(tvsize);
+   dZ_star.Randomize(0x7ed55d16);
+
+   MultiVector mdZ_star{ dZ_star }, mdX_star{ dX_star }, mdY_star{ dY_star };
+   dFu->MultTranspose(mdZ_star, mdX_star); // dX* = (∂F/∂u)^T dZ*
+   dFv->MultTranspose(mdZ_star, mdY_star); // dY* = (∂F/∂v)^T dZ*
+
+   // Tangent/cotangent consistency test:
+   // <dZ*, dZ> = <dX*, dX> + <dY*, dY>
+   REQUIRE(InnerProduct(comm, dZ_star, dZ) ==
+           MFEM_Approx(InnerProduct(comm, dX_star, dX) +
+                       InnerProduct(comm, dY_star, dY)));
+
+   REQUIRE(InnerProduct(comm, dZ_star, dU) ==
+           MFEM_Approx(InnerProduct(comm, dX_star, dX)));
+
+   REQUIRE(InnerProduct(comm, dZ_star, dV) ==
+           MFEM_Approx(InnerProduct(comm, dY_star, dY)));
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM>
+void TestJvpVjp(const char *filename, int p)
+{
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   smesh.Clear();
+
+   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace fes(&pmesh, &fec);
+   const auto *nfes = nodes->ParFESpace();
+
+   const auto geom = pmesh.GetTypicalElementGeometry();
+   const auto ir = &IntRules.Get(geom, 2 * p + 1);
+
+   using vfds_t = std::vector<FieldDescriptor>;
+   const vfds_t in_fds = { { U, &fes }, { V, &fes }, { X, nfes } };
+   const vfds_t out_fds = { { U, &fes } };
+   DifferentiableOperator F(in_fds, out_fds, pmesh);
+
+   using IT = Inputs<Value<U>, Value<V>, Gradient<X>, Weight>;
+   using OT = Outputs<Value<U>>;
+   using DT = Derivatives<U, V>;
+
+   if constexpr (!mfem_use_gpu)
+   {
+      global_qf<DIM> q_gfn{};
+      F.AddDomainIntegrator<GlobalQFBackend>(
+         q_gfn, IT{}, OT{}, *ir, all_domain_attr, DT{});
+   }
+
+   using LQT = local_qf<DIM>;
+   local_qf<DIM> q_lfn{};
+   F.AddDomainIntegrator<LocalQFBackend>(
+      q_lfn, IT{}, OT{}, *ir, all_domain_attr, DT{});
+   AddLocalSpecializations<DIM, 3, LQT, IT, OT, DT>();
+
+   VerifyJvpVjp<DIM>(F, fes, *nodes);
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM JVP-VJP 2D", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({ 1 }, { 2, 3 });
+   const auto meshs = { "../../data/inline-quad.mesh" };
+   const auto extra = { "../../data/star.mesh",
+                        "../../data/star-q3.mesh",
+                        "../../data/rt-2d-q3.mesh",
+                        "../../data/periodic-square.mesh"
+                      };
+   TestJvpVjp<2>(GenAll(meshs, extra), p);
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM JVP-VJP 3D", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({ 1 }, { 2, 3 });
+   const auto meshs = { "../../data/inline-hex.mesh" };
+   const auto extra = { "../../data/fichera.mesh",
+                        "../../data/fichera-q3.mesh",
+                        "../../data/toroid-hex.mesh",
+                        "../../data/periodic-cube.mesh"
+                      };
+   TestJvpVjp<3>(GenAll(meshs, extra), p);
+}
+
+#endif // _WIN32
+#endif // MFEM_USE_MPI

tests/unit/dfem/test_lvector_interface.cpp

@@ -10,37 +10,37 @@
 // CONTRIBUTING.md for details.
 
 #include "../unit_tests.hpp"
+
 #include "mfem.hpp"
 
 #ifdef MFEM_USE_MPI
 
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
 using namespace mfem;
 using namespace mfem::future;
-using mfem::future::tensor;
 
-constexpr int DIM = 3;
-
-namespace kernels
-{
+template <int DIM>
 struct MFApply
 {
-   MFEM_HOST_DEVICE inline auto operator()(const tensor<real_t, DIM> &dudxi,
-                                           const tensor<real_t, DIM, DIM> &J,
-                                           const real_t &w) const
+   MFEM_HOST_DEVICE inline auto operator()(
+      const tensor<real_t, DIM> &dudxi,
+      const tensor<real_t, DIM, DIM> &J,
+      const real_t &w,
+      tensor<real_t, DIM> &dvdxi) const
    {
       const auto invJ = inv(J);
-      return tuple{ (dudxi * invJ) * transpose(invJ) * det(J) * w };
+      const auto invJt = transpose(invJ);
+      dvdxi = (dudxi * invJ) * invJt * det(J) * w;
    }
 };
-}
 
-TEST_CASE("DFEM L-Vector interface", "[Parallel][dFEM][GPU]")
+template <int DIM, typename QFBackend = LocalQFBackend>
+void l_vector_interface(const char *filename, int p)
 {
-   constexpr int p = 2; // Polynomial order
-   constexpr int r = 1;
-   constexpr int q = 2 * p + r;
+   CAPTURE(filename, DIM, p);
 
-   const auto filename = GENERATE("../../data/fichera.mesh");
    Mesh smesh(filename);
    ParMesh pmesh(MPI_COMM_WORLD, smesh);
    MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
@@ -49,6 +49,9 @@ TEST_CASE("DFEM L-Vector interface", "[Parallel][dFEM][GPU]")
    auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
    smesh.Clear();
 
+   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+   const int q = 2 * p + 1;
+
    Array<int> all_domain_attr;
    if (pmesh.attributes.Size() > 0)
    {
@@ -75,18 +78,28 @@ TEST_CASE("DFEM L-Vector interface", "[Parallel][dFEM][GPU]")
 
    static constexpr int U = 0, Coords = 1;
 
-   const auto solution = std::vector{FieldDescriptor{U, &pfes}};
-   DifferentiableOperator dop(solution, {{Coords, mfes}}, pmesh);
+   const auto in_fds = std::vector
+   {
+      FieldDescriptor{ U, &pfes },
+      FieldDescriptor{ Coords, mfes }
+   };
+   const auto out_fds = std::vector{ FieldDescriptor{ U, &pfes } };
 
-   kernels::MFApply mf_apply_qf;
-   dop.AddDomainIntegrator(mf_apply_qf,
-                           tuple{Gradient<U>{}, Gradient<Coords>{}, Weight{}},
-                           tuple{Gradient<U>{}}, *ir, all_domain_attr);
+   DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+   MFApply<DIM> mf_apply;
+   dop.AddDomainIntegrator<LocalQFBackend>(
+      mf_apply,
+      tuple{Gradient<U>{}, Gradient<Coords>{}, Weight{}},
+      tuple{Gradient<U>{}},
+      *ir, all_domain_attr);
 
    // Use the L-vector interface to multiply
    dop.SetMultLevel(DifferentiableOperator::MultLevel::LVECTOR);
-   dop.SetParameters({nodes});
-   dop.Mult(x, z);
+
+   MultiVector mx{x, *nodes};
+   MultiVector mz{z};
+   dop.Mult(mx, mz);
 
    blf_fa.Mult(x, y);
 
@@ -94,4 +107,28 @@ TEST_CASE("DFEM L-Vector interface", "[Parallel][dFEM][GPU]")
    REQUIRE(z.Normlinf() == MFEM_Approx(0.0));
 }
 
-#endif
+TEST_CASE("dFEM L-Vector 2D", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3});
+   const auto meshs = { "../../data/inline-quad.mesh" };
+   const auto extra = { "../../data/star.mesh",
+                        "../../data/star-q3.mesh",
+                        "../../data/rt-2d-q3.mesh",
+                        "../../data/periodic-square.mesh"
+                      };
+   l_vector_interface<2>(GenAll(meshs, extra), p);
+}
+
+TEST_CASE("dFEM L-Vector 3D", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3});
+   const auto meshs = { "../../data/inline-hex.mesh" };
+   const auto extra = { "../../data/fichera.mesh",
+                        "../../data/fichera-q3.mesh",
+                        "../../data/toroid-hex.mesh",
+                        "../../data/periodic-cube.mesh"
+                      };
+   l_vector_interface<3>(GenAll(meshs, extra), p);
+}
+
+#endif // MFEM_USE_MPI

tests/unit/dfem/test_mass.cpp

@@ -10,160 +10,82 @@
 // CONTRIBUTING.md for details.
 
 #include "../unit_tests.hpp"
-#include "../linalg/test_same_matrices.hpp"
+
 #include "mfem.hpp"
 
 #ifdef MFEM_USE_MPI
 
+#include "../linalg/test_same_matrices.hpp"
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
 using namespace mfem;
 using namespace mfem::future;
-using mfem::future::tensor;
 
 #ifdef MFEM_USE_ENZYME
 using dscalar_t = real_t;
 #else
-using mfem::future::dual;
 using dscalar_t = dual<real_t, real_t>;
 #endif
 
+// ────────────────────────────────────────────────────────────────────────────
+template <int DIM> struct global_mf_mass_qf
+{
+   void operator()(tensor_array<const dscalar_t> &u,
+                   tensor_array<const real_t, DIM, DIM> &J,
+                   tensor_array<const real_t> &w,
+                   tensor_array<dscalar_t> &v) const
+   {
+      // Enzyme requires extra support to handle:
+      // mfem::forall(u.size(), [=] MFEM_HOST_DEVICE (int q)
+      for (size_t q = 0; q < u.size(); ++q)
+      {
+         v(q) = (dscalar_t)(u(q)) * w(q) * det(J(q));
+      }
+   }
+};
+
+template <int DIM> struct local_mf_mass_qf
+{
+   inline MFEM_HOST_DEVICE
+   void operator()(const dscalar_t &u,
+                   const tensor<real_t, DIM, DIM> &J,
+                   const real_t &w,
+                   dscalar_t &v) const
+   {
+      v = u * w * det(J);
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template <int DIM> struct local_mf_vector_mass_qf
+{
+   MFEM_HOST_DEVICE inline auto operator()(
+      const tensor<dscalar_t, DIM> &vu,
+      const tensor<real_t, DIM, DIM> &J,
+      const real_t &w,
+      tensor<dscalar_t, DIM> &vv) const
+   {
+      vv = vu * w * det(J);
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
 template <int DIM>
 void mass_action(const char *filename, int p)
 {
-   constexpr int BDIM = DIM - 1;
    CAPTURE(filename, DIM, p);
 
    Mesh smesh(filename);
    ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
+
    pmesh.EnsureNodes();
    auto* nodes = static_cast<ParGridFunction*>(pmesh.GetNodes());
-   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
    smesh.Clear();
 
-   H1_FECollection fec(p, DIM);
-   ParFiniteElementSpace fes(&pmesh, &fec);
-
-   ParGridFunction x(&fes), y(&fes), z(&fes);
-   Vector X(fes.GetTrueVSize()), Y(fes.GetTrueVSize()), Z(fes.GetTrueVSize());
-
-   X.Randomize(1);
-   x.SetFromTrueDofs(X);
-
-   ConstantCoefficient one(1.0);
-
-   SECTION("domain")
-   {
-      const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
-
-      Array<int> all_domain_attr;
-      if (pmesh.attributes.Size() > 0)
-      {
-         all_domain_attr.SetSize(pmesh.attributes.Max());
-         all_domain_attr = 1;
-      }
-
-      ParBilinearForm blf(&fes);
-      blf.AddDomainIntegrator(new MassIntegrator(one, ir));
-      blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      blf.Assemble();
-      blf.Mult(x, y);
-      fes.GetProlongationMatrix()->MultTranspose(y, Y);
-
-      static constexpr int U = 0, Coords = 1;
-      const auto sol = std::vector{ FieldDescriptor{ U, &fes } };
-      DifferentiableOperator dop(sol, {{Coords, nodes->ParFESpace()}}, pmesh);
-      const auto mf_mass_qf =
-         [] MFEM_HOST_DEVICE(const real_t &u,
-                             const tensor<real_t, DIM, DIM> &J, const real_t &w)
-      { return tuple{u * w * det(J)}; };
-      dop.AddDomainIntegrator(mf_mass_qf,
-                              tuple{ Value<U>{}, Gradient<Coords>{}, Weight{} },
-                              tuple{ Value<U>{} },
-                              *ir, all_domain_attr);
-      dop.SetParameters({ nodes });
-
-      fes.GetRestrictionMatrix()->Mult(x, X);
-      dop.Mult(X, Z);
-      Y -= Z;
-
-      real_t norm_g, norm_l = Y.Normlinf();
-      MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-      REQUIRE(norm_g == MFEM_Approx(0.0));
-      MPI_Barrier(MPI_COMM_WORLD);
-   }
-
-   // Test boundary
-   // This ensures that we're not trying to test on fully periodic meshes
-   if (!((std::string("../../data/periodic-square.mesh").compare(filename) == 0) ||
-         (std::string("../../data/periodic-cube.mesh").compare(filename) == 0)))
-   {
-      SECTION("boundary")
-      {
-         const auto *ir = &IntRules.Get(pmesh.GetTypicalFaceGeometry(), 2 * p);
-
-         Array<int> all_bdr_attr;
-         if (pmesh.bdr_attributes.Size() > 0)
-         {
-            all_bdr_attr.SetSize(pmesh.bdr_attributes.Max());
-            all_bdr_attr = 1;
-         }
-
-         ParBilinearForm blf(&fes);
-         blf.AddBoundaryIntegrator(new MassIntegrator(one, ir));
-         blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-         blf.Assemble();
-         blf.Mult(x, y);
-         fes.GetProlongationMatrix()->MultTranspose(y, Y);
-
-         static constexpr int U = 0, Coords = 1;
-         const auto sol = std::vector{FieldDescriptor{U, &fes}};
-         DifferentiableOperator dop(sol, {{Coords, nodes->ParFESpace()}}, pmesh);
-
-         const auto mf_mass_qf =
-            [] MFEM_HOST_DEVICE(const dscalar_t &u,
-                                const tensor<real_t, DIM, BDIM> &J,
-                                const real_t &w)
-         {
-            return tuple{u * weight(J) * w};
-         };
-
-         auto derivatives = std::integer_sequence<size_t, U> {};
-         dop.AddBoundaryIntegrator(mf_mass_qf,
-                                   tuple{ Value<U>{}, Gradient<Coords>{}, Weight{} },
-                                   tuple{ Value<U>{} },
-                                   *ir, all_bdr_attr, derivatives);
-         dop.SetParameters({nodes});
-
-         fes.GetRestrictionMatrix()->Mult(x, X);
-         dop.Mult(X, Z);
-
-         Y -= Z;
-         real_t norm_g, norm_l = Y.Normlinf();
-         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-         REQUIRE(norm_g == MFEM_Approx(0.0));
-
-         auto dRdU = dop.GetDerivative(U, {&x}, {nodes});
-         dRdU->Mult(X, Z);
-
-         fes.GetProlongationMatrix()->MultTranspose(y, Y);
-         Y -= Z;
-         norm_l = Y.Normlinf();
-         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-         REQUIRE(norm_g == MFEM_Approx(0.0));
-         MPI_Barrier(MPI_COMM_WORLD);
-      }
-   }
-}
-
-template <int DIM> void mass_mat_mixed(const char* filename, int p)
-{
-   CAPTURE(filename, DIM, p);
-
-   Mesh smesh(filename);
-   ParMesh pmesh(MPI_COMM_WORLD, smesh);
-   pmesh.EnsureNodes();
-   auto* nodes = static_cast<ParGridFunction*>(pmesh.GetNodes());
    p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
-   smesh.Clear();
 
    Array<int> all_domain_attr;
    if (pmesh.attributes.Size() > 0)
@@ -172,51 +94,394 @@ template <int DIM> void mass_mat_mixed(const char* filename, int p)
       all_domain_attr = 1;
    }
 
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   H1_FECollection fec(p, DIM);
+
+   ParFiniteElementSpace pfes(&pmesh, &fec);
+   const int tvsize = pfes.GetTrueVSize();
+
+   ParGridFunction x(&pfes), y(&pfes), z(&pfes);
+   Vector X(tvsize), Y(tvsize), Z(tvsize), dZ(tvsize);
+
+   X.Randomize(1);
+   x.SetFromTrueDofs(X);
+
+   ConstantCoefficient one(1.0), zero(0.0);
+
+   ParBilinearForm blf(&pfes);
+   // Add two mass integrators as we use both local and global QF backends
+   // If GPU is enabled, we only add the local QF backend
+   blf.AddDomainIntegrator(new MassIntegrator(one, ir));
+   if constexpr(!mfem_use_gpu)
+   {
+      blf.AddDomainIntegrator(new MassIntegrator(one, ir));
+   }
+   blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   blf.Assemble();
+
+   static constexpr int U = 0, Coords = 1;
+   const auto in_fds = std::vector
+   {
+      FieldDescriptor{ U, &pfes },
+      FieldDescriptor{ Coords, mfes }
+   };
+   const auto out_fds = std::vector{ FieldDescriptor{ U, &pfes } };
+
+   Vector N;
+   nodes->GetTrueDofs(N);
+
+   using IT = Inputs<Value<U>, Gradient<Coords>, Weight>;
+   using OT = Outputs<Value<U>>;
+   using DT = Derivatives<U>;
+
+   SECTION("Action")
+   {
+      blf.Mult(x, y);
+      pfes.GetProlongationMatrix()->MultTranspose(y, Y);
+
+      DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+      local_mf_mass_qf<DIM> local_qfn;
+      dop.AddDomainIntegrator<LocalQFBackend>(
+         local_qfn, IT {}, OT {}, *ir, all_domain_attr);
+
+      if constexpr(!mfem_use_gpu)
+      {
+         global_mf_mass_qf<DIM> global_qfn;
+         dop.AddDomainIntegrator<GlobalQFBackend>(
+            global_qfn, IT {}, OT {}, *ir, all_domain_attr);
+      }
+
+      MultiVector MX{X, N}, MZ{Z};
+      dop.Mult(MX, MZ);
+      Y -= Z;
+
+      y.SetFromTrueDofs(Y);
+      REQUIRE(y.ComputeMaxError(zero) == MFEM_Approx(0.0));
+   }
+
+   SECTION("Action Linearized")
+   {
+      // Randomize again s.t. the setup cache cannot
+      // trivially succeed by caching one direction only.
+      X.Randomize(0x9e3779b9);
+      x.SetFromTrueDofs(X);
+
+      DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+      local_mf_mass_qf<DIM> local_qfn;
+      dop.AddDomainIntegrator<LocalQFBackend>(
+         local_qfn, IT {}, OT {}, *ir, all_domain_attr, DT {});
+
+      if constexpr(!mfem_use_gpu)
+      {
+         global_mf_mass_qf<DIM> global_qfn;
+         dop.AddDomainIntegrator<GlobalQFBackend>(
+            global_qfn, IT {}, OT {}, *ir, all_domain_attr, DT {});
+      }
+
+      MultiVector MX{X, N}, MZ{Z}, MdZ{dZ};
+
+      // without cache
+      {
+         blf.Mult(x, y);
+         pfes.GetProlongationMatrix()->MultTranspose(y, Y);
+
+         const bool no_cache = false;
+         auto ddop_nc = dop.GetDerivative(U, MX, no_cache);
+         ddop_nc->Mult(MX[0], MdZ);
+         Y -= dZ;
+
+         y.SetFromTrueDofs(Y);
+         REQUIRE(y.ComputeMaxError(zero) == MFEM_Approx(0.0));
+      }
+
+      // allow using the setup cache
+      {
+         blf.Mult(x, y);
+         pfes.GetProlongationMatrix()->MultTranspose(y, Y);
+
+         const bool use_cache = true;
+         auto ddop_wc = dop.GetDerivative(U, MX, use_cache);
+         ddop_wc->Mult(MX[0], MdZ);
+         Y -= dZ;
+
+         y.SetFromTrueDofs(Y);
+         REQUIRE(y.ComputeMaxError(zero) == MFEM_Approx(0.0));
+      }
+   }
+
+   SECTION("Assemble Diagonal")
+   {
+      DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+      local_mf_mass_qf<DIM> local_qfn;
+      dop.AddDomainIntegrator<LocalQFBackend>(
+         local_qfn, IT {}, OT {}, *ir, all_domain_attr, DT {});
+
+      if constexpr(!mfem_use_gpu)
+      {
+         global_mf_mass_qf<DIM> global_qfn;
+         dop.AddDomainIntegrator<GlobalQFBackend>(
+            global_qfn, IT {}, OT {}, *ir, all_domain_attr, DT {});
+      }
+
+      pfes.GetRestrictionMatrix()->Mult(x, X);
+
+      MultiVector MX{X, N};
+
+      auto dRdU = dop.GetDerivative(U, MX);
+
+      Vector dfem_D(tvsize), mfem_D(tvsize);
+      dRdU->AssembleDiagonal(dfem_D);
+      blf.AssembleDiagonal(mfem_D);
+      Y = mfem_D;
+
+      Y -= dfem_D;
+      y.SetFromTrueDofs(Y);
+      REQUIRE(y.ComputeMaxError(zero) == MFEM_Approx(0.0));
+   }
+
+   // Test boundary
+#if 0 // TODO: Boundary tests
+   // This ensures that we're not trying to test on fully periodic meshes
+   if (!((std::string("../../data/periodic-square.mesh").compare(filename) == 0) ||
+         (std::string("../../data/periodic-cube.mesh").compare(filename) == 0)))
+   {
+      constexpr int BDIM = DIM - 1;
+      SECTION("boundary")
+      {
+         Array<int> all_bdr_attr;
+         if (pmesh.bdr_attributes.Size() > 0)
+         {
+            all_bdr_attr.SetSize(pmesh.bdr_attributes.Max());
+            all_bdr_attr = 1;
+         }
+
+         ParBilinearForm blf(&pfes);
+         blf.AddBoundaryIntegrator(new MassIntegrator(one, ir));
+         blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+         blf.Assemble();
+         blf.Mult(x, y);
+         pfes.GetProlongationMatrix()->MultTranspose(y, Y);
+
+         static constexpr int U = 0, Coords = 1;
+         const auto in_fds = std::vector<FieldDescriptor> {{ U, &pfes }, { Coords, mfes }};
+         const auto out_fds = std::vector<FieldDescriptor> {{ U, &pfes }};
+         DifferentiableOperator dop(in_fds, out_fds, pmesh);
+         const auto mf_mass_qf =
+            [] MFEM_HOST_DEVICE(const dscalar_t &u,
+                                const tensor<real_t, DIM, BDIM> &J,
+                                const real_t &w,
+                                dscalar_t& v)
+         {
+            v = u * weight(J) * w;
+         };
+
+         auto derivatives = std::integer_sequence<size_t, U> {};
+         dop.AddBoundaryIntegrator<QFBackend>(mf_mass_qf,
+                                              tuple{ Value<U>{}, Gradient<Coords>{}, Weight{} },
+                                              tuple{ Value<U>{} },
+                                              *ir, all_bdr_attr, derivatives);
+
+         pfes.GetRestrictionMatrix()->Mult(x, X);
+
+         Vector N;
+         nodes->GetTrueDofs(N);
+         MultiVector MX{X, N}, MZ{Z};
+         dop.Mult(MX, MZ);
+
+         Y -= MZ[0];
+         real_t norm_g, norm_l = Y.Normlinf();
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+
+         nodes->GetTrueDofs(N);
+         auto dRdU = dop.GetDerivative(U, MX);
+         dRdU->Mult(MX[0], MZ);
+
+         pfes.GetProlongationMatrix()->MultTranspose(y, Y);
+         Y -= Z;
+         norm_l = Y.Normlinf();
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+   }
+#endif // TODO: Boundary tests
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template <int DIM>
+void vector_mass_action(const char *filename, int p)
+{
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
+
+   pmesh.EnsureNodes();
+   auto* nodes = static_cast<ParGridFunction*>(pmesh.GetNodes());
+   smesh.Clear();
+
+   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   H1_FECollection fec(p, DIM);
+
+   ParFiniteElementSpace vpfes(&pmesh, &fec, DIM);
+   const int tvsize = vpfes.GetTrueVSize();
+
+   ParGridFunction x(&vpfes), y(&vpfes), z(&vpfes);
+   Vector X(tvsize), Y(tvsize), Z(tvsize), dZ(tvsize);
+
+   X.Randomize(1);
+   x.SetFromTrueDofs(X);
+
+   ConstantCoefficient one(1.0), zero(0.0);
+
+   ParBilinearForm blf(&vpfes);
+   blf.AddDomainIntegrator(new VectorMassIntegrator(one, ir));
+   blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   blf.Assemble();
+
+   static constexpr int U = 0, Coords = 1;
+   const auto in_fds = std::vector
+   {
+      FieldDescriptor{ U, &vpfes },
+      FieldDescriptor{ Coords, mfes }
+   };
+   const auto out_fds = std::vector{ FieldDescriptor{ U, &vpfes } };
+
+   Vector N;
+   nodes->GetTrueDofs(N);
+
+   using IT = Inputs<Value<U>, Gradient<Coords>, Weight>;
+   using OT = Outputs<Value<U>>;
+
+   SECTION("Vector Mass Action")
+   {
+      blf.Mult(x, y);
+      vpfes.GetProlongationMatrix()->MultTranspose(y, Y);
+
+      DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+      local_mf_vector_mass_qf<DIM> local_qfn;
+      dop.AddDomainIntegrator<LocalQFBackend>(
+         local_qfn, IT {}, OT {}, *ir, all_domain_attr);
+
+      MultiVector MX{X, N}, MZ{Z};
+      dop.Mult(MX, MZ);
+      Y -= Z;
+
+      y.SetFromTrueDofs(Y);
+      REQUIRE(y.ComputeMaxError(zero) == MFEM_Approx(0.0));
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template <int DIM>
+void mass_mat_mixed(const char* filename, int p)
+{
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
+
+   pmesh.EnsureNodes();
+   auto* nodes = static_cast<ParGridFunction*>(pmesh.GetNodes());
+   smesh.Clear();
+
+   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+   const auto* ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
    H1_FECollection fec0(p, DIM);
    H1_FECollection fec1(p + 1, DIM);
    ParFiniteElementSpace fes0(&pmesh, &fec0);
    ParFiniteElementSpace fes1(&pmesh, &fec1);
 
-   const auto* ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
-
    ConstantCoefficient one(1.0);
    ParMixedBilinearForm blf(&fes1, &fes0);
    blf.AddDomainIntegrator(new MassIntegrator(one, ir));
    blf.SetAssemblyLevel(AssemblyLevel::FULL);
    blf.Assemble();
    blf.Finalize();
-
    blf.SpMat().Finalize();
 
    static constexpr int U = 0, P = 1, Coords = 2;
-   const auto sol = std::vector{FieldDescriptor{U, &fes1}};
-   DifferentiableOperator dop(sol, {{P, &fes0}, {Coords, nodes->ParFESpace()}},
-   pmesh);
-   const auto mf_mass_qf = [] MFEM_HOST_DEVICE(
-                              const dscalar_t& u,
-                              const tensor<real_t, DIM, DIM>& J,
-                              const real_t& w)
+   DifferentiableOperator dop(
+   { { U, &fes1 }, { Coords, mfes } },
+   { { P, &fes0 } }, pmesh);
+
+   local_mf_mass_qf<DIM> local_qfn;
+   dop.AddDomainIntegrator<LocalQFBackend>(
+      local_qfn,
+      Inputs<Value<U>, Gradient<Coords>, Weight> {},
+      Outputs<Value<P>> {},
+      *ir, all_domain_attr,
+      Derivatives<U> {});
+
+   ParGridFunction ugf(&fes1), pgf(&fes0);
+   ugf = 0.0, pgf = 0.0;
+
+   Vector xtvec(fes1.GetTrueVSize()), ytvec(fes0.GetTrueVSize());
+   Vector nodestv;
+
+   xtvec.Randomize(1);
+   ugf.SetFromTrueDofs(xtvec);
+   nodes->GetTrueDofs(nodestv);
+
+   fes1.GetRestrictionMatrix()->Mult(ugf, xtvec);
+   MultiVector X{xtvec, nodestv};
+
+   auto ddopdu = dop.GetDerivative(U, X, false);
+
+   SECTION("Action Linearized")
    {
-      return tuple{u * w * det(J)};
-   };
+      xtvec.Randomize(567);
+      ugf.SetFromTrueDofs(xtvec);
 
-   auto derivatives = std::integer_sequence<size_t, U> {};
-   dop.AddDomainIntegrator(mf_mass_qf,
-                           tuple{Value<U>{}, Gradient<Coords>{}, Weight{}},
-                           tuple{Value<P>{}},
-                           *ir, all_domain_attr, derivatives);
+      Vector dztvec(fes0.GetTrueVSize());
+      MultiVector DZ{dztvec};
+      ddopdu->Mult(X[0], DZ);
 
-   ParGridFunction ugf(&fes1);
-   ugf = 0.0;
+      blf.Mult(ugf, pgf);
+      fes0.GetProlongationMatrix()->MultTranspose(pgf, ytvec);
 
-   ParGridFunction pgf(&fes0);
-   pgf = 0.0;
+      ytvec -= dztvec;
 
-   dop.SetParameters({&pgf, nodes});
+      real_t norm_global = 0.0;
+      real_t norm_local = ytvec.Normlinf();
+      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
+                    pmesh.GetComm());
 
-   auto ddopdu = dop.GetDerivative(U, {&ugf}, {&pgf, nodes});
+      REQUIRE(norm_global == MFEM_Approx(0.0));
+      MPI_Barrier(MPI_COMM_WORLD);
+   }
 
-   SECTION("spmat")
+   SECTION("MFEM SparseMatrix")
    {
       SparseMatrix *A;
       ddopdu->Assemble(A);
@@ -224,7 +489,8 @@ template <int DIM> void mass_mat_mixed(const char* filename, int p)
       delete A;
    }
 
-   SECTION("hypre parallel mat")
+   // TODO Hypre parallel matrix
+   if constexpr(false && !mfem_use_gpu)
    {
       HypreParMatrix *Amfem = blf.ParallelAssemble();
 
@@ -236,39 +502,52 @@ template <int DIM> void mass_mat_mixed(const char* filename, int p)
    }
 }
 
-// no GPU tag to avoid failing 'hypre parallel mat' section
-TEST_CASE("dFEM Mass", "[Parallel][dFEM]")
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM Mass 2D", "[Parallel][dFEM][GPU]")
 {
-   const bool all_tests = launch_all_non_regression_tests;
+   const auto p = GenAll({1}, {2, 3});
+   const auto meshs = { "../../data/inline-quad.mesh" };
+   const auto extra = { "../../data/star.mesh",
+                        "../../data/star-q3.mesh",
+                        "../../data/rt-2d-q3.mesh",
+                        "../../data/periodic-square.mesh"
+                      };
 
-   const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
-
-   SECTION("2d")
+   SECTION("Scalar Mass")
    {
-      const auto filename2d =
-         GENERATE(
-            "../../data/star.mesh",
-            "../../data/star-q3.mesh",
-            "../../data/rt-2d-q3.mesh",
-            "../../data/inline-quad.mesh",
-            "../../data/periodic-square.mesh"
-         );
-      mass_action<2>(filename2d, p);
-      mass_mat_mixed<2>(filename2d, p);
+      mass_action<2>(GenAll(meshs, extra), p);
    }
-
-   SECTION("3d")
+   SECTION("Scalar Mass Mixed")
    {
-      const auto filename3d =
-         GENERATE(
-            "../../data/fichera.mesh",
-            "../../data/fichera-q3.mesh",
-            "../../data/inline-hex.mesh",
-            "../../data/toroid-hex.mesh",
-            "../../data/periodic-cube.mesh"
-         );
-      mass_action<3>(filename3d, p);
-      mass_mat_mixed<3>(filename3d, p);
+      mass_mat_mixed<2>(GenAll(meshs, extra), p);
+   }
+   SECTION("Vector Mass")
+   {
+      vector_mass_action<2>(GenAll(meshs, extra), p);
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM Mass 3D", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3});
+   const auto meshs = { "../../data/inline-hex.mesh" };
+   const auto extra = { "../../data/fichera.mesh",
+                        "../../data/fichera-q3.mesh",
+                        "../../data/toroid-hex.mesh",
+                        "../../data/periodic-cube.mesh"
+                      };
+   SECTION("Scalar Mass")
+   {
+      mass_action<3>(GenAll(meshs, extra), p);
+   }
+   SECTION("Scalar Mass Mixed")
+   {
+      mass_mat_mixed<3>(GenAll(meshs, extra), p);
+   }
+   SECTION("Vector Mass")
+   {
+      vector_mass_action<3>(GenAll(meshs, extra), p);
    }
 }
 

tests/unit/dfem/test_multiple_inputs.cpp

@@ -0,0 +1,213 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, typename ValSeq, typename GradSeq>
+struct multiply_inputs_qf;
+
+template<int DIM, std::size_t... Vs, std::size_t... Gs>
+struct multiply_inputs_qf<DIM, std::index_sequence<Vs...>,
+          std::index_sequence<Gs...>>
+{
+   template<std::size_t>
+   using value_t = const dscalar_t &;
+
+   template<std::size_t>
+   using gradient_t = const tensor<dscalar_t, DIM> &;
+
+   inline MFEM_HOST_DEVICE void operator()(
+      const tensor<real_t, DIM, DIM> &J,
+      value_t<Vs>... vs,
+      gradient_t<Gs>... gs,
+      const real_t &w,
+      dscalar_t &v) const
+   {
+      v = (vs * ...) * (real_t{1} + (w - w) * (gs(0) + ...)) * w * det(J);
+   }
+};
+
+template<int DIM, std::size_t Nv, std::size_t Ng>
+using multiply_inputs_qf_t =
+   multiply_inputs_qf<DIM,
+   std::make_index_sequence<Nv>,
+   std::make_index_sequence<Ng>>;
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int I>
+using Values = Value<I>;
+
+template<int I>
+using Gradients = Gradient<I>;
+
+template<int... Vs, int... Gs>
+constexpr auto make_inputs(std::integer_sequence<int, Vs...>,
+                           std::integer_sequence<int, Gs...>)
+{
+   return Inputs<Gradient<0>, Values<1 + Vs>...,
+          Gradients<1 + sizeof...(Vs) + Gs>..., Weight> {};
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+struct InputsTestContext
+{
+   ParMesh &pmesh;
+   ParFiniteElementSpace &pfes;
+   ParFiniteElementSpace *mfes;
+   const IntegrationRule &ir;
+   Array<int> &all_domain_attr;
+   const Vector &Y_ref;
+   Vector &N_vec;
+   Vector &input;
+   int tvsize;
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, std::size_t Nv, std::size_t Ng>
+void test_nv_ng_inputs(const InputsTestContext &ctx)
+{
+   constexpr std::size_t Ni = Nv + Ng;
+
+   // inputs & output
+   std::vector<FieldDescriptor> in_fds;
+   in_fds.reserve(Ni + 1);
+   in_fds.emplace_back(0, ctx.mfes); // Coords
+   for (std::size_t i = 1; i <= Ni; ++i)
+   {
+      in_fds.emplace_back(i, &ctx.pfes);
+   }
+   const auto out_fds = std::vector{ FieldDescriptor{ 1, &ctx.pfes } };
+
+   // Prepare the MultiVector inputs
+   Array<int> mx_sizes(Ni + 1);
+   mx_sizes[0] = ctx.N_vec.Size(); // Coords
+   for (std::size_t i = 1; i <= Ni; ++i) { mx_sizes[i] = ctx.tvsize; }
+
+   MultiVector MX;
+   MX.SetSizes(mx_sizes);
+   MX.MakeRef(0, ctx.N_vec);
+   for (std::size_t i = 1; i <= Ni; ++i)
+   {
+      MX.MakeRef(static_cast<int>(i), ctx.input);
+   }
+
+   using IT = decltype(make_inputs(
+                          std::make_integer_sequence<int, Nv> {},
+                          std::make_integer_sequence<int, Ng> {}));
+   using OT = Outputs<Value<1>>;
+
+   multiply_inputs_qf_t<DIM, Nv, Ng> qfn;
+
+   DifferentiableOperator dop(in_fds, out_fds, ctx.pmesh);
+   dop.AddDomainIntegrator<LocalQFBackend>(
+      qfn, IT{}, OT{}, ctx.ir, ctx.all_domain_attr);
+
+   Vector Z(ctx.tvsize);
+   MultiVector MZ{ Z };
+   dop.Mult(MX, MZ);
+
+   Vector Y_diff(ctx.Y_ref);
+   Y_diff -= Z;
+
+   ParGridFunction y(&ctx.pfes);
+   ConstantCoefficient zero(0.0);
+   y.SetFromTrueDofs(Y_diff);
+   REQUIRE(y.ComputeMaxError(zero) == MFEM_Approx(0.0));
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+void test_multiple_inputs(int p)
+{
+   static constexpr int DIM = 2;
+   CAPTURE(DIM, p);
+
+   Mesh smesh("../../data/inline-quad.mesh");
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   smesh.Clear();
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+   const auto &ir = IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace pfes(&pmesh, &fec);
+   const int tvsize = pfes.GetTrueVSize();
+
+   ParGridFunction x(&pfes), y(&pfes);
+   Vector X_ref(tvsize), Y_ref(tvsize);
+
+   X_ref = 1.0;
+   x.SetFromTrueDofs(X_ref);
+
+   ConstantCoefficient one(1.0);
+
+   ParBilinearForm blf(&pfes);
+   blf.AddDomainIntegrator(new MassIntegrator(one, &ir));
+   blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   blf.Assemble();
+   blf.Mult(x, y);
+   pfes.GetProlongationMatrix()->MultTranspose(y, Y_ref);
+
+   Vector N_vec;
+   nodes->GetTrueDofs(N_vec);
+
+   Vector input(tvsize);
+   input = 1.0;
+
+   const InputsTestContext ctx
+   {
+      pmesh, pfes, mfes, ir, all_domain_attr, Y_ref, N_vec, input, tvsize
+   };
+
+   // with current dFEM tuple: max 9 = J + 3 + 3 + weights + output
+#ifndef _WIN32
+   test_nv_ng_inputs<DIM, 3, 3>(ctx);
+#else
+   // avoiding 'number of sections exceeded object file format limit' error
+   test_nv_ng_inputs<DIM, 2, 2>(ctx);
+#endif // _WIN32
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM Inputs", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3, 8});
+   test_multiple_inputs(p);
+}
+
+#endif // MFEM_USE_MPI

tests/unit/dfem/test_multiple_outputs.cpp

@@ -0,0 +1,495 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include <memory>
+
+#include "../unit_tests.hpp"
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+#include "../../../linalg/tensor_arrays.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+constexpr int DIM = 2;
+
+class DummyParameterSpace : public ParameterSpace
+{
+public:
+   class Bimpl : public Operator
+   {
+      void Mult(const Vector &x, Vector &y) const override
+      {
+         const bool use_dev = x.UseDevice() || y.UseDevice();
+         const auto xr = x.Read(use_dev);
+         auto yw = y.Write(use_dev);
+         mfem::forall_switch(use_dev, y.Size(), [=] MFEM_HOST_DEVICE (int i)
+         {
+            yw[i] = xr[0];
+         });
+      }
+   };
+
+   class Btimpl : public Operator
+   {
+      void Mult(const Vector &x, Vector &y) const override
+      {
+         const bool use_dev = x.UseDevice() || y.UseDevice();
+         const auto xr = x.Read(use_dev);
+         auto yw = y.Write(use_dev);
+         mfem::forall_switch(use_dev, 1, [=] MFEM_HOST_DEVICE (int)
+         {
+            yw[0] = xr[0];
+         });
+      }
+   };
+
+   DummyParameterSpace() : ParameterSpace(1) {}
+
+   int GetTrueVSize() const override
+   {
+      return 1;
+   }
+
+   int GetVSize() const override
+   {
+      return 1;
+   }
+
+   const Operator* GetB() const override
+   {
+      if (!B)
+      {
+         B = std::make_unique<Bimpl>();
+      }
+      return B.get();
+   }
+
+   const Operator* GetBt() const override
+   {
+      if (!Bt)
+      {
+         Bt = std::make_unique<Btimpl>();
+      }
+      return Bt.get();
+   }
+};
+
+/*struct mass_global_qf
+{
+   void operator()(
+      tensor_array<const dscalar_t> &u,
+      tensor_array<const real_t, DIM, DIM> &J,
+      tensor_array<const real_t> &w,
+      tensor_array<dscalar_t> &out1,
+      tensor_array<dscalar_t> &out2) const
+   {
+      mfem::forall(u.size(), [=] MFEM_HOST_DEVICE (int q)
+      {
+         const auto v = u(q) * det(J(q)) * w(q);
+         out1(q) = v;
+         out2(q) = v;
+      });
+   }
+};*/
+
+// __enzyme_fwddiff(.....
+struct mass_diffusion_global_qf
+{
+   void operator()(
+      tensor_array<const dscalar_t> &u,
+      tensor_array<const dscalar_t, DIM> &dudxi,
+      tensor_array<const real_t, DIM, DIM> &J,
+      [[maybe_unused]] tensor_array<const real_t, DIM, DIM> &qdata,
+      tensor_array<const real_t> &w,
+      [[maybe_unused]] tensor_array<const real_t> &dummy_parameter,
+      tensor_array<dscalar_t> &out1,
+      tensor_array<dscalar_t, DIM> &out2,
+      tensor_array<real_t, DIM, DIM> &out3) const
+   {
+      mfem::forall<UseEnzyme>(u.size(), [=] MFEM_HOST_DEVICE (int q)
+      {
+         const auto invJq = inv(J(q));
+         const auto detJq = det(J(q));
+         const real_t weight = detJq * w(q);
+         out1(q) = u(q) * weight;
+         out2(q) = (dudxi(q) * invJq) * transpose(invJq) * (detJq * w(q));
+         out3(q) = J(q);
+      });
+   }
+};
+// ..... );
+
+struct mass_local_qf
+{
+   inline MFEM_HOST_DEVICE
+   void operator()(
+      const dscalar_t &u,
+      const tensor<real_t, DIM, DIM> &J,
+      const real_t &w,
+      dscalar_t &out1,
+      dscalar_t &out2) const
+   {
+      const auto v = u * det(J) * w;
+      out1 = v;
+      out2 = v;
+   }
+};
+
+struct mass_diffusion_local_qf
+{
+   inline MFEM_HOST_DEVICE
+   void operator()(
+      const real_t &u,
+      const tensor<real_t, DIM> &dudxi,
+      const tensor<real_t, DIM, DIM> &J,
+      const real_t &w,
+      real_t &out1,
+      tensor<real_t, DIM> &out2) const
+   {
+      const auto invJ = inv(J);
+      const auto detJ = det(J);
+      out1 = u * detJ * w;
+      out2 = (dudxi * invJ) * transpose(invJ) * (detJ * w);
+   }
+};
+
+TEST_CASE("dFEM Multiple Outputs", "[Parallel][dFEM][GPU]")
+{
+   const bool all_tests = launch_all_non_regression_tests;
+
+   const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
+   const char *filename = "../../data/inline-quad.mesh";
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   MFEM_ASSERT(smesh.Dimension() == DIM, "DIM and mesh dimension have to match");
+
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   pmesh.EnsureNodes();
+   auto* nodes = static_cast<ParGridFunction*>(pmesh.GetNodes());
+   smesh.Clear();
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace fes(&pmesh, &fec);
+
+   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   ParGridFunction x(&fes), y(&fes), z(&fes);
+
+   ConstantCoefficient one(1.0);
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   // {
+   //    Array<int> inoffsets(3);
+   // inoffsets[0] = 0;
+   // inoffsets[1] = fes.GetTrueVSize();
+   // inoffsets[2] = nodes->ParFESpace()->GetTrueVSize();
+   // inoffsets.PartialSum();
+
+   //    BlockVector X(inoffsets);
+   //    X.GetBlock(0).Randomize(1);
+   //    X.GetBlock(1) = *nodes;
+   //    x.SetFromTrueDofs(X.GetBlock(0));
+
+   //    Array<int> outoffsets(2);
+   //    outoffsets[0] = 0;
+   //    outoffsets[1] = fes.GetTrueVSize();
+   //    outoffsets.PartialSum();
+   //    BlockVector Z(outoffsets);
+
+   //    ParBilinearForm blf(&fes);
+   //    blf.AddDomainIntegrator(new MassIntegrator(one, ir));
+   //    blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   //    blf.Assemble();
+   //    blf.Mult(x, y);
+   //    Vector Y(fes.GetTrueVSize());
+   //    fes.GetProlongationMatrix()->MultTranspose(y, Y);
+
+   //    static constexpr int U = 0, COORDINATES = 1, V = 2;
+   //    const std::vector<FieldDescriptor> in
+   //    {
+   //       {U, &fes},
+   //       {COORDINATES, nodes->ParFESpace()}
+   //    };
+
+   //    const std::vector<FieldDescriptor> out // test spaces?
+   //    {
+   //       {V, &fes},
+   //    };
+   //    DifferentiableOperator dop(in, out, pmesh);
+
+   //    auto derivatives = std::integer_sequence<size_t, U> {};
+   //    auto mass_qfunc = massqf{};
+   //    dop.AddDomainIntegrator(mass_qfunc,
+   //                            tuple{ Value<U>{}, Gradient<COORDINATES>{}, Weight{} },
+   //                            tuple{ Value<V>{}, Value<V>{} },
+   //                            *ir, all_domain_attr, derivatives);
+
+   //    fes.GetRestrictionMatrix()->Mult(x, X.GetBlock(0));
+   //    dop.Mult(X, Z);
+
+   //    Vector Y0(Y);
+   //    Y0 *= 2.0;
+   //    Y0 -= Z.GetBlock(0);
+
+   //    real_t norm_g, norm_l = Y0.Normlinf();
+   //    MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+   //    REQUIRE(norm_g == MFEM_Approx(0.0));
+   //    MPI_Barrier(MPI_COMM_WORLD);
+
+   //    auto ddop = dop.GetDerivative(U, X);
+
+   //    ddop->Mult(X.GetBlock(0), Z);
+   //    Y0 = Y;
+   //    Y0 *= 2.0;
+   //    Y0 -= Z.GetBlock(0);
+
+   //    norm_l = Y0.Normlinf();
+   //    MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+   //    REQUIRE(norm_g == MFEM_Approx(0.0));
+   //    MPI_Barrier(MPI_COMM_WORLD);
+   // }
+
+   {
+      QuadratureSpace qs(pmesh, *ir);
+      VectorQuadratureSpace vqs(qs, DIM * DIM);
+      QuadratureFunction qdata(vqs);
+
+      DummyParameterSpace dps;
+      ParameterFunction dpf(dps);
+      dpf = 9.12345;
+
+      auto coef_func = [](const Vector &coords)
+      {
+         return coords[0] * coords[1] * (DIM == 3 ? coords[2] : 1.0);
+      };
+      FunctionCoefficient coef(coef_func);
+      x.ProjectCoefficient(coef);
+
+      Vector xtvec, ytvec, ytvecmfem;
+      x.GetTrueDofs(xtvec);
+      ytvec.SetSize(xtvec.Size());
+      ytvecmfem.SetSize(xtvec.Size());
+
+      Vector nodestvec;
+      nodes->GetTrueDofs(nodestvec);
+
+      qdata = 123.0;
+      Vector yqdata(qdata.Size());
+
+      static constexpr int U = 0, COORDINATES = 1, V = 2, S = 3, L = 4;
+
+#if !defined(MFEM_USE_HIP)
+      {
+         xtvec.UseDevice(true);
+         nodestvec.UseDevice(true);
+         qdata.UseDevice(true);
+         dpf.UseDevice(true);
+         ytvec.UseDevice(true);
+         yqdata.UseDevice(true);
+         MultiVector X{xtvec, nodestvec, qdata, dpf};
+         MultiVector Z{ytvec, yqdata};
+
+         ParBilinearForm blf(&fes);
+         blf.AddDomainIntegrator(new MassIntegrator(ir));
+         blf.AddDomainIntegrator(new DiffusionIntegrator(ir));
+         blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+         blf.Assemble();
+         blf.Mult(x, y);
+         fes.GetProlongationMatrix()->MultTranspose(y, ytvecmfem);
+
+         const std::vector<FieldDescriptor> in_fds
+         {
+            {U, &fes},
+            {COORDINATES, nodes->ParFESpace()},
+            {S, &vqs},
+            {L, &dps}
+         };
+
+         const std::vector<FieldDescriptor> out_fds
+         {
+            {V, &fes},
+            {S, &vqs}
+         };
+
+         DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+         dop.SetQLayouts({{Value<U>{}, {1, 0}}}, {});
+
+         auto derivatives = Derivatives<U> {};
+         auto mass_diffusion_qfunc = mass_diffusion_global_qf{};
+         dop.AddDomainIntegrator(
+            mass_diffusion_qfunc,
+            Inputs<Value<U>, Gradient<U>, Gradient<COORDINATES>, Identity<S>, Weight, Value<L>> {},
+            Outputs<Value<V>, Gradient<V>, Identity<S>> {},
+            *ir, all_domain_attr, derivatives);
+
+         fes.GetRestrictionMatrix()->Mult(x, xtvec);
+         dop.Mult(X, Z);
+
+         Vector Y0(ytvecmfem);
+         Y0.UseDevice(true);
+
+         Y0 -= Z[0];
+
+         real_t norm_l = Y0.Normlinf();
+         real_t norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+
+         auto ddop = dop.GetDerivative(U, X);
+
+         ddop->Mult(X[0], Z);
+         Z[0].HostRead();
+         Y0 = ytvecmfem;
+         Y0.HostRead();
+         Y0 -= Z[0];
+
+         norm_l = Y0.Normlinf();
+         norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+#endif
+
+      {
+         static constexpr int W = 0;
+
+         ParBilinearForm blf(&fes);
+         blf.AddDomainIntegrator(new MassIntegrator(ir));
+         blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+         blf.Assemble();
+         blf.Mult(x, y);
+         fes.GetProlongationMatrix()->MultTranspose(y, ytvecmfem);
+
+         const std::vector<FieldDescriptor> in_fds
+         {
+            {U, &fes},
+            {COORDINATES, nodes->ParFESpace()},
+         };
+
+         const std::vector<FieldDescriptor> out_fds
+         {
+            {V, &fes},
+            {W, &fes},
+         };
+
+         DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+         auto mass_qfunclocal = mass_local_qf{};
+         dop.AddDomainIntegrator<LocalQFBackend>(
+            mass_qfunclocal,
+            tuple{Value<U>{}, Gradient<COORDINATES>{}, Weight{}},
+            tuple{Value<V>{}, Value<W>{}},
+            *ir, all_domain_attr);
+
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         fes.GetRestrictionMatrix()->Mult(x, xtvec);
+         Vector ztvec(xtvec.Size());
+         Vector zztvec(xtvec.Size());
+
+         MultiVector X{xtvec, nodestv};
+         MultiVector Z{ztvec, zztvec};
+
+         dop.Mult(X, Z);
+
+         Vector Y0(ytvecmfem);
+         Y0 -= Z[0];
+
+         real_t norm_l = Y0.Normlinf();
+         real_t norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+
+         Vector Y1(ytvecmfem);
+         Y1 -= Z[1];
+
+         norm_l = Y1.Normlinf();
+         norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+
+      {
+         ParBilinearForm blf(&fes);
+         blf.AddDomainIntegrator(new MassIntegrator(ir));
+         blf.AddDomainIntegrator(new DiffusionIntegrator(ir));
+         blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+         blf.Assemble();
+         blf.Mult(x, y);
+         fes.GetProlongationMatrix()->MultTranspose(y, ytvecmfem);
+
+         const std::vector<FieldDescriptor> in_fds
+         {
+            {U, &fes},
+            {COORDINATES, nodes->ParFESpace()},
+         };
+
+         const std::vector<FieldDescriptor> out_fds
+         {
+            {V, &fes},
+         };
+
+         DifferentiableOperator dop(in_fds, out_fds, pmesh);
+
+         auto mass_diffusion_qfunclocal = mass_diffusion_local_qf{};
+         dop.AddDomainIntegrator<LocalQFBackend>(
+            mass_diffusion_qfunclocal,
+            tuple{Value<U>{}, Gradient<U>{}, Gradient<COORDINATES>{}, Weight{}},
+            tuple{Value<V>{}, Gradient<V>{}},
+            *ir, all_domain_attr);
+
+         Vector nodestv;
+         nodes->GetTrueDofs(nodestv);
+         fes.GetRestrictionMatrix()->Mult(x, xtvec);
+         Vector ztvec(xtvec.Size());
+
+         MultiVector X{xtvec, nodestv};
+         MultiVector Z{ztvec};
+
+         dop.Mult(X, Z);
+
+         Vector Y0(ytvecmfem);
+         Y0 -= Z[0];
+
+         real_t norm_l = Y0.Normlinf();
+         real_t norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+   }
+}
+
+#endif // MFEM_USE_MPI

tests/unit/dfem/test_second_derivative.cpp

@@ -0,0 +1,680 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_ENZYME
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+#include "../../../fem/dfem/backends/local_qf/revdiff_transformer.hpp"
+
+#ifdef MFEM_USE_MPI
+
+using namespace mfem;
+using namespace mfem::future;
+using mfem::future::tensor;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using mfem::future::dual;
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+
+namespace second_derivative_test
+{
+
+template <typename dscalar_t, int dim>
+struct MinimalSurfaceEnergyFunctional
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const dscalar_t &u,
+                   const tensor<dscalar_t, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &f /* dfdu, dfddudxi */
+                  ) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      const auto dx = det(J) * w;
+      const auto E = sqrt(1.0_r + sqnorm(dudx));
+      f = E * dx;
+   }
+};
+
+template <typename dscalar_t, int dim>
+struct MinimalSurfaceEnergy
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const tensor<dscalar_t, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &f) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      const auto dx = det(J) * w;
+      const auto E = sqrt(1.0_r + sqnorm(dudx));
+      f = E * dx;
+   }
+};
+
+template <typename dscalar_t, int dim>
+struct MinimalSurfaceResidual
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const tensor<dscalar_t, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   tensor<real_t, dim> &dvdx) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      const auto dx = det(J) * w;
+      dvdx = dudx / (sqrt(1.0_r + sqnorm(dudx))) * transpose(invJ) * dx;
+   }
+};
+
+// Hand-coded action of the second derivative of the minimal surface energy,
+// i.e. the Hessian-vector product integrand d^2 J(u)[delta_u, v].
+template <typename dscalar_t, int dim>
+struct MinimalSurfaceHessianAction
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const tensor<real_t, dim> &ddelta_udxi,
+                   const tensor<dscalar_t, dim> &dudxi,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   tensor<real_t, dim> &dvdx) const
+   {
+      const auto invJ = inv(J);
+      const auto dudx = dudxi * invJ;
+      const auto ddelta_udx = ddelta_udxi * invJ;
+      const auto dx = det(J) * w;
+      const auto c = 1.0_r / sqrt(1.0_r + sqnorm(dudx));
+      const auto term1 = c * ddelta_udx;
+      const auto term2 = c * c * c * dot(dudx, ddelta_udx) * dudx;
+      dvdx = (term1 - term2) * transpose(invJ) * dx;
+   }
+};
+
+
+// Functional for the mixed problem with two fields u and rho, with the energy functional:
+// J(u, rho) = int (rho u^2 + 0.5 rho^2) dx
+template <typename dscalar_t, int dim>
+struct MixedFunctional
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const dscalar_t &u,
+                   const dscalar_t &rho,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &f) const
+   {
+      f = (rho * u * u + 0.5_r * rho * rho) * det(J) * w;
+   }
+};
+
+template <typename dscalar_t, int dim>
+struct MixedFunctionalUUAction
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const real_t &du,
+                   const dscalar_t &rho,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &v) const
+   {
+      v = 2.0_r * rho * du * det(J) * w;
+   }
+};
+
+template <typename dscalar_t, int dim>
+struct MixedFunctionalURhoAction
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const real_t &drho,
+                   const dscalar_t &u,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &v) const
+   {
+      v = 2.0_r * u * drho * det(J) * w;
+   }
+};
+
+template <typename dscalar_t, int dim>
+struct MixedFunctionalRhoUAction
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const real_t &du,
+                   const dscalar_t &u,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &v) const
+   {
+      v = 2.0_r * u * du * det(J) * w;
+   }
+};
+
+template <typename dscalar_t, int dim>
+struct MixedFunctionalRhoRhoAction
+{
+   MFEM_HOST_DEVICE inline __attribute__((always_inline))
+   auto operator()(const real_t &drho,
+                   const tensor<real_t, dim, dim> &J,
+                   const real_t &w,
+                   real_t &v) const
+   {
+      v = drho * det(J) * w;
+   }
+};
+
+template <int dim>
+class MyFunctional
+{
+   static constexpr int U = 0, Coords = 1, Q = 2, DirU = 3;
+
+public:
+   MyFunctional(const ParFiniteElementSpace &fes,
+                const ParFiniteElementSpace &mfes,
+                const IntegrationRule &ir) :
+      comm(fes.GetComm()),
+      mesh(*mfes.GetParMesh()),
+      qspace(*fes.GetParMesh(), ir),
+      qspace_vec(qspace, 1),
+      q(qspace_vec)
+   {
+
+      const auto &pmesh = *fes.GetParMesh();
+      Array<int> all_domain_attr;
+      if (pmesh.attributes.Size() > 0)
+      {
+         all_domain_attr.SetSize(pmesh.attributes.Max());
+         all_domain_attr = 1;
+      }
+
+      // Energy
+      {
+         const auto in = std::vector
+         {
+            FieldDescriptor{U, &fes},
+            FieldDescriptor{Coords, &mfes}
+         };
+         const auto out = std::vector
+         {
+            FieldDescriptor{Q, &qspace_vec}
+         };
+
+         functional_dop = std::make_unique<DifferentiableOperator>(in, out, mesh);
+         MinimalSurfaceEnergyFunctional<real_t, dim> energy;
+         auto derivatives = std::integer_sequence<size_t, U> {};
+         auto second_derivatives = std::integer_sequence<size_t, U> {};
+         functional_dop->AddDomainIntegrator<LocalQFBackend>(
+            energy,
+            Inputs<Value<U>, Gradient<U>, Gradient<Coords>, Weight> {},
+            Outputs<FunctionalValue<Q>> {}, /* Value<U>, Gradient<U> */
+            ir, all_domain_attr, derivatives /* , second_derivatives */);
+      }
+
+      // Manually computed residual
+      {
+         const auto in = std::vector
+         {
+            FieldDescriptor{U, &fes},
+            FieldDescriptor{Coords, &mfes}
+         };
+         const auto out = std::vector
+         {
+            FieldDescriptor{U, &fes}
+         };
+
+         residual_dop = std::make_unique<DifferentiableOperator>(in, out, pmesh);
+         MinimalSurfaceResidual<real_t, dim> residual;
+         auto derivatives = std::integer_sequence<size_t, U> {};
+         residual_dop->AddDomainIntegrator<LocalQFBackend>(
+            residual,
+            tuple{Gradient<U>{}, Gradient<Coords>{}, Weight{}},
+            tuple{Gradient<U>{}},
+            ir, all_domain_attr, derivatives);
+      }
+
+      // Differentiated energy representing the residual
+      {
+         const auto in = std::vector
+         {
+            FieldDescriptor{U, &fes},
+            FieldDescriptor{Coords, &mfes}
+         };
+         const auto out = std::vector
+         {
+            FieldDescriptor{U, &fes}
+         };
+
+         dfunctional_dop = std::make_unique<DifferentiableOperator>(in, out, mesh);
+         // Differentiate output f (argument 3) with respect to dudxi
+         // (argument 0).
+         RevDiff<MinimalSurfaceEnergy<real_t, dim>, tuple<Active, Const, Const>, tuple<Active>>
+               fd;
+
+         auto derivatives = std::integer_sequence<size_t, U> {};
+         dfunctional_dop->AddDomainIntegrator<LocalQFBackend>(
+            fd,
+            tuple{Gradient<U>{}, Gradient<Coords>{}, Weight{}},
+            tuple{Gradient<U>{}},
+            ir, all_domain_attr, derivatives);
+      }
+
+      // Hand-coded Hessian action with the direction as an explicit field
+      {
+         const auto in = std::vector
+         {
+            FieldDescriptor{DirU, &fes},
+            FieldDescriptor{U, &fes},
+            FieldDescriptor{Coords, &mfes}
+         };
+         const auto out = std::vector
+         {
+            FieldDescriptor{U, &fes}
+         };
+
+         hessian_dop = std::make_unique<DifferentiableOperator>(in, out, mesh);
+         MinimalSurfaceHessianAction<real_t, dim> hessian_action;
+         hessian_dop->AddDomainIntegrator<LocalQFBackend>(
+            hessian_action,
+            tuple{Gradient<DirU>{}, Gradient<U>{}, Gradient<Coords>{}, Weight{}},
+            tuple{Gradient<U>{}},
+            ir, all_domain_attr);
+      }
+
+      mesh.GetNodes()->GetTrueDofs(coords);
+   }
+
+   void gradient_exact(const Vector &u, Vector &g) const
+   {
+      MultiVector X{u, coords};
+      MultiVector Y{g};
+      residual_dop->Mult(X, Y);
+   }
+
+   void gradient(const Vector &u, Vector &g) const
+   {
+      MultiVector X{u, coords};
+      MultiVector Y{g};
+      functional_dop->GetDerivative(U)->Mult(X, Y);
+   }
+
+   // Gradient assembled into a Vector from the state captured by
+   // GetDerivative, the functional counterpart of Assemble(SparseMatrix *&).
+   void gradient_assembled(const Vector &u, Vector &g) const
+   {
+      MultiVector X{u, coords};
+      functional_dop->GetDerivative(U, X)->Assemble(g);
+   }
+
+   // Hessian-vector product H(u) v with the hand-coded second derivative.
+   void hvp_exact(const Vector &u, const Vector &v, Vector &Hv) const
+   {
+      MultiVector X{v, u, coords};
+      MultiVector Y{Hv};
+      hessian_dop->Mult(X, Y);
+   }
+
+   // H(u) v as the derivative of the hand-coded residual (single AD).
+   void hvp_dresidual(const Vector &u, const Vector &v, Vector &Hv) const
+   {
+      MultiVector X{u, coords};
+      MultiVector Y{Hv};
+      residual_dop->GetDerivative(U, X)->Mult(v, Y);
+   }
+
+   // H(u) v as the derivative of the differentiated energy
+   // (forward-over-reverse AD).
+   void hvp(const Vector &u, const Vector &v, Vector &Hv) const
+   {
+      MultiVector X{u, coords};
+      MultiVector Y{Hv};
+      dfunctional_dop->GetDerivative(U, X)->Mult(v, Y);
+   }
+
+   // H(u) v from the functional's second-derivative interface.
+   void hvp_functional(const Vector &u, const Vector &v, Vector &Hv) const
+   {
+      MultiVector X{u, coords};
+      MultiVector Y{Hv};
+      functional_dop->GetSecondDerivative(U, X)->Mult(v, Y);
+   }
+
+private:
+   MPI_Comm comm;
+   ParMesh &mesh;
+   std::unique_ptr<DifferentiableOperator> functional_dop;
+   std::unique_ptr<DifferentiableOperator> dfunctional_dop;
+   std::unique_ptr<DifferentiableOperator> residual_dop;
+   std::unique_ptr<DifferentiableOperator> hessian_dop;
+   QuadratureSpace qspace;
+   VectorQuadratureSpace qspace_vec;
+   mutable QuadratureFunction q;
+   Vector coords;
+};
+
+template <int DIM>
+void second_derivative(const char *filename, int p)
+{
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace fes(&pmesh, &fec);
+
+   const IntegrationRule &ir =
+      IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   ParGridFunction u_gf(&fes);
+   FunctionCoefficient u_coeff(
+      [](const auto &x)
+   {
+      return 2_r * M_PI * x[0] * x[0] * 2_r * M_PI * x[1] * x[1];
+   });
+   u_gf.ProjectCoefficient(u_coeff);
+
+   Vector u(fes.GetTrueVSize());
+   u_gf.GetTrueDofs(u);
+
+   MyFunctional<DIM> functional(fes, *mfes, ir);
+
+   Vector exact_g(fes.GetTrueVSize());
+   functional.gradient_exact(u, exact_g);
+
+
+   Vector g(fes.GetTrueVSize());
+   functional.gradient(u, g);
+
+   Vector diff(g);
+   diff -= exact_g;
+   REQUIRE(diff.Norml2() < 1e-12);
+
+   // The functional derivative assembled into a Vector.
+   Vector assembled_g;
+   functional.gradient_assembled(u, assembled_g);
+   REQUIRE(assembled_g.Size() == fes.GetTrueVSize());
+
+   diff = assembled_g;
+   diff -= exact_g;
+   REQUIRE(diff.Norml2() < 1e-12);
+
+   // Direction for the Hessian-vector product
+   ParGridFunction v_gf(&fes);
+   FunctionCoefficient v_coeff(
+      [](const auto &x)
+   {
+      return sin(M_PI * x[0]) * cos(M_PI * x[1]) + 0.5_r * x[0] * x[1];
+   });
+   v_gf.ProjectCoefficient(v_coeff);
+
+   Vector v(fes.GetTrueVSize());
+   v_gf.GetTrueDofs(v);
+
+   Vector exact_Hv(fes.GetTrueVSize());
+   functional.hvp_exact(u, v, exact_Hv);
+
+   Vector Hv_dres(fes.GetTrueVSize());
+   functional.hvp_dresidual(u, v, Hv_dres);
+
+   diff = Hv_dres;
+   diff -= exact_Hv;
+   REQUIRE(MFEM_Approx(diff.Norml2()) == 0.0);
+
+   Vector Hv(fes.GetTrueVSize());
+   functional.hvp(u, v, Hv);
+
+   diff = Hv;
+   diff -= exact_Hv;
+   REQUIRE(MFEM_Approx(diff.Norml2()) == 0.0);
+
+   Vector Hv_functional(fes.GetTrueVSize());
+   functional.hvp_functional(u, v, Hv_functional);
+
+   diff = Hv_functional;
+   diff -= exact_Hv;
+   REQUIRE(MFEM_Approx(diff.Norml2()) == 0.0);
+
+   // std::cout << "Gradient using FwdDiff<f>:\n";
+   // pretty_print(g);
+
+   // std::cout << "Handcoded gradient of f(u):\n";
+   // pretty_print(exact_g);
+
+   // std::cout << "Handcoded H(u) v:\n";
+   // pretty_print(exact_Hv);
+
+   // std::cout << "H(u) v using the derivative of the handcoded residual:\n";
+   // pretty_print(Hv_dres);
+
+   // std::cout << "H(u) v using the derivative of FwdDiff<f>:\n";
+   // pretty_print(Hv);
+}
+
+template <int DIM>
+void mixed_second_derivative(const char *filename, int p)
+{
+   static constexpr int U = 0, Rho = 1, Coords = 2, Q = 3;
+   static constexpr int DU = 4, DRho = 5;
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace fes(&pmesh, &fec);
+   const int tvsize = fes.GetTrueVSize();
+
+   const IntegrationRule &ir =
+      IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   // Use smooth, non-constant fields so all four second-derivative blocks are
+   // nontrivial while keeping the exact expressions simple.
+   ParGridFunction u_gf(&fes), rho_gf(&fes), du_gf(&fes), drho_gf(&fes);
+   FunctionCoefficient u_coeff(
+      [](const auto &x)
+   {
+      return 1.0_r + x[0] + 0.25_r * x[1];
+   });
+   FunctionCoefficient rho_coeff(
+      [](const auto &x)
+   {
+      return 0.5_r + 0.2_r * x[0] * x[0] + 0.1_r * x[1];
+   });
+   FunctionCoefficient du_coeff(
+      [](const auto &x)
+   {
+      return cos(M_PI * x[0]) + 0.25_r * x[0] * x[1];
+   });
+   FunctionCoefficient drho_coeff(
+      [](const auto &x)
+   {
+      return sin(M_PI * x[0]) + 0.5_r * x[1];
+   });
+   u_gf.ProjectCoefficient(u_coeff);
+   rho_gf.ProjectCoefficient(rho_coeff);
+   du_gf.ProjectCoefficient(du_coeff);
+   drho_gf.ProjectCoefficient(drho_coeff);
+
+   Vector u(tvsize), rho(tvsize), du(tvsize), drho(tvsize), coords;
+   u_gf.GetTrueDofs(u);
+   rho_gf.GetTrueDofs(rho);
+   du_gf.GetTrueDofs(du);
+   drho_gf.GetTrueDofs(drho);
+   pmesh.GetNodes()->GetTrueDofs(coords);
+
+   const auto functional_in = std::vector
+   {
+      FieldDescriptor{U, &fes},
+      FieldDescriptor{Rho, &fes},
+      FieldDescriptor{Coords, mfes}
+   };
+   QuadratureSpace qspace(pmesh, ir);
+   VectorQuadratureSpace qspace_vec(qspace, 1);
+   const auto functional_out = std::vector
+   {
+      FieldDescriptor{Q, &qspace_vec}
+   };
+
+   DifferentiableOperator functional_dop(functional_in, functional_out, pmesh);
+   MixedFunctional<real_t, DIM> functional;
+   functional_dop.AddDomainIntegrator<LocalQFBackend>(
+      functional,
+      Inputs<Value<U>, Value<Rho>, Gradient<Coords>, Weight> {},
+      Outputs<FunctionalValue<Q>> {},
+      ir, all_domain_attr,
+      Derivatives<U, Rho> {});
+
+   MultiVector X{u, rho, coords};
+
+   auto check_block = [&](auto gradient_id,
+                          auto direction_id,
+                          const Vector &direction,
+                          auto exact_qfunc,
+                          auto exact_inputs,
+                          auto exact_outputs,
+                          const std::vector<FieldDescriptor> &exact_in,
+                          const std::vector<FieldDescriptor> &exact_out,
+                          MultiVector exact_x)
+   {
+      Vector actual(tvsize);
+      MultiVector Actual{actual};
+      functional_dop.GetSecondDerivative(gradient_id, direction_id, X)->Mult(
+         direction, Actual);
+
+      Vector expected(tvsize);
+      MultiVector Expected{expected};
+      DifferentiableOperator exact_dop(exact_in, exact_out, pmesh);
+      exact_dop.AddDomainIntegrator<LocalQFBackend>(
+         exact_qfunc, exact_inputs, exact_outputs, ir, all_domain_attr);
+      exact_dop.Mult(exact_x, Expected);
+
+      Vector diff(actual);
+      diff -= expected;
+      REQUIRE(MFEM_Approx(diff.Norml2()) == 0.0);
+   };
+
+   check_block(U, U, du,
+               MixedFunctionalUUAction<real_t, DIM> {},
+               Inputs<Value<DU>, Value<Rho>, Gradient<Coords>, Weight> {},
+               Outputs<Value<U>> {},
+               std::vector{FieldDescriptor{DU, &fes},
+                           FieldDescriptor{Rho, &fes},
+                           FieldDescriptor{Coords, mfes}},
+               std::vector{FieldDescriptor{U, &fes}},
+               MultiVector{du, rho, coords});
+
+   check_block(U, Rho, drho,
+               MixedFunctionalURhoAction<real_t, DIM> {},
+               Inputs<Value<DRho>, Value<U>, Gradient<Coords>, Weight> {},
+               Outputs<Value<U>> {},
+               std::vector{FieldDescriptor{DRho, &fes},
+                           FieldDescriptor{U, &fes},
+                           FieldDescriptor{Coords, mfes}},
+               std::vector{FieldDescriptor{U, &fes}},
+               MultiVector{drho, u, coords});
+
+   check_block(Rho, U, du,
+               MixedFunctionalRhoUAction<real_t, DIM> {},
+               Inputs<Value<DU>, Value<U>, Gradient<Coords>, Weight> {},
+               Outputs<Value<Rho>> {},
+               std::vector{FieldDescriptor{DU, &fes},
+                           FieldDescriptor{U, &fes},
+                           FieldDescriptor{Coords, mfes}},
+               std::vector{FieldDescriptor{Rho, &fes}},
+               MultiVector{du, u, coords});
+
+   check_block(Rho, Rho, drho,
+               MixedFunctionalRhoRhoAction<real_t, DIM> {},
+               Inputs<Value<DRho>, Gradient<Coords>, Weight> {},
+               Outputs<Value<Rho>> {},
+               std::vector{FieldDescriptor{DRho, &fes},
+                           FieldDescriptor{Coords, mfes}},
+               std::vector{FieldDescriptor{Rho, &fes}},
+               MultiVector{drho, coords});
+}
+
+} // namespace second_derivative_test
+
+TEST_CASE("dFEM functional second derivative action matches mfem",
+          "[Parallel][dFEM][second-derivative]")
+{
+   const bool all_tests = launch_all_non_regression_tests;
+   const auto p = !all_tests ? 1 : GENERATE(1, 2, 3);
+
+   SECTION("2d")
+   {
+      const auto f =
+         GENERATE(
+            // "../../data/star.mesh",
+            // "../../data/star-q3.mesh",
+            // "../../data/rt-2d-q3.mesh",
+            "../../data/inline-quad.mesh"
+            // "../../data/periodic-square.mesh"
+         );
+      second_derivative_test::second_derivative<2>(f, p);
+   }
+
+   // SECTION("3d")
+   // {
+   //    const auto f =
+   //       GENERATE(
+   //          "../../data/fichera-q3.mesh",
+   //          "../../data/inline-hex.mesh",
+   //          "../../data/toroid-hex.mesh",
+   //          "../../data/periodic-cube.mesh"
+   //       );
+   //    second_derivative_test::second_derivative<3>(f, p);
+   // }
+}
+
+TEST_CASE("dFEM functional mixed second derivative action matches exact action",
+          "[Parallel][dFEM][second-derivative]")
+{
+   const bool all_tests = launch_all_non_regression_tests;
+   const auto p = !all_tests ? 1 : GENERATE(1, 2, 3);
+
+   SECTION("2d")
+   {
+      const auto f = GENERATE("../../data/inline-quad.mesh");
+      second_derivative_test::mixed_second_derivative<2>(f, p);
+   }
+}
+
+#endif // MFEM_USE_MPI
+
+#endif // MFEM_USE_ENZYME

tests/unit/dfem/test_tmop.cpp

@@ -0,0 +1,173 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../../../fem/dfem/doperator.hpp"
+#include "../../../fem/dfem/backends/local_qf/prelude.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+// ────────────────────────────────────────────────────────────────────────────
+namespace
+{
+
+static constexpr int DIM = 2;
+
+struct VectorValueCopy
+{
+   MFEM_HOST_DEVICE inline
+   void operator()(const tensor<real_t, DIM> &x,
+                   tensor<real_t, DIM> &y) const
+   {
+      y(0) = x(0);
+      y(1) = x(1);
+   }
+};
+
+struct VectorMassCopy
+{
+   MFEM_HOST_DEVICE inline
+   void operator()(const tensor<real_t, DIM> &x,
+                   const tensor<real_t, DIM, DIM> &J,
+                   const real_t &w,
+                   tensor<real_t, DIM> &y) const
+   {
+      y = x * w * det(J);
+   }
+};
+
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+void test_vqspace_identity_copy(int order)
+{
+   Mesh serial_mesh =
+      Mesh::MakeCartesian2D(1, 1, Element::QUADRILATERAL, true, 1.0, 1.0);
+   ParMesh pmesh(MPI_COMM_WORLD, serial_mesh);
+
+   const IntegrationRule &ir = IntRules.Get(Geometry::SQUARE, 2 * order + 1);
+   QuadratureSpace qspace(pmesh, ir);
+   VectorQuadratureSpace vqspace(qspace, DIM);
+   const auto qvsize = vqspace.GetVSize();
+
+   Array<int> domain_attr(pmesh.attributes.Max());
+   domain_attr = 1;
+
+   static constexpr int U = 0, V = 1;
+
+   const std::vector fdi { FieldDescriptor{U, &vqspace} };
+   const std::vector fdo { FieldDescriptor{V, &vqspace} };
+
+   DifferentiableOperator dop(fdi, fdo, pmesh);
+
+   VectorValueCopy qfunc;
+   dop.AddDomainIntegrator<LocalQFBackend>(
+      qfunc,
+      Inputs<Identity<U>> {},
+      Outputs<Identity<V>> {},
+      ir, domain_attr);
+
+   Vector vX(qvsize), vY(qvsize);
+   vX.Randomize(0x9e3779b9);
+   vY = 0.0;
+
+   MultiVector Xmv{vX}, Ymv{vY};
+   dop.Mult(Xmv, Ymv);
+
+   Vector diff(vY);
+   diff -= vX;
+   REQUIRE(diff.Normlinf() == MFEM_Approx(0.0));
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+void test_value_vector_mass(int order)
+{
+   Mesh serial_mesh =
+      Mesh::MakeCartesian2D(1, 1, Element::QUADRILATERAL, true, 1.0, 1.0);
+   ParMesh pmesh(MPI_COMM_WORLD, serial_mesh);
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   const auto *nfes = nodes->ParFESpace();
+
+   H1_FECollection fec(order, DIM);
+   ParFiniteElementSpace fes(&pmesh, &fec, DIM, Ordering::byVDIM);
+   const auto tvsize = fes.GetTrueVSize();
+
+   Array<int> domain_attr(pmesh.attributes.Max());
+   domain_attr = 1;
+
+   const IntegrationRule &ir = IntRules.Get(Geometry::SQUARE, 2 * order + 1);
+   ConstantCoefficient one(1.0), zero(0.0);
+
+   ParGridFunction input_gf(&fes);
+   ParGridFunction reference_gf(&fes);
+   ParGridFunction error_gf(&fes);
+   Vector vX(tvsize), vY_ref(tvsize), vY(tvsize), N(nfes->GetTrueVSize());
+   vX.Randomize(0x9e3779b9);
+   input_gf.SetFromTrueDofs(vX);
+   nodes->GetTrueDofs(N);
+   vY = 0.0;
+
+   ParBilinearForm blf(&fes);
+   blf.AddDomainIntegrator(new VectorMassIntegrator(one, &ir));
+   blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   blf.Assemble();
+   blf.Mult(input_gf, reference_gf);
+   fes.GetProlongationMatrix()->MultTranspose(reference_gf, vY_ref);
+
+   static constexpr int U = 0, V = 1, Coords = 2;
+   const std::vector fdi { FieldDescriptor{U, &fes}, FieldDescriptor{Coords, nfes} };
+   const std::vector fdo { FieldDescriptor{V, &fes} };
+
+   DifferentiableOperator dop(fdi, fdo, pmesh);
+   VectorMassCopy qfunc;
+   dop.AddDomainIntegrator<LocalQFBackend>(
+      qfunc,
+      Inputs<Value<U>, Gradient<Coords>, Weight> {},
+      Outputs<Value<V>> {},
+      ir, domain_attr);
+
+   MultiVector Xmv{vX, N}, Ymv{vY};
+   dop.Mult(Xmv, Ymv);
+
+   vY_ref -= vY;
+   error_gf.SetFromTrueDofs(vY_ref);
+   REQUIRE(error_gf.ComputeMaxError(zero) == MFEM_Approx(0.0));
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+TEST_CASE("dFEM TMOP", "[Parallel][dFEM][GPU]")
+{
+   const auto p = GenAll({1}, {2, 3});
+   SECTION("VectorQuadratureSpace identity copy")
+   {
+      test_vqspace_identity_copy(p);
+   }
+   SECTION("Value vector mass matches PA MFEM")
+   {
+      test_value_vector_mass(p);
+   }
+}
+
+#endif // MFEM_USE_MPI

tests/unit/enzyme/compatibility.cpp

@@ -59,4 +59,66 @@ TEST_CASE("AD Vector implementation", "[Enzyme]")
    run_test<std::vector<double>>();
 }
 
+namespace enzyme_test
+{
+
+template <int N>
+void f(const double *x, double *y, double *a)
+{
+   mfem::forall<mfem::UseEnzyme>(N, [=] MFEM_HOST_DEVICE(int q)
+   {
+      y[q] = a[q] * x[q] * x[q];
+   });
+}
+
+} // namespace enzyme_test
+
+TEST_CASE("AD Global qfunction with GPU", "[Enzyme][GPU]")
+{
+   constexpr int N = 10;
+   mfem::Vector x(N), xd(N), y(N), yd(N), a(N), ad(N);
+
+   auto x_w = x.HostWrite();
+   auto xd_w = xd.HostWrite();
+   auto y_w = y.HostWrite();
+   auto yd_w = yd.HostWrite();
+   auto a_w = a.HostWrite();
+   auto ad_w = ad.HostWrite();
+
+   for (int i = 0; i < N; i++)
+   {
+      x_w[i] = i;
+      xd_w[i] = 1.0;
+      y_w[i] = 0.0;
+      yd_w[i] = 0.0;
+      a_w[i] = 2.0;
+      ad_w[i] = 0.0;
+   }
+
+   auto x_d = x.Read();
+   auto xd_d = xd.ReadWrite();
+   auto y_d = y.ReadWrite();
+   auto yd_d = yd.ReadWrite();
+   auto a_d = a.Read();
+
+   __enzyme_fwddiff<void>((void *)enzyme_test::f<N>, enzyme_dup, x_d, xd_d,
+                          enzyme_dup, y_d,
+                          yd_d, enzyme_const, a_d, enzyme_runtime_activity);
+
+   const mfem::real_t *yd_h = yd.HostRead();
+   const mfem::real_t *x_h = x.HostRead();
+   const mfem::real_t *a_h = a.HostRead();
+   bool ok = true;
+   for (int q = 0; q < N; q++)
+   {
+      mfem::real_t exact = 2.0 * a_h[q] * x_h[q];
+      if (yd_h[q] != exact)
+      {
+         ok = false;
+         break;
+      }
+   }
+   REQUIRE(ok);
+}
+
 #endif

tests/unit/enzyme/test_enzyme_reverse_tape.cpp

@@ -0,0 +1,122 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "unit_tests.hpp"
+#include "../../../linalg/tensor_arrays.hpp"
+
+#ifdef MFEM_USE_ENZYME
+
+namespace enzyme_test_reversetape
+{
+
+using namespace mfem;
+using namespace mfem::future;
+
+void myQF(real_t x[2])
+{
+   const real_t u = x[0];
+   const real_t v = x[1];
+   x[0] = sin(u) * cos(v) * (u + v);
+}
+
+struct WrappedQF
+{
+   void operator()(tensor_array<real_t> &x) const
+   {
+      const real_t u = x(0);
+      const real_t v = x(1);
+      x(0) = sin(u) * cos(v) * (u + v);
+   }
+};
+
+template <int N>
+void qfunction_wrapper(real_t *x)
+{
+   auto x_t = make_tensor_array(x, N);
+
+   WrappedQF qf;
+   qf(x_t);
+}
+
+
+template <typename return_type, typename... Args>
+return_type  __enzyme_augmentfwd(Args...);
+
+template <typename return_type, typename... Args>
+return_type  __enzyme_reverse(Args...);
+
+} // namespace enzyme_test_reversetape
+
+TEST_CASE("Enzyme split reverse mode point qfunction VJP",
+          "[Enzyme][SplitReverse]")
+{
+   using namespace enzyme_test_reversetape;
+
+   real_t x[2] = {0.7, 0.2};
+   real_t x_bar[2] = {1.5, 0.0};
+   real_t y[2] = {x[0], x[1]};
+   real_t y_bar[2] = {x_bar[0], x_bar[1]};
+
+   void *tape = __enzyme_augmentfwd<void *>((void *)myQF, x, x_bar);
+   __enzyme_reverse<void>((void *)myQF, x, x_bar, tape);
+
+   __enzyme_autodiff<void>((void *)myQF, y, y_bar);
+
+   const real_t u = 0.7;
+   const real_t v = 0.2;
+   const real_t residual = sin(u) * cos(v) * (u + v);
+   REQUIRE(x[0] == MFEM_Approx(residual));
+   REQUIRE(y[0] == MFEM_Approx(residual));
+   REQUIRE(x[1] == MFEM_Approx(v));
+   REQUIRE(y[1] == MFEM_Approx(v));
+
+   const real_t exact_u_bar = 1.5 * (cos(u) * cos(v) * (u + v) + sin(u) * cos(v));
+   const real_t exact_v_bar = 1.5 * (-sin(u) * sin(v) * (u + v) + sin(u) * cos(v));
+   REQUIRE(x_bar[0] == MFEM_Approx(exact_u_bar));
+   REQUIRE(x_bar[1] == MFEM_Approx(exact_v_bar));
+   REQUIRE(y_bar[0] == MFEM_Approx(x_bar[0]));
+   REQUIRE(y_bar[1] == MFEM_Approx(x_bar[1]));
+}
+
+TEST_CASE("Enzyme split reverse mode wrapped point qfunction VJP",
+          "[Enzyme][SplitReverse][QFunctionWrapper]")
+{
+   using namespace enzyme_test_reversetape;
+
+   real_t x[2] = {0.7, 0.2};
+   real_t x_bar[2] = {1.5, 0.0};
+   real_t y[2] = {x[0], x[1]};
+   real_t y_bar[2] = {x_bar[0], x_bar[1]};
+
+   void *tape = __enzyme_augmentfwd<void *>((void *)qfunction_wrapper<2>, x,
+                                            x_bar);
+   __enzyme_reverse<void>((void *)qfunction_wrapper<2>, x, x_bar, tape);
+
+   __enzyme_autodiff<void>((void *)qfunction_wrapper<2>, y, y_bar);
+
+   const real_t u = 0.7;
+   const real_t v = 0.2;
+   const real_t residual = sin(u) * cos(v) * (u + v);
+   REQUIRE(x[0] == MFEM_Approx(residual));
+   REQUIRE(y[0] == MFEM_Approx(residual));
+   REQUIRE(x[1] == MFEM_Approx(v));
+   REQUIRE(y[1] == MFEM_Approx(v));
+
+   const real_t exact_u_bar = 1.5 * (cos(u) * cos(v) * (u + v) + sin(u) * cos(v));
+   const real_t exact_v_bar = 1.5 * (-sin(u) * sin(v) * (u + v) + sin(u) * cos(v));
+   REQUIRE(x_bar[0] == MFEM_Approx(exact_u_bar));
+   REQUIRE(x_bar[1] == MFEM_Approx(exact_v_bar));
+   REQUIRE(y_bar[0] == MFEM_Approx(x_bar[0]));
+   REQUIRE(y_bar[1] == MFEM_Approx(x_bar[1]));
+}
+
+#endif