clean up pass

MixedVector gradient partial assembly

CHANGELOG

@@ -79,6 +79,11 @@ Linear and nonlinear solvers
   PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
   DPG miniapps).
 
+- Added new class MultiVector: an array of Vectors of different sizes where each
+  Vector can be allocated independently. Also, added associated methods in class
+  Operator: MultMV, MultTransposeMV, and GetGradientMV, that use MultiVector
+  objects for input and/or output parameters. [PR #5249]
+
 GPU computing
 -------------
 - Improved partial assembly for VectorDivergenceIntegrator with shared-memory
@@ -92,6 +97,22 @@ GPU computing
 
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
+- Added partial assembly support for MixedScalarWeakGradientIntegrator.
+
+- Added partial assembly support for MixedDotProductIntegrator.
+
+- Added partial assembly support for MixedScalarCrossProductIntegrator.
+
+- Added partial assembly support for MixedScalarWeakCrossProductIntegrator.
+
+- Added partial assembly support for MixedVectorGradientIntegrator for H1->RT.
+
+- Added support for device partial assembly CurlInterpolator.
+  This supports 2D and 3D variants:
+    2D H1 (out-of-plane) to RT (in-plane)
+    2D ND (in-plane) to Integral L2 (out-of-plane)
+    3D ND to RT
+
 - Added NVIDIA cuDSS library interface. Implementation examples have been
   added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
   details. Supported versions >= 0.6.0.
@@ -104,6 +125,9 @@ GPU computing
 - Added support for FiniteElement::MapType::INTEGRAL spaces to
   QuadratureInterpolator.
 
+- Added support for FiniteElement::MapType::INTEGRAL spaces to
+  MixedScalarCurlIntegrator.
+
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
@@ -118,6 +142,15 @@ Miscellaneous
   using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
   method will return immediately if no sign flips are needed.
 
+- Added support for saving DataCollection output on the node-local storage,
+  instead of requiring that the filesystem is shared among all the ranks.
+
+API changes
+-----------
+- Removed ProjectGrad from 2D RT elements. Users should use ProjectCurl instead.
+  This also fixes a bug where ProjectCurl was returning the negative curl,
+  identical to ProjectGrad.
+
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -89,7 +89,7 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   set(XSDK_ENABLE_Fortran ON)
 endif()
 # RAJA requires C++20:
-if (MFEM_USE_RAJA AND ("${CMAKE_CXX_STANDARD}" LESS "20"))
+if ((MFEM_USE_UMPIRE OR MFEM_USE_RAJA) AND ("${CMAKE_CXX_STANDARD}" LESS "20"))
   set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard to use." FORCE)
 endif()
 

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -962,7 +962,7 @@ function(mfem_export_mk_files)
     set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
   endforeach()
   # TODO: MFEM_TPLFLAGS: add other TPL flags, in addition to the -I flags.
-  set(MFEM_INCFLAGS "-I\$(MFEM_INC_DIR) \$(MFEM_TPLFLAGS)")
+  set(MFEM_INCFLAGS "-I\$(MFEM_INC_DIR) -I\$(MFEM_SOURCE_DIR) \$(MFEM_TPLFLAGS)")
   set(MFEM_PICFLAG "")
   if (BUILD_SHARED_LIBS)
     set(MFEM_PICFLAG "${CMAKE_SHARED_LIBRARY_CXX_FLAGS}")
@@ -1098,6 +1098,7 @@ function(mfem_export_mk_files)
     "${INSTALL_LIB_DIR}" "${CMAKE_INSTALL_PREFIX}" MFEM_LIB_DIR)
   mfem_path_to_fullpath(
     "${INSTALL_SHARE_DIR}/mfem/test.mk" "${CMAKE_INSTALL_PREFIX}" MFEM_TEST_MK)
+  set(MFEM_INCFLAGS "-I\$(MFEM_INC_DIR) \$(MFEM_TPLFLAGS)")
   set(MFEM_CONFIG_EXTRA "")
 
   # Create the install-tree version of 'config.mk'

config/defaults.mk

@@ -28,11 +28,8 @@ MPICXX = mpicxx
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
 
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-SHADOW_WARNING_FLAG = $(if $(findstring clang,\
- $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
-WARNING_FLAGS = -pedantic -Wall $(SHADOW_WARNING_FLAG)
-
+# The variable WARNING_FLAGS depends on which compiler is used, and is defined
+# later in this file.
 DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
@@ -52,6 +49,10 @@ SHARED = NO
 #
 # If you set MFEM_USE_ENZYME=YES, must use CUDA_CXX=clang++
 CUDA_CXX = nvcc
+# CUDA compute capability used during compilation, e.g. sm_60. Multiple
+# architectures can be requested as a comma-separated list, e.g. sm_70,sm_80.
+# A single value may also be one of the nvcc special values "all",
+# "all-major", or "native".
 CUDA_ARCH = sm_60
 # Base CUDA install directory, only needed if building with clang+cuda:
 # The default setting is:
@@ -60,11 +61,23 @@ CUDA_ARCH = sm_60
 # 3. Use /usr/local/cuda
 CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
  $(patsubst %/,%,$(dir $(shell command -v nvcc))))),/usr/local/cuda)
+# Derive nvcc/clang architecture flags from CUDA_ARCH. A comma-separated list
+# expands into one -gencode / --cuda-gpu-arch flag per architecture; otherwise
+# use the -arch / --cuda-gpu-arch shorthand.
+MFEM_COMMA := ,
+CUDA_ARCH_NUMS = $(patsubst sm_%,%,$(subst $(MFEM_COMMA), ,$(CUDA_ARCH)))
+NVCC_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
+ $(foreach arch,$(CUDA_ARCH_NUMS),\
+ -gencode arch=compute_$(arch)$(MFEM_COMMA)code=sm_$(arch)),\
+ -arch=$(CUDA_ARCH)))
+CLANG_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
+ $(foreach arch,$(CUDA_ARCH_NUMS),--cuda-gpu-arch=sm_$(arch)),\
+ --cuda-gpu-arch=$(CUDA_ARCH)))
 # flags for clang+cuda
-CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
+CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) $(CLANG_ARCH_FLAGS)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
+ $(NVCC_ARCH_FLAGS) -isystem "$(CUDA_DIR)/include"
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=
@@ -382,7 +395,7 @@ CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
 CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
 CUDSS_LIB         = \
  $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
-# The cuDSS communication and threading libraries. 
+# The cuDSS communication and threading libraries.
 MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
    $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
 MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
@@ -665,3 +678,15 @@ VERBOSE = NO
 
 # Optional build tag
 MFEM_BUILD_TAG = $(shell uname -snm)
+
+# Enable -pedantic flag only for gcc or clang. nvcc complains with -pedantic
+# because of line directives.
+PEDANTIC_FLAG = $(if \
+   $(findstring NVIDIA,$(shell $(MFEM_CXX) --version 2>&1)),, \
+   $(if $(or \
+      $(findstring gcc version,$(shell $(MFEM_CXX) -v 2>&1)), \
+      $(findstring clang version,$(shell $(MFEM_CXX) -v 2>&1))),-pedantic,))
+# Enable shadow warnings for clang only; GCC's -Wshadow flags more.
+SHADOW_WARNING_FLAG = $(if $(findstring clang,\
+ $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
+WARNING_FLAGS = $(PEDANTIC_FLAG) -Wall $(SHADOW_WARNING_FLAG)

doc/CodeDocumentation.conf.in

@@ -1083,7 +1083,8 @@ EXCLUDE_PATTERNS       =
 # ANamespace::AClass, ANamespace::*Test
 
 EXCLUDE_SYMBOLS        = mfem::internal \
-                         mfem::kernels::internal
+                         mfem::kernels::internal \
+                         mfem::future::detail
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

examples/CMakeLists.txt

@@ -262,3 +262,8 @@ endif()
 if(MFEM_USE_MOONOLITH)
   add_subdirectory(moonolith)
 endif()
+
+# Include the examples/magma directory if MAGMA is enabled.
+if (MFEM_USE_MAGMA)
+  add_subdirectory(magma)
+endif()

examples/magma/CMakeLists.txt

@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(MAGMA_EXAMPLES_SRCS)
+list(APPEND MAGMA_EXAMPLES_SRCS
+  batched_mass_bench.cpp
+  )
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_magma" target (build-only by default).
+add_custom_target(test_magma
+  COMMENT "Building MAGMA examples ...")
+
+# Add one executable per cpp file, adding "magma_" as prefix to avoid
+# collisions in a unified build tree.
+set(PFX magma_)
+add_mfem_examples(MAGMA_EXAMPLES_SRCS ${PFX} "" test_magma)

examples/magma/README

@@ -0,0 +1,12 @@
+This directory contains an MFEM example focused on benchmarking MAGMA batched
+element-mass-matrix inverses/solves on GPU backends (CUDA/HIP).
+
+- `batched_mass_bench.cpp`: Assemble packed element mass matrices and compare:
+  - MFEM tripack inverse apply (packed lower-triangular)
+  - MAGMA packed Cholesky factor + solve
+  - MAGMA full (dense) batched Cholesky factor + solve
+
+Build and run from this directory (after building MFEM with MAGMA enabled):
+
+- `make batched_mass_bench`
+- `./batched_mass_bench -m ../../data/beam-hex.mesh -o 3 -r 2 -d hip -reps 100`

examples/magma/batched_mass_bench.cpp

@@ -0,0 +1,798 @@
+//                    MFEM Batched Mass Benchmark
+//
+// Compile with: make batched_mass_bench
+//
+// Sample runs:  batched_mass_bench -m ../../data/beam-hex.mesh -o 3 -r 2 -d hip -reps 100
+//               batched_mass_bench -m ../../data/beam-hex.mesh -o 3 -r 3 -d cuda -reps 50
+//
+// Description:  This example isolates the element mass-matrix
+//               inverse paths used by element matrix kernels. It
+//               assembles packed lower-triangular L2 element mass matrices,
+//               then times repeated mass inverse applications using MFEM's
+//               tripack path and MAGMA (packed and full-matrix variants).
+
+#include "mfem.hpp"
+#include <cmath>
+#include <iomanip>
+#include <iostream>
+#include <limits>
+#include <memory>
+
+using namespace mfem;
+using namespace std;
+
+namespace
+{
+
+#ifdef MFEM_USE_MAGMA
+inline void MagmaSetPointer(float **output, float *input,
+                            magma_int_t inc, magma_int_t row, magma_int_t col,
+                            magma_int_t lda, magma_int_t batch_count,
+                            magma_queue_t queue)
+{
+   magma_sset_pointer(output, input, inc, row, col, lda, batch_count, queue);
+}
+
+inline void MagmaSetPointer(double **output, double *input,
+                            magma_int_t inc, magma_int_t row, magma_int_t col,
+                            magma_int_t lda, magma_int_t batch_count,
+                            magma_queue_t queue)
+{
+   magma_dset_pointer(output, input, inc, row, col, lda, batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrfBatched(magma_uplo_t uplo, magma_int_t n,
+                                     float **dA, magma_int_t ldda,
+                                     magma_int_t *info, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_spotrf_batched(uplo, n, dA, ldda, info, batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrfBatched(magma_uplo_t uplo, magma_int_t n,
+                                     double **dA, magma_int_t ldda,
+                                     magma_int_t *info, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_dpotrf_batched(uplo, n, dA, ldda, info, batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrsBatched(magma_uplo_t uplo, magma_int_t n,
+                                     magma_int_t nrhs, float **dA,
+                                     magma_int_t ldda, float **dB,
+                                     magma_int_t lddb, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_spotrs_batched(uplo, n, nrhs, dA, ldda, dB, lddb,
+                              batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrsBatched(magma_uplo_t uplo, magma_int_t n,
+                                     magma_int_t nrhs, double **dA,
+                                     magma_int_t ldda, double **dB,
+                                     magma_int_t lddb, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_dpotrs_batched(uplo, n, nrhs, dA, ldda, dB, lddb,
+                              batch_count, queue);
+}
+
+void PrintMagmaFasterCondition(const double eq_fixed_ms,
+                               const double eq_apply_ms,
+                               const double magma_fixed_ms,
+                               const double magma_apply_ms)
+{
+   cout << "MAGMA faster condition (assembly+setup+N applies): ";
+
+   if (magma_fixed_ms <= eq_fixed_ms && magma_apply_ms <= eq_apply_ms)
+   {
+      if (magma_fixed_ms == eq_fixed_ms && magma_apply_ms == eq_apply_ms)
+      {
+         cout << "modeled totals are equal for all positive integer N.\n";
+      }
+      else
+      {
+         cout << "faster for every positive integer N.\n";
+      }
+      return;
+   }
+
+   if (magma_fixed_ms >= eq_fixed_ms && magma_apply_ms >= eq_apply_ms)
+   {
+      cout << "not faster for any positive integer N.\n";
+      return;
+   }
+
+   if (magma_apply_ms > eq_apply_ms)
+   {
+      const double crossover =
+         (eq_fixed_ms - magma_fixed_ms)/(magma_apply_ms - eq_apply_ms);
+      if (crossover <= 1.0)
+      {
+         cout << "only for N < " << crossover
+              << ", so not for any positive integer N.\n";
+      }
+      else
+      {
+         const double last_n =
+            floor(nextafter(crossover, -numeric_limits<double>::infinity()));
+         cout << "faster for N < " << crossover
+              << " applies (positive integer N <= " << (long long)last_n
+              << "); eq-iter is faster above that.\n";
+      }
+   }
+   else
+   {
+      const double crossover =
+         (magma_fixed_ms - eq_fixed_ms)/(eq_apply_ms - magma_apply_ms);
+      const double first_n = floor(crossover) + 1.0;
+      cout << "faster for N > " << crossover
+           << " applies (positive integer N >= " << (long long)first_n
+           << "); eq-iter is faster below that.\n";
+	   }
+	}
+
+real_t **SetMagmaPackedPointerArray(Array<real_t *> &ptrs, real_t *data,
+                                    const int stride,
+                                    const int batch_size,
+                                    const magma_queue_t queue)
+{
+   if (ptrs.Size() != batch_size)
+   {
+      if (ptrs.Size() != 0) { magma_queue_sync(queue); }
+      ptrs.SetSize(batch_size, Device::GetDeviceMemoryType());
+   }
+
+   real_t **d_ptrs = ptrs.Write();
+   MagmaSetPointer(d_ptrs, data, 1, 0, 0, stride, batch_size, queue);
+   return d_ptrs;
+}
+
+void ComputeMagmaFullCholeskyLower(
+   const Vector &full,
+   const int n,
+   Vector &factor,
+   Array<real_t *> &factor_ptrs,
+   const magma_queue_t queue)
+{
+   const int batch_size = full.Size()/(n*n);
+   MFEM_VERIFY(full.Size() == batch_size*n*n, "Invalid full matrix storage.");
+
+   factor = full;
+
+   if (batch_size == 0) { return; }
+
+   real_t *factor_data = factor.ReadWrite();
+   real_t **dA = SetMagmaPackedPointerArray(factor_ptrs, factor_data, n*n,
+                                            batch_size, queue);
+
+   Array<magma_int_t> info_array(batch_size, Device::GetDeviceMemoryType());
+   magma_int_t *d_info = info_array.Write();
+   magma_memset(d_info, 0, batch_size*sizeof(magma_int_t));
+
+   const magma_int_t status =
+      MagmaPotrfBatched(MagmaLower, n, dA, n, d_info, batch_size, queue);
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA full potrf batched failed.");
+
+   magma_queue_sync(queue);
+   const magma_int_t *info = info_array.HostRead();
+   for (int e = 0; e < batch_size; ++e)
+   {
+      MFEM_VERIFY(info[e] == 0, "MAGMA full potrf failed on matrix " << e << '.');
+   }
+}
+
+void SolveMagmaFullCholeskyLowerInPlace(
+   const Vector &full_factor,
+   const int n,
+   const Array<real_t *> &factor_ptrs,
+   Array<real_t *> &rhs_ptrs,
+   Vector &rhs_sol,
+   const magma_queue_t queue)
+{
+   const int batch_size = rhs_sol.Size()/n;
+   MFEM_VERIFY(rhs_sol.Size() == batch_size*n, "Invalid RHS size.");
+   MFEM_VERIFY(factor_ptrs.Size() == batch_size,
+               "Factor pointer array has the wrong size.");
+
+   if (batch_size == 0) { return; }
+
+   real_t *factor_data = const_cast<real_t *>(full_factor.Read());
+   real_t **dA = const_cast<real_t **>(factor_ptrs.Read());
+   (void)factor_data; // Factor data is referenced by dA (for clarity).
+
+   real_t *rhs_data = rhs_sol.ReadWrite();
+   real_t **dB = SetMagmaPackedPointerArray(rhs_ptrs, rhs_data, n, batch_size,
+                                            queue);
+
+   const magma_int_t status =
+      MagmaPotrsBatched(MagmaLower, n, 1, dA, n, dB, n, batch_size, queue);
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA full potrs batched failed.");
+}
+#endif
+
+void FillRHS(Vector &rhs)
+{
+   rhs.UseDevice(true);
+   real_t *x = rhs.HostWrite();
+   for (int i = 0; i < rhs.Size(); ++i)
+   {
+      x[i] = 1.0 + real_t((13*i + 7) % 29)/real_t(29);
+   }
+}
+
+void ComputeLowerPackedResidual(
+   const TriPackLowerMatrix &lower,
+   const Vector &x,
+   const Vector &rhs,
+   double &l2_residual,
+   double &relative_l2_residual,
+   real_t &max_residual,
+   real_t &relative_max_residual)
+{
+   const int n = lower.GetNumRows();
+   const int batch_size = lower.GetNumMatrices();
+   const int packed_size = lower.GetPackedSize();
+   MFEM_VERIFY(x.Size() == batch_size*n, "Solution vector has the wrong size.");
+   MFEM_VERIFY(rhs.Size() == batch_size*n, "Right-hand side has the wrong size.");
+
+   const real_t *A = lower.Data().HostRead();
+   const real_t *X = x.HostRead();
+   const real_t *B = rhs.HostRead();
+
+   long double l2_sum = 0.0;
+   long double rhs_l2_sum = 0.0;
+   real_t max_abs = 0.0;
+   real_t rhs_max_abs = 0.0;
+
+   for (int e = 0; e < batch_size; ++e)
+   {
+      const real_t *Ae = A + e*packed_size;
+      const real_t *Xe = X + e*n;
+      const real_t *Be = B + e*n;
+      for (int i = 0; i < n; ++i)
+      {
+         long double ax = 0.0;
+         for (int j = 0; j < n; ++j)
+         {
+         const real_t aij =
+            (i >= j) ?
+               Ae[TriPackLowerMatrix::LowerIndex(i, j, n)] :
+               Ae[TriPackLowerMatrix::LowerIndex(j, i, n)];
+            ax += (long double)aij * (long double)Xe[j];
+         }
+         const long double residual = ax - (long double)Be[i];
+         l2_sum += residual*residual;
+         rhs_l2_sum += (long double)Be[i]*(long double)Be[i];
+         max_abs = max(max_abs, (real_t)fabs((double)residual));
+         rhs_max_abs = max(rhs_max_abs, fabs(Be[i]));
+      }
+   }
+
+   l2_residual = sqrt((double)l2_sum);
+   const double rhs_l2_norm = sqrt((double)rhs_l2_sum);
+   relative_l2_residual =
+      (rhs_l2_norm > 0.0) ? l2_residual/rhs_l2_norm : l2_residual;
+   max_residual = max_abs;
+   relative_max_residual =
+      (rhs_max_abs > 0.0) ? max_residual/rhs_max_abs : max_residual;
+}
+
+void ApplyLowerInverseInPlace(
+   const TriPackLowerMatrix &lower_inverse,
+   Vector &x,
+   Vector &work)
+{
+   const int n = lower_inverse.GetNumRows();
+   const int batch_size = lower_inverse.GetNumMatrices();
+   const int packed_size = lower_inverse.GetPackedSize();
+   MFEM_VERIFY(x.Size() == batch_size*n, "Input vector has the wrong size.");
+
+   work.SetSize(batch_size*n);
+   work.UseDevice(true);
+
+   const real_t *L = lower_inverse.Data().Read();
+   const real_t *X = x.Read();
+   real_t *T = work.Write();
+
+   mfem::forall(batch_size*n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int i = idx % n;
+      const int e = idx / n;
+      const real_t *Le = L + e*packed_size;
+      const real_t *Xe = X + e*n;
+      real_t sum = 0.0;
+      for (int j = 0; j <= i; ++j)
+      {
+         sum += Le[TriPackLowerMatrix::LowerIndex(i, j, n)]*Xe[j];
+      }
+      T[idx] = sum;
+   });
+
+   const real_t *T_in = work.Read();
+   real_t *Y = x.Write();
+   mfem::forall(batch_size*n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int i = idx % n;
+      const int e = idx / n;
+      const real_t *Le = L + e*packed_size;
+      const real_t *Te = T_in + e*n;
+      real_t sum = 0.0;
+      for (int j = i; j < n; ++j)
+      {
+         sum += Le[TriPackLowerMatrix::LowerIndex(j, i, n)]*Te[j];
+      }
+      Y[idx] = sum;
+   });
+}
+
+double TimeLowerInverseApply(
+   const TriPackLowerMatrix &inverse,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   Vector &work)
+{
+   StopWatch sw;
+
+   // Dry run to remove first-use kernel and workspace allocation costs.
+   x = rhs;
+   ApplyLowerInverseInPlace(inverse, x, work);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      ApplyLowerInverseInPlace(inverse, x, work);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+
+#ifdef MFEM_USE_MAGMA
+double TimeMagmaSolve(
+   const TriPackLowerMatrix &lower_factor,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   MagmaPackedLowerCholesky &ws)
+{
+   StopWatch sw;
+
+   // Dry run to remove first-use MAGMA and RHS pointer-array setup costs.
+   x = rhs;
+   ws.SolveInPlace(lower_factor, x);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      ws.SolveInPlace(lower_factor, x);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+
+double TimeMagmaInverseApply(
+   const TriPackLowerMatrix &lower_inverse,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   MagmaPackedLowerInverse &ws)
+{
+   StopWatch sw;
+
+   // Dry run to remove first-use MAGMA and RHS pointer-array setup costs.
+   x = rhs;
+   ws.ApplyInPlace(lower_inverse, x);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      ws.ApplyInPlace(lower_inverse, x);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+
+double TimeMagmaFullSolve(
+   const Vector &full_factor,
+   const int n,
+   const Array<real_t *> &factor_ptrs,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   const magma_queue_t queue)
+{
+   StopWatch sw;
+   Array<real_t *> rhs_ptrs;
+
+   x = rhs;
+   SolveMagmaFullCholeskyLowerInPlace(full_factor, n, factor_ptrs, rhs_ptrs, x,
+                                      queue);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      SolveMagmaFullCholeskyLowerInPlace(full_factor, n, factor_ptrs, rhs_ptrs,
+                                         x, queue);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+#endif
+
+} // namespace
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../data/beam-hex.mesh";
+   int order = 3;
+   int ref_levels = 1;
+   int reps = 100;
+   int setup_reps = 10;
+   const char *device_config = "cpu";
+   bool use_magma = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of uniform refinements.");
+   args.AddOption(&reps, "-reps", "--apply-repetitions",
+                  "Number of mass inverse applications to time.");
+   args.AddOption(&setup_reps, "-sreps", "--setup-repetitions",
+                  "Number of setup repetitions to time.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&use_magma, "-magma", "--magma-solve",
+                  "-no-magma", "--no-magma-solve",
+                  "Time the MAGMA packed Cholesky solve when available.");
+   args.ParseCheck();
+
+   MFEM_VERIFY(reps > 0, "The number of apply repetitions must be positive.");
+   MFEM_VERIFY(setup_reps > 0,
+               "The number of setup repetitions must be positive.");
+
+   Device device(device_config);
+   device.Print();
+
+#ifndef MFEM_USE_MAGMA
+   MFEM_VERIFY(!use_magma, "MFEM was built without MAGMA support.");
+#endif
+
+   Mesh mesh(mesh_file, 1, 1);
+   for (int l = 0; l < ref_levels; ++l)
+   {
+      mesh.UniformRefinement();
+   }
+
+   const int dim = mesh.Dimension();
+   L2_FECollection fec(order, dim, BasisType::Positive);
+   FiniteElementSpace fespace(&mesh, &fec);
+
+   MFEM_VERIFY(UsesTensorBasis(fespace),
+               "This example requires a tensor-product finite element space.");
+
+   const int ne = mesh.GetNE();
+   const int elem_dofs = fespace.GetTypicalFE()->GetDof();
+
+   MassIntegrator mass;
+   StopWatch sw;
+
+   // Dry run assembly before timing steady-state work.
+   TriPackLowerMatrix packed_ea;
+   mass.AssembleEATriangular(fespace, packed_ea, false);
+   MFEM_DEVICE_SYNC;
+
+   sw.Clear();
+   sw.Start();
+   mass.AssembleEATriangular(fespace, packed_ea, false);
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   const double assemble_ms = 1000.0*sw.RealTime();
+
+   TriPackLowerMatrix tripack_inverse;
+   // Dry run setup before timing steady-state setup work.
+   tripack::ComputeCholeskyLowerInverse(packed_ea, tripack_inverse);
+   MFEM_DEVICE_SYNC;
+
+   sw.Clear();
+   sw.Start();
+   for (int r = 0; r < setup_reps; ++r)
+   {
+      tripack::ComputeCholeskyLowerInverse(packed_ea, tripack_inverse);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   const double tripack_inverse_setup_ms = 1000.0*sw.RealTime()/setup_reps;
+
+#ifdef MFEM_USE_MAGMA
+   magma_queue_t magma_queue = nullptr;
+   if (use_magma) { magma_queue = Magma::Queue(); }
+
+   TriPackLowerMatrix magma_factor;
+   std::unique_ptr<MagmaPackedLowerCholesky> magma_chol_ws;
+   double magma_factor_ms = 0.0;
+
+   TriPackLowerMatrix magma_inverse;
+   std::unique_ptr<MagmaPackedLowerInverse> magma_inv_ws;
+   double magma_inverse_ms = 0.0;
+   bool magma_ppinv_enabled = false;
+   Vector magma_full_factor;
+   Array<real_t *> magma_full_factor_ptrs;
+   double magma_full_factor_ms = 0.0;
+
+   if (use_magma)
+   {
+      magma_chol_ws.reset(new MagmaPackedLowerCholesky());
+      magma_chol_ws->SetQueue(magma_queue);
+
+      // Dry run setup before timing steady-state setup work.
+      magma_chol_ws->Factor(packed_ea, magma_factor);
+      MFEM_DEVICE_SYNC;
+
+      sw.Clear();
+      sw.Start();
+      for (int r = 0; r < setup_reps; ++r)
+      {
+         magma_chol_ws->Factor(packed_ea, magma_factor);
+      }
+      MFEM_DEVICE_SYNC;
+      sw.Stop();
+      magma_factor_ms = 1000.0*sw.RealTime()/setup_reps;
+
+      // Benchmark packed inverse (ppinv) only for sizes supported by MAGMA's
+      // current packed-inverse apply kernel.
+      if (elem_dofs <= 64)
+      {
+         magma_ppinv_enabled = true;
+         magma_inv_ws.reset(new MagmaPackedLowerInverse());
+         magma_inv_ws->SetQueue(magma_queue);
+
+         // Dry run setup before timing steady-state setup work.
+         magma_inv_ws->Compute(packed_ea, magma_inverse);
+         MFEM_DEVICE_SYNC;
+
+         sw.Clear();
+         sw.Start();
+         for (int r = 0; r < setup_reps; ++r)
+         {
+            magma_inv_ws->Compute(packed_ea, magma_inverse);
+         }
+         MFEM_DEVICE_SYNC;
+         sw.Stop();
+         magma_inverse_ms = 1000.0*sw.RealTime()/setup_reps;
+      }
+
+      // Full (dense) batched Cholesky factorization for comparison.
+      Vector full_ea(ne*elem_dofs*elem_dofs);
+      full_ea.UseDevice(true);
+      mass.AssembleEA(fespace, full_ea, false);
+
+      magma_full_factor.SetSize(full_ea.Size(), Device::GetDeviceMemoryType());
+      magma_full_factor.UseDevice(true);
+
+      ComputeMagmaFullCholeskyLower(full_ea, elem_dofs, magma_full_factor,
+                                    magma_full_factor_ptrs, magma_queue);
+      MFEM_DEVICE_SYNC;
+
+      sw.Clear();
+      sw.Start();
+      for (int r = 0; r < setup_reps; ++r)
+      {
+         ComputeMagmaFullCholeskyLower(full_ea, elem_dofs, magma_full_factor,
+                                       magma_full_factor_ptrs, magma_queue);
+      }
+      MFEM_DEVICE_SYNC;
+      sw.Stop();
+      magma_full_factor_ms = 1000.0*sw.RealTime()/setup_reps;
+   }
+#endif
+
+   Vector rhs(ne*elem_dofs);
+   FillRHS(rhs);
+
+   Vector tripack_x(rhs.Size()), work;
+   tripack_x.UseDevice(true);
+
+   const double tripack_apply_ms =
+      TimeLowerInverseApply(tripack_inverse, rhs, reps, tripack_x, work);
+
+   double tripack_res_l2 = 0.0, tripack_rel_res_l2 = 0.0;
+   real_t tripack_res_max = 0.0, tripack_rel_res_max = 0.0;
+   ComputeLowerPackedResidual(packed_ea, tripack_x, rhs,
+                              tripack_res_l2, tripack_rel_res_l2,
+                              tripack_res_max, tripack_rel_res_max);
+
+#ifdef MFEM_USE_MAGMA
+   double magma_solve_ms = 0.0;
+   double magma_full_solve_ms = 0.0;
+   double magma_res_l2 = 0.0, magma_rel_res_l2 = 0.0;
+   real_t magma_res_max = 0.0, magma_rel_res_max = 0.0;
+   double magma_full_res_l2 = 0.0, magma_full_rel_res_l2 = 0.0;
+   real_t magma_full_res_max = 0.0, magma_full_rel_res_max = 0.0;
+   Vector magma_x;
+   Vector magma_full_x;
+   if (use_magma)
+   {
+      magma_x.SetSize(rhs.Size());
+      magma_x.UseDevice(true);
+      magma_solve_ms =
+         TimeMagmaSolve(magma_factor, rhs, reps, magma_x, *magma_chol_ws);
+      ComputeLowerPackedResidual(packed_ea, magma_x, rhs,
+                                 magma_res_l2, magma_rel_res_l2,
+                                 magma_res_max, magma_rel_res_max);
+
+      magma_full_x.SetSize(rhs.Size());
+      magma_full_x.UseDevice(true);
+      magma_full_solve_ms =
+         TimeMagmaFullSolve(magma_full_factor, elem_dofs,
+                            magma_full_factor_ptrs, rhs, reps, magma_full_x,
+                            magma_queue);
+      ComputeLowerPackedResidual(packed_ea, magma_full_x, rhs,
+                                 magma_full_res_l2, magma_full_rel_res_l2,
+                                 magma_full_res_max, magma_full_rel_res_max);
+   }
+
+   double magma_ppinv_apply_ms = 0.0;
+   double magma_ppinv_res_l2 = 0.0, magma_ppinv_rel_res_l2 = 0.0;
+   real_t magma_ppinv_res_max = 0.0, magma_ppinv_rel_res_max = 0.0;
+   Vector magma_ppinv_x;
+   if (use_magma && magma_ppinv_enabled)
+   {
+      magma_ppinv_x.SetSize(rhs.Size());
+      magma_ppinv_x.UseDevice(true);
+      magma_ppinv_apply_ms =
+         TimeMagmaInverseApply(magma_inverse, rhs, reps, magma_ppinv_x,
+                               *magma_inv_ws);
+      ComputeLowerPackedResidual(packed_ea, magma_ppinv_x, rhs,
+                                 magma_ppinv_res_l2, magma_ppinv_rel_res_l2,
+                                 magma_ppinv_res_max,
+                                 magma_ppinv_rel_res_max);
+   }
+#endif
+
+   cout << fixed << setprecision(6);
+   cout << "Mass matrix inverse microbenchmark" << '\n';
+   cout << "Mesh: " << mesh_file << '\n';
+   cout << "Dimension: " << dim << '\n';
+   cout << "Elements: " << ne << '\n';
+   cout << "Element dofs: " << elem_dofs << '\n';
+   cout << "Scalar element unknowns: " << ne*elem_dofs << '\n';
+   cout << "Apply repetitions: " << reps << '\n';
+   cout << "Setup repetitions: " << setup_reps << '\n';
+   cout << '\n';
+
+   cout << "Assembly packed EA (ms): " << assemble_ms << '\n';
+   cout << "Setup MFEM tripack inverse (ms): " << tripack_inverse_setup_ms
+        << '\n';
+#ifdef MFEM_USE_MAGMA
+   if (use_magma)
+   {
+      cout << "Setup MAGMA packed Cholesky factor (ms): " << magma_factor_ms
+           << '\n';
+      cout << "Setup MAGMA full Cholesky factor (ms): " << magma_full_factor_ms
+           << '\n';
+      if (magma_ppinv_enabled)
+      {
+         cout << "Setup MAGMA packed inverse (ppinv) (ms): " << magma_inverse_ms
+              << '\n';
+      }
+      else
+      {
+         cout << "Setup MAGMA packed inverse (ppinv) (ms): skipped "
+              << "(requires element dofs <= 64)\n";
+      }
+   }
+#endif
+   cout << '\n';
+
+   cout << "Apply MFEM tripack inverse (ms/apply): " << tripack_apply_ms
+        << '\n';
+#ifdef MFEM_USE_MAGMA
+   if (use_magma)
+   {
+      cout << "Apply MAGMA packed Cholesky solve (ms/apply): "
+           << magma_solve_ms << '\n';
+      cout << "Apply MAGMA full Cholesky solve (ms/apply): "
+           << magma_full_solve_ms << '\n';
+      cout << "MAGMA solve / MFEM tripack inverse apply: "
+           << magma_solve_ms/tripack_apply_ms << '\n';
+      cout << "MAGMA full solve / MFEM tripack inverse apply: "
+           << magma_full_solve_ms/tripack_apply_ms << '\n';
+
+      if (magma_ppinv_enabled)
+      {
+         cout << "Apply MAGMA packed inverse (ppinv) (ms/apply): "
+              << magma_ppinv_apply_ms << '\n';
+         cout << "MAGMA ppinv apply / MFEM tripack inverse apply: "
+              << magma_ppinv_apply_ms/tripack_apply_ms << '\n';
+      }
+      else
+      {
+         cout << "Apply MAGMA packed inverse (ppinv) (ms/apply): skipped "
+              << "(requires element dofs <= 64)\n";
+      }
+
+      const double tripack_fixed_ms = assemble_ms + tripack_inverse_setup_ms;
+      const double magma_fixed_ms = assemble_ms + magma_factor_ms;
+      const double magma_full_fixed_ms = assemble_ms + magma_full_factor_ms;
+      const double tripack_total_ms =
+         tripack_fixed_ms + reps*tripack_apply_ms;
+      const double magma_total_ms = magma_fixed_ms + reps*magma_solve_ms;
+      const double magma_full_total_ms =
+         magma_full_fixed_ms + reps*magma_full_solve_ms;
+      cout << "Total MFEM tripack inverse for current repetitions "
+           << "(assembly+setup+applies, ms): " << tripack_total_ms << '\n';
+      cout << "Total MAGMA packed Cholesky solve for current repetitions "
+           << "(assembly+setup+applies, ms): " << magma_total_ms << '\n';
+      cout << "Total MAGMA full Cholesky solve for current repetitions "
+           << "(assembly+setup+applies, ms): " << magma_full_total_ms << '\n';
+      cout << "Faster approach for current repetitions: "
+           << ((magma_total_ms < tripack_total_ms &&
+                magma_total_ms <= magma_full_total_ms) ? "MAGMA packed" :
+               ((magma_full_total_ms < tripack_total_ms &&
+                 magma_full_total_ms < magma_total_ms) ? "MAGMA full" :
+                ((tripack_total_ms < magma_total_ms &&
+                  tripack_total_ms <= magma_full_total_ms) ? "tripack" :
+                 "tie")))
+           << '\n';
+      PrintMagmaFasterCondition(tripack_fixed_ms, tripack_apply_ms,
+                                magma_fixed_ms, magma_solve_ms);
+      PrintMagmaFasterCondition(tripack_fixed_ms, tripack_apply_ms,
+                                magma_full_fixed_ms, magma_full_solve_ms);
+
+      if (magma_ppinv_enabled)
+      {
+         const double magma_ppinv_fixed_ms =
+            assemble_ms + magma_inverse_ms;
+         const double magma_ppinv_total_ms =
+            magma_ppinv_fixed_ms + reps*magma_ppinv_apply_ms;
+         cout << "Total MAGMA packed inverse (ppinv) for current "
+              << "repetitions (assembly+setup+applies, ms): "
+              << magma_ppinv_total_ms << '\n';
+      }
+   }
+#endif
+   cout << '\n';
+
+   cout << scientific << setprecision(12);
+   cout << "Residual, MFEM tripack inverse, max: "
+        << tripack_res_max << " (relative "
+        << tripack_rel_res_max << "), L2: "
+        << tripack_res_l2 << " (relative "
+        << tripack_rel_res_l2 << ")\n";
+#ifdef MFEM_USE_MAGMA
+   if (use_magma)
+   {
+      cout << "Residual, MAGMA packed Cholesky solve, max: "
+           << magma_res_max << " (relative "
+           << magma_rel_res_max << "), L2: "
+           << magma_res_l2 << " (relative "
+           << magma_rel_res_l2 << ")\n";
+      cout << "Residual, MAGMA full Cholesky solve, max: "
+           << magma_full_res_max << " (relative "
+           << magma_full_rel_res_max << "), L2: "
+           << magma_full_res_l2 << " (relative "
+           << magma_full_rel_res_l2 << ")\n";
+      if (magma_ppinv_enabled)
+      {
+         cout << "Residual, MAGMA packed inverse (ppinv), max: "
+              << magma_ppinv_res_max << " (relative "
+              << magma_ppinv_rel_res_max << "), L2: "
+              << magma_ppinv_res_l2 << " (relative "
+              << magma_ppinv_rel_res_l2 << ")\n";
+      }
+   }
+#endif
+
+   return 0;
+}

examples/magma/makefile

@@ -0,0 +1,55 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+MFEM_INSTALL_DIR ?= ../../mfem
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/magma/,)
+CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
+   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = batched_mass_bench
+EXAMPLES = $(SEQ_EXAMPLES)
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_MAGMA),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with MAGMA)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints

examples/makefile

@@ -68,6 +68,9 @@ endif
 ifeq ($(MFEM_USE_CALIPER),YES)
    SUBDIRS += caliper
 endif
+ifeq ($(MFEM_USE_MAGMA),YES)
+   SUBDIRS += magma
+endif
 
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
@@ -84,7 +87,8 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
 
 # Replace the default implicit rule for *.cpp files
 %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@.o
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) $@.o -o $@ $(MFEM_LIBS)
 
 all: $(EXAMPLES) $(SUBDIRS_ALL)
 

fem/bilinearform.cpp

@@ -1255,6 +1255,31 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
    }
 }
 
+void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMult(x, y, a);
+   }
+   else
+   {
+      mat->AddMult(x, y, a);
+   }
+}
+
+void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
+                                    const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMultTranspose(x, y, a);
+   }
+   else
+   {
+      mat->AddMultTranspose(x, y, a);
+   }
+}
+
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
    if (ext)

fem/bilinearform.hpp

@@ -307,8 +307,8 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
-   { mat -> AddMult (x, y, a); }
+   void AddMult(const Vector &x, Vector &y,
+                const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
        The original matrix is $ M + Me $ so we have:
@@ -318,8 +318,7 @@ public:
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
    void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override
-   { mat->AddMultTranspose(x, y, a); }
+                         const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix transpose vector
        multiple to a vector. The original matrix is $ M + M_e $

fem/bilinearform_ext.cpp

@@ -1997,7 +1997,11 @@ void PADiscreteLinearOperatorExtension::Assemble()
    }
    else
    {
-      mfem_error("A real ElementRestriction is required in this setting!");
+      const L2ElementRestriction* l2_elem_restrict =
+         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
+      MFEM_VERIFY(l2_elem_restrict,
+                  "A real ElementRestriction is required in this setting!");
+      test_multiplicity = 1.0;
    }
 
    auto tm = test_multiplicity.ReadWrite();
@@ -2036,7 +2040,13 @@ void PADiscreteLinearOperatorExtension::AddMult(
    }
    else
    {
-      mfem_error("In this setting you need a real ElementRestriction!");
+      const L2ElementRestriction* l2_elem_restrict =
+         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
+      MFEM_VERIFY(l2_elem_restrict,
+                  "In this setting you need a real ElementRestriction!");
+      tempY.SetSize(y.Size());
+      l2_elem_restrict->MultTranspose(localTest, tempY);
+      y += tempY;
    }
 }
 

fem/coefficient.hpp

@@ -1055,7 +1055,8 @@ public:
 
 typedef VectorCoefficient DiagonalMatrixCoefficient;
 
-/// Base class for Matrix Coefficients that optionally depend on time and space.
+/** Base class for matrix-valued coefficients that optionally depend on time
+    and space. */
 class MatrixCoefficient
 {
 protected:
@@ -1102,6 +1103,9 @@ public:
    /// the quadrature points. The matrix will be transposed or not according to
    /// the boolean argument @a transpose.
    ///
+   /// The stored entries use the same row/column convention as `Eval()`,
+   /// unless `transpose == true`, in which case `K^T` is stored instead.
+   ///
    /// The @a vdim of the QuadratureFunction should be equal to the height times
    /// the width of the matrix.
    virtual void Project(QuadratureFunction &qf, bool transpose=false);

fem/complex_fem.cpp

@@ -588,6 +588,38 @@ SesquilinearForm::AssembleComplexSparseMatrix()
                                   false, false, conv);
 }
 
+void
+SesquilinearForm::BuildComplexOperator(OperatorHandle &A_r,
+                                       OperatorHandle &A_i,
+                                       OperatorHandle &A) const
+{
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   {
+      ComplexSparseMatrix * A_sp =
+         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
+                                 A_i.As<SparseMatrix>(),
+                                 A_r.OwnsOperator(),
+                                 A_i.OwnsOperator(),
+                                 conv);
+      A.Reset<ComplexSparseMatrix>(A_sp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.Ptr(),
+                             A_i.Ptr(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
 void
 SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                    Vector &x, Vector &b,
@@ -716,31 +748,7 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
    B_r.SyncAliasMemory(B);
    B_i.SyncAliasMemory(B);
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_sp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_sp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.Ptr(),
-                             A_i.Ptr(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void
@@ -777,31 +785,7 @@ SesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
       }
    }
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_sp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_sp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.Ptr(),
-                             A_i.Ptr(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void
@@ -1893,6 +1877,81 @@ ParSesquilinearForm::ParallelAssemble()
                                     true, true, conv);
 }
 
+void
+ParSesquilinearForm::BuildComplexOperator(OperatorHandle &A_r,
+                                          OperatorHandle &A_i,
+                                          OperatorHandle &A) const
+{
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   {
+      ComplexHypreParMatrix * A_hyp =
+         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
+                                   A_i.As<HypreParMatrix>(),
+                                   A_r.OwnsOperator(),
+                                   A_i.OwnsOperator(),
+                                   conv);
+      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.As<Operator>(),
+                             A_i.As<Operator>(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+namespace
+{
+struct ZeroDiagonalHypreKernel
+{
+   const int *ess_tdof_list;
+   const HYPRE_Int *diag_i;
+   real_t *diag_data;
+
+   void MFEM_HOST_DEVICE operator()(int k) const
+   {
+      const int j = ess_tdof_list[k];
+      diag_data[diag_i[j]] = 0.0;
+   }
+};
+}
+
+void
+ParSesquilinearForm::SetImaginaryEssentialDiagonalToZero(
+   const Array<int> &ess_tdof_list, OperatorHandle &A)
+{
+   if (A.Type() == Operator::Hypre_ParCSR)
+   {
+      const int n = ess_tdof_list.Size();
+      HypreParMatrix *Ah;
+      A.Get(Ah);
+      hypre_ParCSRMatrix *Aih = *Ah;
+      Ah->HypreReadWrite();
+      const int *d_ess_tdof_list =
+         ess_tdof_list.GetMemory().Read(GetHypreForallMemoryClass(), n);
+      HYPRE_Int *d_diag_i = Aih->diag->i;
+      real_t *d_diag_data = Aih->diag->data;
+      mfem::hypre_forall(n, ZeroDiagonalHypreKernel
+      {
+         d_ess_tdof_list, d_diag_i, d_diag_data
+      });
+   }
+   else
+   {
+      A.As<ConstrainedOperator>()->SetDiagonalPolicy
+      (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
+   }
+}
+
 void
 ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                       Vector &x, Vector &b,
@@ -1993,27 +2052,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       });
       // Modify off-diagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      if (A_i.Type() == Operator::Hypre_ParCSR)
-      {
-         HypreParMatrix * Ah;
-         A_i.Get(Ah);
-         hypre_ParCSRMatrix *Aih = *Ah;
-         Ah->HypreReadWrite();
-         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(GetHypreForallMemoryClass(), n);
-         HYPRE_Int *d_diag_i = Aih->diag->i;
-         real_t *d_diag_data = Aih->diag->data;
-         mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)
-         {
-            const int j = d_ess_tdof_list[k];
-            d_diag_data[d_diag_i[j]] = 0.0;
-         });
-      }
-      else
-      {
-         A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
-         (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
-      }
+      SetImaginaryEssentialDiagonalToZero(ess_tdof_list, A_i);
    }
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC)
@@ -2032,31 +2071,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
    B_r.SyncAliasMemory(B);
    B_i.SyncAliasMemory(B);
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.As<Operator>(),
-                             A_i.As<Operator>(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void
@@ -2081,50 +2096,10 @@ ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    {
       // Modify off-diagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      if ( A_i.Type() == Operator::Hypre_ParCSR )
-      {
-         int n = ess_tdof_list.Size();
-         HypreParMatrix * Ah;
-         A_i.Get(Ah);
-         hypre_ParCSRMatrix * Aih = *Ah;
-         for (int k = 0; k < n; k++)
-         {
-            int j = ess_tdof_list[k];
-            Aih->diag->data[Aih->diag->i[j]] = 0.0;
-         }
-      }
-      else
-      {
-         A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
-         (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
-      }
+      SetImaginaryEssentialDiagonalToZero(ess_tdof_list, A_i);
    }
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.As<Operator>(),
-                             A_i.As<Operator>(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void

fem/complex_fem.hpp

@@ -392,6 +392,9 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
+   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
+                             OperatorHandle &A) const;
+
 public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
@@ -986,6 +989,12 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
+   void SetImaginaryEssentialDiagonalToZero(
+      const Array<int> &ess_tdof_list, OperatorHandle &A);
+
+   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
+                             OperatorHandle &A) const;
+
 public:
    ParSesquilinearForm(ParFiniteElementSpace *pf,
                        ComplexOperator::Convention

fem/datacollection.cpp

@@ -38,9 +38,24 @@ int DataCollection::create_directory(const std::string &dir_name,
    // create directories recursively
    const char path_delim = '/';
    std::string::size_type pos = 0;
-   int err_flag;
+   int err_flag = 0;
 #ifdef MFEM_USE_MPI
    const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
+   // In addition to the global root, let the lowest rank on each shared-memory
+   // node create the directory too, so that node-local (non-shared) filesystems
+   // get it on every node rather than only where the global root lives. On a
+   // shared filesystem the extra mkdir() hits EEXIST and is tolerated below.
+   bool node_root = true;
+   if (pmesh)
+   {
+      MPI_Comm node_comm;
+      MPI_Comm_split_type(pmesh->GetComm(), MPI_COMM_TYPE_SHARED, myid,
+                          MPI_INFO_NULL, &node_comm);
+      int node_rank;
+      MPI_Comm_rank(node_comm, &node_rank);
+      node_root = (node_rank == 0);
+      MPI_Comm_free(&node_comm);
+   }
 #endif
 
    do
@@ -52,7 +67,7 @@ int DataCollection::create_directory(const std::string &dir_name,
       err_flag = mkdir(subdir.c_str(), 0777);
       err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
 #else
-      if (myid == 0 || pmesh == NULL)
+      if (node_root || pmesh == NULL)
       {
          err_flag = mkdir(subdir.c_str(), 0777);
          err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
@@ -64,7 +79,8 @@ int DataCollection::create_directory(const std::string &dir_name,
 #ifdef MFEM_USE_MPI
    if (pmesh)
    {
-      MPI_Bcast(&err_flag, 1, MPI_INT, 0, pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE, &err_flag, 1, MPI_INT, MPI_MAX,
+                    pmesh->GetComm());
    }
 #endif
 

fem/dfem/doperator.cpp

@@ -51,4 +51,52 @@ DifferentiableOperator::DifferentiableOperator(
    }
 }
 
+
+void FDJacobian::Mult(const Vector &v, Vector &y) const
+{
+   // See [1] for choice of eps.
+   //
+   // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
+   // finite difference matrix-vector products in Newton-Krylov solvers for
+   // implicit climate dynamics with spectral elements. Procedia Computer
+   // Science, 51, pp.2036-2045.
+   real_t eps;
+   if (fixed_eps > 0.0)
+   {
+      eps = fixed_eps;
+   }
+   else
+   {
+      const real_t vnorm_local = v.Norml2();
+      real_t vnorm;
+      MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                    MPI_COMM_WORLD);
+      eps = lambda * (lambda + xnorm / vnorm);
+   }
+
+   // x + eps * v
+   {
+      const auto d_v = v.Read();
+      const auto d_x = x.Read();
+      auto d_xpev = xpev.Write();
+      mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_xpev[i] = d_x[i] + eps * d_v[i];
+      });
+   }
+
+   // y = f(x + eps * v)
+   op.Mult(xpev, y);
+
+   // y = (f(x + eps * v) - f(x)) / eps
+   {
+      const auto d_f = f.Read();
+      auto d_y = y.ReadWrite();
+      mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_y[i] = (d_y[i] - d_f[i]) / eps;
+      });
+   }
+}
+
 #endif // MFEM_USE_MPI

fem/dfem/doperator.hpp

@@ -697,17 +697,18 @@ void DifferentiableOperator::AddIntegrator(
 
    // The explicit captures are necessary to avoid dependency on
    // the specific instance of this class (this pointer).
-   restriction_callback =
-      [=, solutions = this->solutions, parameters = this->parameters]
-      (std::vector<Vector> &sol,
-       const std::vector<Vector> &par,
-       std::vector<Vector> &f)
+   restriction_callback = [element_dof_ordering,
+                           solutions_ = this->solutions,
+                           parameters_ = this->parameters]
+   (std::vector<Vector> &sol,
+    const std::vector<Vector> &par,
+    std::vector<Vector> &f)
    {
-      restriction<entity_t>(solutions, sol, f,
+      restriction<entity_t>(solutions_, sol, f,
                             element_dof_ordering);
-      restriction<entity_t>(parameters, par, f,
+      restriction<entity_t>(parameters_, par, f,
                             element_dof_ordering,
-                            solutions.size());
+                            solutions_.size());
    };
 
    prolongation_transpose = get_prolongation_transpose(
@@ -835,19 +836,19 @@ void DifferentiableOperator::AddIntegrator(
 
          // capture by ref:
          &restriction_cb = this->restriction_callback,
-         &fields_e = this->fields_e,
-         &residual_e = this->residual_e,
-         &output_restriction_transpose = this->output_restriction_transpose
+         &fields_e_ = this->fields_e,
+         &residual_e_ = this->residual_e,
+         &output_restriction_transpose_ = this->output_restriction_transpose
       ]
       (std::vector<Vector> &sol, const std::vector<Vector> &par, Vector &res)
       mutable // mutable: needed to modify 'shmem_cache'
    {
-      restriction_cb(sol, par, fields_e);
+      restriction_cb(sol, par, fields_e_);
 
-      residual_e = 0.0;
-      auto ye = Reshape(residual_e.ReadWrite(), test_vdim, num_test_dof, num_entities);
+      residual_e_ = 0.0;
+      auto ye = Reshape(residual_e_.ReadWrite(), test_vdim, num_test_dof, num_entities);
 
-      auto wrapped_fields_e = wrap_fields(fields_e,
+      auto wrapped_fields_e = wrap_fields(fields_e_,
                                           action_shmem_info.field_sizes,
                                           num_entities);
 
@@ -878,7 +879,7 @@ void DifferentiableOperator::AddIntegrator(
             y, fhat, output_fop, output_dtq_shmem[0],
             scratch_shmem, dimension, use_sum_factorization);
       }, num_entities, thread_blocks, action_shmem_info.total_size, shmem_cache.ReadWrite());
-      output_restriction_transpose(residual_e, res);
+      output_restriction_transpose_(residual_e_, res);
    });
 
    // Without this compile-time check, some valid instantiations of this method
@@ -1193,7 +1194,7 @@ void DifferentiableOperator::AddIntegrator(
 
                // capture by ref:
                &qpdc_mem = derivative_qp_caches_ref,
-               &fields = fields_ref
+               &fields_ = fields_ref
             ](std::vector<Vector> &f_e, SparseMatrix *&A) mutable
          {
             auto wrapped_fields_e = wrap_fields(f_e, shmem_info.field_sizes,
@@ -1241,14 +1242,14 @@ void DifferentiableOperator::AddIntegrator(
             {
                if (input_is_dependent[s])
                {
-                  trial_field = &fields[input_to_field[s]];
+                  trial_field = &fields_[input_to_field[s]];
                }
             }
 
             auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
                              (&trial_field->data);
             auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields[output_to_field[0]].data);
+                            (&fields_[output_to_field[0]].data);
 
             A = new SparseMatrix(test_fes->GetVSize(), trial_fes->GetVSize());
 
@@ -1334,7 +1335,7 @@ void DifferentiableOperator::AddIntegrator(
                input_to_field,
                output_to_field,
                &spmatcb = assemble_derivative_sparsematrix_callbacks_ref,
-               &fields = fields_ref
+               &fields_ = fields_ref
             ](std::vector<Vector> &f_e, HypreParMatrix *&A) mutable
          {
             SparseMatrix *spmat = nullptr;
@@ -1366,14 +1367,14 @@ void DifferentiableOperator::AddIntegrator(
             {
                if (input_is_dependent[s])
                {
-                  trial_field = &fields[input_to_field[s]];
+                  trial_field = &fields_[input_to_field[s]];
                }
             }
 
             auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
                              (&trial_field->data);
             auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields[output_to_field[0]].data);
+                            (&fields_[output_to_field[0]].data);
 
             if (same_test_and_trial)
             {

fem/dfem/util.hpp

@@ -597,7 +597,7 @@ struct ThreadBlocks
    int z = 1;
 };
 
-#if defined(MFEM_USE_CUDA_OR_HIP)
+#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
 template <typename func_t>
 __global__ void forall_kernel_shmem(func_t f, int n)
 {
@@ -617,10 +617,11 @@ void forall(func_t f,
             int num_shmem = 0,
             real_t *shmem = nullptr)
 {
-   if (Device::Allows(Backend::CUDA_MASK) ||
-       Device::Allows(Backend::HIP_MASK))
+   internal::RequireKernelCompilation();
+
+#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
+   if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
-#if defined(MFEM_USE_CUDA_OR_HIP)
       // int gridsize = (N + Z - 1) / Z;
       int num_bytes = num_shmem * sizeof(decltype(shmem));
       dim3 block_size(blocks.x, blocks.y, blocks.z);
@@ -631,9 +632,10 @@ void forall(func_t f,
       MFEM_GPU_CHECK(hipGetLastError());
 #endif
       MFEM_DEVICE_SYNC;
-#endif
+      return;
    }
-   else if (Device::Allows(Backend::CPU_MASK))
+#endif
+   if (Device::Allows(Backend::CPU_MASK))
    {
       MFEM_ASSERT(!((bool)num_shmem != (bool)shmem),
                   "Backend::CPU needs a pre-allocated shared memory block");
@@ -671,52 +673,7 @@ public:
                     MPI_COMM_WORLD);
    }
 
-   void Mult(const Vector &v, Vector &y) const override
-   {
-      // See [1] for choice of eps.
-      //
-      // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
-      // finite difference matrix-vector products in Newton-Krylov solvers for
-      // implicit climate dynamics with spectral elements. Procedia Computer
-      // Science, 51, pp.2036-2045.
-      real_t eps;
-      if (fixed_eps > 0.0)
-      {
-         eps = fixed_eps;
-      }
-      else
-      {
-         const real_t vnorm_local = v.Norml2();
-         real_t vnorm;
-         MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                       MPI_COMM_WORLD);
-         eps = lambda * (lambda + xnorm / vnorm);
-      }
-
-      // x + eps * v
-      {
-         const auto d_v = v.Read();
-         const auto d_x = x.Read();
-         auto d_xpev = xpev.Write();
-         mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
-         {
-            d_xpev[i] = d_x[i] + eps * d_v[i];
-         });
-      }
-
-      // y = f(x + eps * v)
-      op.Mult(xpev, y);
-
-      // y = (f(x + eps * v) - f(x)) / eps
-      {
-         const auto d_f = f.Read();
-         auto d_y = y.ReadWrite();
-         mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
-         {
-            d_y[i] = (d_y[i] - d_f[i]) / eps;
-         });
-      }
-   }
+   void Mult(const Vector &v, Vector &y) const override;
 
    virtual MemoryClass GetMemoryClass() const override
    {

fem/fe/fe_base.cpp

@@ -1316,13 +1316,14 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
-void VectorFiniteElement::ProjectGrad_RT(
+void VectorFiniteElement::ProjectCurl2D_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &grad) const
 {
+   // 2D "ProjectCurl_RT"
    if (dim != 2)
    {
-      mfem_error("VectorFiniteElement::ProjectGrad_RT works only in 2D!");
+      mfem_error("VectorFiniteElement::ProjectCurl2D_RT works only in 2D!");
    }
 
    DenseMatrix dshape(fe.GetDof(), fe.GetDim());
@@ -1333,8 +1334,8 @@ void VectorFiniteElement::ProjectGrad_RT(
    for (int k = 0; k < dof; k++)
    {
       fe.CalcDShape(Nodes.IntPoint(k), dshape);
-      tk[0] = nk[d2n[k]*dim+1];
-      tk[1] = -nk[d2n[k]*dim];
+      tk[0] = -nk[d2n[k]*dim+1];
+      tk[1] = nk[d2n[k]*dim];
       dshape.Mult(tk, grad_k);
       for (int j = 0; j < grad_k.Size(); j++)
       {
@@ -1381,7 +1382,7 @@ void VectorFiniteElement::ProjectCurl_ND(
    }
 }
 
-void VectorFiniteElement::ProjectCurl_RT(
+void VectorFiniteElement::ProjectCurl3D_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &curl) const
 {

fem/fe/fe_base.hpp

@@ -957,10 +957,11 @@ protected:
                    const FiniteElement &fe, ElementTransformation &Trans,
                    DenseMatrix &I) const;
 
-   // rotated gradient in 2D
-   void ProjectGrad_RT(const real_t *nk, const Array<int> &d2n,
-                       const FiniteElement &fe, ElementTransformation &Trans,
-                       DenseMatrix &grad) const;
+   // Input is a scalar representing the Z (out of plane) component, Output is
+   // the X-Y (in-plane) RT curl
+   void ProjectCurl2D_RT(const real_t *nk, const Array<int> &d2n,
+                         const FiniteElement &fe, ElementTransformation &Trans,
+                         DenseMatrix &grad) const;
 
    // Compute the curl as a discrete operator from ND FE (fe) to ND FE (this).
    // The natural FE for the range is RT, so this is an approximation.
@@ -968,9 +969,9 @@ protected:
                        const FiniteElement &fe, ElementTransformation &Trans,
                        DenseMatrix &curl) const;
 
-   void ProjectCurl_RT(const real_t *nk, const Array<int> &d2n,
-                       const FiniteElement &fe, ElementTransformation &Trans,
-                       DenseMatrix &curl) const;
+   void ProjectCurl3D_RT(const real_t *nk, const Array<int> &d2n,
+                         const FiniteElement &fe, ElementTransformation &Trans,
+                         DenseMatrix &curl) const;
 
    /** @brief Project a vector coefficient onto the ND basis functions
        @param tk    Edge tangent vectors for this element type
@@ -1446,6 +1447,8 @@ public:
                                 dof2quad_array_open);
    }
 
+   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
+
    virtual ~VectorTensorFiniteElement();
 };
 

fem/fe/fe_rt.hpp

@@ -73,16 +73,11 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
-   // Gradient + rotation = Curl: H1 -> H(div)
-   void ProjectGrad(const FiniteElement &fe,
-                    ElementTransformation &Trans,
-                    DenseMatrix &grad) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
 
    void GetFaceMap(const int face_id, Array<int> &face_map) const override;
 
@@ -148,7 +143,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 
    /// @brief Return the mapping from lexicographically ordered face DOFs to
    /// lexicographically ordered element DOFs corresponding to local face
@@ -210,16 +205,11 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
-   // Gradient + rotation = Curl: H1 -> H(div)
-   void ProjectGrad(const FiniteElement &fe,
-                    ElementTransformation &Trans,
-                    DenseMatrix &grad) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 
@@ -274,7 +264,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 class RT_WedgeElement : public VectorFiniteElement
@@ -332,7 +322,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 /** Arbitrary order H(Div) basis functions defined on pyramid-shaped elements
@@ -428,7 +418,7 @@ public:
    virtual void ProjectCurl(const FiniteElement &fe,
                             ElementTransformation &Trans,
                             DenseMatrix &curl) const
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 
    void CalcRawVShape(const IntegrationPoint &ip,
                       DenseMatrix &shape) const;

fem/gslib.cpp

@@ -556,7 +556,7 @@ void obboxsurf_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, n);
    lag(I0, work, n, 1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=n2,y+=n2,z+=n2)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=n2,y+=n2,z+=n2)
    {
       struct gslib::dbl_range ab[3];
       struct gslib::dbl_range tb[3];
@@ -780,7 +780,7 @@ void obboxedge_calc_2(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr,1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr)
    {
       double x0[2], A[4];
       struct gslib::dbl_range ab[2], tb[2];
@@ -892,7 +892,7 @@ void obboxedge_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr, 1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr,z+=nr)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr,z+=nr)
    {
       double x0[3], A[9], Ai[9];
       struct gslib::dbl_range ab[3], tb[3];
@@ -4518,7 +4518,7 @@ Mesh* FindPointsGSLIB::GetBoundingBoxMesh(int type)
    int eidx = 0;
    if (myid == save_rank)
    {
-      for (int p = 0; p < gsl_comm->np; p++)
+      for (int p = 0; (unsigned)p < gsl_comm->np; p++)
       {
          if (static_cast<unsigned int>(p) != save_rank)
          {

fem/integ/bilininteg_convection_pa.cpp

@@ -178,6 +178,8 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
+   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
+               "Only value map type currently supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
    if (DeviceCanUseCeed())

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -785,6 +785,23 @@ void PAHcurlL2Setup2D(const int Q1D,
    });
 }
 
+void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
+                         Vector &coeff, const Vector &detJ, Vector &op)
+{
+   const int NQ = Q1D*Q1D;
+   auto W = w.Read();
+   auto C = Reshape(coeff.Read(), NQ, NE);
+   auto J = Reshape(detJ.Read(), NQ, NE);
+   auto y = Reshape(op.Write(), NQ, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         y(q,e) = W[q] * C(q,e) / J(q,e);
+      }
+   });
+}
+
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,
                       const int NE,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -1889,13 +1889,17 @@ inline void SmemPACurlCurlApply3D(const int d1d,
    ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }
 
-// PA H(curl)-L2 Assemble 2D kernel
+// PA H(curl)-L2 value Assemble 2D kernel
 void PAHcurlL2Setup2D(const int Q1D,
                       const int NE,
                       const Array<real_t> &w,
                       Vector &coeff,
                       Vector &op);
 
+// PA H(curl)-L2 integral Assemble 2D kernel
+void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
+                         Vector &coeff, const Vector &detJ, Vector &op);
+
 // PA H(curl)-L2 Assemble 3D kernel
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -864,8 +864,656 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
    }); // end of element loop
 }
 
+namespace curlinterp
+{
+constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
+{
+   if (ndof_o <= 0 || nquad_o <= 0)
+   {
+      return 1;
+   }
+   int ndof_c = ndof_o + 1;
+   int nquad_c = nquad_o + 1;
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp =
+      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
+       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
+                                 const int nquad_o, const Vector &pa,
+                                 const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_O, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
+                                  const int nquad_o, const Vector &pa,
+                                  const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 } // namespace internal
 
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
 } // namespace mfem
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_interp_pa.cpp

@@ -14,9 +14,218 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
 namespace mfem
 {
 
+namespace
+{
+
+void PAHcurlApplyCurl2D(const int c_dofs1D,
+                        const int o_dofs1D,
+                        const int NE,
+                        const Array<real_t> &Bo_,
+                        const Array<real_t> &Gc_,
+                        const Vector &x_,
+                        Vector &y_)
+{
+   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), o_dofs1D, o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < o_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix + iy * o_dofs1D, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(ox, oy, e) -= Bo(ox, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int iy = 0; iy < o_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(y_nd + ix + iy * c_dofs1D, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t by = Bo(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(ox, oy, e) += Gc(ox, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlApplyCurlTranspose2D(const int c_dofs1D,
+                                 const int o_dofs1D,
+                                 const int NE,
+                                 const Array<real_t> &Bo_,
+                                 const Array<real_t> &Gc_,
+                                 const Vector &x_,
+                                 Vector &y_)
+{
+   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), o_dofs1D, o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int dy = 0; dy < c_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < o_dofs1D; ++dx)
+         {
+            real_t sum = 0.0;
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, dy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  sum -= Bo(ox, dx) * gy * X(ox, oy, e);
+               }
+            }
+            Y(dx + dy * o_dofs1D, e) += sum;
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int dy = 0; dy < o_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < c_dofs1D; ++dx)
+         {
+            real_t sum = 0.0;
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t by = Bo(oy, dy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  sum += Gc(ox, dx) * by * X(ox, oy, e);
+               }
+            }
+            Y(y_nd + dx + dy * c_dofs1D, e) += sum;
+         }
+      }
+   });
+}
+
+void PAHdivApplyCurl2D(const int c_dofs1D,
+                       const int o_dofs1D,
+                       const int NE,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Gc_,
+                       const Vector &x_,
+                       Vector &y_)
+{
+   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), c_dofs1D, c_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix, iy, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, iy);
+               for (int ox = 0; ox < c_dofs1D; ++ox)
+               {
+                  Y(ox + oy * c_dofs1D, e) += Bc(ox, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix, iy, e);
+            for (int oy = 0; oy < c_dofs1D; ++oy)
+            {
+               const real_t by = Bc(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(y_nd + ox + oy * o_dofs1D, e) -= Gc(ox, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHdivApplyCurlTranspose2D(const int c_dofs1D,
+                                const int o_dofs1D,
+                                const int NE,
+                                const Array<real_t> &Bc_,
+                                const Array<real_t> &Gc_,
+                                const Vector &x_,
+                                Vector &y_)
+{
+   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int dy = 0; dy < o_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < c_dofs1D; ++dx)
+         {
+            const real_t xv = X(dx + dy * c_dofs1D, e);
+            for (int iy = 0; iy < c_dofs1D; ++iy)
+            {
+               const real_t gy = Gc(dy, iy);
+               for (int ix = 0; ix < c_dofs1D; ++ix)
+               {
+                  Y(ix, iy, e) += Bc(dx, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int dy = 0; dy < c_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < o_dofs1D; ++dx)
+         {
+            const real_t xv = X(y_nd + dx + dy * o_dofs1D, e);
+            for (int iy = 0; iy < c_dofs1D; ++iy)
+            {
+               const real_t by = Bc(dy, iy);
+               for (int ix = 0; ix < c_dofs1D; ++ix)
+               {
+                  Y(ix, iy, e) -= Gc(dx, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+}
+
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
@@ -1950,4 +2159,266 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
+void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
+                                  const FiniteElementSpace &ran_fes)
+{
+   Mesh *mesh = dom_fes.GetMesh();
+   dim = mesh->Dimension();
+   ne = dom_fes.GetNE();
+   pa_mode_2d = 0;
+   MFEM_VERIFY(ne == ran_fes.GetNE(),
+               "Different meshes for domain and range spaces");
+
+   if (dim == 2)
+   {
+      pa_data.SetSize(0);
+      const FiniteElement *dom_fel = dom_fes.GetTypicalFE();
+      const FiniteElement *ran_fel = ran_fes.GetTypicalFE();
+      const bool hcurl_to_scalar =
+         dynamic_cast<const VectorTensorFiniteElement*>(dom_fel) != NULL &&
+         dom_fel->GetDerivType() == FiniteElement::CURL &&
+         dynamic_cast<const TensorBasisElement*>(ran_fel) != NULL &&
+         ran_fel->GetRangeType() == FiniteElement::SCALAR;
+      const bool scalar_to_hdiv =
+         dynamic_cast<const TensorBasisElement*>(dom_fel) != NULL &&
+         dom_fel->GetRangeType() == FiniteElement::SCALAR &&
+         dynamic_cast<const VectorTensorFiniteElement*>(ran_fel) != NULL &&
+         ran_fel->GetDerivType() == FiniteElement::DIV;
+
+      MFEM_VERIFY(hcurl_to_scalar || scalar_to_hdiv,
+                  "2D CurlInterpolator PA supports H(curl)->scalar and scalar->H(div) only.");
+
+      int closed_basis_type = -1;
+      int open_basis_type = -1;
+      if (hcurl_to_scalar)
+      {
+         const auto *trial_fec = dynamic_cast<const ND_FECollection*>(dom_fes.FEColl());
+         const auto *range_fec = dynamic_cast<const L2_FECollection*>(ran_fes.FEColl());
+         MFEM_VERIFY(trial_fec != NULL, "H(curl) domain must use ND_FECollection.");
+         MFEM_VERIFY(range_fec != NULL, "Scalar range must use L2_FECollection.");
+         MFEM_VERIFY(ran_fel->GetMapType() == FiniteElement::INTEGRAL,
+                     "2D H(curl)->scalar CurlInterpolator PA supports integral-map scalar range spaces only.");
+         closed_basis_type = trial_fec->GetClosedBasisType();
+         open_basis_type = trial_fec->GetOpenBasisType();
+         MFEM_VERIFY(range_fec->GetBasisType() == open_basis_type,
+                     "Domain/range open basis types do not match.");
+         pa_mode_2d = 1;
+      }
+      else
+      {
+         const auto *trial_fec = dynamic_cast<const H1_FECollection*>(dom_fes.FEColl());
+         const auto *range_fec = dynamic_cast<const RT_FECollection*>(ran_fes.FEColl());
+         MFEM_VERIFY(trial_fec != NULL, "Scalar domain must use H1_FECollection.");
+         MFEM_VERIFY(range_fec != NULL, "H(div) range must use RT_FECollection.");
+         closed_basis_type = trial_fec->GetBasisType();
+         open_basis_type = range_fec->GetOpenBasisType();
+         MFEM_VERIFY(range_fec->GetClosedBasisType() == closed_basis_type,
+                     "Domain/range closed basis types do not match.");
+         pa_mode_2d = 2;
+      }
+
+      const int order = hcurl_to_scalar
+                        ? dynamic_cast<const VectorTensorFiniteElement*>(dom_fel)->GetOrder()
+                        : dynamic_cast<const NodalTensorFiniteElement*>(dom_fel)->GetOrder();
+      c_dofs1D = order + 1;
+      o_dofs1D = order;
+
+      closed_dofquad_fe.reset(new H1_SegmentElement(order, closed_basis_type));
+      open_dofquad_fe.reset(new L2_SegmentElement(order - 1, open_basis_type));
+
+      mfem::QuadratureFunctions1D qf1d;
+      mfem::IntegrationRule closed_ir;
+      closed_ir.SetSize(c_dofs1D);
+      qf1d.GaussLobatto(c_dofs1D, &closed_ir);
+
+      mfem::IntegrationRule open_ir;
+      open_ir.SetSize(o_dofs1D);
+      qf1d.GaussLegendre(o_dofs1D, &open_ir);
+
+      maps_C_C = &closed_dofquad_fe->GetDofToQuad(closed_ir, DofToQuad::TENSOR);
+      maps_O_C = &closed_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
+      maps_O_O = &open_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
+
+      MFEM_VERIFY(maps_C_C->ndof == c_dofs1D && maps_C_C->nqpt == c_dofs1D, "");
+      MFEM_VERIFY(maps_O_C->ndof == c_dofs1D && maps_O_C->nqpt == o_dofs1D, "");
+      MFEM_VERIFY(maps_O_O->ndof == o_dofs1D && maps_O_O->nqpt == o_dofs1D, "");
+      return;
+   }
+
+   closed_dofquad_fe.reset();
+   open_dofquad_fe.reset();
+   maps_C_C = nullptr;
+   maps_O_C = nullptr;
+   maps_O_O = nullptr;
+
+   const VectorTensorFiniteElement *dom_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
+   const VectorTensorFiniteElement *ran_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
+   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
+               "Domain space must be H(curl)");
+   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
+               "Range space must be H(div)");
+
+   const int dims = dom_el->GetDim();
+   MFEM_VERIFY(dims == 3, "");
+
+   ndof_o = dom_el->GetOrder();
+   int ndof_c = ndof_o + 1;
+   nquad_o = ran_el->GetOrder();
+   int nquad_c = nquad_o + 1;
+
+   // extract the tensor product range dof locations
+   std::vector<real_t> qc(nquad_c);
+   std::vector<real_t> qo(nquad_o);
+   {
+      const IntegrationRule &ran_nodes = ran_el->GetNodes();
+      const Array<int> &quad_map = ran_el->GetDofMap();
+      for (int i = 0; i < nquad_c; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i]);
+         qc[i] = ran_nodes.IntPoint(idx).x;
+      }
+      int offset = ndof_c * ndof_o * ndof_o;
+      for (int i = 0; i < nquad_o; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i + offset]);
+         qo[i] = ran_nodes.IntPoint(idx).x;
+      }
+   }
+
+   // evaluate closed/open 1D basis (and their derivatives) at closed and
+   // open quads
+   // storage order: GCO, BCC, BOO
+   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   auto ptr = pa_data.HostWrite();
+   auto &cbasis1d = dom_el->GetBasis1D();
+   auto &obasis1d = dom_el->GetOpenBasis1D();
+   Vector b, g;
+   b.SetSize(ndof_c);
+   g.SetSize(ndof_c);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      cbasis1d.Eval(qo[j], b, g);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_o] = g[i];
+      }
+   }
+   ptr += nquad_o * ndof_c;
+
+   for (int j = 0; j < nquad_c; ++j)
+   {
+      cbasis1d.Eval(qc[j], b);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_c] = b[i];
+      }
+   }
+   ptr += ndof_c * nquad_c;
+
+   b.SetSize(ndof_o);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      obasis1d.Eval(qo[j], b);
+      for (int i = 0; i < ndof_o; ++i)
+      {
+         ptr[j + i * nquad_o] = b[i];
+      }
+   }
+}
+
+CurlInterpolator::Kernels::Kernels()
+{
+   CurlInterpolator::AddSpecialization<3, 1, 1>();
+   CurlInterpolator::AddSpecialization<3, 2, 2>();
+   CurlInterpolator::AddSpecialization<3, 3, 3>();
+   CurlInterpolator::AddSpecialization<3, 4, 4>();
+   CurlInterpolator::AddSpecialization<3, 5, 5>();
+}
+
+CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
+
+void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
+                  "2D CurlInterpolator PA data is not assembled.");
+      if (pa_mode_2d == 1)
+      {
+         MFEM_VERIFY(maps_O_O != nullptr,
+                     "2D CurlInterpolator scalar curl map is not assembled.");
+         PAHcurlApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B, maps_O_C->G,
+                            x, y);
+      }
+      else if (pa_mode_2d == 2)
+      {
+         PAHdivApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B, maps_O_C->G,
+                           x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
+      }
+      return;
+   }
+
+   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
+}
+
+void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
+                  "2D CurlInterpolator PA data is not assembled.");
+      if (pa_mode_2d == 1)
+      {
+         MFEM_VERIFY(maps_O_O != nullptr,
+                     "2D CurlInterpolator scalar curl map is not assembled.");
+         PAHcurlApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B,
+                                     maps_O_C->G, x, y);
+      }
+      else if (pa_mode_2d == 2)
+      {
+         PAHdivApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B,
+                                    maps_O_C->G, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
+      }
+      return;
+   }
+
+   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
+}
+
+/// \cond DO_NOT_DOCUMENT
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem

fem/integ/bilininteg_mass_ea.cpp

@@ -16,6 +16,71 @@
 
 namespace mfem
 {
+namespace
+{
+
+void AssembleEATriangularImpl(const int dim, const int ne,
+                              const int dofs1D, const int quad1D,
+                              const Array<real_t> &B,
+                              const Vector &pa_data,
+                              Vector &data,
+                              const bool add)
+{
+   using internal::EAMassAssembleTriangular1DLower;
+   using internal::EAMassAssembleTriangular2DLower;
+   using internal::EAMassAssembleTriangular3DLower;
+
+   if (dim == 1)
+   {
+      auto kernel = EAMassAssembleTriangular1DLower<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x22: kernel = EAMassAssembleTriangular1DLower<2,2>; break;
+         case 0x33: kernel = EAMassAssembleTriangular1DLower<3,3>; break;
+         case 0x44: kernel = EAMassAssembleTriangular1DLower<4,4>; break;
+         case 0x55: kernel = EAMassAssembleTriangular1DLower<5,5>; break;
+         case 0x66: kernel = EAMassAssembleTriangular1DLower<6,6>; break;
+         case 0x77: kernel = EAMassAssembleTriangular1DLower<7,7>; break;
+         case 0x88: kernel = EAMassAssembleTriangular1DLower<8,8>; break;
+         case 0x99: kernel = EAMassAssembleTriangular1DLower<9,9>; break;
+      }
+      return kernel(ne, B, pa_data, data, add, dofs1D, quad1D);
+   }
+   else if (dim == 2)
+   {
+      auto kernel = EAMassAssembleTriangular2DLower<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x22: kernel = EAMassAssembleTriangular2DLower<2,2>; break;
+         case 0x33: kernel = EAMassAssembleTriangular2DLower<3,3>; break;
+         case 0x44: kernel = EAMassAssembleTriangular2DLower<4,4>; break;
+         case 0x55: kernel = EAMassAssembleTriangular2DLower<5,5>; break;
+         case 0x66: kernel = EAMassAssembleTriangular2DLower<6,6>; break;
+         case 0x77: kernel = EAMassAssembleTriangular2DLower<7,7>; break;
+         case 0x88: kernel = EAMassAssembleTriangular2DLower<8,8>; break;
+         case 0x99: kernel = EAMassAssembleTriangular2DLower<9,9>; break;
+      }
+      return kernel(ne, B, pa_data, data, add, dofs1D, quad1D);
+   }
+   else if (dim == 3)
+   {
+      auto kernel = EAMassAssembleTriangular3DLower<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x23: kernel = EAMassAssembleTriangular3DLower<2,3>; break;
+         case 0x34: kernel = EAMassAssembleTriangular3DLower<3,4>; break;
+         case 0x45: kernel = EAMassAssembleTriangular3DLower<4,5>; break;
+         case 0x56: kernel = EAMassAssembleTriangular3DLower<5,6>; break;
+         case 0x67: kernel = EAMassAssembleTriangular3DLower<6,7>; break;
+         case 0x78: kernel = EAMassAssembleTriangular3DLower<7,8>; break;
+         case 0x89: kernel = EAMassAssembleTriangular3DLower<8,9>; break;
+      }
+      return kernel(ne, B, pa_data, data, add, dofs1D, quad1D);
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+}
 
 void MassIntegrator::AssembleEA_(Vector &ea_data,
                                  const bool add)
@@ -75,6 +140,16 @@ void MassIntegrator::AssembleEA_(Vector &ea_data,
    MFEM_ABORT("Unknown kernel.");
 }
 
+void MassIntegrator::AssembleEATriangular_(
+   TriPackLowerMatrix &ea_data,
+   const bool add)
+{
+   Vector &data = ea_data.Data();
+   const Array<real_t> &B = maps->B;
+   return AssembleEATriangularImpl(dim, ne, dofs1D, quad1D, B, pa_data, data,
+                                   add);
+}
+
 void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
                                 Vector &ea_data,
                                 const bool add)
@@ -83,6 +158,35 @@ void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
    if (ne > 0) { AssembleEA_(ea_data, add); }
 }
 
+void MassIntegrator::AssembleEATriangular(const FiniteElementSpace &fes,
+                                          TriPackLowerMatrix &ea_data,
+                                          const bool add)
+{
+   AssemblePA(fes);
+   if (ne == 0) { return; }
+
+   int elem_dofs = 1;
+   for (int d = 0; d < dim; ++d)
+   {
+      elem_dofs *= dofs1D;
+   }
+
+   if (add)
+   {
+      MFEM_VERIFY(ea_data.GetNumRows() == elem_dofs,
+                  "Invalid triangular EA element size.");
+      MFEM_VERIFY(ea_data.GetNumMatrices() == ne,
+                  "Invalid triangular EA element count.");
+   }
+   else
+   {
+      ea_data.SetSize(elem_dofs, ne);
+      ea_data.UseDevice(true);
+   }
+
+   AssembleEATriangular_(ea_data, add);
+}
+
 void MassIntegrator::AssembleEABoundary(const FiniteElementSpace &fes,
                                         Vector &ea_data,
                                         const bool add)

fem/integ/bilininteg_mass_kernels.hpp

@@ -1402,6 +1402,480 @@ inline void EAMassAssemble3D(const int NE,
    });
 }
 
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssembleTriangular1DLower(const int NE,
+                                            const Array<real_t> &basis,
+                                            const Vector &padata,
+                                            Vector &eadata,
+                                            const bool add,
+                                            const int d1d = 0,
+                                            const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(D1D), NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      for (int i1 = 0; i1 < D1D; ++i1)
+      {
+         for (int j1 = 0; j1 <= i1; ++j1)
+         {
+            real_t val = 0.0;
+            for (int k1 = 0; k1 < Q1D; ++k1)
+            {
+               val += B(k1, i1) * B(k1, j1) * D(k1, e);
+            }
+            const int idx = TriPackLowerMatrix::LowerIndex(i1, j1, D1D);
+            if (add)
+            {
+               M(idx, e) += val;
+            }
+            else
+            {
+               M(idx, e) = val;
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssembleTriangular2DLower(const int NE,
+                                            const Array<real_t> &basis,
+                                            const Vector &padata,
+                                            Vector &eadata,
+                                            const bool add,
+                                            const int d1d = 0,
+                                            const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const int ndofs = D1D*D1D;
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(ndofs), NE);
+
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      const int ndofs = D1D*D1D;
+      real_t r_B[MQ1][MD1];
+      for (int d = 0; d < D1D; d++)
+      {
+         for (int q = 0; q < Q1D; q++)
+         {
+            r_B[q][d] = B(q,d);
+         }
+      }
+      MFEM_SHARED real_t s_D[MQ1][MQ1];
+      MFEM_FOREACH_THREAD(k1,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(k2,y,Q1D)
+         {
+            s_D[k1][k2] = D(k1,k2,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(i1,x,D1D)
+      {
+         MFEM_FOREACH_THREAD(i2,y,D1D)
+         {
+            const int row = i1 + D1D*i2;
+            for (int j2 = 0; j2 < D1D; ++j2)
+            {
+               for (int j1 = 0; j1 < D1D; ++j1)
+               {
+                  const int col = j1 + D1D*j2;
+                  if (row < col)
+                  {
+                     continue;
+                  }
+                  real_t val = 0.0;
+                  for (int k1 = 0; k1 < Q1D; ++k1)
+                  {
+                     for (int k2 = 0; k2 < Q1D; ++k2)
+                     {
+                        val += r_B[k1][i1] * r_B[k1][j1]
+                               * r_B[k2][i2] * r_B[k2][j2]
+                               * s_D[k1][k2];
+                     }
+                  }
+                  const int idx = TriPackLowerMatrix::LowerIndex(row, col, ndofs);
+                  if (add)
+                  {
+                     M(idx, e) += val;
+                  }
+                  else
+                  {
+                     M(idx, e) = val;
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D, int T_Q1D, int T_COLB, int T_NT = 32>
+inline void EAMassAssembleTriangular3D_LowerBlockCols_Impl(
+   const int NE,
+   const Array<real_t> &basis,
+   const Vector &padata,
+   Vector &eadata,
+   const bool add,
+   const int,
+   const int)
+{
+   static_assert(T_D1D > 0 && T_Q1D > 0, "");
+   // Specialized packed lower-triangular hex mass assembly using block-column
+   // sum-factorization.
+   constexpr int D1D = T_D1D;
+   constexpr int Q1D = T_Q1D;
+   constexpr int COLB = T_COLB;
+   constexpr int NT = T_NT;
+   constexpr int ND = D1D*D1D*D1D;
+   constexpr int NQ = Q1D*Q1D*Q1D;
+
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(ND), NE);
+
+   mfem::forall_3D_grid(NE, NT, 1, 1, 0, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int tid = MFEM_THREAD_ID(x);
+
+      MFEM_SHARED real_t s_B[Q1D][D1D];
+      MFEM_SHARED real_t uW[NQ*COLB];
+      MFEM_SHARED real_t t1[D1D*Q1D*Q1D*COLB];
+      MFEM_SHARED real_t t2[D1D*D1D*Q1D*COLB];
+
+      for (int qb = tid; qb < Q1D*D1D; qb += NT)
+      {
+         const int q = qb % Q1D;
+         const int d = qb / Q1D;
+         s_B[q][d] = B(q, d);
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int j0 = 0; j0 < ND; j0 += COLB)
+      {
+         const int b = (j0 + COLB <= ND) ? COLB : (ND - j0);
+         int j1[COLB], j2[COLB], j3[COLB];
+         for (int c = 0; c < COLB; ++c)
+         {
+            if (c < b)
+            {
+               const int jj = j0 + c;
+               j1[c] = jj % D1D;
+               const int tmp = jj / D1D;
+               j2[c] = tmp % D1D;
+               j3[c] = tmp / D1D;
+            }
+         }
+
+         for (int q = tid; q < NQ; q += NT)
+         {
+            const int q1 = q % Q1D;
+            const int tmp = q / Q1D;
+            const int q2 = tmp % Q1D;
+            const int q3 = tmp / Q1D;
+            const real_t Dq = D(q1, q2, q3, e);
+
+            for (int c = 0; c < b; ++c)
+            {
+               uW[q + NQ*c] = s_B[q1][j1[c]] * s_B[q2][j2[c]]
+                              * s_B[q3][j3[c]] * Dq;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         constexpr int T1S = D1D*Q1D*Q1D;
+         for (int a = tid; a < T1S; a += NT)
+         {
+            const int i1 = a % D1D;
+            const int tmp = a / D1D;
+            const int q2 = tmp % Q1D;
+            const int q3 = tmp / Q1D;
+
+            for (int c = 0; c < b; ++c)
+            {
+               real_t sum = 0.0;
+               for (int q1 = 0; q1 < Q1D; ++q1)
+               {
+                  const int q = q1 + Q1D*(q2 + Q1D*q3);
+                  sum += s_B[q1][i1] * uW[q + NQ*c];
+               }
+               t1[a + T1S*c] = sum;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         constexpr int T2S = D1D*D1D*Q1D;
+         for (int a = tid; a < T2S; a += NT)
+         {
+            const int i1 = a % D1D;
+            const int tmp = a / D1D;
+            const int i2 = tmp % D1D;
+            const int q3 = tmp / D1D;
+
+            for (int c = 0; c < b; ++c)
+            {
+               real_t sum = 0.0;
+               for (int q2 = 0; q2 < Q1D; ++q2)
+               {
+                  const int a1 = i1 + D1D*(q2 + Q1D*q3);
+                  sum += s_B[q2][i2] * t1[a1 + T1S*c];
+               }
+               t2[a + T2S*c] = sum;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         for (int c = 0; c < b; ++c)
+         {
+            const int col = j0 + c;
+            const int jj1 = j1[c];
+            const int jj2 = j2[c];
+            const int jj3 = j3[c];
+
+            for (int i3 = jj3 + 1; i3 < D1D; ++i3)
+            {
+               for (int a = tid; a < D1D*D1D; a += NT)
+               {
+                  const int i1 = a % D1D;
+                  const int i2 = a / D1D;
+                  real_t sum = 0.0;
+                  for (int q3 = 0; q3 < Q1D; ++q3)
+                  {
+                     const int a2 = i1 + D1D*(i2 + D1D*q3);
+                     sum += s_B[q3][i3] * t2[a2 + T2S*c];
+                  }
+                  const int row = i1 + D1D*(i2 + D1D*i3);
+                  const int idx =
+                     TriPackLowerMatrix::LowerIndex(row, col, ND);
+                  if (add) { M(idx, e) += sum; }
+                  else { M(idx, e) = sum; }
+               }
+               MFEM_SYNC_THREAD;
+            }
+
+            for (int i2 = jj2 + 1; i2 < D1D; ++i2)
+            {
+               const int i3 = jj3;
+               for (int i1 = tid; i1 < D1D; i1 += NT)
+               {
+                  real_t sum = 0.0;
+                  for (int q3 = 0; q3 < Q1D; ++q3)
+                  {
+                     const int a2 = i1 + D1D*(i2 + D1D*q3);
+                     sum += s_B[q3][i3] * t2[a2 + T2S*c];
+                  }
+                  const int row = i1 + D1D*(i2 + D1D*i3);
+                  const int idx =
+                     TriPackLowerMatrix::LowerIndex(row, col, ND);
+                  if (add) { M(idx, e) += sum; }
+                  else { M(idx, e) = sum; }
+               }
+               MFEM_SYNC_THREAD;
+            }
+
+            {
+               const int i3 = jj3;
+               const int i2 = jj2;
+               for (int i1 = tid + jj1; i1 < D1D; i1 += NT)
+               {
+                  real_t sum = 0.0;
+                  for (int q3 = 0; q3 < Q1D; ++q3)
+                  {
+                     const int a2 = i1 + D1D*(i2 + D1D*q3);
+                     sum += s_B[q3][i3] * t2[a2 + T2S*c];
+                  }
+                  const int row = i1 + D1D*(i2 + D1D*i3);
+                  const int idx =
+                     TriPackLowerMatrix::LowerIndex(row, col, ND);
+                  if (add) { M(idx, e) += sum; }
+                  else { M(idx, e) = sum; }
+               }
+               MFEM_SYNC_THREAD;
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D, int T_Q1D, int T_NT = 32>
+inline void EAMassAssembleTriangular3D_LowerBlockCols(
+   const int NE,
+   const Array<real_t> &basis,
+   const Vector &padata,
+   Vector &eadata,
+   const bool add,
+   const int d1d = 0,
+   const int q1d = 0)
+{
+   static_assert(T_D1D > 0 && T_Q1D > 0, "");
+
+   constexpr int D1D = T_D1D;
+   constexpr int Q1D = T_Q1D;
+   constexpr int NQ = Q1D*Q1D*Q1D;
+   constexpr int SharedBytesPerCol =
+      sizeof(real_t)*(NQ + D1D*Q1D*Q1D + D1D*D1D*Q1D);
+   constexpr int SharedBytesBase = sizeof(real_t)*(Q1D*D1D);
+   constexpr int MaxSharedBytes = 48*1024;
+   constexpr int COLB =
+      (SharedBytesBase + 4*SharedBytesPerCol <= MaxSharedBytes) ? 4 :
+      (SharedBytesBase + 2*SharedBytesPerCol <= MaxSharedBytes) ? 2 : 1;
+
+   return EAMassAssembleTriangular3D_LowerBlockCols_Impl<T_D1D, T_Q1D, COLB, T_NT>(
+             NE, basis, padata, eadata, add, d1d, q1d);
+}
+
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssembleTriangular3DLower(const int NE,
+                                            const Array<real_t> &basis,
+                                            const Vector &padata,
+                                            Vector &eadata,
+                                            const bool add,
+                                            const int d1d = 0,
+                                            const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const int ndofs = D1D*D1D*D1D;
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(ndofs), NE);
+
+   if constexpr (T_D1D > 0 && T_Q1D > 0)
+   {
+      // Use the sum-factorized packed paths when the tensor dimensions are
+      // known at compile time. The generic path below handles dynamic sizes.
+      return EAMassAssembleTriangular3D_LowerBlockCols<T_D1D, T_Q1D>(
+                NE, basis, padata, eadata, add, d1d, q1d);
+   }
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int DQ = T_D1D * T_Q1D;
+      const int ndofs = D1D*D1D*D1D;
+
+      constexpr bool USE_REG = DQ != 0 && DQ <= 12;
+      constexpr int MD1r = USE_REG ? MD1 : 1;
+      constexpr int MQ1r = USE_REG ? MQ1 : 1;
+      constexpr int MD1s = USE_REG ? 1 : MD1;
+      constexpr int MQ1s = USE_REG ? 1 : MQ1;
+
+      MFEM_SHARED real_t s_B[MQ1s][MD1s];
+      real_t r_B[MQ1r][MD1r];
+      real_t (*l_B)[MD1] = nullptr;
+      if (USE_REG)
+      {
+         for (int d = 0; d < D1D; d++)
+         {
+            for (int q = 0; q < Q1D; q++)
+            {
+               r_B[q][d] = B(q,d);
+            }
+         }
+         l_B = (real_t (*)[MD1])r_B;
+      }
+      else
+      {
+         if (MFEM_THREAD_ID(z) == 0)
+         {
+            MFEM_FOREACH_THREAD(d,x,D1D)
+            {
+               MFEM_FOREACH_THREAD(q,y,Q1D)
+               {
+                  s_B[q][d] = B(q,d);
+               }
+            }
+         }
+         l_B = (real_t (*)[MD1])s_B;
+      }
+
+      MFEM_SHARED real_t s_D[MQ1][MQ1][MQ1];
+      MFEM_FOREACH_THREAD(k1,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(k2,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(k3,z,Q1D)
+            {
+               s_D[k1][k2][k3] = D(k1,k2,k3,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(i1,x,D1D)
+      {
+         MFEM_FOREACH_THREAD(i2,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(i3,z,D1D)
+            {
+               const int row = i1 + D1D*(i2 + D1D*i3);
+               for (int j3 = 0; j3 < D1D; ++j3)
+               {
+                  for (int j2 = 0; j2 < D1D; ++j2)
+                  {
+                     for (int j1 = 0; j1 < D1D; ++j1)
+                     {
+                        const int col = j1 + D1D*(j2 + D1D*j3);
+                        if (row < col)
+                        {
+                           continue;
+                        }
+                        real_t val = 0.0;
+                        for (int k1 = 0; k1 < Q1D; ++k1)
+                        {
+                           for (int k2 = 0; k2 < Q1D; ++k2)
+                           {
+                              for (int k3 = 0; k3 < Q1D; ++k3)
+                              {
+                                 val += l_B[k1][i1] * l_B[k1][j1]
+                                        * l_B[k2][i2] * l_B[k2][j2]
+                                        * l_B[k3][i3] * l_B[k3][j3]
+                                        * s_D[k1][k2][k3];
+                              }
+                           }
+                        }
+                        const int idx = TriPackLowerMatrix::LowerIndex(row, col, ndofs);
+                        if (add)
+                        {
+                           M(idx, e) += val;
+                        }
+                        else
+                        {
+                           M(idx, e) = val;
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
 } // namespace internal
 
 namespace

fem/integ/bilininteg_vecmass_pa.cpp

@@ -22,6 +22,8 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
+   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
+               "Only value map type supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const auto *ir = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, Trans);
 

fem/lor/lor_ams.cpp

@@ -94,10 +94,10 @@ void BatchedLOR_AMS::Form2DEdgeToVertex_RT(Array<int> &edge2vert)
          const int iv0 = ix + iy*op1;
          const int iv1 = ix1 + iy1*op1;
 
-         // Rotated gradient in 2D (-dy, dx), so flip the sign for the first
-         // component (c == 0).
-         e2v(0, iedge) = (c == 1) ? iv0 : iv1;
-         e2v(1, iedge) = (c == 1) ? iv1 : iv0;
+         // 2D curl (dy, -dx), so flip the sign for the second
+         // component (c == 1).
+         e2v(0, iedge) = (c == 0) ? iv0 : iv1;
+         e2v(1, iedge) = (c == 0) ? iv1 : iv0;
       }
    }
 }

fem/lor/lor_batched.cpp

@@ -142,8 +142,6 @@ static MFEM_HOST_DEVICE int GetAndIncrementNnzIndex(const int i_L, int* I)
 
 int BatchedLORAssembly::FillI(SparseMatrix &A) const
 {
-   static constexpr int Max = 16;
-
    const int nvdof = fes_ho.GetVSize();
 
    const int ndof_per_el = fes_ho.GetTypicalFE()->GetDof();
@@ -165,6 +163,8 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
    const auto K = dof_glob2loc_offsets_.Read();
    const auto map = Reshape(sparse_mapping.Read(), nnz_per_row, ndof_per_el);
 
+   Array<int> ij_elts(dof_glob2loc_.Size() * 2);
+   auto d_ij_elts = Reshape(ij_elts.Write(), dof_glob2loc_.Size(), 2);
 
    auto I = A.WriteI();
 
@@ -176,10 +176,10 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho);
       const int ii = (sii >= 0) ? sii : -1 -sii;
       // Get number and list of elements containing this DOF
-      int i_elts[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
+      int *i_elts = &d_ij_elts(i_offset, 0);
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -202,7 +202,7 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int j_elts[Max];
+            int *j_elts = &d_ij_elts(j_offset, 1);
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed
@@ -269,7 +269,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       mfem::forall(nvdof + 1, [=] MFEM_HOST_DEVICE (int i) { I[i] = I2[i]; });
    }
 
-   static constexpr int Max = 16;
+   Array<int> ij_B_el(dof_glob2loc_.Size() * 4);
+   auto d_ij_B_el = Reshape(ij_B_el.Write(), dof_glob2loc_.Size(), 4);
 
    mfem::forall(ndof_per_el*nel_ho, [=] MFEM_HOST_DEVICE (int i)
    {
@@ -279,11 +280,13 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho); // signed
       const int ii = (sii >= 0) ? sii : -1 - sii;
       // Get number and list of elements containing this DOF
-      int i_elts[Max];
-      int i_B[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
+
+      int *i_elts = &d_ij_B_el(i_offset, 0);
+      int *i_B = &d_ij_B_el(i_offset, 1);
+
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -312,8 +315,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int j_elts[Max];
-            int j_B[Max];
+            int *j_elts = &d_ij_B_el(j_offset, 2);
+            int *j_B = &d_ij_B_el(j_offset, 3);
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed

general/backends.hpp

@@ -14,7 +14,7 @@
 
 #include "../config/config.hpp"
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
@@ -22,7 +22,7 @@
 #endif
 #include "cuda.hpp"
 
-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#if defined(MFEM_USE_HIP)
 #include <hip/hip_runtime.h>
 #endif
 #include "hip.hpp"
@@ -45,15 +45,17 @@
 #endif
 
 #if !defined(MFEM_USE_CUDA_OR_HIP)
-constexpr bool mfem_use_gpu = false;
-#define MFEM_DEVICE
-#define MFEM_HOST
-#define MFEM_LAMBDA
-// #define MFEM_HOST_DEVICE // defined in config/config.hpp
 // MFEM_DEVICE_SYNC is made available for debugging purposes
 #define MFEM_DEVICE_SYNC
 // MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
 #define MFEM_STREAM_SYNC
+#endif
+
+#if !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+#define MFEM_DEVICE
+#define MFEM_HOST
+#define MFEM_LAMBDA
+// #define MFEM_HOST_DEVICE // defined in config/config.hpp
 #define MFEM_LAUNCH_BOUNDS(...)
 #endif
 
@@ -126,4 +128,23 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 #endif
 }
 
+namespace mfem::internal
+{
+
+#if defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+static constexpr bool can_compile_kernels = false;
+#else
+static constexpr bool can_compile_kernels = true;
+#endif
+
+template <bool can_compile_kernels = can_compile_kernels>
+void RequireKernelCompilation()
+{
+   static_assert(
+      can_compile_kernels,
+      "The calling function needs to be compiled with CUDA/HIP language!");
+}
+
+}
+
 #endif // MFEM_BACKENDS_HPP

general/cuda.hpp

@@ -18,14 +18,8 @@
 // CUDA block size used by MFEM.
 #define MFEM_CUDA_BLOCKS 256
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 #define MFEM_USE_CUDA_OR_HIP
-constexpr bool mfem_use_gpu = true;
-#define MFEM_DEVICE __device__
-#define MFEM_HOST __host__
-#define MFEM_LAMBDA __host__
-#define MFEM_LAUNCH_BOUNDS __launch_bounds__
-// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(cudaDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(cudaStreamSynchronize(0))
 // Define a CUDA error check macro, MFEM_GPU_CHECK(x), where x returns/is of
@@ -40,6 +34,15 @@ constexpr bool mfem_use_gpu = true;
     }                                                                          \
   } while (0)
 
+// Macros defined only when compiling with CUDA language
+#if defined(__CUDACC__)
+#define MFEM_USE_CUDA_OR_HIP_LANG
+#define MFEM_DEVICE __device__
+#define MFEM_HOST __host__
+#define MFEM_LAMBDA __host__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
+// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
+
 // Define the MFEM inner threading macros
 #if defined(__CUDA_ARCH__)
 #define MFEM_SHARED __shared__
@@ -67,12 +70,13 @@ constexpr bool mfem_use_gpu = true;
    if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \
        (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__CUDA_ARCH__)
-#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#endif // defined(__CUDACC__)
+#endif // defined(MFEM_USE_CUDA)
 
 namespace mfem
 {
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 // Function used by the macro MFEM_GPU_CHECK.
 void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                      const char *file, int line);

general/error.cpp

@@ -171,7 +171,7 @@ void mfem_error(const char *msg)
 #ifdef MFEM_USE_EXCEPTIONS
    if (mfem_error_action == MFEM_ERROR_THROW)
    {
-      throw ErrorException(msg);
+      throw ErrorException(msg ? msg : "");
    }
 #endif
 

general/error.hpp

@@ -15,7 +15,7 @@
 #include "../config/config.hpp"
 #include <iomanip>
 #include <sstream>
-#ifdef MFEM_USE_HIP
+#if defined(MFEM_USE_HIP)
 #include <hip/hip_runtime.h>
 #endif
 
@@ -153,21 +153,13 @@ void mfem_warning(const char *msg = NULL);
 
 
 // Additional abort functions for HIP
-#if defined(MFEM_USE_HIP)
-#ifndef __HIP_DEVICE_COMPILE__
-template<typename T>
-__host__ void abort_msg(T & msg)
-{
-   MFEM_ABORT(msg);
-}
-#else
+#if defined(__HIP_DEVICE_COMPILE__)
 template<typename T>
 __device__ void abort_msg(T & msg)
 {
    abort();
 }
 #endif
-#endif
 
 // Abort inside a device kernel
 #if defined(__CUDA_ARCH__)

general/forall.hpp

@@ -1044,6 +1044,8 @@ inline void ForallWrap(const bool use_dev, const int N,
                        const int X=0, const int Y=0, const int Z=0,
                        const int G=0)
 {
+   internal::RequireKernelCompilation();
+
    MFEM_CONTRACT_VAR(X);
    MFEM_CONTRACT_VAR(Y);
    MFEM_CONTRACT_VAR(Z);
@@ -1276,6 +1278,9 @@ inline void hypre_forall_cpu(int N, lambda &&body)
 template<typename lambda>
 inline void hypre_forall_gpu(int N, lambda &&body)
 {
+   internal::RequireKernelCompilation();
+
+#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
 #if defined(HYPRE_USING_CUDA)
    CuWrap1D(N, body);
 #elif defined(HYPRE_USING_HIP)
@@ -1283,6 +1288,7 @@ inline void hypre_forall_gpu(int N, lambda &&body)
 #else
 #error Unknown HYPRE GPU backend!
 #endif
+#endif
 }
 #endif
 

general/hip.hpp

@@ -18,14 +18,8 @@
 // HIP block size used by MFEM.
 #define MFEM_HIP_BLOCKS 256
 
-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#if defined(MFEM_USE_HIP)
 #define MFEM_USE_CUDA_OR_HIP
-constexpr bool mfem_use_gpu = true;
-#define MFEM_DEVICE __device__
-#define MFEM_HOST __host__
-#define MFEM_LAMBDA __host__ __device__
-#define MFEM_LAUNCH_BOUNDS __launch_bounds__
-// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(hipDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(hipStreamSynchronize(0))
 // Define a HIP error check macro, MFEM_GPU_CHECK(x), where x returns/is of
@@ -40,6 +34,15 @@ constexpr bool mfem_use_gpu = true;
     }                                                                          \
   } while (0)
 
+// Macros defined only when compiling with HIP language
+#if defined(__HIP__)
+#define MFEM_USE_CUDA_OR_HIP_LANG
+#define MFEM_DEVICE __device__
+#define MFEM_HOST __host__
+#define MFEM_LAMBDA __host__ __device__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
+// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
+
 // Define the MFEM inner threading macros
 #if defined(__HIP_DEVICE_COMPILE__)
 #define MFEM_SHARED __shared__
@@ -71,7 +74,8 @@ constexpr bool mfem_use_gpu = true;
        iz = iy / (OY);                                                         \
        (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__HIP_DEVICE_COMPILE__)
-#endif // defined(MFEM_USE_HIP) && defined(__HIP__)
+#endif // defined(__HIP__)
+#endif // defined(MFEM_USE_HIP)
 
 namespace mfem
 {

general/reducers.hpp

@@ -550,10 +550,10 @@ void reduce(int N, T &res, B &&body, const R &reducer, bool use_dev,
 
       int num_mp = Device::NumMultiprocessors(Device::GetId());
 #if defined(MFEM_USE_CUDA)
-      // good value of mp_sat found experimentally on Lassen
+      // good value of mp_sat found experimentally on Lassen (V100)
       constexpr int mp_sat = 8;
 #elif defined(MFEM_USE_HIP)
-      // good value of mp_sat found experimentally on Tuolumne
+      // good value of mp_sat found experimentally on Tuolumne (MI300A)
       constexpr int mp_sat = 4;
 #else
       num_mp = 1;

general/scan.hpp

@@ -15,6 +15,10 @@
 #include "backends.hpp"
 #include "forall.hpp"
 
+#if defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+#error "This header requires compilation with CUDA/HIP language!"
+#else
+
 #ifdef MFEM_USE_CUDA
 #include <cub/device/device_scan.cuh>
 #include <cub/device/device_select.cuh>
@@ -406,4 +410,6 @@ void CopyUnique(bool use_dev, InputIt d_in, OutputIt d_out,
 
 #undef MFEM_CUB_NAMESPACE
 
-#endif
+#endif // defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+
+#endif // MFEM_SCAN_HPP

linalg/CMakeLists.txt

@@ -27,6 +27,7 @@ list(APPEND SRCS
   handle.cpp
   matrix.cpp
   mma.cpp
+  multivector.cpp
   ode.cpp
   operator.cpp
   ordering.cpp
@@ -35,6 +36,8 @@ list(APPEND SRCS
   sparsemat.cpp
   sparsesmoothers.cpp
   symmat.cpp
+  tripack.cpp
+  tripack_magma.cpp
   vector.cpp
   )
 
@@ -63,6 +66,7 @@ list(APPEND HDRS
   linalg.hpp
   matrix.hpp
   mma.hpp
+  multivector.hpp
   ode.hpp
   operator.hpp
   ordering.hpp
@@ -71,6 +75,8 @@ list(APPEND HDRS
   sparsemat.hpp
   sparsesmoothers.hpp
   symmat.hpp
+  tripack.hpp
+  tripack_magma.hpp
   tlayout.hpp
   tmatrix.hpp
   ttensor.hpp

linalg/densemat.hpp

@@ -1136,6 +1136,17 @@ private:
 public:
    DenseTensor() : ni(0), nj(0), nk(0) { }
 
+   DenseTensor(const DenseTensor &other)
+      : tdata(other.tdata), ni(other.ni), nj(other.nj), nk(other.nk) { }
+
+   DenseTensor(DenseTensor &&other)
+      : tdata(std::move(other.tdata)), ni(other.ni), nj(other.nj), nk(other.nk)
+   {
+      // Reset other; other.tdata is reset in Array<T> move constructror.
+      other.Mk.ClearExternalData();
+      other.ni = other.nj = other.nk = 0;
+   }
+
    DenseTensor(int i, int j, int k) : tdata(i*j*k), ni(i), nj(j), nk(k) { }
 
    DenseTensor(real_t *d, int i, int j, int k)
@@ -1144,6 +1155,33 @@ public:
    DenseTensor(int i, int j, int k, MemoryType mt)
       : tdata(i*j*k, mt), ni(i), nj(j), nk(k) { }
 
+   DenseTensor &operator=(const DenseTensor &other)
+   {
+      if (this == &other) { return *this; }
+      Mk.ClearExternalData();
+      tdata = other.tdata;
+      ni = other.ni;
+      nj = other.nj;
+      nk = other.nk;
+      return *this;
+   }
+
+   DenseTensor &operator=(DenseTensor &&other)
+   {
+      if (this == &other) { return *this; }
+      Mk.ClearExternalData();
+      tdata = std::move(other.tdata);
+      ni = other.ni;
+      nj = other.nj;
+      nk = other.nk;
+
+      // Reset other; other.tdata is reset in Array<T> move assignment.
+      other.Mk.ClearExternalData();
+      other.ni = other.nj = other.nk = 0;
+
+      return *this;
+   }
+
    int SizeI() const { return ni; }
    int SizeJ() const { return nj; }
    int SizeK() const { return nk; }

linalg/hypre.cpp

@@ -5842,6 +5842,10 @@ void HypreAMS::MakeGradientAndInterpolation(
    {
       grad->AddTraceFaceInterpolator(new GradientInterpolator);
    }
+   else if (dynamic_cast<const RT_FECollection *>(edge_fec))
+   {
+      grad->AddDomainInterpolator(new CurlInterpolator);
+   }
    else
    {
       grad->AddDomainInterpolator(new GradientInterpolator);

linalg/linalg.hpp

@@ -15,6 +15,7 @@
 // Linear algebra header file
 
 #include "vector.hpp"
+#include "multivector.hpp"
 #include "operator.hpp"
 #include "matrix.hpp"
 #include "sparsemat.hpp"
@@ -26,6 +27,10 @@
 #include "sparsesmoothers.hpp"
 #include "densemat.hpp"
 #include "symmat.hpp"
+#include "tripack.hpp"
+#ifdef MFEM_USE_MAGMA
+#include "tripack_magma.hpp"
+#endif
 #include "ode.hpp"
 #include "solvers.hpp"
 #include "handle.hpp"

linalg/multivector.cpp

@@ -0,0 +1,60 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "multivector.hpp"
+
+namespace mfem
+{
+
+MultiVector::MultiVector(const Array<int> &vector_sizes)
+{
+   SetSizes(vector_sizes);
+}
+
+MultiVector::MultiVector(const Array<int> &vector_sizes, MemoryType mt)
+{
+   SetSizes(vector_sizes, mt);
+}
+
+MultiVector::MultiVector(Vector &base, const Array<int> &vector_sizes)
+{
+   MakeRef(base, vector_sizes);
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i]);
+   }
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes, MemoryType mt)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i], mt);
+   }
+}
+
+void MultiVector::MakeRef(Vector &base, const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int offset = 0, i = 0; i < vector_sizes.Size(); i++)
+   {
+      blocks[i].emplace<0>(base, offset, vector_sizes[i]);
+      offset += vector_sizes[i];
+   }
+}
+
+} // namespace mfem

linalg/multivector.hpp

@@ -0,0 +1,251 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MULTIVECTOR_HPP
+#define MFEM_MULTIVECTOR_HPP
+
+#include "../general/array.hpp"
+#include "vector.hpp"
+#include <vector>
+#include <array>
+#include <variant>
+
+namespace mfem
+{
+
+/// Class representing an array of Vectors with generally different sizes.
+/** This class is similar to BlockVector with the following two main
+    differences:
+    - the data for the individual Vector blocks does not need to be part of one
+      big contiguous memory allocation;
+    - this class does not inherit from class Vector (as a consequence of the
+      first bullet).
+
+    Internally, each Vector block is represented as one of the following
+    three options:
+    - (default) a Vector object constructed and owned by this class; this
+      object, in turn, as any Vector object, can own its Memory allocation or
+      refer to a sub-Memory of another Memory object; or
+    - a pointer to an externally allocated Vector or classes derived from
+      Vector.
+    - a pointer to an externally allocated const Vector or classes derived from
+      Vector. This option is helpful for wrapping const Vector objects as a
+      MultiVector that will be then used as a const MultiVector. */
+class MultiVector
+{
+private:
+   std::vector<std::variant<Vector,Vector*,const Vector*>> blocks;
+
+public:
+   /// Create an empty MultiVector with zero blocks.
+   MultiVector() = default;
+
+   /** @brief Create a MultiVector with @a num_blocks blocks. The individual
+       Vector blocks are default initialized, i.e. they all have size zero. */
+   MultiVector(int num_blocks)
+      : blocks(num_blocks) { }
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes. All Vector blocks use the
+       MemoryType @a mt.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Construct a MultiVector referencing data within a given monolithic
+       Vector @a base.
+
+       With this constructor, the Memory flags of @a base and of the individual
+       Vector blocks may need to be explicitly synchronized when data is moved
+       between host and device. */
+   MultiVector(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector referencing multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       With this constructor, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   MultiVector(VectorTypes &...vs) { MakeRef(vs...); }
+
+   /** @brief Construct a MultiVector referencing multiple const Vectors given
+       as arguments. Individual blocks are read-only; non-const operator[]
+       will generate an error. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<const VectorTypes&,const Vector&>...>,
+                bool> = true>
+   MultiVector(const VectorTypes &...vs) { MakeRef(vs...); }
+
+   /// Return the number of Vectors in the MultiVector.
+   int NumBlocks() const { return blocks.size(); }
+
+   /** @brief Set the number of Vectors in the MultiVector. Existing Vector
+       blocks will remain unmodified. New Vector blocks will be default
+       initialized, i.e. they all have size zero. */
+   void SetNumBlocks(int num_blocks) { blocks.resize(num_blocks); }
+
+   /** @brief Read-write access to the i-th Vector. Generates an error if the
+       i-th block is read-only, i.e. it is a pointer to a const Vector. */
+   inline Vector &operator[](int i);
+
+   /// Read-only access to the i-th Vector.
+   inline const Vector &operator[](int i) const;
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes are updated using the method Vector::SetSize(int). */
+   void SetSizes(const Array<int> &vector_sizes);
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes and
+       MemoryType @a mt.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes and MemoryType are updated using the method
+       Vector::SetSize(int, MemoryType). */
+   void SetSizes(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Update the MultiVector to reference data within a given monolithic
+       Vector @a base.
+
+       After calling this method, the Memory flags of @a base and of the
+       individual Vector blocks may need to be explicitly synchronized when data
+       is moved between host and device.*/
+   void MakeRef(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Update the @a i-th MultiVector block to reference data within the
+       given monolithic Vector @a base at the given @a offset and with the given
+       @a size.
+
+       After calling this method, the Memory flags of @a base and of the @a i-th
+       Vector block may need to be explicitly synchronized when data is moved
+       between host and device.*/
+   inline void MakeRef(int i, Vector &base, int offset, int size)
+   {
+      blocks[i].emplace<0>(base, offset, size);
+   }
+
+   /** @brief Update the MultiVector to reference multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       After calling this method, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   inline void MakeRef(VectorTypes &...vs);
+
+   /** @brief Update the MultiVector to reference multiple const Vectors given
+       as arguments. Individual blocks are read-only; non-const operator[]
+       will generate an error. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<const VectorTypes&,const Vector&>...>,
+                bool> = true>
+   inline void MakeRef(const VectorTypes &...vs);
+
+   /** @brief Update the @a i-th MultiVector block to reference the given
+       Vector @a v.
+
+       After calling this method, operations on the @a i-th Vector block are
+       performed directly on the Vector @a v. In particular, there is no need
+       to synchronize the Memory flags of @a v and the ones of the @a i-th
+       Vector blocks when data is moved between host and device. */
+   inline void MakeRef(int i, Vector &v) { blocks[i] = &v; }
+
+   /** @brief Update the @a i-th MultiVector block to reference the given
+       const Vector @a v. The block becomes read-only. */
+   inline void MakeRef(int i, const Vector &v) { blocks[i] = &v; }
+};
+
+// Inline and template methods
+
+inline Vector &MultiVector::operator[](int i)
+{
+   auto &bi = blocks[i];
+   const auto idx = bi.index();
+   if (idx == 0) { return std::get<0>(bi); }
+   if (idx == 1) { return *std::get<1>(bi); }
+   MFEM_ABORT("Non-const access to a const Vector block!");
+}
+
+inline const Vector &MultiVector::operator[](int i) const
+{
+   auto &bi = blocks[i];
+   const auto idx = bi.index();
+   return (idx == 0) ? std::get<0>(bi) :
+          (idx == 1) ? *std::get<1>(bi) :
+          /**/         *std::get<2>(bi);
+}
+
+template <typename... VectorTypes,
+          std::enable_if_t<
+             std::conjunction_v<
+                std::is_convertible<VectorTypes&,Vector&>...>, bool>>
+inline void MultiVector::MakeRef(VectorTypes &...vs)
+{
+   blocks.resize(sizeof...(vs));
+   if constexpr (sizeof...(vs) > 0)
+   {
+      const std::array vs_p{&static_cast<Vector&>(vs)...};
+      for (std::size_t i = 0; i < sizeof...(vs); i++)
+      {
+         blocks[i] = vs_p[i];
+      }
+   }
+}
+
+template <typename... VectorTypes,
+          std::enable_if_t<
+             std::conjunction_v<
+                std::is_convertible<const VectorTypes&,const Vector&>...>,
+             bool>>
+inline void MultiVector::MakeRef(const VectorTypes &...vs)
+{
+   blocks.resize(sizeof...(vs));
+   if constexpr (sizeof...(vs) > 0)
+   {
+      const std::array vs_p{&static_cast<const Vector&>(vs)...};
+      for (std::size_t i = 0; i < sizeof...(vs); i++)
+      {
+         blocks[i] = vs_p[i];
+      }
+   }
+}
+
+} // namespace mfem
+
+#endif // MFEM_MULTIVECTOR_HPP

linalg/operator.cpp

@@ -111,6 +111,21 @@ void Operator::ArrayAddMultTranspose(const Array<const Vector *> &X,
    }
 }
 
+void Operator::MultMV(const MultiVector &, MultiVector &) const
+{
+   MFEM_ABORT("this method is not overridden for this class!");
+}
+
+void Operator::MultTransposeMV(const MultiVector &x, MultiVector &y) const
+{
+   MFEM_ABORT("this method is not overridden for this class!");
+}
+
+Operator &Operator::GetGradientMV(const MultiVector &) const
+{
+   MFEM_ABORT("this method is not overridden for this class!");
+}
+
 void Operator::FormLinearSystem(const Array<int> &ess_tdof_list,
                                 Vector &x, Vector &b,
                                 Operator* &Aout, Vector &X, Vector &B,

linalg/operator.hpp

@@ -13,6 +13,7 @@
 #define MFEM_OPERATOR
 
 #include "vector.hpp"
+#include "multivector.hpp"
 
 namespace mfem
 {
@@ -129,6 +130,20 @@ public:
    virtual void ArrayAddMultTranspose(const Array<const Vector *> &X,
                                       Array<Vector *> &Y, const real_t a = 1.0) const;
 
+   /** @brief Operator application, y = A(x), where the input @a x and the
+       output @a y are MultiVector objects, i.e. they generally use
+       non-contiguous memory representation.
+
+       The base class implementation for the method is to generate an error. */
+   virtual void MultMV(const MultiVector &x, MultiVector &y) const;
+
+   /** @brief Action of the transpose operator, y = A^t(x), where the input @a x
+       and the output @a y are MultiVector objects, i.e. they generally use
+       non-contiguous memory representation.
+
+       The base class implementation for this method is to generate an error. */
+   virtual void MultTransposeMV(const MultiVector &x, MultiVector &y) const;
+
    /** @brief Evaluate the gradient operator at the point @a x. The default
        behavior in class Operator is to generate an error. */
    virtual Operator &GetGradient(const Vector &x) const
@@ -137,6 +152,13 @@ public:
       return const_cast<Operator &>(*this);
    }
 
+   /** @brief Evaluate the gradient operator at the point @a x. The input @a x
+       is provided as a MultiVector, i.e. it generally uses non-contiguous
+       memory representation.
+
+       The base class implementation for the method is to generate an error. */
+   virtual Operator &GetGradientMV(const MultiVector &x) const;
+
    /** @brief Computes the diagonal entries into @a diag. Typically, this
        operation only makes sense for linear Operator%s. In some cases, only an
        approximation of the diagonal is computed. */

linalg/tripack.cpp

@@ -0,0 +1,471 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tripack.hpp"
+
+#include "../general/forall.hpp"
+
+#include <cmath>
+#include <limits>
+
+namespace mfem
+{
+namespace
+{
+
+MFEM_HOST_DEVICE inline int SymmetricIndex(const int i,
+                                           const int j,
+                                           const int n)
+{
+   return TriPackLowerMatrix::Index(i, j, n);
+}
+
+MFEM_HOST_DEVICE inline bool TriPackIsFinite(const real_t val)
+{
+#ifdef isfinite
+   return isfinite(val);
+#else
+   return std::isfinite(val);
+#endif
+}
+
+void ComputeCholeskyFactorsLowerDevice(
+   const TriPackLowerMatrix &packed_lower,
+   Vector &factor)
+{
+   const int n = packed_lower.GetNumRows();
+   const int batch_size = packed_lower.GetNumMatrices();
+   const int packed_size = packed_lower.GetPackedSize();
+   const real_t nan = std::numeric_limits<real_t>::quiet_NaN();
+
+   const real_t *A = packed_lower.Data().Read();
+   factor.SetSize(batch_size*packed_size);
+   factor.UseDevice(true);
+   real_t *L = factor.Write();
+
+   mfem::forall(batch_size, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int eoff = e*packed_size;
+      const real_t eps = std::numeric_limits<real_t>::epsilon();
+      bool bad = false;
+
+      // Copy packed-lower input into factor storage (no scaling).
+      for (int j = 0; j < n; ++j)
+      {
+         for (int i = j; i < n; ++i)
+         {
+            const int t = TriPackLowerMatrix::LowerIndex(i, j, n);
+            const real_t Aij = A[eoff + t];
+            if (!TriPackIsFinite(Aij)) { bad = true; }
+            L[eoff + t] = Aij;
+         }
+      }
+
+      if (bad)
+      {
+         for (int t = 0; t < packed_size; ++t) { L[eoff + t] = nan; }
+         return;
+      }
+
+      for (int k = 0; k < n; ++k)
+      {
+         const int kk = eoff + TriPackLowerMatrix::LowerIndex(k, k, n);
+         const real_t Lkk0 = L[kk];
+         real_t Lkk = Lkk0;
+
+         for (int s = 0; s < k; ++s)
+         {
+            const real_t Lks = L[eoff + TriPackLowerMatrix::LowerIndex(k, s, n)];
+            Lkk -= Lks*Lks;
+         }
+
+         const real_t tol = 64.0*eps*fabs(Lkk0);
+         if (!TriPackIsFinite(Lkk) || Lkk < -tol)
+         {
+            bad = true;
+            break;
+         }
+
+         if (Lkk < 0.0) { Lkk = 0.0; }
+         L[kk] = sqrt(Lkk);
+
+         const real_t Ldiag = L[kk];
+         for (int i = k + 1; i < n; ++i)
+         {
+            const int ik = eoff + TriPackLowerMatrix::LowerIndex(i, k, n);
+            real_t Aik = L[ik];
+            for (int s = 0; s < k; ++s)
+            {
+               Aik -= L[eoff + TriPackLowerMatrix::LowerIndex(i, s, n)] *
+                      L[eoff + TriPackLowerMatrix::LowerIndex(k, s, n)];
+            }
+            L[ik] = Aik/Ldiag;
+         }
+      }
+
+      if (bad)
+      {
+         for (int t = 0; t < packed_size; ++t) { L[eoff + t] = nan; }
+      }
+   });
+}
+
+void ComputeCholeskyFactorsLower(
+   const TriPackLowerMatrix &packed_lower,
+   Vector &factor)
+{
+   const int n = packed_lower.GetNumRows();
+   const int batch_size = packed_lower.GetNumMatrices();
+   const int packed_size = packed_lower.GetPackedSize();
+
+   factor.SetSize(batch_size*packed_size);
+   factor.UseDevice(true);
+
+   const real_t *A = packed_lower.Data().HostRead();
+   real_t *L = factor.HostWrite();
+
+   for (int e = 0; e < batch_size; ++e)
+   {
+      const int eoff = e*packed_size;
+      for (int j = 0; j < n; ++j)
+      {
+         for (int i = j; i < n; ++i)
+         {
+            L[eoff + TriPackLowerMatrix::LowerIndex(i, j, n)] =
+               A[eoff + TriPackLowerMatrix::LowerIndex(i, j, n)];
+         }
+      }
+
+      for (int k = 0; k < n; ++k)
+      {
+         const int kk = eoff + TriPackLowerMatrix::LowerIndex(k, k, n);
+         real_t Akk = L[kk];
+         for (int s = 0; s < k; ++s)
+         {
+            const real_t Lks = L[eoff + TriPackLowerMatrix::LowerIndex(k, s, n)];
+            Akk -= Lks*Lks;
+         }
+         MFEM_VERIFY(Akk > 0.0, "Matrix is not SPD.");
+         L[kk] = std::sqrt(Akk);
+
+         const real_t Lkk = L[kk];
+         for (int i = k + 1; i < n; ++i)
+         {
+            real_t Aik = L[eoff + TriPackLowerMatrix::LowerIndex(i, k, n)];
+            for (int s = 0; s < k; ++s)
+            {
+               Aik -= L[eoff + TriPackLowerMatrix::LowerIndex(i, s, n)] *
+                      L[eoff + TriPackLowerMatrix::LowerIndex(k, s, n)];
+            }
+            L[eoff + TriPackLowerMatrix::LowerIndex(i, k, n)] = Aik / Lkk;
+         }
+      }
+   }
+}
+
+}
+
+namespace tripack
+{
+
+bool CompareWithFull(const TriPackLowerMatrix &packed, const Vector &full,
+                     real_t tol)
+{
+   const int n = packed.GetNumRows();
+   const int batch_size = packed.GetNumMatrices();
+   const int packed_size = packed.GetPackedSize();
+
+   MFEM_VERIFY(full.Size() == batch_size*n*n,
+               "Full matrix data has the wrong size.");
+
+   const real_t *packed_data = packed.Data().HostRead();
+   const real_t *full_data = full.HostRead();
+
+   if (tol == 0.0)
+   {
+      tol = 256.0*std::numeric_limits<real_t>::epsilon();
+   }
+
+   for (int e = 0; e < batch_size; ++e)
+   {
+      const int packed_offset = e*packed_size;
+      const int full_offset = e*n*n;
+      for (int i = 0; i < n; ++i)
+      {
+         for (int j = 0; j < n; ++j)
+         {
+            const real_t packed_val =
+               packed_data[packed_offset + SymmetricIndex(i, j, n)];
+            const real_t full_val = full_data[full_offset + i + n*j];
+            if (std::fabs(full_val - packed_val) > tol)
+            {
+               return false;
+            }
+         }
+      }
+      for (int i = 0; i < n; ++i)
+      {
+         for (int j = i + 1; j < n; ++j)
+         {
+            const real_t a = full_data[full_offset + i + n*j];
+            const real_t b = full_data[full_offset + j + n*i];
+            if (std::fabs(a - b) > tol)
+            {
+               return false;
+            }
+         }
+      }
+   }
+
+   return true;
+}
+
+void Mult(const TriPackLowerMatrix &packed, const Vector &x, Vector &y)
+{
+   const int n = packed.GetNumRows();
+   const int batch_size = packed.GetNumMatrices();
+
+   MFEM_VERIFY(x.Size() == batch_size*n, "Input vector has the wrong size.");
+
+   y.SetSize(batch_size*n);
+   y.UseDevice(true);
+
+   const real_t *A = packed.Data().Read();
+   const real_t *X = x.Read();
+   real_t *Y = y.Write();
+
+   mfem::forall(batch_size*n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int i = idx % n;
+      const int e = idx / n;
+      const real_t *Ae = A + e*TriPackLowerMatrix::PackedSize(n);
+      const real_t *Xe = X + e*n;
+      real_t sum = 0.0;
+      for (int j = 0; j < n; ++j)
+      {
+         sum += Ae[SymmetricIndex(i, j, n)] * Xe[j];
+      }
+      Y[idx] = sum;
+   });
+}
+
+void Lump(const TriPackLowerMatrix &packed, Vector &lump)
+{
+   const int n = packed.GetNumRows();
+   const int batch_size = packed.GetNumMatrices();
+
+   lump.SetSize(batch_size*n);
+   lump.UseDevice(true);
+
+   const real_t *A = packed.Data().Read();
+   real_t *L = lump.Write();
+
+   mfem::forall(batch_size*n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int i = idx % n;
+      const int e = idx / n;
+      const real_t *Ae = A + e*TriPackLowerMatrix::PackedSize(n);
+      real_t sum = 0.0;
+      for (int j = 0; j < n; ++j)
+      {
+         sum += Ae[SymmetricIndex(i, j, n)];
+      }
+      L[idx] = sum;
+   });
+}
+
+void ComputeCholeskyLower(const TriPackLowerMatrix &packed_lower,
+                          TriPackLowerMatrix &lower_factor)
+{
+   const int n = packed_lower.GetNumRows();
+   const int batch_size = packed_lower.GetNumMatrices();
+
+   MFEM_VERIFY(&packed_lower != &lower_factor,
+               "Input and output TriPackLowerMatrix objects must be distinct.");
+   if (batch_size == 0)
+   {
+      lower_factor.SetSize(n, batch_size);
+      return;
+   }
+
+   Vector factored;
+   ComputeCholeskyFactorsLower(packed_lower, factored);
+
+   lower_factor.SetSize(n, batch_size);
+   lower_factor.UseDevice(true);
+
+   lower_factor.Data() = factored;
+}
+
+void ComputeCholeskyLowerInverse(const TriPackLowerMatrix &packed_lower,
+                                 TriPackLowerMatrix &lower_inverse)
+{
+   const int n = packed_lower.GetNumRows();
+   const int batch_size = packed_lower.GetNumMatrices();
+   const int packed_size = packed_lower.GetPackedSize();
+   const real_t nan = std::numeric_limits<real_t>::quiet_NaN();
+
+   MFEM_VERIFY(&packed_lower != &lower_inverse,
+               "Input and output TriPackLowerMatrix objects must be distinct.");
+   if (batch_size == 0)
+   {
+      lower_inverse.SetSize(n, batch_size);
+      return;
+   }
+
+   lower_inverse.SetSize(n, batch_size);
+   lower_inverse.UseDevice(true);
+
+   Vector factored;
+   Vector work(batch_size*packed_size);
+   work.UseDevice(true);
+
+   ComputeCholeskyFactorsLowerDevice(packed_lower, factored);
+
+   const real_t *L = factored.Read();
+   real_t *X = work.Write();
+   real_t *Linv = lower_inverse.Data().Write();
+
+   mfem::forall(batch_size, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int eoff = e*packed_size;
+      bool bad = false;
+
+      for (int t = 0; t < packed_size; ++t)
+      {
+         if (!TriPackIsFinite(L[eoff + t]))
+         {
+            bad = true;
+            break;
+         }
+      }
+
+      if (bad)
+      {
+         for (int t = 0; t < packed_size; ++t) { Linv[eoff + t] = nan; }
+         return;
+      }
+
+      for (int t = 0; t < packed_size; ++t) { X[eoff + t] = 0.0; }
+
+      // Compute X = L^{-1} (packed lower).
+      for (int j = 0; j < n; ++j)
+      {
+         const int jj = eoff + TriPackLowerMatrix::LowerIndex(j, j, n);
+         X[jj] = 1.0/L[jj];
+
+         const real_t invLjj = X[jj];
+         for (int i = j + 1; i < n; ++i)
+         {
+            real_t sum = 0.0;
+            for (int k = j; k < i; ++k)
+            {
+               sum += L[eoff + TriPackLowerMatrix::LowerIndex(i, k, n)] *
+                      X[eoff + TriPackLowerMatrix::LowerIndex(k, j, n)];
+            }
+            X[eoff + TriPackLowerMatrix::LowerIndex(i, j, n)] = -invLjj*sum;
+         }
+      }
+
+      for (int t = 0; t < packed_size; ++t)
+      {
+         Linv[eoff + t] = X[eoff + t];
+      }
+   });
+}
+
+void SolveLower(const TriPackLowerMatrix &lower_factor,
+                const Vector &rhs,
+                Vector &sol)
+{
+   const int n = lower_factor.GetNumRows();
+   const int batch_size = lower_factor.GetNumMatrices();
+   const int packed_size = lower_factor.GetPackedSize();
+
+   MFEM_VERIFY(rhs.Size() == batch_size*n, "Right-hand side has the wrong size.");
+
+   Vector out(batch_size*n);
+   out.UseDevice(true);
+
+   const real_t *L = lower_factor.Data().Read();
+   const real_t *B = rhs.Read();
+   real_t *X = out.Write();
+
+   mfem::forall(batch_size, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const real_t *Le = L + e*packed_size;
+      const real_t *Be = B + e*n;
+      real_t *Xe = X + e*n;
+
+      for (int i = 0; i < n; ++i)
+      {
+         real_t sum = Be[i];
+         for (int j = 0; j < i; ++j)
+         {
+            sum -= Le[TriPackLowerMatrix::LowerIndex(i, j, n)] * Xe[j];
+         }
+         Xe[i] = sum / Le[TriPackLowerMatrix::LowerIndex(i, i, n)];
+      }
+   });
+
+   sol.SetSize(batch_size*n);
+   sol = out;
+}
+
+void SolveLowerTranspose(const TriPackLowerMatrix &lower_factor,
+                         const Vector &rhs,
+                         Vector &sol)
+{
+   const int n = lower_factor.GetNumRows();
+   const int batch_size = lower_factor.GetNumMatrices();
+   const int packed_size = lower_factor.GetPackedSize();
+
+   MFEM_VERIFY(rhs.Size() == batch_size*n, "Right-hand side has the wrong size.");
+
+   Vector out(batch_size*n);
+   out.UseDevice(true);
+
+   const real_t *L = lower_factor.Data().Read();
+   const real_t *B = rhs.Read();
+   real_t *X = out.Write();
+
+   mfem::forall(batch_size, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const real_t *Le = L + e*packed_size;
+      const real_t *Be = B + e*n;
+      real_t *Xe = X + e*n;
+
+      for (int i = n - 1; i >= 0; --i)
+      {
+         real_t sum = Be[i];
+         for (int j = i + 1; j < n; ++j)
+         {
+            sum -= Le[TriPackLowerMatrix::LowerIndex(j, i, n)] * Xe[j];
+         }
+         Xe[i] = sum / Le[TriPackLowerMatrix::LowerIndex(i, i, n)];
+      }
+   });
+
+   sol.SetSize(batch_size*n);
+   sol = out;
+}
+
+void SolveCholeskyLower(const TriPackLowerMatrix &lower_factor,
+                        const Vector &rhs,
+                        Vector &sol)
+{
+   Vector tmp;
+   SolveLower(lower_factor, rhs, tmp);
+   SolveLowerTranspose(lower_factor, tmp, sol);
+}
+
+} // namespace tripack
+} // namespace mfem

linalg/tripack.hpp

@@ -0,0 +1,120 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_TRIPACK
+#define MFEM_TRIPACK
+
+#include "../config/config.hpp"
+#include "vector.hpp"
+
+namespace mfem
+{
+
+/// Packed storage for a batch of symmetric matrices of fixed size.
+/// Storage is packed lower-triangular in LAPACK/MAGMA column-major convention.
+class TriPackLowerMatrix
+{
+private:
+   Vector data;
+   int nrows = 0;
+   int nmats = 0;
+
+public:
+   TriPackLowerMatrix() = default;
+
+   TriPackLowerMatrix(int n, int batch_size)
+   {
+      SetSize(n, batch_size);
+   }
+
+   MFEM_HOST_DEVICE static int PackedSize(const int n)
+   {
+      return n*(n + 1)/2;
+   }
+
+   /// Packed index for (i,j) in the lower triangle (requires i >= j).
+   MFEM_HOST_DEVICE static int LowerIndex(const int i, const int j, const int n)
+   {
+      return j*(2*n + 1 - j)/2 + (i - j);
+   }
+
+   /// Packed index for (i,j) in symmetric storage (maps to lower triangle).
+   MFEM_HOST_DEVICE static int Index(const int i, const int j, const int n)
+   {
+      return (i >= j) ? LowerIndex(i, j, n) : LowerIndex(j, i, n);
+   }
+
+   void SetSize(const int n, const int batch_size)
+   {
+      nrows = n;
+      nmats = batch_size;
+      data.SetSize(batch_size*PackedSize(n));
+   }
+
+   int GetNumRows() const { return nrows; }
+
+   int GetNumMatrices() const { return nmats; }
+
+   int GetPackedSize() const { return PackedSize(nrows); }
+
+   int Size() const { return data.Size(); }
+
+   void UseDevice(bool use_dev) { data.UseDevice(use_dev); }
+
+   TriPackLowerMatrix &operator=(real_t value)
+   {
+      data = value;
+      return *this;
+   }
+
+   Vector &Data() { return data; }
+   const Vector &Data() const { return data; }
+};
+
+namespace tripack
+{
+
+bool CompareWithFull(const TriPackLowerMatrix &packed, const Vector &full,
+                     real_t tol = 0.0);
+
+void Mult(const TriPackLowerMatrix &packed, const Vector &x, Vector &y);
+
+void Lump(const TriPackLowerMatrix &packed, Vector &lump);
+
+void ComputeCholeskyLower(const TriPackLowerMatrix &packed_lower,
+                          TriPackLowerMatrix &lower_factor);
+
+void SolveLower(const TriPackLowerMatrix &lower_factor,
+                const Vector &rhs,
+                Vector &sol);
+
+void SolveLowerTranspose(const TriPackLowerMatrix &lower_factor,
+                         const Vector &rhs,
+                         Vector &sol);
+
+void SolveCholeskyLower(const TriPackLowerMatrix &lower_factor,
+                        const Vector &rhs,
+                        Vector &sol);
+
+/// Compute the inverse of the Cholesky lower factor for a batch of SPD matrices.
+///
+/// Given packed lower-triangular matrices A (SPD), this routine computes the
+/// packed lower-triangular matrices L^{-1}, where A = L L^T.
+///
+/// This is intended for fast inverse applications using (L^{-1})^T (L^{-1}).
+void ComputeCholeskyLowerInverse(const TriPackLowerMatrix &packed_lower,
+                                 TriPackLowerMatrix &lower_inverse);
+
+} // namespace tripack
+
+} // namespace mfem
+
+#endif

linalg/tripack_magma.cpp

@@ -0,0 +1,267 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tripack_magma.hpp"
+
+#ifdef MFEM_USE_MAGMA
+
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+namespace
+{
+
+#ifdef MFEM_USE_SINGLE
+#define MFEM_TRIPACK_MAGMA_PREFIX(stub) magma_s##stub
+#define MFEM_TRIPACK_MAGMA_SET_POINTER magma_sset_pointer
+#elif defined(MFEM_USE_DOUBLE)
+#define MFEM_TRIPACK_MAGMA_PREFIX(stub) magma_d##stub
+#define MFEM_TRIPACK_MAGMA_SET_POINTER magma_dset_pointer
+#else
+#error "Unsupported MFEM precision for MAGMA packed routines."
+#endif
+
+real_t **SetPackedPointerArray(Array<real_t *> &ptrs,
+                               real_t *data,
+                               const int stride,
+                               const int batch_size,
+                               const magma_queue_t queue)
+{
+   if (ptrs.Size() != batch_size)
+   {
+      if (ptrs.Size() != 0) { magma_queue_sync(queue); }
+      ptrs.SetSize(batch_size, Device::GetDeviceMemoryType());
+   }
+
+   real_t **d_ptrs = ptrs.Write();
+   MFEM_TRIPACK_MAGMA_SET_POINTER(d_ptrs, data, 1, 0, 0, stride,
+                                  batch_size, queue);
+   return d_ptrs;
+}
+
+} // namespace
+
+MagmaPackedLowerCholesky::MagmaPackedLowerCholesky()
+{
+   queue = Magma::Queue();
+}
+
+void MagmaPackedLowerCholesky::Factor(
+   const TriPackLowerMatrix &A,
+   TriPackLowerMatrix &L)
+{
+   MFEM_VERIFY(queue != nullptr, "MAGMA queue is not set.");
+
+   n = A.GetNumRows();
+   batch_size = A.GetNumMatrices();
+   packed_size = A.GetPackedSize();
+
+   L.SetSize(n, batch_size);
+   L.UseDevice(true);
+
+   if (batch_size == 0) { return; }
+
+   L.Data() = A.Data();
+
+   real_t *factor_data = L.Data().ReadWrite();
+   real_t **d_factor_ptrs =
+      SetPackedPointerArray(factor_ptrs, factor_data, packed_size,
+                            batch_size, queue);
+
+   info.SetSize(batch_size, Device::GetDeviceMemoryType());
+   magma_int_t *d_info = info.Write();
+   magma_memset(d_info, 0, batch_size*sizeof(magma_int_t));
+
+   const magma_int_t status =
+      (n <= 8) ?
+      MFEM_TRIPACK_MAGMA_PREFIX(pptrf_batched_small)(
+         MagmaLower, n, d_factor_ptrs, d_info, batch_size, queue) :
+      MFEM_TRIPACK_MAGMA_PREFIX(pptf2_batched_small)(
+         MagmaLower, n, d_factor_ptrs, d_info, batch_size, queue);
+
+   MFEM_VERIFY(status == MAGMA_SUCCESS,
+               "MAGMA packed Cholesky factorization failed.");
+
+   magma_queue_sync(queue);
+
+   const magma_int_t *h_info = info.HostRead();
+   for (int e = 0; e < batch_size; ++e)
+   {
+      MFEM_VERIFY(h_info[e] == 0,
+                  "MAGMA packed Cholesky factorization failed on matrix "
+                  << e << '.');
+   }
+}
+
+void MagmaPackedLowerCholesky::SolveInPlace(
+   const TriPackLowerMatrix &L,
+   Vector &rhs_sol) const
+{
+   MFEM_VERIFY(queue != nullptr, "MAGMA queue is not set.");
+   MFEM_VERIFY(L.GetNumRows() > 0 || L.GetNumMatrices() == 0,
+               "Invalid factor dimensions.");
+
+   const int solve_n = L.GetNumRows();
+   const int solve_batch = L.GetNumMatrices();
+   const int solve_packed = L.GetPackedSize();
+
+   MFEM_VERIFY(rhs_sol.Size() == solve_batch*solve_n,
+               "Right-hand side has the wrong size.");
+   if (solve_batch == 0) { return; }
+
+   real_t *factor_data = const_cast<real_t *>(L.Data().Read());
+   real_t **d_factor_ptrs =
+      SetPackedPointerArray(factor_ptrs, factor_data, solve_packed,
+                            solve_batch, queue);
+
+   real_t *rhs_data = rhs_sol.ReadWrite();
+   real_t **d_rhs_ptrs =
+      SetPackedPointerArray(rhs_ptrs, rhs_data, solve_n, solve_batch, queue);
+
+   const magma_int_t status =
+      MFEM_TRIPACK_MAGMA_PREFIX(pptrs_batched)(
+         MagmaLower, solve_n, 1, d_factor_ptrs, d_rhs_ptrs, solve_n,
+         solve_batch, queue);
+
+   MFEM_VERIFY(status == MAGMA_SUCCESS,
+               "MAGMA packed Cholesky solve failed.");
+}
+
+MagmaPackedLowerInverse::MagmaPackedLowerInverse()
+{
+   queue = Magma::Queue();
+}
+
+void MagmaPackedLowerInverse::Compute(
+   const TriPackLowerMatrix &A,
+   TriPackLowerMatrix &A_inv)
+{
+   MFEM_VERIFY(queue != nullptr, "MAGMA queue is not set.");
+
+   n = A.GetNumRows();
+   batch_size = A.GetNumMatrices();
+   packed_size = A.GetPackedSize();
+
+   MFEM_VERIFY(n <= 64, "MAGMA packed inverse supports n <= 64.");
+
+   A_inv.SetSize(n, batch_size);
+   A_inv.UseDevice(true);
+
+   if (batch_size == 0) { return; }
+
+   A_inv.Data() = A.Data();
+
+   real_t *inv_data = A_inv.Data().ReadWrite();
+   real_t **d_inv_ptrs =
+      SetPackedPointerArray(inv_ptrs, inv_data, packed_size, batch_size, queue);
+
+   info.SetSize(batch_size, Device::GetDeviceMemoryType());
+   magma_int_t *d_info = info.Write();
+   magma_memset(d_info, 0, batch_size*sizeof(magma_int_t));
+
+   // MAGMA currently expects a valid pointer for device_lwork even when the
+   // required workspace is 0 bytes.
+   int64_t device_lwork[1] = {0};
+   const magma_int_t status =
+      MFEM_TRIPACK_MAGMA_PREFIX(ppinv_batched)(
+         MagmaLower, n, d_inv_ptrs,
+         /*device_work*/ nullptr, device_lwork,
+         d_info, batch_size, queue);
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA packed inverse failed.");
+
+   magma_queue_sync(queue);
+
+   const magma_int_t *h_info = info.HostRead();
+   for (int e = 0; e < batch_size; ++e)
+   {
+      MFEM_VERIFY(h_info[e] == 0,
+                  "MAGMA packed inverse failed on matrix " << e << '.');
+   }
+}
+
+void MagmaPackedLowerInverse::ApplyInPlace(
+   const TriPackLowerMatrix &A_inv,
+   Vector &rhs_sol) const
+{
+   MFEM_VERIFY(queue != nullptr, "MAGMA queue is not set.");
+   MFEM_VERIFY(A_inv.GetNumRows() > 0 || A_inv.GetNumMatrices() == 0,
+               "Invalid inverse dimensions.");
+
+   const int apply_n = A_inv.GetNumRows();
+   const int apply_batch = A_inv.GetNumMatrices();
+   const int apply_packed = A_inv.GetPackedSize();
+
+   MFEM_VERIFY(rhs_sol.Size() == apply_batch*apply_n,
+               "Right-hand side has the wrong size.");
+
+   if (apply_batch == 0) { return; }
+
+   // Prefer MAGMA's tuned packed-symmetric matvec when available (n <= 32).
+   // Fall back to an MFEM device kernel for larger n.
+   if (apply_n <= 32)
+   {
+      real_t *inv_data = const_cast<real_t *>(A_inv.Data().Read());
+      real_t **d_inv_ptrs =
+         SetPackedPointerArray(inv_ptrs, inv_data, apply_packed, apply_batch,
+                               queue);
+
+      real_t *rhs_data = rhs_sol.ReadWrite();
+      real_t **d_rhs_ptrs =
+         SetPackedPointerArray(rhs_ptrs, rhs_data, apply_n, apply_batch, queue);
+
+      // Note: MAGMA's symv_packed_inplace_batched_small returns void; it will
+      // report argument errors via magma_xerbla.
+      MFEM_TRIPACK_MAGMA_PREFIX(symv_packed_inplace_batched_small)(
+         MagmaLower, apply_n, d_inv_ptrs, d_rhs_ptrs, apply_n, apply_batch,
+         queue);
+      return;
+   }
+
+   work.SetSize(apply_batch*apply_n);
+   work.UseDevice(true);
+
+   const real_t *AP = A_inv.Data().Read();
+   const real_t *X = rhs_sol.Read();
+   real_t *Y = work.Write();
+
+   mfem::forall(apply_batch*apply_n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int i = idx % apply_n;
+      const int e = idx / apply_n;
+      const real_t *APe = AP + e*apply_packed;
+      const real_t *Xe = X + e*apply_n;
+      real_t sum = 0.0;
+      for (int j = 0; j < apply_n; ++j)
+      {
+         const real_t aij =
+            (i >= j) ?
+            APe[TriPackLowerMatrix::LowerIndex(i, j, apply_n)] :
+            APe[TriPackLowerMatrix::LowerIndex(j, i, apply_n)];
+         sum += aij * Xe[j];
+      }
+      Y[idx] = sum;
+   });
+
+   const real_t *Y_in = work.Read();
+   real_t *X_out = rhs_sol.Write();
+   mfem::forall(apply_batch*apply_n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      X_out[idx] = Y_in[idx];
+   });
+}
+
+#undef MFEM_TRIPACK_MAGMA_SET_POINTER
+#undef MFEM_TRIPACK_MAGMA_PREFIX
+
+} // namespace mfem
+
+#endif // MFEM_USE_MAGMA

linalg/tripack_magma.hpp

@@ -0,0 +1,144 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_TRIPACK_MAGMA
+#define MFEM_TRIPACK_MAGMA
+
+#include "../config/config.hpp"
+#include "tripack.hpp"
+
+#ifdef MFEM_USE_MAGMA
+
+#include "batched/magma.hpp"
+
+namespace mfem
+{
+
+/// Workspace + operations for MAGMA packed-lower batched Cholesky and solve.
+///
+/// This class factors a batch of SPD matrices stored in packed lower-triangular
+/// format (LAPACK/MAGMA column-major packed storage) and applies the inverse via
+/// MAGMA batched triangular solves.
+class MagmaPackedLowerCholesky
+{
+private:
+   int n = 0;
+   int batch_size = 0;
+   int packed_size = 0;
+
+   mutable Array<real_t *> factor_ptrs;
+   mutable Array<real_t *> rhs_ptrs;
+   Array<magma_int_t> info;
+
+   magma_queue_t queue = nullptr;
+
+public:
+   MagmaPackedLowerCholesky();
+
+   void SetQueue(magma_queue_t q) { queue = q; }
+
+   int GetNumRows() const { return n; }
+   int GetNumMatrices() const { return batch_size; }
+   int GetPackedSize() const { return packed_size; }
+
+   /// Factor packed-lower matrices A into L (in-place copy then factor).
+   void Factor(const TriPackLowerMatrix &A,
+               TriPackLowerMatrix &L);
+
+   /// Solve A x = b using L from Factor(), overwriting rhs_sol with x.
+   void SolveInPlace(const TriPackLowerMatrix &L,
+                     Vector &rhs_sol) const;
+};
+
+/// Workspace + operations for MAGMA packed-lower batched inverse and apply.
+///
+/// This class computes the inverse of a batch of SPD matrices stored in packed
+/// lower-triangular format (LAPACK/MAGMA column-major packed storage) using
+/// MAGMA's `ppinv_batched`. The resulting packed inverse can be applied to a
+/// batch of vectors using MAGMA's packed-symmetric batched matvec when
+/// available, falling back to an MFEM device kernel for larger sizes.
+class MagmaPackedLowerInverse
+{
+private:
+   int n = 0;
+   int batch_size = 0;
+   int packed_size = 0;
+
+   mutable Array<real_t *> inv_ptrs;
+   mutable Array<real_t *> rhs_ptrs;
+   mutable Vector work;
+   Array<magma_int_t> info;
+
+   magma_queue_t queue = nullptr;
+
+public:
+   MagmaPackedLowerInverse();
+
+   void SetQueue(magma_queue_t q) { queue = q; }
+
+   int GetNumRows() const { return n; }
+   int GetNumMatrices() const { return batch_size; }
+   int GetPackedSize() const { return packed_size; }
+
+   /// Compute packed inverse of A into A_inv (in-place copy then invert).
+   void Compute(const TriPackLowerMatrix &A,
+                TriPackLowerMatrix &A_inv);
+
+   /// Apply packed inverse to rhs_sol, overwriting rhs_sol with the result.
+   void ApplyInPlace(const TriPackLowerMatrix &A_inv,
+                     Vector &rhs_sol) const;
+};
+
+namespace tripack
+{
+namespace magma
+{
+
+inline void ComputeCholeskyLower(
+   const TriPackLowerMatrix &packed_lower,
+   TriPackLowerMatrix &lower_factor,
+   MagmaPackedLowerCholesky &ws)
+{
+   ws.Factor(packed_lower, lower_factor);
+}
+
+inline void SolveCholeskyLowerInPlace(
+   const TriPackLowerMatrix &lower_factor,
+   Vector &rhs_sol,
+   MagmaPackedLowerCholesky &ws)
+{
+   ws.SolveInPlace(lower_factor, rhs_sol);
+}
+
+inline void ComputeInverseLower(
+   const TriPackLowerMatrix &packed_lower,
+   TriPackLowerMatrix &lower_inverse,
+   MagmaPackedLowerInverse &ws)
+{
+   ws.Compute(packed_lower, lower_inverse);
+}
+
+inline void ApplyInverseLowerInPlace(
+   const TriPackLowerMatrix &lower_inverse,
+   Vector &rhs_sol,
+   MagmaPackedLowerInverse &ws)
+{
+   ws.ApplyInPlace(lower_inverse, rhs_sol);
+}
+
+} // namespace magma
+} // namespace tripack
+
+} // namespace mfem
+
+#endif // MFEM_USE_MAGMA
+
+#endif // MFEM_TRIPACK_MAGMA

mesh/pncmesh.cpp

@@ -2624,7 +2624,8 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
          for (int i = 0; i < rank_neighbors.Size(); i++)
          {
             int elem = rank_neighbors[i];
-            msg.AddElementRank(elem, new_ranks[elements[elem].index]);
+            const Element &el = elements[elem];
+            msg.AddElement(elem, new_ranks[el.index], el.attribute);
          }
 
          msg.Isend(rank, MyComm);
@@ -2647,7 +2648,9 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
       {
          int ghost_index = elements[msg.elements[i]].index;
          MFEM_ASSERT(element_type[ghost_index] == 2, "");
-         new_ranks[ghost_index] = msg.values[i];
+         const ElementRankAndAttribute &value = msg.values[i];
+         new_ranks[ghost_index] = value.rank;
+         elements[msg.elements[i]].attribute = value.attribute;
       }
    }
 
@@ -2718,7 +2721,7 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
 
                if ((element_type[el.index] & 1) || el.rank != rank)
                {
-                  msg.AddElementRank(elem, el.rank);
+                  msg.AddElement(elem, el.rank, el.attribute);
                }
                // NOTE: we skip 'ghosts' that are of the receiver's rank because
                // they are not really ghosts and would get sent multiple times,
@@ -2770,10 +2773,12 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
 
          for (int i = 0; i < msg.Size(); i++)
          {
-            int elem_rank = msg.values[i];
-            elements[msg.elements[i]].rank = elem_rank;
+            const ElementRankAndAttribute &value = msg.values[i];
+            Element &el = elements[msg.elements[i]];
+            el.rank = value.rank;
+            el.attribute = value.attribute;
 
-            if (elem_rank == MyRank) { received_elements++; }
+            if (value.rank == MyRank) { received_elements++; }
          }
 
          // save the ranks we received from, for later use in RecvRebalanceDofs
@@ -2809,7 +2814,10 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
 
             for (int i = 0; i < msg.Size(); i++)
             {
-               elements[msg.elements[i]].rank = msg.values[i];
+               const ElementRankAndAttribute &value = msg.values[i];
+               Element &el = elements[msg.elements[i]];
+               el.rank = value.rank;
+               el.attribute = value.attribute;
             }
 
             // save the ranks we received from, for later use in RecvRebalanceDofs

mesh/pncmesh.hpp

@@ -531,26 +531,36 @@ protected: // implementation
       typedef std::map<int, NeighborDerefinementMessage> Map;
    };
 
-   /** Used in Step 2 of Rebalance() to synchronize new rank assignments in
-    *  the ghost layer.
+   struct ElementRankAndAttribute
+   {
+      int rank;
+      int attribute;
+   };
+
+   /** Used in RedistributeElements() to synchronize new rank assignments and
+    *  element attributes in the ghost layer.
     */
-   class NeighborElementRankMessage : public ElementValueMessage<int, false,
+   class NeighborElementRankMessage :
+      public ElementValueMessage<ElementRankAndAttribute, false,
       VarMessageTag::NEIGHBOR_ELEMENT_RANK_VM>
    {
    public:
-      void AddElementRank(int elem, int rank) { Add(elem, rank); }
+      void AddElement(int elem, int rank, int attribute)
+      { Add(elem, {rank, attribute}); }
       typedef std::map<int, NeighborElementRankMessage> Map;
    };
 
-   /** Used by Rebalance() to send elements and their ranks. Note that
+   /** Used by Rebalance() to send elements, ranks, and attributes. Note that
     *  RefTypes == true which means the refinement hierarchy will be recreated
     *  on the receiving side.
     */
-   class RebalanceMessage : public ElementValueMessage<int, true,
+   class RebalanceMessage :
+      public ElementValueMessage<ElementRankAndAttribute, true,
       VarMessageTag::REBALANCE_VM>
    {
    public:
-      void AddElementRank(int elem, int rank) { Add(elem, rank); }
+      void AddElement(int elem, int rank, int attribute)
+      { Add(elem, {rank, attribute}); }
       typedef std::map<int, RebalanceMessage> Map;
    };
 

mfem.hpp

@@ -44,6 +44,9 @@
 #endif // MFEM_USE_MPI
 
 #include "linalg/linalg.hpp"
+#ifdef MFEM_USE_MAGMA
+#include "linalg/tripack_magma.hpp"
+#endif
 
 #include "mesh/mesh_headers.hpp"
 

miniapps/fluids/navier/makefile

@@ -52,6 +52,8 @@ endif
 .SUFFIXES:
 .SUFFIXES: .o .cpp .mk
 .PHONY: all lib-common clean clean-build clean-exec
+# Keeping the *.o files fixes an issue with the MacOS version of 'make'.
+.PRECIOUS: %.o
 
 # Remove built-in rules
 %: %.cpp

miniapps/hdiv-linear-solver/darcy.cpp

@@ -40,9 +40,15 @@
 //    mpirun -np 4 darcy -m ../../data/fichera-q2.mesh
 
 #include "mfem.hpp"
+#include <cstring>
 #include <iostream>
 #include <memory>
 
+#ifdef MFEM_USE_UMPIRE
+#include <umpire/Allocator.hpp>
+#include <umpire/ResourceManager.hpp>
+#endif
+
 #include "discrete_divergence.hpp"
 #include "hdiv_linear_solver.hpp"
 
@@ -53,6 +59,78 @@ using namespace mfem;
 
 ParMesh LoadParMesh(const char *mesh_file, int ser_ref = 0, int par_ref = 0);
 
+namespace
+{
+
+HdivSaddlePointSolver::L2InverseType ParseL2InverseType(const char *name)
+{
+   if (!name || strcmp(name, "cg") == 0)
+   {
+      return HdivSaddlePointSolver::L2InverseType::CG;
+   }
+   if (strcmp(name, "magma-packed") == 0)
+   {
+      return HdivSaddlePointSolver::L2InverseType::MAGMA_PACKED;
+   }
+   if (strcmp(name, "magma-full") == 0)
+   {
+      return HdivSaddlePointSolver::L2InverseType::MAGMA_FULL;
+   }
+   MFEM_ABORT("Unknown -l2inv value: " << name
+              << " (expected: cg | magma-packed | magma-full)");
+   return HdivSaddlePointSolver::L2InverseType::CG;
+}
+
+#ifdef MFEM_USE_UMPIRE
+void ReportUmpireAllocator(const char *label, const char *alloc_name)
+{
+   auto &rm = umpire::ResourceManager::getInstance();
+   if (!rm.isAllocator(alloc_name))
+   {
+      if (Mpi::Root())
+      {
+         cout << label << ": allocator '" << alloc_name
+              << "' not found (no allocations yet?)\n";
+      }
+      return;
+   }
+
+   auto alloc = rm.getAllocator(alloc_name);
+   const unsigned long long cur = alloc.getCurrentSize();
+   const unsigned long long hwm = alloc.getHighWatermark();
+   unsigned long long cur_sum = 0, cur_max = 0;
+   unsigned long long hwm_sum = 0, hwm_max = 0;
+
+   MPI_Reduce(&cur, &cur_sum, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0,
+              MPI_COMM_WORLD);
+   MPI_Reduce(&cur, &cur_max, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, 0,
+              MPI_COMM_WORLD);
+   MPI_Reduce(&hwm, &hwm_sum, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0,
+              MPI_COMM_WORLD);
+   MPI_Reduce(&hwm, &hwm_max, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, 0,
+              MPI_COMM_WORLD);
+
+   if (Mpi::Root())
+   {
+      cout << label << " (Umpire '" << alloc_name << "'): "
+           << "current(sum/max)=(" << cur_sum << "/" << cur_max << ") bytes, "
+           << "hwm(sum/max)=(" << hwm_sum << "/" << hwm_max << ") bytes\n";
+   }
+}
+
+void ReportUmpireMemory(const char *label)
+{
+   if (Mpi::Root()) { cout << label << '\n'; }
+   ReportUmpireAllocator("  host", MemoryManager::GetUmpireHostAllocatorName());
+   ReportUmpireAllocator("  device",
+                         MemoryManager::GetUmpireDeviceAllocatorName());
+}
+#else
+void ReportUmpireMemory(const char *) { }
+#endif
+
+} // namespace
+
 int main(int argc, char *argv[])
 {
    Mpi::Init(argc, argv);
@@ -64,6 +142,11 @@ int main(int argc, char *argv[])
    int par_ref = 1;
    int order = 3;
    real_t alpha = 0.0;
+   const char *l2inv = "cg";
+   bool bench_l2inv = false;
+   int l2inv_reps = 100;
+   bool use_umpire_pool = false;
+   bool report_umpire_mem = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&device_config, "-d", "--device",
@@ -75,8 +158,32 @@ int main(int argc, char *argv[])
                   "Number of times to refine the mesh in parallel.");
    args.AddOption(&order, "-o", "--order", "Polynomial degree.");
    args.AddOption(&alpha, "-a", "--alpha", "Value of alpha coefficient.");
+   args.AddOption(&l2inv, "-l2inv", "--l2-inverse",
+                  "Local L2 mass inverse: cg | magma-packed | magma-full.");
+   args.AddOption(&bench_l2inv, "-l2bench", "--l2-bench",
+                  "-no-l2bench", "--no-l2-bench",
+                  "Benchmark the local L2 inverse apply.");
+   args.AddOption(&l2inv_reps, "-l2reps", "--l2-repetitions",
+                  "Repetitions for -l2bench timing.");
+   args.AddOption(&use_umpire_pool, "-umpire-pool", "--umpire-pool",
+                  "-no-umpire-pool", "--no-umpire-pool",
+                  "Use Umpire QuickPool allocators for MFEM allocations.");
+   args.AddOption(&report_umpire_mem, "-mem", "--report-memory",
+                  "-no-mem", "--no-report-memory",
+                  "Report Umpire allocator memory usage.");
    args.ParseCheck();
 
+#ifdef MFEM_USE_UMPIRE
+   if (use_umpire_pool)
+   {
+      MemoryManager::SetUmpireHostAllocatorName("mfem_host_pool");
+      MemoryManager::SetUmpireDeviceAllocatorName("mfem_device_pool");
+   }
+#else
+   MFEM_VERIFY(!use_umpire_pool, "MFEM was built without Umpire support.");
+   MFEM_VERIFY(!report_umpire_mem, "MFEM was built without Umpire support.");
+#endif
+
    Device device(device_config);
    if (Mpi::Root()) { device.Print(); }
 
@@ -140,8 +247,26 @@ int main(int argc, char *argv[])
    ConstantCoefficient one(1.0);
    ConstantCoefficient alpha_coeff(alpha);
    const auto solver_mode = HdivSaddlePointSolver::Mode::DARCY;
+   const auto l2inv_type = ParseL2InverseType(l2inv);
+
+   StopWatch setup_sw;
+   setup_sw.Start();
    HdivSaddlePointSolver saddle_point_solver(
-      mesh, fes_rt, fes_l2, alpha_coeff, one, ess_rt_dofs, solver_mode);
+      mesh, fes_rt, fes_l2, alpha_coeff, one, ess_rt_dofs, solver_mode, l2inv_type);
+   MFEM_DEVICE_SYNC;
+   setup_sw.Stop();
+
+   if (Mpi::Root())
+   {
+      const int n = fes_l2.GetTypicalFE()->GetDof();
+      const int ne = mesh.GetNE();
+      const size_t full_bytes = static_cast<size_t>(ne)*n*n*sizeof(real_t);
+      const size_t packed_bytes = static_cast<size_t>(ne)*n*(n+1)/2*sizeof(real_t);
+      cout << "Setup time: " << setup_sw.RealTime() << " s\n"
+           << "Local element matrices (theoretical): full=" << full_bytes
+           << " bytes, packed=" << packed_bytes << " bytes\n";
+   }
+   if (report_umpire_mem) { ReportUmpireMemory("After setup"); }
 
    const Array<int> &offsets = saddle_point_solver.GetOffsets();
    BlockVector X_block(offsets), B_block(offsets);
@@ -160,6 +285,37 @@ int main(int argc, char *argv[])
            << saddle_point_solver.GetNumIterations()
            << "\nElapsed: " << tic_toc.RealTime() << endl;
    }
+   if (report_umpire_mem) { ReportUmpireMemory("After solve"); }
+
+   if (bench_l2inv)
+   {
+      const int n_l2 = offsets[1];
+      Vector rhs(n_l2), x_l2(n_l2);
+      rhs.UseDevice(true);
+      x_l2.UseDevice(true);
+      rhs.Randomize(1);
+
+      // Warm up and time repeated applications.
+      saddle_point_solver.GetL2Inverse().Mult(rhs, x_l2);
+      MFEM_DEVICE_SYNC;
+
+      StopWatch sw;
+      sw.Start();
+      for (int r = 0; r < l2inv_reps; ++r)
+      {
+         saddle_point_solver.GetL2Inverse().Mult(rhs, x_l2);
+      }
+      MFEM_DEVICE_SYNC;
+      sw.Stop();
+
+      const double local_ms = 1000.0*sw.RealTime()/l2inv_reps;
+      double max_ms = 0.0;
+      MPI_Reduce(&local_ms, &max_ms, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+      if (Mpi::Root())
+      {
+         cout << "L2 inverse apply (ms/apply, max over ranks): " << max_ms << '\n';
+      }
+   }
 
    ParGridFunction x(&fes_l2);
    x.SetFromTrueDofs(X_block.GetBlock(0));

miniapps/hdiv-linear-solver/grad_div.cpp

@@ -32,8 +32,13 @@
 //    mpirun -np 4 grad_div -sp -ams -lor -hb -m ../../data/fichera-q2.mesh -rp 0
 
 #include "mfem.hpp"
+#include <cstring>
 #include <iostream>
 #include <memory>
+#ifdef MFEM_USE_UMPIRE
+#include <umpire/Allocator.hpp>
+#include <umpire/ResourceManager.hpp>
+#endif
 #include "hdiv_linear_solver.hpp"
 #include "../solvers/lor_mms.hpp"
 
@@ -43,6 +48,67 @@ using namespace mfem;
 ParMesh LoadParMesh(const char *mesh_file, int ser_ref = 0, int par_ref = 0);
 void SolveCG(Operator &A, Solver &P, const Vector &B, Vector &X);
 
+namespace
+{
+
+HdivSaddlePointSolver::L2InverseType ParseL2InverseType(const char *name)
+{
+   if (!name || strcmp(name, "cg") == 0)
+   {
+      return HdivSaddlePointSolver::L2InverseType::CG;
+   }
+   if (strcmp(name, "magma-packed") == 0)
+   {
+      return HdivSaddlePointSolver::L2InverseType::MAGMA_PACKED;
+   }
+   if (strcmp(name, "magma-full") == 0)
+   {
+      return HdivSaddlePointSolver::L2InverseType::MAGMA_FULL;
+   }
+   MFEM_ABORT("Unknown -l2inv value: " << name
+              << " (expected: cg | magma-packed | magma-full)");
+   return HdivSaddlePointSolver::L2InverseType::CG;
+}
+
+#ifdef MFEM_USE_UMPIRE
+void ReportUmpireAllocator(const char *label, const char *alloc_name)
+{
+   auto &rm = umpire::ResourceManager::getInstance();
+   if (!rm.isAllocator(alloc_name)) { return; }
+   auto alloc = rm.getAllocator(alloc_name);
+   const unsigned long long cur = alloc.getCurrentSize();
+   const unsigned long long hwm = alloc.getHighWatermark();
+   unsigned long long cur_sum = 0, cur_max = 0;
+   unsigned long long hwm_sum = 0, hwm_max = 0;
+   MPI_Reduce(&cur, &cur_sum, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0,
+              MPI_COMM_WORLD);
+   MPI_Reduce(&cur, &cur_max, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, 0,
+              MPI_COMM_WORLD);
+   MPI_Reduce(&hwm, &hwm_sum, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0,
+              MPI_COMM_WORLD);
+   MPI_Reduce(&hwm, &hwm_max, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, 0,
+              MPI_COMM_WORLD);
+   if (Mpi::Root())
+   {
+      cout << label << " (Umpire '" << alloc_name << "'): "
+           << "current(sum/max)=(" << cur_sum << "/" << cur_max << ") bytes, "
+           << "hwm(sum/max)=(" << hwm_sum << "/" << hwm_max << ") bytes\n";
+   }
+}
+
+void ReportUmpireMemory(const char *label)
+{
+   if (Mpi::Root()) { cout << label << '\n'; }
+   ReportUmpireAllocator("  host", MemoryManager::GetUmpireHostAllocatorName());
+   ReportUmpireAllocator("  device",
+                         MemoryManager::GetUmpireDeviceAllocatorName());
+}
+#else
+void ReportUmpireMemory(const char *) { }
+#endif
+
+} // namespace
+
 int main(int argc, char *argv[])
 {
    Mpi::Init(argc, argv);
@@ -57,6 +123,9 @@ int main(int argc, char *argv[])
    bool use_ams = false;
    bool use_lor_ams = false;
    bool use_hybridization = false;
+   const char *l2inv = "cg";
+   bool use_umpire_pool = false;
+   bool report_umpire_mem = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&device_config, "-d", "--device",
@@ -77,8 +146,27 @@ int main(int argc, char *argv[])
    args.AddOption(&use_hybridization,
                   "-hb", "--hybridization", "-no-hb", "--no-hybridization",
                   "Enable or disable hybridization solver.");
+   args.AddOption(&l2inv, "-l2inv", "--l2-inverse",
+                  "Local L2 mass inverse: cg | magma-packed | magma-full.");
+   args.AddOption(&use_umpire_pool, "-umpire-pool", "--umpire-pool",
+                  "-no-umpire-pool", "--no-umpire-pool",
+                  "Use Umpire QuickPool allocators for MFEM allocations.");
+   args.AddOption(&report_umpire_mem, "-mem", "--report-memory",
+                  "-no-mem", "--no-report-memory",
+                  "Report Umpire allocator memory usage.");
    args.ParseCheck();
 
+#ifdef MFEM_USE_UMPIRE
+   if (use_umpire_pool)
+   {
+      MemoryManager::SetUmpireHostAllocatorName("mfem_host_pool");
+      MemoryManager::SetUmpireDeviceAllocatorName("mfem_device_pool");
+   }
+#else
+   MFEM_VERIFY(!use_umpire_pool, "MFEM was built without Umpire support.");
+   MFEM_VERIFY(!report_umpire_mem, "MFEM was built without Umpire support.");
+#endif
+
    if (!use_saddle_point && !use_ams && !use_lor_ams && !use_hybridization)
    {
       if (Mpi::Root()) { cout << "No solver enabled. Exiting.\n"; }
@@ -125,9 +213,11 @@ int main(int argc, char *argv[])
       L2_FECollection fec_l2(order-1, dim, b2, mt);
       ParFiniteElementSpace fes_l2(&mesh, &fec_l2);
 
+      const auto l2inv_type = ParseL2InverseType(l2inv);
       HdivSaddlePointSolver saddle_point_solver(
          mesh, fes_rt, fes_l2, alpha_coeff, beta_coeff, ess_rt_dofs,
-         HdivSaddlePointSolver::Mode::GRAD_DIV);
+         HdivSaddlePointSolver::Mode::GRAD_DIV, l2inv_type);
+      if (report_umpire_mem) { ReportUmpireMemory("After saddle-point setup"); }
 
       const Array<int> &offsets = saddle_point_solver.GetOffsets();
 

miniapps/hdiv-linear-solver/hdiv_linear_solver.cpp

@@ -15,6 +15,199 @@
 namespace mfem
 {
 
+namespace
+{
+
+#ifdef MFEM_USE_MAGMA
+
+#ifdef MFEM_USE_SINGLE
+#define MFEM_HDIV_MAGMA_PREFIX(stub) magma_s##stub
+#define MFEM_HDIV_MAGMA_SET_POINTER magma_sset_pointer
+#elif defined(MFEM_USE_DOUBLE)
+#define MFEM_HDIV_MAGMA_PREFIX(stub) magma_d##stub
+#define MFEM_HDIV_MAGMA_SET_POINTER magma_dset_pointer
+#else
+#error "Unsupported MFEM precision for MAGMA in hdiv-linear-solver."
+#endif
+
+real_t **SetMagmaPointerArray(Array<real_t *> &ptrs,
+                              real_t *data,
+                              const int stride,
+                              const int batch_size,
+                              const magma_queue_t queue)
+{
+   if (ptrs.Size() != batch_size)
+   {
+      if (ptrs.Size() != 0) { magma_queue_sync(queue); }
+      ptrs.SetSize(batch_size, Device::GetDeviceMemoryType());
+   }
+
+   real_t **d_ptrs = ptrs.Write();
+   MFEM_HDIV_MAGMA_SET_POINTER(d_ptrs, data, 1, 0, 0, stride,
+                               batch_size, queue);
+   return d_ptrs;
+}
+
+class MagmaPackedL2MassInverse final : public Solver
+{
+private:
+   const FiniteElementSpace &fes;
+   Coefficient &coeff;
+   const IntegrationRule &ir;
+
+   TriPackLowerMatrix L_factor;
+   mutable MagmaPackedLowerCholesky ws;
+
+public:
+   MagmaPackedL2MassInverse(const FiniteElementSpace &fes_,
+                            Coefficient &coeff_,
+                            const IntegrationRule &ir_)
+      : Solver(fes_.GetTrueVSize()),
+        fes(fes_),
+        coeff(coeff_),
+        ir(ir_)
+   {
+      MFEM_VERIFY(fes.IsDGSpace(), "MagmaPackedL2MassInverse requires DG.");
+      MFEM_VERIFY(UsesTensorBasis(fes),
+                  "MagmaPackedL2MassInverse requires a tensor basis.");
+      MFEM_VERIFY(Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK),
+                  "MAGMA L2 inverse requires CUDA or HIP device backend.");
+      Update();
+   }
+
+   void Update()
+   {
+      MassIntegrator mass(coeff, &ir);
+      mass.AssembleEATriangular(fes, L_factor, false);
+      tripack::magma::ComputeCholeskyLower(L_factor, L_factor, ws);
+   }
+
+   void Mult(const Vector &b, Vector &u) const override
+   {
+      u = b;
+      u.UseDevice(true);
+      tripack::magma::SolveCholeskyLowerInPlace(L_factor, u, ws);
+   }
+
+   void SetOperator(const Operator &) override
+   {
+      MFEM_ABORT("SetOperator not supported with MagmaPackedL2MassInverse.");
+   }
+};
+
+class MagmaFullL2MassInverse final : public Solver
+{
+private:
+   const FiniteElementSpace &fes;
+   Coefficient &coeff;
+   const IntegrationRule &ir;
+
+   Vector A_factor;
+   int n = 0;
+   int batch_size = 0;
+
+   mutable Array<real_t *> mat_ptrs;
+   mutable Array<real_t *> rhs_ptrs;
+   Array<magma_int_t> info;
+   magma_queue_t queue = nullptr;
+
+public:
+   MagmaFullL2MassInverse(const FiniteElementSpace &fes_,
+                          Coefficient &coeff_,
+                          const IntegrationRule &ir_)
+      : Solver(fes_.GetTrueVSize()),
+        fes(fes_),
+        coeff(coeff_),
+        ir(ir_)
+   {
+      MFEM_VERIFY(fes.IsDGSpace(), "MagmaFullL2MassInverse requires DG.");
+      MFEM_VERIFY(UsesTensorBasis(fes),
+                  "MagmaFullL2MassInverse requires a tensor basis.");
+      MFEM_VERIFY(Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK),
+                  "MAGMA L2 inverse requires CUDA or HIP device backend.");
+      queue = Magma::Queue();
+      Update();
+   }
+
+   void Update()
+   {
+      MassIntegrator mass(coeff, &ir);
+
+      n = fes.GetTypicalFE()->GetDof();
+      batch_size = fes.GetMesh()->GetNE();
+
+      // MassIntegrator::AssembleEA expects the output Vector to be sized by
+      // the caller (unlike AssembleEATriangular which sizes its output).
+      A_factor.SetSize(batch_size*n*n, Device::GetDeviceMemoryType());
+      A_factor.UseDevice(true);
+      mass.AssembleEA(fes, A_factor, false);
+
+      MFEM_VERIFY(A_factor.Size() == batch_size*n*n,
+                  "Unexpected element matrix storage size.");
+
+      if (batch_size == 0) { return; }
+
+      real_t *A_data = A_factor.ReadWrite();
+      real_t **dA =
+         SetMagmaPointerArray(mat_ptrs, A_data, n*n, batch_size, queue);
+
+      info.SetSize(batch_size, Device::GetDeviceMemoryType());
+      magma_int_t *d_info = info.Write();
+      magma_memset(d_info, 0, batch_size*sizeof(magma_int_t));
+
+      const magma_int_t status =
+         MFEM_HDIV_MAGMA_PREFIX(potrf_batched)(
+            MagmaLower, n, dA, n, d_info, batch_size, queue);
+
+      MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA full potrf batched failed.");
+      magma_queue_sync(queue);
+
+      const magma_int_t *h_info = info.HostRead();
+      for (int e = 0; e < batch_size; ++e)
+      {
+         MFEM_VERIFY(h_info[e] == 0,
+                     "MAGMA full potrf failed on matrix " << e << '.');
+      }
+   }
+
+   void Mult(const Vector &b, Vector &u) const override
+   {
+      MFEM_VERIFY(queue != nullptr, "MAGMA queue is not set.");
+      MFEM_VERIFY(b.Size() == height, "Invalid RHS size.");
+
+      u = b;
+      u.UseDevice(true);
+
+      if (batch_size == 0) { return; }
+
+      real_t *A_data = const_cast<real_t *>(A_factor.Read());
+      real_t **dA =
+         SetMagmaPointerArray(mat_ptrs, A_data, n*n, batch_size, queue);
+
+      real_t *rhs_data = u.ReadWrite();
+      real_t **dB =
+         SetMagmaPointerArray(rhs_ptrs, rhs_data, n, batch_size, queue);
+
+      const magma_int_t status =
+         MFEM_HDIV_MAGMA_PREFIX(potrs_batched)(
+            MagmaLower, n, 1, dA, n, dB, n, batch_size, queue);
+
+      MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA full potrs batched failed.");
+   }
+
+   void SetOperator(const Operator &) override
+   {
+      MFEM_ABORT("SetOperator not supported with MagmaFullL2MassInverse.");
+   }
+};
+
+#undef MFEM_HDIV_MAGMA_SET_POINTER
+#undef MFEM_HDIV_MAGMA_PREFIX
+
+#endif // MFEM_USE_MAGMA
+
+} // namespace
+
 /// Replace x[i] with 1.0/x[i] for all i.
 void Reciprocal(Vector &x)
 {
@@ -65,7 +258,7 @@ const IntegrationRule &GetMassIntRule(FiniteElementSpace &fes_l2)
 HdivSaddlePointSolver::HdivSaddlePointSolver(
    ParMesh &mesh, ParFiniteElementSpace &fes_rt_, ParFiniteElementSpace &fes_l2_,
    Coefficient &L_coeff_, Coefficient &R_coeff_, const Array<int> &ess_rt_dofs_,
-   Mode mode_)
+   Mode mode_, L2InverseType l2_inv_type_)
    : minres(mesh.GetComm()),
      order(fes_rt_.GetMaxElementOrder()),
      fec_l2(order - 1, mesh.Dimension(), b2, mt),
@@ -81,6 +274,7 @@ HdivSaddlePointSolver::HdivSaddlePointSolver(
      L_coeff(L_coeff_),
      R_coeff(R_coeff_),
      mode(mode_),
+     l2_inv_type(l2_inv_type_),
      qs(mesh, GetMassIntRule(fes_l2)),
      W_coeff_qf(qs),
      W_mix_coeff_qf(qs),
@@ -154,9 +348,10 @@ HdivSaddlePointSolver::HdivSaddlePointSolver(
 
 HdivSaddlePointSolver::HdivSaddlePointSolver(
    ParMesh &mesh, ParFiniteElementSpace &fes_rt_, ParFiniteElementSpace &fes_l2_,
-   Coefficient &R_coeff_, const Array<int> &ess_rt_dofs_)
+   Coefficient &R_coeff_, const Array<int> &ess_rt_dofs_,
+   L2InverseType l2_inv_type_)
    : HdivSaddlePointSolver(mesh, fes_rt_, fes_l2_, zero, R_coeff_,
-                           ess_rt_dofs_, Mode::DARCY)
+                           ess_rt_dofs_, Mode::DARCY, l2_inv_type_)
 { }
 
 void HdivSaddlePointSolver::Setup()
@@ -189,7 +384,28 @@ void HdivSaddlePointSolver::Setup()
       });
    }
 
-   L_inv.reset(new DGMassInverse(fes_l2, W_mix_coeff));
+   switch (l2_inv_type)
+   {
+      case L2InverseType::CG:
+         L_inv.reset(new DGMassInverse(fes_l2, W_mix_coeff));
+         break;
+      case L2InverseType::MAGMA_PACKED:
+#ifdef MFEM_USE_MAGMA
+         L_inv.reset(new MagmaPackedL2MassInverse(fes_l2, W_mix_coeff,
+                                                  qs.GetIntRule(0)));
+#else
+         MFEM_ABORT("MFEM was built without MAGMA support.");
+#endif
+         break;
+      case L2InverseType::MAGMA_FULL:
+#ifdef MFEM_USE_MAGMA
+         L_inv.reset(new MagmaFullL2MassInverse(fes_l2, W_mix_coeff,
+                                                qs.GetIntRule(0)));
+#else
+         MFEM_ABORT("MFEM was built without MAGMA support.");
+#endif
+         break;
+   }
 
    if (zero_l2_block)
    {

miniapps/hdiv-linear-solver/hdiv_linear_solver.hpp

@@ -33,6 +33,14 @@ public:
       GRAD_DIV, ///< Grad-div problem.
       DARCY     ///< Darcy/mixed Poisson problem.
    };
+
+   /// How to apply the local DG L2 mass inverse.
+   enum class L2InverseType
+   {
+      CG,           ///< DGMassInverse (default).
+      MAGMA_PACKED, ///< MAGMA packed-lower batched Cholesky+solve.
+      MAGMA_FULL    ///< MAGMA full dense batched Cholesky+solve.
+   };
 private:
    MINRESSolver minres;
 
@@ -63,7 +71,7 @@ private:
    // Components needed for the block operator
    OperatorHandle L, R, R_e; ///< Mass matrices.
    std::unique_ptr<HypreParMatrix> D, Dt, D_e; ///< Divergence matrices.
-   std::shared_ptr<DGMassInverse> L_inv; ///< Inverse of the DG mass matrix.
+   std::shared_ptr<Solver> L_inv; ///< Inverse of the DG mass matrix.
    std::shared_ptr<Operator> A_11; ///< (1,1)-block of the matrix
 
    /// Diagonals of the mass matrices
@@ -85,6 +93,7 @@ private:
    Coefficient &L_coeff, &R_coeff;
 
    const Mode mode;
+   const L2InverseType l2_inv_type;
    bool zero_l2_block = false;
    QuadratureSpace qs;
    QuadratureFunction W_coeff_qf, W_mix_coeff_qf;
@@ -135,7 +144,8 @@ public:
                          Coefficient &L_coeff_,
                          Coefficient &R_coeff_,
                          const Array<int> &ess_rt_dofs_,
-                         Mode mode_);
+                         Mode mode_,
+                         L2InverseType l2_inv_type_ = L2InverseType::CG);
 
    /// @brief Creates a linear solver for the case when the L2 diagonal block is
    /// zero (for Darcy problems).
@@ -146,7 +156,8 @@ public:
                          ParFiniteElementSpace &fes_rt_,
                          ParFiniteElementSpace &fes_l2_,
                          Coefficient &R_coeff_,
-                         const Array<int> &ess_rt_dofs_);
+                         const Array<int> &ess_rt_dofs_,
+                         L2InverseType l2_inv_type_ = L2InverseType::CG);
 
    /// @brief Build the linear operator and solver. Must be called when the
    /// coefficients change.
@@ -168,6 +179,12 @@ public:
    const Array<int> &GetOffsets() const { return offsets; }
    /// Returns the internal MINRES solver.
    MINRESSolver &GetMINRES() { return minres; }
+
+   /// Returns the internal L2 DG mass inverse operator.
+   const Solver &GetL2Inverse() const { return *L_inv; }
+
+   /// Returns the selected local L2 inverse method.
+   L2InverseType GetL2InverseType() const { return l2_inv_type; }
 };
 
 } // namespace mfem

miniapps/multidomain/makefile

@@ -68,7 +68,7 @@ multidomain-test-par: multidomain
 multidomain_nd-test-par: multidomain_nd
 	@$(call mfem-test,$<, $(RUN_MPI), Multidomain ND miniapp,-tf 0.001)
 multidomain_rt-test-par: multidomain_rt
-	@$(call mfem-test,$<, $(RUN_MPI), Multidomain RT iniapp,-tf 0.001)
+	@$(call mfem-test,$<, $(RUN_MPI), Multidomain RT miniapp,-tf 0.001)
 
 # Generate an error message if the MFEM library is not built and exit
 $(MFEM_LIB_FILE):

miniapps/parelag/MultilevelHcurlHdivSolver.cpp

@@ -761,7 +761,7 @@ int main(int argc, char *argv[])
       if (visualize)
       {
          hcurlhdiv_dofTrueDof.Distribute(X, x);
-         MultiVector tmp(x.GetData(), 1, x.Size());
+         parelag::MultiVector tmp(x.GetData(), 1, x.Size());
          sequence[0]->show(jform, tmp);
       }
       post_timer.Stop();

tests/benchmarks/bench_ceed.cpp

@@ -32,7 +32,11 @@
 // Custom benchmark arguments generator
 static void CustomArguments(bm::Benchmark *b) noexcept
 {
-   constexpr int MAX_NDOFS = 16 * 1024 * (mfem_use_gpu ? 1024 : 8);
+#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
+   constexpr int MAX_NDOFS = 16 * 1024 * 1024;
+#else
+   constexpr int MAX_NDOFS = 16 * 1024 * 8;
+#endif
 
    const auto orders = { 7, 6, 5, 4, 3, 2, 1 };
 

tests/unit/CMakeLists.txt

@@ -39,6 +39,7 @@ set(UNIT_TESTS_SRCS
   dfem/test_divergence.cpp
   dfem/test_lvector_interface.cpp
   dfem/test_mass.cpp
+  dfem/test_tuple.cpp
   general/test_array.cpp
   general/test_scan.cpp
   general/test_arrays_by_name.cpp
@@ -73,6 +74,8 @@ set(UNIT_TESTS_SRCS
   linalg/test_particlevector.cpp
   linalg/test_petsc_nonlinear.cpp
   linalg/test_sparsesmoothers.cpp
+  linalg/test_tripack.cpp
+  linalg/test_tripack_magma.cpp
   linalg/test_vector.cpp
   mesh/mesh_test_utils.cpp
   mesh/test_exodus_reader.cpp

tests/unit/dfem/test_tuple.cpp

@@ -0,0 +1,274 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+#include "mfem.hpp"
+#ifndef MFEM_USE_MPI
+#include "../../../fem/dfem/tuple.hpp"
+#endif
+
+using namespace mfem;
+using namespace mfem::future;
+
+namespace tuple_test
+{
+
+// A payload that is not a scalar, mimicking what dFEM kernels actually store.
+using vec3 = tensor<real_t, 3>;
+using tuple3 = tuple<real_t, int, vec3>;
+
+// mfem::future::tuple is no longer an aggregate: it derives from tuple_leaf
+// bases so that it can be defined for an arbitrary number of elements. These
+// checks pin down the properties that the aggregate used to provide for free
+// and that device kernels (which capture tuples by value) depend on.
+static_assert(std::is_trivially_copyable<tuple3>::value,
+              "tuple must be trivially copyable to be captured by value in device kernels");
+static_assert(std::is_trivially_destructible<tuple3>::value,
+              "tuple must be trivially destructible");
+static_assert(std::is_trivially_default_constructible<tuple3>::value,
+              "tuple must be trivially default constructible");
+static_assert(std::is_trivially_copy_assignable<tuple3>::value,
+              "tuple must be trivially copy assignable");
+static_assert(sizeof(tuple3) == sizeof(real_t) + sizeof(int) + sizeof(vec3) +
+              (alignof(real_t) - sizeof(int)),
+              "tuple must not be larger than the sum of its (padded) members");
+
+// Size and element types, both through mfem::future and through the std
+// specializations that drive structured bindings.
+static_assert(tuple_size<tuple3>::value == 3, "");
+static_assert(std::tuple_size<tuple3>::value == 3, "");
+static_assert(std::is_same<tuple_element<0, tuple3>::type, real_t>::value, "");
+static_assert(std::is_same<tuple_element<1, tuple3>::type, int>::value, "");
+static_assert(std::is_same<tuple_element<2, tuple3>::type, vec3>::value, "");
+static_assert(std::is_same<std::tuple_element_t<0, tuple3>, real_t>::value, "");
+static_assert(std::is_same<std::tuple_element_t<2, tuple3>, vec3>::value, "");
+
+// get must preserve the value category and constness of its argument.
+static_assert(std::is_same<decltype(get<1>(std::declval<tuple3&>())),
+              int&>::value, "get on an lvalue must return an lvalue reference");
+static_assert(std::is_same<decltype(get<1>(std::declval<const tuple3&>())),
+              const int&>::value,
+              "get on a const lvalue must return a const lvalue reference");
+static_assert(std::is_same<decltype(get<1>(std::declval<tuple3&&>())),
+              int&&>::value, "get on an rvalue must return an rvalue reference");
+static_assert(std::is_same<decltype(get<1>(std::declval<const tuple3&&>())),
+              const int&&>::value,
+              "get on a const rvalue must return a const rvalue reference");
+
+// += and -= must return a reference, not a copy of the whole tuple.
+using tuple2 = tuple<real_t, vec3>;
+static_assert(std::is_same<decltype(std::declval<tuple2&>() +=
+                                       std::declval<const tuple2&>()), tuple2&>::value,
+              "operator+= must return a reference");
+static_assert(std::is_same<decltype(std::declval<tuple2&>() -=
+                                       std::declval<const tuple2&>()), tuple2&>::value,
+              "operator-= must return a reference");
+
+// The element-wise constructor must stay implicit, so that the
+// copy-list-initialization forms that worked with the aggregate keep working.
+static_assert(std::is_convertible<int, tuple<int>>::value,
+              "tuple's element-wise constructor must not be explicit");
+
+// Constructing from an incompatible type must SFINAE out rather than hard-error,
+// so that the constructor does not poison type traits.
+struct not_a_number { };
+static_assert(!std::is_constructible<tuple<int, int>, int, not_a_number>::value,
+              "");
+static_assert(!std::is_constructible<tuple<int, int>, int>::value,
+              "arity mismatch must not be constructible");
+
+// Usable at compile time.
+constexpr tuple<int, real_t> const_tuple {2, 3.0};
+static_assert(get<0>(const_tuple) == 2, "");
+
+// Copy-list-initialization in a return statement (broken by an explicit ctor).
+tuple<int, real_t> returns_braced_init_list() { return {7, 8.0}; }
+
+} // namespace tuple_test
+
+using namespace tuple_test;
+
+TEST_CASE("dFEM tuple structured bindings", "[dFEM]")
+{
+   tuple3 t {1.0, 2, vec3{{3.0, 4.0, 5.0}}};
+
+   SECTION("binding by reference writes through")
+   {
+      auto &[a, b, c] = t;
+      a = 10.0;
+      b = 20;
+      c(0) = 30.0;
+      REQUIRE(get<0>(t) == 10.0_r);
+      REQUIRE(get<1>(t) == 20);
+      REQUIRE(get<2>(t)(0) == 30.0_r);
+   }
+
+   SECTION("binding by value copies")
+   {
+      auto [a, b, c] = t;
+      a = 10.0;
+      b = 20;
+      c(0) = 30.0;
+      REQUIRE(get<0>(t) == 1.0_r);
+      REQUIRE(get<1>(t) == 2);
+      REQUIRE(get<2>(t)(0) == 3.0_r);
+   }
+
+   SECTION("binding to const")
+   {
+      const auto &[a, b, c] = t;
+      REQUIRE(a == 1.0_r);
+      REQUIRE(b == 2);
+      REQUIRE(c(2) == 5.0_r);
+      static_assert(std::is_same<decltype(a), const real_t>::value, "");
+      static_assert(std::is_same<decltype(c), const vec3>::value, "");
+   }
+
+   SECTION("the bindings alias the tuple storage")
+   {
+      auto &[a, b, c] = t;
+      REQUIRE(&a == &get<0>(t));
+      REQUIRE(&b == &get<1>(t));
+      REQUIRE(&c == &get<2>(t));
+   }
+}
+
+TEST_CASE("dFEM tuple construction", "[dFEM]")
+{
+   SECTION("copy-list-initialization")
+   {
+      tuple<int, real_t> a = {1, 2.0};
+      REQUIRE(get<0>(a) == 1);
+      REQUIRE(get<1>(a) == 2.0_r);
+
+      const auto b = returns_braced_init_list();
+      REQUIRE(get<0>(b) == 7);
+      REQUIRE(get<1>(b) == 8.0_r);
+   }
+
+   SECTION("direct initialization and CTAD")
+   {
+      tuple c {1, 2.0_r, vec3{{1.0, 2.0, 3.0}}};
+      static_assert(std::is_same<decltype(c), tuple<int, real_t, vec3>>::value,
+                    "CTAD must decay the arguments");
+      REQUIRE(get<1>(c) == 2.0_r);
+   }
+
+   SECTION("make_tuple")
+   {
+      const auto d = make_tuple(1, 2.0_r);
+      static_assert(std::is_same<decltype(d), const tuple<int, real_t>>::value, "");
+      REQUIRE(get<0>(d) == 1);
+   }
+
+   SECTION("copy and move construction preserve values")
+   {
+      tuple3 t {1.0, 2, vec3{{3.0, 4.0, 5.0}}};
+      tuple3 copy(t);
+      tuple3 moved(std::move(t));
+      REQUIRE(get<1>(copy) == 2);
+      REQUIRE(get<2>(moved)(1) == 4.0_r);
+   }
+
+   SECTION("value initialization zeroes trivial members")
+   {
+      tuple<int, real_t> z {};
+      REQUIRE(get<0>(z) == 0);
+      REQUIRE(get<1>(z) == 0.0_r);
+   }
+}
+
+TEST_CASE("dFEM tuple arithmetic", "[dFEM]")
+{
+   const tuple2 x {1.0, vec3{{1.0, 2.0, 3.0}}};
+   const tuple2 y {2.0, vec3{{4.0, 5.0, 6.0}}};
+
+   SECTION("element-wise binary operators")
+   {
+      const auto sum = x + y;
+      REQUIRE(get<0>(sum) == 3.0_r);
+      REQUIRE(get<1>(sum)(2) == 9.0_r);
+
+      const auto diff = y - x;
+      REQUIRE(get<0>(diff) == 1.0_r);
+      REQUIRE(get<1>(diff)(0) == 3.0_r);
+   }
+
+   SECTION("compound assignment mutates in place and returns a reference")
+   {
+      tuple2 z = x;
+      auto &ref = (z += y);
+      REQUIRE(&ref == &z);
+      REQUIRE(get<0>(z) == 3.0_r);
+      REQUIRE(get<1>(z)(1) == 7.0_r);
+
+      auto &ref2 = (z -= y);
+      REQUIRE(&ref2 == &z);
+      REQUIRE(get<0>(z) == 1.0_r);
+      REQUIRE(get<1>(z)(1) == 2.0_r);
+   }
+
+   SECTION("scalar operators and unary minus")
+   {
+      const auto scaled = 2.0_r * x;
+      REQUIRE(get<0>(scaled) == 2.0_r);
+      REQUIRE(get<1>(scaled)(2) == 6.0_r);
+
+      const auto halved = x / 2.0_r;
+      REQUIRE(get<0>(halved) == 0.5_r);
+
+      const auto negated = -x;
+      REQUIRE(get<0>(negated) == -1.0_r);
+      REQUIRE(get<1>(negated)(0) == -1.0_r);
+   }
+
+   SECTION("apply")
+   {
+      const auto s = apply([](const real_t &a, const vec3 &b) { return a + b(0); },
+      x);
+      REQUIRE(s == 2.0_r);
+   }
+}
+
+// The tuples are captured by value in device kernels, so exercise a round trip
+// through device memory: construct, mutate through structured bindings and read
+// back on the device.
+TEST_CASE("dFEM tuple on device", "[dFEM][GPU]")
+{
+   Vector res(4);
+   auto d_res = res.Write();
+
+   forall(1, [=] MFEM_HOST_DEVICE (int)
+   {
+      tuple3 t {1.0, 2, vec3{{3.0, 4.0, 5.0}}};
+      auto &[a, b, c] = t;
+      a += static_cast<real_t>(b);
+      c(0) = a;
+
+      tuple2 u {get<0>(t), get<2>(t)};
+      u += tuple2 {1.0, vec3{{1.0, 1.0, 1.0}}};
+
+      d_res[0] = get<0>(u);
+      d_res[1] = get<1>(u)(0);
+      d_res[2] = get<1>(u)(1);
+      d_res[3] = static_cast<real_t>(get<1>(t));
+
+      tuple2 v1{0_r, vec3{0_r, 0_r, 0_r}};
+      tuple2 v2{0_r, vec3{0_r, 0_r, 0_r}};
+      [[maybe_unused]] auto v = v1 + v2;
+   });
+
+   res.HostRead();
+   REQUIRE(std::as_const(res)(0) == 4.0_r);
+   REQUIRE(std::as_const(res)(1) == 4.0_r);
+   REQUIRE(std::as_const(res)(2) == 5.0_r);
+   REQUIRE(std::as_const(res)(3) == 2.0_r);
+}

tests/unit/fem/test_2d_bilininteg.cpp

@@ -3451,4 +3451,81 @@ TEST_CASE("2D Bilinear Scalar Weak Curl Cross Integrators",
    }
 }
 
+TEST_CASE("2D Bilinear Scalar Curl Integrator PartialAssembly",
+          "[MixedScalarCurlIntegrator]"
+          "[BilinearFormIntegrator]"
+          "[NonlinearFormIntegrator]"
+          "[GPU]")
+{
+   int order = 2, n = 1, dim = 2;
+   double tol = 1e-9;
+
+   Mesh mesh = Mesh::MakeCartesian2D(n, n, Element::QUADRILATERAL, 1, 2.0, 3.0);
+
+   VectorFunctionCoefficient   F2_coef(dim, F2);
+   FunctionCoefficient         q2_coef(q2);
+
+   SECTION("Operators on ND")
+   {
+      ND_FECollection    fec_nd(order, dim);
+      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+
+      GridFunction f_nd(&fespace_nd); f_nd.ProjectCoefficient(F2_coef);
+
+      for (int map_type = (int)FiniteElement::VALUE;
+           map_type <= (int)FiniteElement::INTEGRAL; map_type++)
+      {
+         SECTION("Mapping ND to L2 (" +
+                 MapTypeName((FiniteElement::MapType)map_type) + ")")
+         {
+            L2_FECollection    fec_l2(order - 1, dim,
+                                      BasisType::GaussLegendre,
+                                      (FiniteElement::MapType)map_type);
+            FiniteElementSpace fespace_l2(&mesh, &fec_l2);
+
+            Vector tmp_l2(fespace_l2.GetNDofs());
+            Vector tmp_l2_pa(fespace_l2.GetNDofs());
+
+            SECTION("Without Coefficient")
+            {
+               MixedBilinearForm blf_fa(&fespace_nd, &fespace_l2);
+               blf_fa.AddDomainIntegrator(new MixedScalarCurlIntegrator());
+               blf_fa.Assemble();
+               blf_fa.Finalize();
+
+               blf_fa.Mult(f_nd, tmp_l2);
+
+               MixedBilinearForm blf_pa(&fespace_nd, &fespace_l2);
+               blf_pa.SetAssemblyLevel(mfem::AssemblyLevel::PARTIAL);
+               blf_pa.AddDomainIntegrator(new MixedScalarCurlIntegrator());
+               blf_pa.Assemble();
+
+               blf_pa.Mult(f_nd, tmp_l2_pa);
+               tmp_l2_pa -= tmp_l2;
+               REQUIRE(tmp_l2_pa.Normlinf() < tol);
+            }
+            SECTION("With Scalar Coefficient")
+            {
+               MixedBilinearForm blf_fa(&fespace_nd, &fespace_l2);
+               blf_fa.AddDomainIntegrator(
+                  new MixedScalarCurlIntegrator(q2_coef));
+               blf_fa.Assemble();
+               blf_fa.Finalize();
+
+               blf_fa.Mult(f_nd, tmp_l2);
+
+               MixedBilinearForm blf_pa(&fespace_nd, &fespace_l2);
+               blf_pa.SetAssemblyLevel(mfem::AssemblyLevel::PARTIAL);
+               blf_pa.AddDomainIntegrator(new MixedScalarCurlIntegrator(q2_coef));
+               blf_pa.Assemble();
+
+               blf_pa.Mult(f_nd, tmp_l2_pa);
+               tmp_l2_pa -= tmp_l2;
+               REQUIRE(tmp_l2_pa.Normlinf() < tol);
+            }
+         }
+      }
+   }
+}
+
 } // namespace bilininteg_2d

tests/unit/fem/test_lin_interp.cpp

@@ -1069,4 +1069,238 @@ TEST_CASE("Exact Sequence Properties: d(df)=0",
    }
 }
 
+template <class A, class B>
+static void TestCurl(FiniteElementSpace &dom_fes, FiniteElementSpace &ran_fes,
+                     A coeff, B dcoeff)
+{
+   real_t tol = 1e-10;
+   DiscreteLinearOperator CurlFA(&dom_fes, &ran_fes);
+   CurlFA.AddDomainInterpolator(new CurlInterpolator());
+   CurlFA.Assemble();
+   CurlFA.Finalize();
+
+   SparseMatrix &Curl = CurlFA.SpMat();
+   GridFunction x(&dom_fes), y_fa(&ran_fes), y(&ran_fes);
+   x.ProjectCoefficient(coeff);
+   y.ProjectCoefficient(dcoeff);
+   REQUIRE(x.Size() == Curl.Width());
+   REQUIRE(y_fa.Size() == Curl.Height());
+   Curl.Mult(x, y_fa);
+   y_fa -= y;
+   REQUIRE(y_fa.Normlinf() < tol);
+}
+
+template<class Coeff, class TCoeff>
+static void CompareCurlPA(FiniteElementSpace& dom_fes,
+                          FiniteElementSpace &ran_fes,
+                          Coeff coeff, TCoeff tcoeff)
+{
+   real_t tol = 1e-10;
+   DiscreteLinearOperator CurlFA(&dom_fes, &ran_fes);
+   CurlFA.AddDomainInterpolator(new CurlInterpolator());
+   CurlFA.Assemble();
+   CurlFA.Finalize();
+   DiscreteLinearOperator CurlPA(&dom_fes, &ran_fes);
+   CurlPA.AddDomainInterpolator(new CurlInterpolator());
+   CurlPA.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   CurlPA.Assemble();
+
+   SparseMatrix &Curl = CurlFA.SpMat();
+   GridFunction x(&dom_fes), y_fa(&ran_fes), y_pa(&ran_fes);
+   x.ProjectCoefficient(coeff);
+   REQUIRE(x.Size() == Curl.Width());
+   REQUIRE(y_fa.Size() == Curl.Height());
+   REQUIRE(x.Size() == CurlPA.Width());
+   REQUIRE(y_pa.Size() == CurlPA.Height());
+   Curl.Mult(x, y_fa);
+   CurlPA.Mult(x, y_pa);
+   y_pa -= y_fa;
+   REQUIRE(y_pa.Normlinf() < tol);
+   // transpose
+   y_fa.ProjectCoefficient(tcoeff);
+   GridFunction x_fa(&dom_fes), x_pa(&dom_fes);
+   Curl.MultTranspose(y_fa, x_fa);
+   CurlPA.MultTranspose(y_fa, x_pa);
+   x_pa -= x_fa;
+   REQUIRE(x_pa.Normlinf() < tol);
+}
+
+TEST_CASE("Partial Assemble Linear Interpolator",
+          "[CurlInterpolator]"
+          "[GPU]")
+{
+   constexpr int maxOrder = 3;
+   auto order = GENERATE_COPY(range(1, maxOrder + 1));
+   CAPTURE(order);
+
+   auto dim = GENERATE(2, 3);
+   CAPTURE(dim);
+
+   int n = 3;
+
+   Mesh mesh;
+
+   switch (dim)
+   {
+      case 2:
+         mesh =
+            Mesh::MakeCartesian2D(n, n, Element::QUADRILATERAL, true, 2.0, 3.0);
+         break;
+      case 3:
+         mesh = Mesh::MakeCartesian3D(n, n, n, Element::HEXAHEDRON, 2.0, 3.0, 5.0);
+         break;
+   }
+
+   // domain spaces
+   H1_FECollection fec_h1(order, dim);
+   FiniteElementSpace fespace_h1(&mesh, &fec_h1);
+   ND_FECollection fec_nd(order, dim);
+   FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+
+   // range spaces
+   RT_FECollection fec_rt(order - 1, dim);
+   FiniteElementSpace fespace_rt(&mesh, &fec_rt);
+   L2_FECollection fec_l2(order - 1, dim, BasisType::GaussLegendre,
+                          FiniteElement::INTEGRAL);
+   FiniteElementSpace fespace_l2(&mesh, &fec_l2);
+
+   switch (dim)
+   {
+      case 2:
+      {
+         FunctionCoefficient coeff([](const Vector &x)
+         { return sin(2 * M_PI * x[1] / 3) - cos(2 * M_PI * x[0] / 2); });
+         VectorFunctionCoefficient vcoeff(2, [](const Vector &x, Vector &y)
+         {
+            y.SetSize(2);
+            y[0] = -cos(2 * M_PI * x[1] / 3);
+            y[1] = sin(2 * M_PI * x[0] / 2);
+         });
+         // out of plane H1 -> in-plane RT
+         SECTION("H1 to RT")
+         {
+            CompareCurlPA(fespace_h1, fespace_rt, coeff, vcoeff);
+         }
+         // in-plane ND -> out of plane L2
+         SECTION("ND to L2")
+         {
+            CompareCurlPA(fespace_nd, fespace_l2, vcoeff, coeff);
+         }
+         break;
+      }
+      case 3:
+      {
+         VectorFunctionCoefficient coeff(3, [](const Vector &x, Vector &y)
+         {
+            y.SetSize(3);
+            y[0] = sin(2 * M_PI * x[2] / 5) - cos(2 * M_PI * x[1] / 3);
+            y[1] = sin(2 * M_PI * x[0] / 2) - cos(2 * M_PI * x[2] / 5);
+            y[2] = sin(2 * M_PI * x[1] / 3) - cos(2 * M_PI * x[0] / 2);
+         });
+         CompareCurlPA(fespace_nd, fespace_rt, coeff, coeff);
+         break;
+      }
+   }
+}
+
+TEST_CASE("Curl Linear Interpolator",
+          "[CurlInterpolator]"
+          "[GPU]")
+{
+   int order = 2;
+
+   auto type = (Element::Type)GENERATE(range((int)Element::TRIANGLE,
+                                             (int)Element::PYRAMID + 1));
+   CAPTURE(type);
+
+   int n = 3;
+
+   Mesh mesh;
+
+   int dim;
+
+   if (type < (int)Element::TETRAHEDRON)
+   {
+      dim = 2;
+      mesh = Mesh::MakeCartesian2D(n, n, (Element::Type)type, 1, 2.0, 3.0);
+   }
+   else
+   {
+      dim = 3;
+      mesh = Mesh::MakeCartesian3D(n, n, n, (Element::Type)type,
+                                   2.0, 3.0, 5.0);
+   }
+
+   // domain spaces
+   H1_FECollection fec_h1(order, dim);
+   FiniteElementSpace fespace_h1(&mesh, &fec_h1);
+   ND_FECollection fec_nd(order, dim);
+   FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+
+   // range spaces
+   RT_FECollection fec_rt(order - 1, dim);
+   FiniteElementSpace fespace_rt(&mesh, &fec_rt);
+   L2_FECollection fec_l2(order - 1, dim, BasisType::GaussLegendre,
+                          FiniteElement::INTEGRAL);
+   FiniteElementSpace fespace_l2(&mesh, &fec_l2);
+
+   switch (dim)
+   {
+      case 2:
+      {
+         // out of plane H1 -> in-plane RT
+         SECTION("H1 to RT")
+         {
+            FunctionCoefficient coeff([](const Vector &x)
+            {
+               return 1 - 2 * x[0] + 3 * x[1];
+            });
+            VectorFunctionCoefficient dcoeff(2, [](const Vector &x, Vector &y)
+            {
+               y.SetSize(2);
+               // d Ez/dy
+               y[0] = 3;
+               // -d Ez/dx
+               y[1] = 2;
+            });
+
+            TestCurl(fespace_h1, fespace_rt, coeff, dcoeff);
+         }
+         // in-plane ND -> out of plane L2
+         SECTION("ND to L2")
+         {
+            VectorFunctionCoefficient coeff(2, [](const Vector &x, Vector &y)
+            {
+               y.SetSize(2);
+               y[0] = 1 - 2 * x[0] + 3 * x[1];
+               y[1] = 2 * (1 - 2 * x[0] + 3 * x[1]);
+            });
+            FunctionCoefficient dcoeff([](const Vector &x)
+            { return 2 * (-2) - 3; });
+            TestCurl(fespace_nd, fespace_l2, coeff, dcoeff);
+         }
+         break;
+      }
+      case 3:
+      {
+         VectorFunctionCoefficient coeff(3, [](const Vector &x, Vector &y)
+         {
+            y.SetSize(3);
+            y[0] = 1 + 2 * x[0] - 3 * x[1] + 4 * x[2];
+            y[1] = 4 + 3 * x[0] - 2 * x[1] + 1 * x[2];
+            y[2] = 2 - 1 * x[0] + 4 * x[1] - 3 * x[2];
+         });
+         VectorFunctionCoefficient dcoeff(3, [](const Vector &x, Vector &y)
+         {
+            y.SetSize(3);
+            y[0] = 4 - 1;
+            y[1] = 4 + 1;
+            y[2] = 3 + 3;
+         });
+         TestCurl(fespace_nd, fespace_rt, coeff, dcoeff);
+         break;
+      }
+   }
+}
+
 } // namespace lin_interp

tests/unit/fem/test_lor_batched.cpp

@@ -214,7 +214,14 @@ TEST_CASE("LOR AMS", "[LOR][BatchedLOR][AMS][Parallel][GPU]")
    ParFiniteElementSpace vert_fespace(edge_fespace.GetParMesh(), &vert_fec);
 
    ParDiscreteLinearOperator grad(&vert_fespace, &edge_fespace);
-   grad.AddDomainInterpolator(new GradientInterpolator);
+   if (space_type == RT)
+   {
+      grad.AddDomainInterpolator(new CurlInterpolator);
+   }
+   else
+   {
+      grad.AddDomainInterpolator(new GradientInterpolator);
+   }
    grad.Assemble();
    grad.Finalize();
    std::unique_ptr<HypreParMatrix> G(grad.ParallelAssemble());

tests/unit/fem/test_pa_coeff.cpp

@@ -750,6 +750,89 @@ TEST_CASE("Hcurl/Hdiv Mixed PA Coefficient",
    }
 }
 
+TEST_CASE("Hcurl/Hdiv MixedVectorGradientPA",
+          "[GPU][PartialAssembly][Coefficient]")
+{
+   constexpr real_t tol = 4e-12;
+   dimension = GENERATE(2, 3);
+   // no coeff, scalar coeff, diagonal matrix coeff, full matrix coeff
+   auto coeffType = GENERATE(0, 1, 2, 3);
+   auto order = GENERATE(1, 2, 3);
+   // RT, ND
+   auto vFEType = GENERATE(0, 1);
+   CAPTURE(dimension, coeffType, order, vFEType);
+
+   const int ne = 3;
+   Mesh mesh = MakeCartesianNonaligned(dimension, ne);
+
+   H1_FECollection scalar_fec(order, dimension);
+   FiniteElementSpace s_fespace(&mesh, &scalar_fec);
+
+   std::unique_ptr<FiniteElementCollection> vector_fec;
+
+   switch (vFEType)
+   {
+      case 0:
+         vector_fec.reset(new RT_FECollection(order - 1, dimension));
+         break;
+      case 1:
+         vector_fec.reset(new ND_FECollection(order, dimension));
+         break;
+   }
+   FiniteElementSpace v_fespace(&mesh, vector_fec.get());
+
+   MixedBilinearForm pa_form(&s_fespace, &v_fespace);
+   pa_form.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   MixedBilinearForm fa_form(&s_fespace, &v_fespace);
+
+   std::unique_ptr<Coefficient> coeff;
+   std::unique_ptr<DiagonalMatrixCoefficient> dq_coeff;
+   std::unique_ptr<MatrixCoefficient> mq_coeff;
+   switch (coeffType)
+   {
+      case 0:
+         pa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator);
+         fa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator);
+         break;
+      case 1:
+         coeff.reset(new FunctionCoefficient(&coeffFunction));
+         pa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator(*coeff));
+         fa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator(*coeff));
+         break;
+      case 2:
+         dq_coeff.reset(new VectorFunctionCoefficient(dimension, &vectorCoeffFunction));
+         pa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator(*dq_coeff));
+         fa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator(*dq_coeff));
+         break;
+      case 3:
+         mq_coeff.reset(new MatrixFunctionCoefficient(
+                           dimension, &asymmetricMatrixCoeffFunction));
+         pa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator(*mq_coeff));
+         fa_form.AddDomainIntegrator(new MixedVectorGradientIntegrator(*mq_coeff));
+         break;
+   }
+   pa_form.Assemble();
+   fa_form.Assemble();
+
+   GridFunction x_fa(&s_fespace), y_fa(&v_fespace), y_pa(&v_fespace);
+   x_fa.Randomize(1234);
+   REQUIRE(x_fa.Size() == pa_form.Width());
+   REQUIRE(x_fa.Size() == fa_form.Width());
+   REQUIRE(y_fa.Size() == fa_form.Height());
+   REQUIRE(y_pa.Size() == pa_form.Height());
+   pa_form.Mult(x_fa, y_pa);
+   fa_form.Mult(x_fa, y_fa);
+   y_pa -= y_fa;
+   REQUIRE(y_pa.Normlinf() <= tol);
+
+   GridFunction x_pa(&s_fespace);
+   y_fa.Randomize(1234);
+   pa_form.MultTranspose(y_fa, x_pa);
+   fa_form.MultTranspose(y_fa, x_fa);
+   x_pa -= x_fa;
+   REQUIRE(x_pa.Normlinf() <= tol);
+}
+
 TEST_CASE("3D Bilinear VectorFE Integrators PartialAssembly",
           "[BilinearFormIntegrator]"
           "[PartialAssembly]"
@@ -1059,4 +1142,111 @@ TEST_CASE("3D Bilinear VectorFE Integrators PartialAssembly",
    }
 }
 
+TEST_CASE("3D Bilinear Weak Curl Integrators Partial Assembly",
+          "[MixedVectorWeakCurlIntegrator]"
+          "[BilinearFormIntegrator]"
+          "[PartialAssembly]"
+          "[GPU]")
+{
+   auto order = GENERATE(1, 2);
+   CAPTURE(order);
+   int dim = 3;
+
+   FunctionCoefficient q3_coeff(coeffFunction);
+   VectorFunctionCoefficient F3_coeff(dim, vectorCoeffFunction);
+
+   auto mesh_fname =
+      GENERATE("../../data/fichera-amr.mesh", "../../data/ball-nurbs.mesh");
+   CAPTURE(mesh_fname);
+   Mesh mesh(mesh_fname);
+   REQUIRE(mesh.Dimension() == dim);
+   REQUIRE(mesh.SpaceDimension() == dim);
+
+   // convert nurbs into piecewise-quadratic curved mesh
+   if (mesh.NURBSext)
+   {
+      mesh.UniformRefinement();
+      mesh.SetCurvature(2);
+   }
+
+   SECTION("RT to ND No Coeff")
+   {
+      ND_FECollection fec_nd(order, dim);
+      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+      RT_FECollection fec_rt(order - 1, dim);
+      FiniteElementSpace fespace_rt(&mesh, &fec_rt);
+
+      MixedBilinearForm bfa(&fespace_rt, &fespace_nd);
+      bfa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator);
+      bfa.Assemble();
+      bfa.Finalize();
+
+      MixedBilinearForm bpa(&fespace_rt, &fespace_nd);
+      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      bpa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator);
+      bpa.Assemble();
+
+      GridFunction x(&fespace_rt), y_fa(&fespace_nd), y_pa(&fespace_nd);
+      x.Randomize(1234);
+      REQUIRE(bfa.Height() == y_fa.Size());
+      REQUIRE(bfa.Width() == x.Size());
+      REQUIRE(bpa.Height() == y_fa.Size());
+      REQUIRE(bpa.Width() == x.Size());
+      bfa.Mult(x, y_fa);
+      bpa.Mult(x, y_pa);
+      y_pa -= y_fa;
+      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
+   }
+
+   SECTION("RT to ND Scalar Coeff")
+   {
+      ND_FECollection fec_nd(order, dim);
+      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+      RT_FECollection fec_rt(order - 1, dim);
+      FiniteElementSpace fespace_rt(&mesh, &fec_rt);
+
+      MixedBilinearForm bfa(&fespace_rt, &fespace_nd);
+      bfa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(q3_coeff));
+      bfa.Assemble();
+      bfa.Finalize();
+
+      MixedBilinearForm bpa(&fespace_rt, &fespace_nd);
+      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      bpa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(q3_coeff));
+      bpa.Assemble();
+
+      GridFunction x(&fespace_rt), y_fa(&fespace_nd), y_pa(&fespace_nd);
+      x.Randomize(1234);
+      bfa.Mult(x, y_fa);
+      bpa.Mult(x, y_pa);
+      y_pa -= y_fa;
+      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
+   }
+
+   SECTION("RT to ND Diagonal Matrix Coeff")
+   {
+      ND_FECollection fec_nd(order, dim);
+      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+      RT_FECollection fec_rt(order - 1, dim);
+      FiniteElementSpace fespace_rt(&mesh, &fec_rt);
+
+      MixedBilinearForm bfa(&fespace_rt, &fespace_nd);
+      bfa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(F3_coeff));
+      bfa.Assemble();
+      bfa.Finalize();
+
+      MixedBilinearForm bpa(&fespace_rt, &fespace_nd);
+      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      bpa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(F3_coeff));
+      bpa.Assemble();
+
+      GridFunction x(&fespace_rt), y_fa(&fespace_nd), y_pa(&fespace_nd);
+      x.Randomize(1234);
+      bfa.Mult(x, y_fa);
+      bpa.Mult(x, y_pa);
+      y_pa -= y_fa;
+      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
+   }
+}
+
 } // namespace pa_coeff

tests/unit/linalg/test_complex_operator.cpp

@@ -164,12 +164,29 @@ TEST_CASE("ComplexHypreParMatrix GetSystemMatrix",
    a.AddDomainIntegrator(new VectorFEMassIntegrator(one),
                          new VectorFEMassIntegrator(one));
    a.Assemble();
+
+   // 2. Test ParSesquilinearForm::FormSystemMatrix directly and verify that
+   //    essential entries on the imaginary diagonal are zero.
    OperatorPtr Ah;
+   a.FormSystemMatrix(ess_tdof_list, Ah);
+   ComplexHypreParMatrix *A_complex = Ah.Is<ComplexHypreParMatrix>();
+   REQUIRE(A_complex != nullptr);
+   Vector diag;
+   A_complex->imag().GetDiag(diag);
+   const Array<int> &ess_tdofs = ess_tdof_list;
+   const Vector &diag_h = diag;
+   ess_tdofs.HostRead();
+   diag_h.HostRead();
+   for (const int tdof : ess_tdofs)
+   {
+      REQUIRE(diag_h[tdof] == 0.0);
+   }
+
+   // 3. Test the call to ComplexHypreParMatrix::GetSystemMatrix and destroying
+   //    the returned matrix.
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
 
-   // 2. Test the call to ComplexHypreParMatrix::GetSystemMatrix and destroying
-   //    the returned matrix.
    HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
    delete A;
 }

tests/unit/linalg/test_tripack.cpp

@@ -0,0 +1,243 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "unit_tests.hpp"
+
+using namespace mfem;
+
+namespace
+{
+
+void PackLower(const DenseMatrix &mat, real_t *packed)
+{
+   const int n = mat.Height();
+   for (int j = 0; j < n; ++j)
+   {
+      for (int i = j; i < n; ++i)
+      {
+         packed[TriPackLowerMatrix::LowerIndex(i, j, n)] = mat(i, j);
+      }
+   }
+}
+
+void FillFullBatch(const DenseMatrix &mat, real_t *full)
+{
+   const int n = mat.Height();
+   for (int j = 0; j < n; ++j)
+   {
+      for (int i = 0; i < n; ++i)
+      {
+         full[i + n*j] = mat(i, j);
+      }
+   }
+}
+
+void BuildLowerDense(const TriPackLowerMatrix &packed,
+                     int e, DenseMatrix &mat)
+{
+   const int n = packed.GetNumRows();
+   mat.SetSize(n);
+   mat = 0.0;
+   const real_t *data = packed.Data().HostRead() + e*packed.GetPackedSize();
+   for (int j = 0; j < n; ++j)
+   {
+      for (int i = j; i < n; ++i)
+      {
+         mat(i, j) = data[TriPackLowerMatrix::LowerIndex(i, j, n)];
+      }
+   }
+}
+
+}
+
+TEST_CASE("TriPackLowerMatrix operations", "[TriPackLowerMatrix]")
+{
+   constexpr int n = 3;
+   constexpr int batch_size = 2;
+   constexpr real_t tol = 1e-12;
+
+   DenseMatrix A0(n), A1(n);
+   A0 = 0.0;
+   A1 = 0.0;
+
+   A0(0,0) = 4.0; A0(0,1) = 1.0; A0(0,2) = 2.0;
+   A0(1,0) = 1.0; A0(1,1) = 5.0; A0(1,2) = 3.0;
+   A0(2,0) = 2.0; A0(2,1) = 3.0; A0(2,2) = 6.0;
+
+   A1(0,0) = 7.0; A1(0,1) = 2.0; A1(0,2) = 1.0;
+   A1(1,0) = 2.0; A1(1,1) = 8.0; A1(1,2) = 2.0;
+   A1(2,0) = 1.0; A1(2,1) = 2.0; A1(2,2) = 5.0;
+
+   TriPackLowerMatrix packed(n, batch_size);
+   packed = 0.0;
+   real_t *packed_data = packed.Data().HostWrite();
+   PackLower(A0, packed_data);
+   PackLower(A1, packed_data + packed.GetPackedSize());
+
+   Vector full(batch_size*n*n);
+   real_t *full_data = full.HostWrite();
+   FillFullBatch(A0, full_data);
+   FillFullBatch(A1, full_data + n*n);
+
+   SECTION("Compare with full symmetric matrices")
+   {
+      REQUIRE(tripack::CompareWithFull(packed, full, tol));
+   }
+
+   SECTION("Symmetric multiply and lumping")
+   {
+      Vector x({1.0, 2.0, 3.0, 4.0, 5.0, 6.0});
+      Vector y, lump;
+
+      tripack::Mult(packed, x, y);
+      tripack::Lump(packed, lump);
+
+      Vector y_expected(batch_size*n);
+      Vector lump_expected(batch_size*n);
+      y_expected = 0.0;
+      lump_expected = 0.0;
+
+      const DenseMatrix *mats[batch_size] = { &A0, &A1 };
+      for (int e = 0; e < batch_size; ++e)
+      {
+         const DenseMatrix &M = *mats[e];
+         for (int i = 0; i < n; ++i)
+         {
+            real_t rowsum = 0.0;
+            real_t val = 0.0;
+            for (int j = 0; j < n; ++j)
+            {
+               rowsum += M(i, j);
+               val += M(i, j) * x(e*n + j);
+            }
+            lump_expected(e*n + i) = rowsum;
+            y_expected(e*n + i) = val;
+         }
+      }
+
+      for (int i = 0; i < y.Size(); ++i)
+      {
+         REQUIRE(y(i) == MFEM_Approx(y_expected(i)));
+         REQUIRE(lump(i) == MFEM_Approx(lump_expected(i)));
+      }
+   }
+
+   SECTION("Lower Cholesky factor and solves")
+   {
+      TriPackLowerMatrix lfac;
+      Vector rhs({1.0, -1.0, 2.0, 0.5, 1.5, -2.0});
+      Vector y, t, x;
+
+      tripack::ComputeCholeskyLower(packed, lfac);
+      tripack::SolveLowerTranspose(lfac, rhs, t);
+      tripack::SolveLower(lfac, t, x);
+      tripack::SolveCholeskyLower(lfac, rhs, y);
+
+      const DenseMatrix *mats[batch_size] = { &A0, &A1 };
+      for (int e = 0; e < batch_size; ++e)
+      {
+         DenseMatrix L;
+         DenseMatrix recon(n);
+         BuildLowerDense(lfac, e, L);
+         MultABt(L, L, recon);
+         recon -= *mats[e];
+         REQUIRE(recon.MaxMaxNorm() == MFEM_Approx(0.0, tol, tol));
+      }
+
+      Vector x_expected(batch_size*n);
+      for (int e = 0; e < batch_size; ++e)
+      {
+         DenseMatrix inv(n);
+         CalcInverse(*mats[e], inv);
+         for (int i = 0; i < n; ++i)
+         {
+            real_t sum = 0.0;
+            for (int j = 0; j < n; ++j)
+            {
+               sum += inv(i, j) * rhs(e*n + j);
+            }
+            x_expected(e*n + i) = sum;
+         }
+      }
+
+      for (int i = 0; i < x.Size(); ++i)
+      {
+         REQUIRE(x(i) == MFEM_Approx(x_expected(i)).epsilon(tol));
+         REQUIRE(y(i) == MFEM_Approx(x_expected(i)).epsilon(tol));
+      }
+   }
+
+   SECTION("Cholesky lower inverse")
+   {
+      TriPackLowerMatrix linv;
+      Vector rhs({1.0, -1.0, 2.0, 0.5, 1.5, -2.0});
+
+      tripack::ComputeCholeskyLowerInverse(packed, linv);
+      const DenseMatrix *mats[batch_size] = { &A0, &A1 };
+      for (int e = 0; e < batch_size; ++e)
+      {
+         DenseMatrix L, recon(n), inv(n);
+         BuildLowerDense(linv, e, L);
+         MultAtB(L, L, recon);
+         CalcInverse(*mats[e], inv);
+         recon -= inv;
+         REQUIRE(recon.MaxMaxNorm() == MFEM_Approx(0.0, tol, tol));
+      }
+
+      Vector y(batch_size*n);
+      Vector y_expected(batch_size*n);
+      for (int e = 0; e < batch_size; ++e)
+      {
+         DenseMatrix Linv;
+         DenseMatrix inv(n);
+         CalcInverse(*mats[e], inv);
+
+         BuildLowerDense(linv, e, Linv);
+         Vector tmp(n);
+         tmp = 0.0;
+         for (int i = 0; i < n; ++i)
+         {
+            real_t sum = 0.0;
+            for (int j = 0; j <= i; ++j)
+            {
+               sum += Linv(i, j) * rhs(e*n + j);
+            }
+            tmp(i) = sum;
+         }
+
+         for (int i = 0; i < n; ++i)
+         {
+            real_t sum = 0.0;
+            for (int j = 0; j < n; ++j)
+            {
+               sum += inv(i, j) * rhs(e*n + j);
+            }
+            y_expected(e*n + i) = sum;
+         }
+
+         for (int i = 0; i < n; ++i)
+         {
+            real_t sum = 0.0;
+            for (int j = i; j < n; ++j)
+            {
+               sum += Linv(j, i) * tmp(j);
+            }
+            y(e*n + i) = sum;
+         }
+      }
+
+      for (int i = 0; i < y.Size(); ++i)
+      {
+         REQUIRE(y(i) == MFEM_Approx(y_expected(i)).epsilon(tol));
+      }
+   }
+}

tests/unit/linalg/test_tripack_magma.cpp

@@ -0,0 +1,166 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "unit_tests.hpp"
+
+using namespace mfem;
+
+#if defined(MFEM_USE_MAGMA) && (defined(MFEM_USE_HIP) || defined(MFEM_USE_CUDA))
+
+#include <vector>
+
+#if defined(MFEM_USE_HIP)
+#include <hip/hip_runtime.h>
+#elif defined(MFEM_USE_CUDA)
+#include <cuda_runtime.h>
+#endif
+
+namespace
+{
+
+bool HasGpuDevice()
+{
+#if defined(MFEM_USE_HIP)
+   int count = 0;
+   const hipError_t err = hipGetDeviceCount(&count);
+   return err == hipSuccess && count > 0;
+#elif defined(MFEM_USE_CUDA)
+   int count = 0;
+   const cudaError_t err = cudaGetDeviceCount(&count);
+   return err == cudaSuccess && count > 0;
+#else
+   return false;
+#endif
+}
+
+DenseMatrix MakeSPD(const int n, const int seed)
+{
+   DenseMatrix B(n), A(n);
+   for (int j = 0; j < n; ++j)
+   {
+      for (int i = 0; i < n; ++i)
+      {
+         const int v = (17*(i + 1) + 31*(j + 1) + 7*seed) % 23;
+         B(i, j) = 0.05 * real_t(v);
+      }
+   }
+   MultAtB(B, B, A);
+   for (int i = 0; i < n; ++i) { A(i, i) += 1.0 + 0.1*i; }
+   return A;
+}
+
+void PackLower(const DenseMatrix &mat, real_t *packed)
+{
+   const int n = mat.Height();
+   for (int j = 0; j < n; ++j)
+   {
+      for (int i = j; i < n; ++i)
+      {
+         packed[TriPackLowerMatrix::LowerIndex(i, j, n)] =
+            mat(i, j);
+      }
+   }
+}
+
+real_t MaxResidual(const DenseMatrix &A, const Vector &x, const Vector &b)
+{
+   const int n = A.Height();
+   real_t max_abs = 0.0;
+   for (int i = 0; i < n; ++i)
+   {
+      real_t sum = 0.0;
+      for (int j = 0; j < n; ++j) { sum += A(i, j) * x(j); }
+      max_abs = std::max(max_abs, std::abs(sum - b(i)));
+   }
+   return max_abs;
+}
+
+} // namespace
+
+TEST_CASE("MAGMA packed-lower Cholesky factor+solve",
+          "[MAGMA][TriPackLowerMatrix]")
+{
+   if (!HasGpuDevice())
+   {
+      WARN("No GPU device visible; skipping MAGMA packed-lower tests.");
+      return;
+   }
+
+   Device device(
+#if defined(MFEM_USE_HIP)
+      "hip"
+#elif defined(MFEM_USE_CUDA)
+      "cuda"
+#else
+      "cpu"
+#endif
+   );
+
+   constexpr int n = 8;
+   constexpr int batch_size = 17;
+   constexpr double tol = 5e-9;
+
+   TriPackLowerMatrix A_packed(n, batch_size);
+   A_packed.UseDevice(true);
+   A_packed = 0.0;
+
+   std::vector<DenseMatrix> A_dense;
+   A_dense.reserve(batch_size);
+
+   real_t *h_packed = A_packed.Data().HostWrite();
+   const int ps = A_packed.GetPackedSize();
+   for (int e = 0; e < batch_size; ++e)
+   {
+      A_dense.emplace_back(MakeSPD(n, e + 1));
+      PackLower(A_dense.back(), h_packed + e*ps);
+   }
+
+   Vector b(batch_size*n);
+   Vector x(batch_size*n);
+   real_t *h_b = b.HostWrite();
+   for (int e = 0; e < batch_size; ++e)
+   {
+      for (int i = 0; i < n; ++i)
+      {
+         h_b[e*n + i] = 1.0 + real_t((13*(i + 1) + 7*(e + 1)) % 29)/real_t(29);
+      }
+   }
+   b.UseDevice(true);
+   x.UseDevice(true);
+
+   TriPackLowerMatrix L;
+   MagmaPackedLowerCholesky ws;
+   ws.Factor(A_packed, L);
+
+   x = b;
+   ws.SolveInPlace(L, x);
+   MFEM_DEVICE_SYNC;
+
+   const real_t *h_x = x.HostRead();
+   const real_t *h_b_ro = b.HostRead();
+   for (int e = 0; e < batch_size; ++e)
+   {
+      Vector xe(const_cast<real_t *>(h_x) + e*n, n);
+      Vector be(const_cast<real_t *>(h_b_ro) + e*n, n);
+      const real_t res = MaxResidual(A_dense[e], xe, be);
+      REQUIRE(res == MFEM_Approx(0.0, tol, tol));
+   }
+}
+
+#else
+
+TEST_CASE("MAGMA packed-lower tests disabled", "[MAGMA][TriPackLowerMatrix]")
+{
+   SUCCEED("MFEM was built without MAGMA+GPU support.");
+}
+
+#endif

tests/unit/mesh/test_bb_grid_map.cpp

@@ -152,7 +152,7 @@ TEST_CASE("GlobalBBoxTensorGridMap Parallel",
    std::map<int, std::vector<int>> pt_to_procs;
    map.MapPointsToProcs(centers, 1, pt_to_procs);
 
-   REQUIRE(pt_to_procs.size() == nel + 1);
+   REQUIRE(pt_to_procs.size() == (unsigned)nel + 1);
    for (int i = 0; i < nel; i++)
    {
       std::vector<int> procs = pt_to_procs[i];

tests/unit/mesh/test_ncmesh.cpp

@@ -304,6 +304,66 @@ TEST_CASE("pNCMesh PA diagonal",  "[Parallel], [NCMesh]")
    }
 } // test case
 
+TEST_CASE("ParNCMesh Rebalance preserves element attributes",
+          "[Parallel], [NCMesh]")
+{
+   const int rank = Mpi::WorldRank();
+   const int nranks = Mpi::WorldSize();
+   if (nranks < 2) { return; }
+
+   auto mesh_fname = GENERATE("../../data/star.mesh",
+                              "../../data/fichera.mesh");
+   CAPTURE(mesh_fname);
+
+   auto CheckRebalance = [rank, nranks, mesh_fname](bool refine,
+                                                    bool custom_partition)
+   {
+      Mesh mesh(mesh_fname);
+      mesh.EnsureNCMesh();
+      ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+      const int attribute = 1234 + (custom_partition ? rank : 0);
+      for (int i = 0; i < pmesh.GetNE(); i++)
+      {
+         pmesh.SetAttribute(i, attribute);
+      }
+      pmesh.SetAttributes();
+
+      if (refine)
+      {
+         Array<int> refinements;
+         if (pmesh.GetNE() && (custom_partition || rank == 0))
+         {
+            refinements.Append(0);
+         }
+         pmesh.GeneralRefinement(refinements);
+      }
+
+      int expected_attribute = attribute;
+      if (custom_partition)
+      {
+         // Move every element to the next rank, as in GitHub issue #4009.
+         Array<int> partition(pmesh.GetNE());
+         partition = (rank + 1) % nranks;
+         pmesh.Rebalance(partition);
+         expected_attribute = 1234 + (rank + nranks - 1) % nranks;
+      }
+      else
+      {
+         pmesh.Rebalance();
+      }
+
+      for (int i = 0; i < pmesh.GetNE(); i++)
+      {
+         CHECK(pmesh.GetAttribute(i) == expected_attribute);
+      }
+   };
+
+   SECTION("Custom partition, unrefined") { CheckRebalance(false, true); }
+   SECTION("Custom partition, refined") { CheckRebalance(true, true); }
+   SECTION("Default partition, refined") { CheckRebalance(true, false); }
+}
+
 TEST_CASE("EdgeFaceConstraint", "[Parallel], [NCMesh]")
 {
    auto exact_soln = [](const Vector& x)