Merge branch 'master' into dfem-lvector-interface

fem/bilinearform.cpp

@@ -1275,22 +1275,6 @@ void BilinearForm::Update(FiniteElementSpace *nfes)
    height = width = fes->GetVSize();
 
    if (ext) { ext->Update(); }
-   for (int k = 0; k < domain_integs.Size(); ++k)
-   {
-      domain_integs[k]->Update();
-   }
-   for (int k = 0; k < boundary_integs.Size(); ++k)
-   {
-      boundary_integs[k]->Update();
-   }
-   for (int k = 0; k < interior_face_integs.Size(); ++k)
-   {
-      interior_face_integs[k]->Update();
-   }
-   for (int k = 0; k < boundary_integs.Size(); ++k)
-   {
-      boundary_face_integs[k]->Update();
-   }
 }
 
 void BilinearForm::SetDiagonalPolicy(DiagonalPolicy policy)
@@ -2353,31 +2337,6 @@ void MixedBilinearForm::Update()
    height = test_fes->GetVSize();
    width = trial_fes->GetVSize();
    if (ext) { ext->Update(); }
-
-   for (int k = 0; k < domain_integs.Size(); ++k)
-   {
-      domain_integs[k]->Update();
-   }
-   for (int k = 0; k < boundary_integs.Size(); ++k)
-   {
-      boundary_integs[k]->Update();
-   }
-   for (int k = 0; k < interior_face_integs.Size(); ++k)
-   {
-      interior_face_integs[k]->Update();
-   }
-   for (int k = 0; k < boundary_integs.Size(); ++k)
-   {
-      boundary_face_integs[k]->Update();
-   }
-   for (int k = 0; k < trace_face_integs.Size(); ++k)
-   {
-      trace_face_integs[k]->Update();
-   }
-   for (int k = 0; k < boundary_trace_face_integs.Size(); ++k)
-   {
-      boundary_trace_face_integs[k]->Update();
-   }
 }
 
 MixedBilinearForm::~MixedBilinearForm()

fem/bilininteg.cpp

@@ -21,11 +21,6 @@ using namespace std;
 namespace mfem
 {
 
-void BilinearFormIntegrator::Update()
-{
-   // default no-op
-}
-
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
    MFEM_ABORT("BilinearFormIntegrator::AssemblePA(fes)\n"
@@ -3465,12 +3460,6 @@ real_t ElasticityIntegrator::ComputeFluxEnergy(const FiniteElement &fluxelem,
    return energy;
 }
 
-void DGTraceIntegrator::Update()
-{
-   qspace[0].reset();
-   qspace[1].reset();
-}
-
 void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
                                            const FiniteElement &el2,
                                            FaceElementTransformations &Trans,

fem/bilininteg.hpp

@@ -23,8 +23,6 @@
 
 namespace mfem
 {
-class QuadratureSpace;
-class FaceQuadratureSpace;
 
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
@@ -46,10 +44,6 @@ public:
    // make sense for the action of the nonlinear operator (but they all make
    // sense for its Jacobian).
 
-   /// Signal this integrator that something about either the trial or test space has changed.
-   virtual void Update();
-
-
    /// Method defining partial assembly.
    /** The result of the partial assembly is stored internally so that it can be
        used later in the methods AddMultPA() and AddMultTransposePA(). */
@@ -3317,7 +3311,6 @@ protected:
    VectorCoefficient *u;
    real_t alpha, beta;
    // PA extension
-   std::unique_ptr<FaceQuadratureSpace> qspace[2];
    Vector pa_data;
    const DofToQuad *maps;             ///< Not owned
    const FaceGeometricFactors *geom;  ///< Not owned
@@ -3340,8 +3333,6 @@ public:
                      real_t a, real_t b)
    { rho = &rho_; u = &u_; alpha = a; beta = b; }
 
-   void Update() override;
-
    using BilinearFormIntegrator::AssembleFaceMatrix;
    void AssembleFaceMatrix(const FiniteElement &el1,
                            const FiniteElement &el2,

fem/dfem/doperator.hpp

@@ -231,22 +231,53 @@ public:
       const std::vector<FieldDescriptor> &parameters,
       const ParMesh &mesh);
 
+   /// MultLevel enum to indicate if the T->L Operators are used in the
+   /// Mult method.
+   enum MultLevel
+   {
+      TVECTOR,
+      LVECTOR
+   };
+
+   /// @brief Set the MultLevel mode for the DifferentiableOperator.
+   /// The default is TVECTOR, which means that the Operator will use
+   /// T->L before Mult and L->T Operators after.
+   void SetMultLevel(MultLevel level)
+   {
+      mult_level = level;
+   }
+
    /// @brief Compute the action of the operator on a given vector.
    ///
-   /// @param solutions_t The solution vector in which to compute the action.
-   /// This has to be a T-dof vector.
-   /// @param result_t Result vector of the action of the operator on
-   /// solutions_t. The result is a T-dof vector.
-   void Mult(const Vector &solutions_t, Vector &result_t) const override
+   /// @param solutions_in The solution vector in which to compute the action.
+   /// This has to be a T-dof vector if MultLevel is set to TVECTOR, or L-dof
+   /// Vector if MultLevel is set to LVECTOR.
+   /// @param result_in Result vector of the action of the operator on
+   /// solutions. The result is a T-dof vector or L-dof vector depending on
+   /// the MultLevel.
+   void Mult(const Vector &solutions_in, Vector &result_in) const override
    {
       MFEM_ASSERT(!action_callbacks.empty(), "no integrators have been set");
-      prolongation(solutions, solutions_t, solutions_l);
-      residual_l = 0.0;
-      for (auto &action : action_callbacks)
+
+      if (mult_level == MultLevel::LVECTOR)
       {
-         action(solutions_l, parameters_l, residual_l);
+         get_lvectors(solutions, solutions_in, solutions_l);
+         result_in = 0.0;
+         for (auto &action : action_callbacks)
+         {
+            action(solutions_l, parameters_l, result_in);
+         }
+      }
+      else
+      {
+         prolongation(solutions, solutions_in, solutions_l);
+         residual_l = 0.0;
+         for (auto &action : action_callbacks)
+         {
+            action(solutions_l, parameters_l, residual_l);
+         }
+         prolongation_transpose(residual_l, result_in);
       }
-      prolongation_transpose(residual_l, result_t);
    }
 
    /// @brief Add a domain integrator to the operator.
@@ -345,6 +376,8 @@ public:
 private:
    const ParMesh &mesh;
 
+   MultLevel mult_level = TVECTOR;
+
    std::vector<action_t> action_callbacks;
    std::map<size_t,
        std::vector<derivative_action_t>> derivative_action_callbacks;
@@ -354,7 +387,6 @@ private:
        std::vector<assemble_derivative_hypreparmatrix_callback_t>>
        assemble_derivative_hypreparmatrix_callbacks;
 
-
    std::vector<FieldDescriptor> solutions;
    std::vector<FieldDescriptor> parameters;
    // solutions and parameters

fem/dfem/util.hpp

@@ -1076,6 +1076,24 @@ void prolongation(const std::vector<FieldDescriptor> fields,
    }
 }
 
+inline
+void get_lvectors(const std::vector<FieldDescriptor> fields,
+                  const Vector &x,
+                  std::vector<Vector> &fields_l)
+{
+   int data_offset = 0;
+   for (std::size_t i = 0; i < fields.size(); i++)
+   {
+      const int sz = GetVSize(fields[i]);
+      fields_l[i].SetSize(sz);
+
+      const Vector x_i(const_cast<Vector&>(x), data_offset, sz);
+      fields_l[i] = x_i;
+
+      data_offset += sz;
+   }
+}
+
 /// @brief Get a transpose prolongation callback for a field descriptor.
 ///
 /// In the special case of a one field operator, the transpose prolongation

fem/integ/bilininteg_dgtrace_pa.cpp

@@ -139,6 +139,8 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
    const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
                          Device::GetDeviceMemoryType() : pa_mt;
 
+   nf = fes.GetNFbyType(type);
+   if (nf==0) { return; }
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalTraceElement();
@@ -146,17 +148,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
                                IntRule:
                                &GetRule(el.GetGeomType(), el.GetOrder(),
                                         *mesh->GetTypicalElementTransformation());
-
-   if (!qspace[static_cast<int>(type)])
-   {
-      qspace[static_cast<int>(type)].reset(
-         new FaceQuadratureSpace(*mesh, *ir, type));
-   }
-
-   FaceQuadratureSpace& qs = *qspace[static_cast<int>(type)];
-   nf = qs.GetNumFaces();
-   if (nf==0) { return; }
-
    const int symmDims = 4;
    nq = ir->GetNPoints();
    dim = mesh->Dimension();
@@ -168,6 +159,8 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
+
+   FaceQuadratureSpace qs(*mesh, *ir, type);
    CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED);
 
    CoefficientVector r(qs, CoefficientStorage::COMPRESSED);

fem/qspace.cpp

@@ -17,9 +17,8 @@ namespace mfem
 {
 
 QuadratureSpaceBase::QuadratureSpaceBase(Mesh &mesh_, Geometry::Type geom,
-                                         const IntegrationRule &ir,
-                                         QSpaceStorage storage)
-   : mesh(mesh_), order(ir.GetOrder()), storage(storage)
+                                         const IntegrationRule &ir)
+   : mesh(mesh_), order(ir.GetOrder())
 {
    for (int g = 0; g < Geometry::NumGeom; g++)
    {
@@ -97,10 +96,10 @@ void QuadratureSpaceBase::Integrate(VectorCoefficient &coeff,
 
 void QuadratureSpace::ConstructOffsets()
 {
-   const int num_elem = ne;
+   const int num_elem = mesh.GetNE();
+   ne = num_elem;
 
-   if (storage == QSpaceStorage::COMPRESSED &&
-       mesh.GetNumGeometries(mesh.Dimension()) == 1)
+   if (mesh.GetNumGeometries(mesh.Dimension()) == 1)
    {
       Array<Geometry::Type> geoms;
       mesh.GetGeometries(mesh.Dimension(), geoms);
@@ -125,9 +124,14 @@ void QuadratureSpace::ConstructOffsets()
    }
 }
 
-QuadratureSpace::QuadratureSpace(Mesh *mesh_, std::istream &in,
-                                 QSpaceStorage storage)
-   : QuadratureSpaceBase(*mesh_, 0, storage)
+void QuadratureSpace::Construct()
+{
+   ConstructIntRules(mesh.Dimension());
+   ConstructOffsets();
+}
+
+QuadratureSpace::QuadratureSpace(Mesh *mesh_, std::istream &in)
+   : QuadratureSpaceBase(*mesh_)
 {
    const char *msg = "invalid input stream";
    std::string ident;
@@ -146,24 +150,15 @@ QuadratureSpace::QuadratureSpace(Mesh *mesh_, std::istream &in,
       return;
    }
 
-   ne = mesh.GetNE();
-   ConstructIntRules(mesh.Dimension());
+   Construct();
 }
 
-QuadratureSpace::QuadratureSpace(Mesh *mesh_, int order_, QSpaceStorage storage)
-   : QuadratureSpaceBase(*mesh_, order_, storage)
-{
-   ne = mesh.GetNE();
-   ConstructIntRules(mesh.Dimension());
-}
-
-QuadratureSpace::QuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,
-                                 QSpaceStorage storage)
-   : QuadratureSpaceBase(mesh_, mesh_.GetTypicalElementGeometry(), ir, storage)
+QuadratureSpace::QuadratureSpace(Mesh &mesh_, const IntegrationRule &ir)
+   : QuadratureSpaceBase(mesh_, mesh_.GetTypicalElementGeometry(), ir)
 {
    MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension()) <= 1,
                "Constructor not valid for mixed meshes");
-   ne = mesh.GetNE();
+   ConstructOffsets();
 }
 
 void QuadratureSpace::Save(std::ostream &os) const
@@ -185,53 +180,55 @@ const Vector &QuadratureSpace::GetGeometricFactorWeights() const
 }
 
 FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, int order_,
-                                         FaceType face_type_,
-                                         QSpaceStorage storage)
-   : QuadratureSpaceBase(mesh_, order_, storage), face_type(face_type_),
-     face_indices(mesh.GetFaceIndices(face_type_)),
-     face_indices_inv(mesh.GetInvFaceIndices(face_type_))
+                                         FaceType face_type_)
+   : QuadratureSpaceBase(mesh_, order_),
+     face_type(face_type_),
+     num_faces(mesh.GetNFbyType(face_type))
 {
-   ne = face_indices.Size();
-   ConstructIntRules(mesh.Dimension() - 1);
+   Construct();
 }
 
 FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,
-                                         FaceType face_type_,
-                                         QSpaceStorage storage)
-   : QuadratureSpaceBase(mesh_, mesh_.GetTypicalFaceGeometry(), ir, storage),
-     face_type(face_type_), face_indices(mesh.GetFaceIndices(face_type_)),
-     face_indices_inv(mesh.GetInvFaceIndices(face_type_))
+                                         FaceType face_type_)
+   : QuadratureSpaceBase(mesh_, mesh_.GetTypicalFaceGeometry(), ir),
+     face_type(face_type_),
+     num_faces(mesh.GetNFbyType(face_type))
 {
    MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension() - 1) <= 1,
                "Constructor not valid for mixed meshes");
-   ne = face_indices.Size();
+   ConstructOffsets();
 }
 
 void FaceQuadratureSpace::ConstructOffsets()
 {
-   if (storage == QSpaceStorage::COMPRESSED &&
-       mesh.GetNumGeometries(mesh.Dimension() - 1) == 1)
+   face_indices.SetSize(num_faces);
+   offsets.SetSize(num_faces + 1);
+   ne = num_faces;
+   int offset = 0;
+   int f_idx = 0;
+   for (int i = 0; i < mesh.GetNumFacesWithGhost(); i++)
    {
-      Array<Geometry::Type> geoms;
-      mesh.GetGeometries(mesh.Dimension() - 1, geoms);
-      offsets.SetSize(1);
-      offsets.HostWrite();
-      offsets[0] = int_rule[geoms[0]]->GetNPoints();
-      size = ne * offsets[0];
-   }
-   else
-   {
-      offsets.SetSize(face_indices.Size() + 1);
-      int offset = 0;
-      for (int i = 0; i < mesh.GetNFbyType(face_type); ++i)
+      const Mesh::FaceInformation face = mesh.GetFaceInformation(i);
+      if (face.IsNonconformingCoarse() || !face.IsOfFaceType(face_type))
       {
-         offsets[i] = offset;
-         Geometry::Type geom = mesh.GetFaceGeometry(face_indices[i]);
-         MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule");
-         offset += int_rule[geom]->GetNPoints();
+         continue;
       }
-      offsets[face_indices.Size()] = size = offset;
+      face_indices[f_idx] = i;
+      face_indices_inv[i] = f_idx;
+      offsets[f_idx] = offset;
+      Geometry::Type geom = mesh.GetFaceGeometry(i);
+      MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule");
+      offset += int_rule[geom]->GetNPoints();
+
+      f_idx++;
    }
+   offsets[num_faces] = size = offset;
+}
+
+void FaceQuadratureSpace::Construct()
+{
+   ConstructIntRules(mesh.Dimension() - 1);
+   ConstructOffsets();
 }
 
 int FaceQuadratureSpace::GetPermutedIndex(int idx, int iq) const

fem/qspace.hpp

@@ -19,49 +19,39 @@
 namespace mfem
 {
 
-enum class QSpaceStorage
-{
-   FULL,
-   COMPRESSED
-};
-
 /// Abstract base class for QuadratureSpace and FaceQuadratureSpace.
 /** This class represents the storage layout for QuadratureFunction%s, that may
     be defined either on mesh elements or mesh faces. */
 class QuadratureSpaceBase
 {
 protected:
+   friend class QuadratureFunction; // Uses the offsets.
+
    Mesh &mesh; ///< The underlying mesh.
    int order; ///< The order of integration rule.
-   int size = -1; ///< Total number of quadrature points. -1 indicates
-   ///< offsets/size not computed yet.
+   int size; ///< Total number of quadrature points.
    int ne; ///< Actual number of entities
    mutable Vector weights; ///< Integration weights.
    mutable long nodes_sequence = 0; ///< Nodes counter for cache invalidation.
 
-   QSpaceStorage storage;
-
    /// @brief Entity quadrature point offset array.
    ///
    /// Supports a constant compression scheme for meshes which have a single
    /// geometry type. When compressed, will have a single value. The true offset
    /// can be computed as i * offsets[0], where i is the entity index. Otherwise
-   /// has size num_entities + 1. Lazily constructed.
+   /// has size num_entities + 1.
    ///
    Array<int> offsets;
    /// The quadrature rules used for each geometry type.
    const IntegrationRule *int_rule[Geometry::NumGeom];
 
    /// Protected constructor. Used by derived classes.
-   QuadratureSpaceBase(Mesh &mesh_, int order_ = 0,
-                       QSpaceStorage storage = QSpaceStorage::COMPRESSED)
-      : mesh(mesh_), order(order_), storage(storage)
-   {}
+   QuadratureSpaceBase(Mesh &mesh_, int order_ = 0)
+      : mesh(mesh_), order(order_) { }
 
    /// Protected constructor. Used by derived classes.
    QuadratureSpaceBase(Mesh &mesh_, Geometry::Type geom,
-                       const IntegrationRule &ir,
-                       QSpaceStorage storage = QSpaceStorage::COMPRESSED);
+                       const IntegrationRule &ir);
 
    /// Fill the @ref int_rule array for each geometry type using @ref order.
    void ConstructIntRules(int dim);
@@ -72,21 +62,13 @@ protected:
    /// Compute the integration weights.
    void ConstructWeights() const;
 
-   virtual void ConstructOffsets() = 0;
-
 public:
-   QSpaceStorage StorageType() const { return storage; }
-
    /// @brief Gets the offset for a given entity @a idx.
    ///
    /// The quadrature point values for entity i are stored in the indices
    /// between Offset(i) and Offset(i+1)
    int Offset(int idx) const
    {
-      if (size < 0)
-      {
-         const_cast<QuadratureSpaceBase *>(this)->ConstructOffsets();
-      }
       return (offsets.Size() == 1) ? (idx * offsets[0]) : offsets[idx];
    }
 
@@ -97,24 +79,10 @@ public:
    /// can be computed as i * offsets[0], where i is the entity index. Otherwise
    /// has size num_entities + 1.
    ///
-   const Array<int> &Offsets() const
-   {
-      if (size < 0)
-      {
-         const_cast<QuadratureSpaceBase *>(this)->ConstructOffsets();
-      }
-      return offsets;
-   }
+   const Array<int> &Offsets() const { return offsets; }
 
    /// Return the total number of quadrature points.
-   int GetSize() const
-   {
-      if (size < 0)
-      {
-         const_cast<QuadratureSpaceBase *>(this)->ConstructOffsets();
-      }
-      return size;
-   }
+   int GetSize() const { return size; }
 
    /// Return the order of the quadrature rule(s) used by all elements.
    int GetOrder() const { return order; }
@@ -174,20 +142,19 @@ class QuadratureSpace : public QuadratureSpaceBase
 {
 protected:
    const Vector &GetGeometricFactorWeights() const override;
-   void ConstructOffsets() override;
+   void ConstructOffsets();
+   void Construct();
 public:
    /// Create a QuadratureSpace based on the global rules from #IntRules.
-   QuadratureSpace(Mesh *mesh_, int order_,
-                   QSpaceStorage storage = QSpaceStorage::COMPRESSED);
+   QuadratureSpace(Mesh *mesh_, int order_)
+      : QuadratureSpaceBase(*mesh_, order_) { Construct(); }
 
    /// @brief Create a QuadratureSpace with an IntegrationRule, valid only when
    /// the mesh has one element type.
-   QuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,
-                   QSpaceStorage storage = QSpaceStorage::COMPRESSED);
+   QuadratureSpace(Mesh &mesh_, const IntegrationRule &ir);
 
    /// Read a QuadratureSpace from the stream @a in.
-   QuadratureSpace(Mesh *mesh_, std::istream &in,
-                   QSpaceStorage storage = QSpaceStorage::COMPRESSED);
+   QuadratureSpace(Mesh *mesh_, std::istream &in);
 
    /// Returns number of elements in the mesh.
    inline int GetNE() const { return mesh.GetNE(); }
@@ -224,29 +191,29 @@ public:
 class FaceQuadratureSpace : public QuadratureSpaceBase
 {
    FaceType face_type; ///< Is the space defined on interior or boundary faces?
+   const int num_faces; ///< Number of faces.
 
    /// Map from boundary or interior face indices to mesh face indices.
-   const Array<int> &face_indices;
+   Array<int> face_indices;
 
    /// Inverse of the map @a face_indices.
-   const std::unordered_map<int,int> &face_indices_inv;
+   std::unordered_map<int,int> face_indices_inv;
 
    const Vector &GetGeometricFactorWeights() const override;
-   void ConstructOffsets() override;
+   void ConstructOffsets();
+   void Construct();
 
 public:
    /// Create a FaceQuadratureSpace based on the global rules from #IntRules.
-   FaceQuadratureSpace(Mesh &mesh_, int order_, FaceType face_type_,
-                       QSpaceStorage storage = QSpaceStorage::COMPRESSED);
+   FaceQuadratureSpace(Mesh &mesh_, int order_, FaceType face_type_);
 
    /// @brief Create a FaceQuadratureSpace with an IntegrationRule, valid only
    /// when the mesh has one type of face geometry.
    FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,
-                       FaceType face_type_,
-                       QSpaceStorage storage = QSpaceStorage::COMPRESSED);
+                       FaceType face_type_);
 
    /// Returns number of faces in the mesh.
-   inline int GetNumFaces() const { return face_indices.Size(); }
+   inline int GetNumFaces() const { return num_faces; }
 
    /// Returns the face type (boundary or interior).
    FaceType GetFaceType() const { return face_type; }

general/array.hpp

@@ -211,9 +211,6 @@ public:
    /// Delete the first entry with value == 'el'.
    inline void DeleteFirst(const T &el);
 
-   /// Delete entries at @a indices, and resize.
-   inline void DeleteAt(const Array<int> &indices);
-
    /// Delete the whole array.
    inline void DeleteAll();
 
@@ -938,30 +935,6 @@ inline void Array<T>::DeleteFirst(const T &el)
    }
 }
 
-template <class T>
-inline void Array<T>::DeleteAt(const Array<int> &indices)
-{
-   // Make a copy of the indices, sorted.
-   Array<int> sorted_indices(indices);
-   sorted_indices.Sort();
-
-   int rm_count = 0;
-   for (int i = 0; i < size; i++)
-   {
-      if (rm_count < sorted_indices.Size() && i == sorted_indices[rm_count])
-      {
-         rm_count++;
-      }
-      else
-      {
-         data[i-rm_count] = data[i]; // shift data rm_count
-      }
-   }
-
-   // Resize to remove tail
-   size -= rm_count;
-}
-
 template <class T>
 inline void Array<T>::DeleteAll()
 {

general/scan.hpp

@@ -1,176 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_SCAN_HPP
-#define MFEM_SCAN_HPP
-
-#ifdef MFEM_USE_CUDA
-#include <cub/device/device_scan.cuh>
-#define MFEM_CUB_NAMESPACE cub
-#elif MFEM_USE_HIP
-#include <hipcub/device/device_scan.hpp>
-#define MFEM_CUB_NAMESPACE hipcub
-#endif
-
-#include <functional>
-#include <numeric>
-
-namespace mfem
-{
-/// Equivalent to InclusiveScan(use_dev, d_in, d_out, num_items, workspace,
-/// std::plus<>{})
-template <class InputIt, class OutputIt>
-void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
-                   Array<char> &workspace)
-{
-   // forward to InclusiveSum for potentially faster kernels
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-   if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
-   {
-      size_t bytes = workspace.Size();
-      if (bytes)
-      {
-         auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                       workspace.Write(), bytes, d_in, d_out, num_items);
-#if defined(MFEM_USE_CUDA)
-         if (err == cudaSuccess)
-         {
-            return;
-         }
-#elif defined(MFEM_USE_HIP)
-         if (err == hipSuccess)
-         {
-            return;
-         }
-#endif
-      }
-      // try allocating a larger buffer
-      bytes = 0;
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                        nullptr, bytes, d_in, d_out, num_items));
-      workspace.SetSize(bytes);
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                        workspace.Write(), bytes, d_in, d_out, num_items));
-      return;
-   }
-#endif
-   std::inclusive_scan(d_in, d_in + num_items, d_out);
-}
-
-/// Performs an inclusive scan of [d_in, d_in+num_items) -> [d_out,
-/// d_out+num_items). This call is potentially asynchronous on the device.
-/// @a d_in input start.
-/// @a d_out output start. Can perform in-place scans with d_out = d_in
-/// @a workspace temporary workspace used for device scans. TODO: replace with
-/// internal temporary workspace once that's added to the memory manager.
-/// @a scan_op binary scan functor. Must be associative. If only weakly
-/// associative (i.e. floating point addition) results are not deterministic. On
-/// device this must also be commutative.
-template <class InputIt, class OutputIt, class ScanOp>
-void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
-                   Array<char> &workspace, ScanOp scan_op)
-{
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-   if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
-   {
-      size_t bytes = workspace.Size();
-      if (bytes)
-      {
-         auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                       workspace.Write(), bytes, d_in, d_out, scan_op, num_items);
-#if defined(MFEM_USE_CUDA)
-         if (err == cudaSuccess)
-         {
-            return;
-         }
-#elif defined(MFEM_USE_HIP)
-         if (err == hipSuccess)
-         {
-            return;
-         }
-#endif
-      }
-      // try allocating a larger buffer
-      bytes = 0;
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                        nullptr, bytes, d_in, d_out, scan_op, num_items));
-      workspace.SetSize(bytes);
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                        workspace.Write(), bytes, d_in, d_out, scan_op, num_items));
-      return;
-   }
-#endif
-   std::inclusive_scan(d_in, d_in + num_items, d_out, scan_op);
-}
-
-/// Performs an exclusive scan of [d_in, d_in+num_items) -> [d_out,
-/// d_out+num_items). This call is potentially asynchronous on the device.
-/// @a d_in input start.
-/// @a d_out output start. Can perform in-place scans with d_out = d_in
-/// @a workspace temporary workspace used for device scans. TODO: replace with
-/// internal temporary workspace once that's added to the memory manager.
-/// @a scan_op binary scan functor. Must be associative. If only weakly
-/// associative (i.e. floating point addition) results are not deterministic. On
-/// device this must also be commutative.
-template <class InputIt, class OutputIt, class T, class ScanOp>
-void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
-                   T init_value, Array<char> &workspace, ScanOp scan_op)
-{
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-   if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
-   {
-      size_t bytes = workspace.Size();
-      if (bytes)
-      {
-         auto err = MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                       workspace.Write(), bytes, d_in, d_out, scan_op, init_value,
-                       num_items);
-#if defined(MFEM_USE_CUDA)
-         if (err == cudaSuccess)
-         {
-            return;
-         }
-#elif defined(MFEM_USE_HIP)
-         if (err == hipSuccess)
-         {
-            return;
-         }
-#endif
-      }
-      // try allocating a larger buffer
-      bytes = 0;
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                        nullptr, bytes, d_in, d_out, scan_op, init_value, num_items));
-      workspace.SetSize(bytes);
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                        workspace.Write(), bytes, d_in, d_out, scan_op, init_value,
-                        num_items));
-      return;
-   }
-#endif
-   std::exclusive_scan(d_in, d_in + num_items, d_out, init_value, scan_op);
-}
-
-/// Equivalent to ExclusiveScan(use_dev, d_in, d_out, num_items, init_value,
-/// workspace, std::plus<>{})
-template <class InputIt, class OutputIt, class T>
-void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
-                   T init_value, Array<char> &workspace)
-{
-   ExclusiveScan(use_dev, d_in, d_out, num_items, init_value, workspace,
-                 std::plus<> {});
-}
-
-} // namespace mfem
-
-#undef MFEM_CUB_NAMESPACE
-
-#endif

linalg/vector.cpp

@@ -14,7 +14,6 @@
 #include "../general/forall.hpp"
 #include "../general/reducers.hpp"
 #include "../general/hash.hpp"
-#include "../general/scan.hpp"
 #include "vector.hpp"
 
 #ifdef MFEM_USE_OPENMP
@@ -1253,44 +1252,4 @@ real_t Vector::Sum() const
    return res;
 }
 
-void Vector::DeleteAt(const Array<int> &indices)
-{
-   const bool use_dev = UseDevice();
-
-   Array<int> flag(size);
-   const auto d_flag = flag.Write(use_dev);
-   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (int i)
-   {
-      d_flag[i] = true;
-   });
-   const auto d_indices = indices.Read(use_dev);
-   mfem::forall_switch(use_dev, indices.Size(), [=] MFEM_HOST_DEVICE (int i)
-   {
-      d_flag[d_indices[i]] = false;
-   });
-
-   Array<int> out_idx(size);
-   auto d_out_idx = out_idx.Write(use_dev);
-   Array<char> workspace;
-   // Perform inclusive scan so that the last entry is the new size.
-   InclusiveScan(use_dev, d_flag, d_out_idx, size, workspace);
-
-   Vector copy(*this);
-   auto d_in = copy.Read(use_dev);
-   auto d_out = Write(use_dev);
-   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (int i)
-   {
-      if (d_flag[i])
-      {
-         // Transform inclusive scan to exclusive by shifting.
-         const int j = (i > 0) ? d_out_idx[i - 1] : 0;
-         d_out[j] = d_in[i];
-      }
-   });
-
-   // Get the new size of the vector. Copy only the last entry.
-   Memory<int> submem(out_idx.GetMemory(), out_idx.Size() - 1, 1);
-   size = submem.Read(MemoryClass::HOST, 1)[0];
-}
-
 } // namespace mfem

linalg/vector.hpp

@@ -171,12 +171,6 @@ public:
    /// Resize the vector to size @a s using the MemoryType of @a v.
    void SetSize(int s, const Vector &v) { SetSize(s, v.GetMemory().GetMemoryType()); }
 
-   /// Update \ref Capacity() to @a res (if less than current), keeping existing entries.
-   void Reserve(int res);
-
-   /// Delete entries at @a indices and resize vector accordingly.
-   void DeleteAt(const Array<int> &indices);
-
    /// Set the Vector data.
    /// @warning This method should be called only when OwnsData() is false.
    void SetData(real_t *d) { data.Wrap(d, data.Capacity(), false); }
@@ -627,18 +621,6 @@ inline void Vector::SetSize(int s, MemoryType mt)
    data.UseDevice(use_dev);
 }
 
-inline void Vector::Reserve(int res)
-{
-   if (res > Capacity())
-   {
-      Memory<real_t> p(res, data.GetMemoryType());
-      p.CopyFrom(data, size);
-      p.UseDevice(data.UseDevice());
-      data.Delete();
-      data = p;
-   }
-}
-
 inline void Vector::NewMemoryAndSize(const Memory<real_t> &mem, int s,
                                      bool own_mem)
 {

mesh/mesh.cpp

@@ -979,46 +979,6 @@ const Array<int>& Mesh::GetElementAttributes() const
    return elem_attrs_cache;
 }
 
-void Mesh::ComputeFaceInfo(FaceType ftype) const
-{
-   auto &fidcs = face_indices[static_cast<int>(ftype)];
-   auto &ifidcs = inv_face_indices[static_cast<int>(ftype)];
-   fidcs.SetSize(GetNFbyType(ftype));
-   fidcs.HostWrite();
-   ifidcs.reserve(fidcs.Size());
-   int f_idx = 0;
-   for (int i = 0; i < GetNumFacesWithGhost(); ++i)
-   {
-      const FaceInformation face = GetFaceInformation(i);
-      if (face.IsNonconformingCoarse() || !face.IsOfFaceType(ftype))
-      {
-         continue;
-      }
-      fidcs[f_idx] = i;
-      ifidcs[i] = f_idx;
-      ++f_idx;
-   }
-}
-
-const Array<int> &Mesh::GetFaceIndices(FaceType ftype) const
-{
-   if (face_indices[static_cast<int>(ftype)].Size() == 0)
-   {
-      ComputeFaceInfo(ftype);
-   }
-   return face_indices[static_cast<int>(ftype)];
-}
-
-const std::unordered_map<int, int> &
-Mesh::GetInvFaceIndices(FaceType ftype) const
-{
-   if (inv_face_indices[static_cast<int>(ftype)].empty())
-   {
-      ComputeFaceInfo(ftype);
-   }
-   return inv_face_indices[static_cast<int>(ftype)];
-}
-
 void Mesh::DeleteGeometricFactors()
 {
    for (int i = 0; i < geom_factors.Size(); i++)
@@ -1906,11 +1866,6 @@ void Mesh::Destroy()
    bdr_face_attrs_cache.DeleteAll();
    attributes.DeleteAll();
    bdr_attributes.DeleteAll();
-
-   face_indices[0].DeleteAll();
-   face_indices[1].DeleteAll();
-   inv_face_indices[0] = std::unordered_map<int, int>();
-   inv_face_indices[1] = std::unordered_map<int, int>();
 }
 
 void Mesh::ResetLazyData()
@@ -8182,12 +8137,6 @@ void Mesh::GenerateFaces()
       FreeElement(f);
    }
 
-   // delete caches
-   face_indices[0].SetSize(0);
-   face_indices[1].SetSize(0);
-   inv_face_indices[0].clear();
-   inv_face_indices[1].clear();
-
    // (re)generate the interior faces and the info for them
    faces.SetSize(nfaces);
    faces_info.SetSize(nfaces);
@@ -10987,11 +10936,6 @@ void Mesh::Swap(Mesh& other, bool non_geometry)
    // copy attribute caches
    mfem::Swap(elem_attrs_cache, other.elem_attrs_cache);
    mfem::Swap(bdr_face_attrs_cache, other.bdr_face_attrs_cache);
-
-   mfem::Swap(face_indices[0], other.face_indices[0]);
-   mfem::Swap(face_indices[1], other.face_indices[1]);
-   inv_face_indices[0].swap(other.inv_face_indices[0]);
-   inv_face_indices[1].swap(other.inv_face_indices[1]);
 }
 
 void Mesh::GetElementData(const Array<Element*> &elem_array, int geom,

mesh/mesh.hpp

@@ -278,13 +278,6 @@ protected:
 
    // used during NC mesh initialization only
    Array<Triple<int, int, int> > tmp_vertex_parents;
-   /// cache for FaceIndices(ftype)
-   mutable Array<int> face_indices[2];
-   /// cache for FaceIndices(ftype)
-   mutable std::unordered_map<int, int> inv_face_indices[2];
-
-   /// compute face_indices[ftype] and inv_face_indices[type]
-   void ComputeFaceInfo(FaceType ftype) const;
 
 public:
    typedef Geometry::Constants<Geometry::SEGMENT>     seg_t;
@@ -319,11 +312,6 @@ public:
    // (true) is set in mesh_readers.cpp.
    static bool remove_unused_vertices;
 
-   /// Map from boundary or interior face indices to mesh face indices.
-   const Array<int>& GetFaceIndices(FaceType ftype) const;
-   /// Inverse of the map FaceIndices(ftype)
-   const std::unordered_map<int, int>& GetInvFaceIndices(FaceType ftype) const;
-
 protected:
    Operation last_operation;
 

tests/unit/CMakeLists.txt

@@ -20,9 +20,9 @@ set(UNIT_TESTS_SRCS
   dfem/test_diffusion.cpp
   dfem/test_divergence.cpp
   dfem/test_mass.cpp
+  dfem/test_lvector_interface.cpp
   general/test_array.cpp
   general/test_reduction.cpp
-  general/test_scan.cpp
   general/test_arrays_by_name.cpp
   general/test_error.cpp
   general/test_mem.cpp
@@ -145,11 +145,11 @@ set(UNIT_TESTS_SRCS
 # SERIAL CPU TESTS: unit_tests
 #-----------------------------------------------------------
 if (MFEM_USE_CUDA)
-   set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
+    set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
                 PROPERTY LANGUAGE CUDA)
 endif()
 if (MFEM_USE_HIP)
-  set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
+    set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS}
                PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 
@@ -176,7 +176,7 @@ COMMAND ${CMAKE_COMMAND} -E copy_directory
 #   make unit_tests
 #   ctest -R unit_tests [-V]
 if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-  add_test(NAME unit_tests COMMAND unit_tests)
+    add_test(NAME unit_tests COMMAND unit_tests)
 endif()
 
 #-----------------------------------------------------------
@@ -184,16 +184,16 @@ endif()
 #-----------------------------------------------------------
 # Create CUDA executable and test
 if (MFEM_USE_CUDA)
-   # gpu_unit_tests
-   set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
-   set_property(SOURCE ${GPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-   mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
-   target_link_libraries(gpu_unit_tests mfem)
-   add_dependencies(gpu_unit_tests copy_data)
-   add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
-   if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-      add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
-   endif()
+    # gpu_unit_tests
+    set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
+    set_property(SOURCE ${GPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+    mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+    target_link_libraries(gpu_unit_tests mfem)
+    add_dependencies(gpu_unit_tests copy_data)
+    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
+    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+        add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
+    endif()
 endif()
 
 #-----------------------------------------------------------
@@ -201,15 +201,15 @@ endif()
 #-----------------------------------------------------------
 # Create HIP 'gpu_unit_tests' executable and test
 if (MFEM_USE_HIP)
-   # gpu_unit_tests
-   set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
-   mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
-   target_link_libraries(gpu_unit_tests mfem)
-   add_dependencies(gpu_unit_tests copy_data)
-   add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
-   if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-      add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
-   endif()
+    # gpu_unit_tests
+    set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
+    mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+    target_link_libraries(gpu_unit_tests mfem)
+    add_dependencies(gpu_unit_tests copy_data)
+    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
+    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+        add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
+    endif()
 endif()
 
 #-----------------------------------------------------------
@@ -225,7 +225,7 @@ function(add_serial_miniapp_test name test_uvm)
 
     set(${NAME}_TESTS_SRCS miniapps/test_${name}.cpp)
     if (MFEM_USE_CUDA)
-       set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+        set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
     endif(MFEM_USE_CUDA)
 
     mfem_add_executable(${name}_tests_cpu ${${NAME}_TESTS_SRCS})
@@ -245,25 +245,25 @@ function(add_serial_miniapp_test name test_uvm)
     endif()
 
     if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-       mfem_add_executable(${name}_tests_gpu ${${NAME}_TESTS_SRCS})
-       target_compile_definitions(${name}_tests_gpu PUBLIC MFEM_${NAME}_DEVICE="gpu")
-       target_link_libraries(${name}_tests_gpu mfem)
-       add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu)
-       if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-           add_test(NAME ${name}_tests_gpu COMMAND ${name}_tests_gpu)
-       endif()
+        mfem_add_executable(${name}_tests_gpu ${${NAME}_TESTS_SRCS})
+        target_compile_definitions(${name}_tests_gpu PUBLIC MFEM_${NAME}_DEVICE="gpu")
+        target_link_libraries(${name}_tests_gpu mfem)
+        add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu)
+        if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+            add_test(NAME ${name}_tests_gpu COMMAND ${name}_tests_gpu)
+        endif()
 
-       if (test_uvm)
-           mfem_add_executable(${name}_tests_gpu_uvm ${${NAME}_TESTS_SRCS})
-           target_compile_definitions(${name}_tests_gpu_uvm PUBLIC
+        if (test_uvm)
+            mfem_add_executable(${name}_tests_gpu_uvm ${${NAME}_TESTS_SRCS})
+            target_compile_definitions(${name}_tests_gpu_uvm PUBLIC
                MFEM_${NAME}_DEVICE="gpu:uvm")
-           target_link_libraries(${name}_tests_gpu_uvm mfem)
-           add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME}
+            target_link_libraries(${name}_tests_gpu_uvm mfem)
+            add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME}
                ${name}_tests_gpu_uvm)
-           if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-               add_test(NAME ${name}_tests_gpu_uvm COMMAND ${name}_tests_gpu_uvm)
-           endif()
-       endif()
+            if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+                add_test(NAME ${name}_tests_gpu_uvm COMMAND ${name}_tests_gpu_uvm)
+            endif()
+        endif()
     endif()
 endfunction(add_serial_miniapp_test)
 
@@ -278,25 +278,25 @@ add_dependencies(tmop_pa_tests_cpu copy_miniapps_meshing_data)
 #-----------------------------------------------------------
 # Add 'ceed_tests' executable and test; add extra tests 'ceed_test_*'
 if (MFEM_USE_CEED)
-   set(CEED_TESTS_SRCS
+    set(CEED_TESTS_SRCS
       ceed/test_ceed.cpp
       ceed/test_ceed_main.cpp)
-   if (MFEM_USE_CUDA)
-      set_property(SOURCE ${CEED_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-   endif(MFEM_USE_CUDA)
-   mfem_add_executable(ceed_tests ${CEED_TESTS_SRCS})
-   target_link_libraries(ceed_tests mfem)
-   add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ceed_tests)
-   # Add CEED tests
-   add_test(NAME ceed_tests COMMAND ceed_tests)
-   if (MFEM_USE_CUDA)
-      add_test(NAME ceed_tests_cuda_ref
+    if (MFEM_USE_CUDA)
+        set_property(SOURCE ${CEED_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+    endif(MFEM_USE_CUDA)
+    mfem_add_executable(ceed_tests ${CEED_TESTS_SRCS})
+    target_link_libraries(ceed_tests mfem)
+    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ceed_tests)
+    # Add CEED tests
+    add_test(NAME ceed_tests COMMAND ceed_tests)
+    if (MFEM_USE_CUDA)
+        add_test(NAME ceed_tests_cuda_ref
                COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/ref)
-      add_test(NAME ceed_tests_cuda_shared
+        add_test(NAME ceed_tests_cuda_shared
                COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/shared)
-      add_test(NAME ceed_tests_cuda_gen
+        add_test(NAME ceed_tests_cuda_gen
                COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/gen)
-   endif()
+    endif()
 endif()
 
 #-----------------------------------------------------------
@@ -304,54 +304,54 @@ endif()
 #-----------------------------------------------------------
 # Define executables and tests
 if (MFEM_USE_MPI)
-   # punit_tests
-   if (MFEM_USE_CUDA)
-      set_property(SOURCE punit_test_main.cpp PROPERTY LANGUAGE CUDA)
-   endif()
-   mfem_add_executable(punit_tests punit_test_main.cpp ${UNIT_TESTS_SRCS})
-   target_link_libraries(punit_tests mfem)
-   add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} punit_tests)
-   foreach(np 1 ${MFEM_MPI_NP})
-      if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-         add_test(NAME punit_tests_np=${np}
+    # punit_tests
+    if (MFEM_USE_CUDA)
+        set_property(SOURCE punit_test_main.cpp PROPERTY LANGUAGE CUDA)
+    endif()
+    mfem_add_executable(punit_tests punit_test_main.cpp ${UNIT_TESTS_SRCS})
+    target_link_libraries(punit_tests mfem)
+    add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} punit_tests)
+    foreach(np 1 ${MFEM_MPI_NP})
+        if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+            add_test(NAME punit_tests_np=${np}
                   COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
                   ${MPIEXEC_PREFLAGS} $<TARGET_FILE:punit_tests>
                   ${MPIEXEC_POSTFLAGS})
-      endif()
-   endforeach()
-   if (MFEM_USE_CUDA)
-       # pgpu_unit_tests
-       set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
-       set_property(SOURCE ${PGPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-       mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
-       add_dependencies(pgpu_unit_tests copy_data)
-       target_link_libraries(pgpu_unit_tests mfem)
-       add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
-       foreach(np 1 ${MFEM_MPI_NP})
-          if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-             add_test(NAME pgpu_unit_tests_np=${np}
+        endif()
+    endforeach()
+    if (MFEM_USE_CUDA)
+        # pgpu_unit_tests
+        set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
+        set_property(SOURCE ${PGPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+        mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+        add_dependencies(pgpu_unit_tests copy_data)
+        target_link_libraries(pgpu_unit_tests mfem)
+        add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
+        foreach(np 1 ${MFEM_MPI_NP})
+            if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+                add_test(NAME pgpu_unit_tests_np=${np}
                       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
                       ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pgpu_unit_tests>
                       ${MPIEXEC_POSTFLAGS})
-          endif()
-       endforeach()
+            endif()
+        endforeach()
     endif()
     if (MFEM_USE_HIP)
-       # pgpu_unit_tests
-       set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
-       mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
-       add_dependencies(pgpu_unit_tests copy_data)
-       target_link_libraries(pgpu_unit_tests mfem)
-       add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
-       foreach(np 1 ${MFEM_MPI_NP})
-          if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-             add_test(NAME pgpu_unit_tests_np=${np}
+        # pgpu_unit_tests
+        set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
+        mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+        add_dependencies(pgpu_unit_tests copy_data)
+        target_link_libraries(pgpu_unit_tests mfem)
+        add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
+        foreach(np 1 ${MFEM_MPI_NP})
+            if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+                add_test(NAME pgpu_unit_tests_np=${np}
                       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
                       ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pgpu_unit_tests>
                       ${MPIEXEC_POSTFLAGS})
-          endif()
-       endforeach()
-   endif()
+            endif()
+        endforeach()
+    endif()
 endif(MFEM_USE_MPI)
 
 #-----------------------------------------------------------
@@ -425,8 +425,8 @@ endfunction(add_parallel_miniapp_test)
 
 # Additional MPI unit tests
 if (MFEM_USE_MPI)
-   add_parallel_miniapp_test(sedov TRUE)
-   add_parallel_miniapp_test(tmop_pa FALSE)
+    add_parallel_miniapp_test(sedov TRUE)
+    add_parallel_miniapp_test(tmop_pa FALSE)
 endif(MFEM_USE_MPI)
 
 #-----------------------------------------------------------
@@ -435,10 +435,10 @@ endif(MFEM_USE_MPI)
 #-----------------------------------------------------------
 set(DEBUG_DEVICE_SRCS miniapps/test_debug_device.cpp)
 if (MFEM_USE_CUDA)
-   set_property(SOURCE ${DEBUG_DEVICE_SRCS} PROPERTY LANGUAGE CUDA)
+    set_property(SOURCE ${DEBUG_DEVICE_SRCS} PROPERTY LANGUAGE CUDA)
 endif()
 if (MFEM_USE_HIP)
-   set_property(SOURCE ${DEBUG_DEVICE_SRCS}
+    set_property(SOURCE ${DEBUG_DEVICE_SRCS}
                 PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 mfem_add_executable(debug_device_tests ${DEBUG_DEVICE_SRCS})

tests/unit/dfem/test_diffusion.cpp

@@ -255,8 +255,8 @@ void DFemDiffusion(const char *filename, int p, const int r)
          DOperator dop_mf(vsol, {{Coords, mfes}}, pmesh);
          const auto mf_vector_diffusion_qf =
             [] MFEM_HOST_DEVICE (const tensor<dscalar_t, DIM, DIM> &dudxi,
-               const tensor<real_t, DIM, DIM> &J,
-               const real_t &w)
+                                 const tensor<real_t, DIM, DIM> &J,
+                                 const real_t &w)
          {
             const auto invJ = inv(J), TinJ = transpose(invJ);
             return tuple{ (dudxi * invJ) * TinJ * det(J) * w };

tests/unit/dfem/test_lvector_interface.cpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+#include "mfem.hpp"
+#include <utility>
+
+#ifdef MFEM_USE_MPI
+
+using namespace mfem;
+using namespace mfem::future;
+using mfem::future::tensor;
+
+constexpr int DIM = 3;
+
+namespace kernels
+{
+struct MFApply
+{
+   MFEM_HOST_DEVICE inline auto operator()(const tensor<real_t, DIM> &dudxi,
+                                           const tensor<real_t, DIM, DIM> &J,
+                                           const real_t &w) const
+   {
+      const auto invJ = inv(J);
+      return tuple{ (dudxi * invJ) * transpose(invJ) * det(J) * w };
+   }
+};
+}
+
+TEST_CASE("DFEM L-Vector interface", "[Parallel][DFEM]")
+{
+   constexpr int p = 2; // Polynomial order
+   constexpr int r = 1;
+   constexpr int q = 2 * p + r;
+
+   const auto filename = GENERATE("../../data/fichera.mesh");
+   Mesh smesh(filename);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
+
+   pmesh.EnsureNodes();
+   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
+   smesh.Clear();
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace pfes(&pmesh, &fec);
+   ParFiniteElementSpace *mfes = nodes->ParFESpace();
+
+   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), q);
+
+   ParGridFunction x(&pfes), y(&pfes), z(&pfes);
+   Vector X(pfes.GetTrueVSize()), Y(pfes.GetTrueVSize()), Z(pfes.GetTrueVSize());
+
+   X.Randomize(1);
+   x.SetFromTrueDofs(X);
+
+   ParBilinearForm blf_fa(&pfes);
+   blf_fa.AddDomainIntegrator(new DiffusionIntegrator(ir));
+   blf_fa.Assemble();
+   blf_fa.Finalize();
+
+   static constexpr int U = 0, Coords = 1;
+
+   const auto solution = std::vector{FieldDescriptor{U, &pfes}};
+   DifferentiableOperator dop(solution, {{Coords, mfes}}, pmesh);
+
+   kernels::MFApply mf_apply_qf;
+   dop.AddDomainIntegrator(mf_apply_qf,
+                           tuple{Gradient<U>{}, Gradient<Coords>{}, Weight{}},
+                           tuple{Gradient<U>{}}, *ir, all_domain_attr);
+
+   // Use the L-vector interface to multiply
+   dop.SetMultLevel(DifferentiableOperator::MultLevel::LVECTOR);
+   dop.SetParameters({nodes});
+   dop.Mult(x, z);
+
+   blf_fa.Mult(x, y);
+
+   z -= y;
+   REQUIRE(z.Normlinf() == MFEM_Approx(0.0));
+}
+
+#endif

tests/unit/general/test_array.cpp

@@ -124,19 +124,3 @@ TEST_CASE("Array stl-interactions", "[Array]")
       CHECK(x[i] == y[i]);
    }
 }
-
-TEST_CASE("Array delete at indices", "[Array]")
-{
-   Array<int>       test({0,1,2,3,4,5,6,7,8});
-   Array<int> rm_indices({0,    3,4,  6,  8});
-   Array<int>     result({  1,2,    5,  7  });
-
-   test.DeleteAt(rm_indices);
-
-   REQUIRE(test.Size() == result.Size());
-
-   for (int i = 0; i < test.Size(); i++)
-   {
-      CHECK(test[i] == result[i]);
-   }
-}

tests/unit/general/test_scan.cpp

@@ -1,102 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include <algorithm>
-#include <limits>
-
-#include "mfem.hpp"
-#include "unit_tests.hpp"
-
-// must be included after mfem.hpp
-#include "general/scan.hpp"
-
-using namespace mfem;
-
-TEST_CASE("Inclusive Scan", "[Scan],[GPU]")
-{
-   Array<char> workspace;
-   Array<int> a(10);
-
-   for (int use_dev = 0; use_dev < 2; ++use_dev)
-   {
-      CAPTURE(use_dev);
-      a.HostReadWrite();
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         a[i] = i;
-      }
-      auto dptr = a.ReadWrite(use_dev);
-      InclusiveScan(use_dev, dptr, dptr, a.Size(), workspace);
-      a.HostRead();
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         int expected = (i + 1) * i / 2;
-         CAPTURE(i);
-         REQUIRE(a[i] == expected);
-      }
-      a.HostReadWrite();
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         a[i] = i + 1;
-      }
-      a.ReadWrite(use_dev);
-      InclusiveScan(use_dev, dptr, dptr, a.Size(), workspace, std::multiplies<> {});
-      a.HostRead();
-      int expected = 1;
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         expected *= i + 1;
-         CAPTURE(i);
-         REQUIRE(a[i] == expected);
-      }
-   }
-}
-
-TEST_CASE("Exclusive Scan", "[Scan],[GPU]")
-{
-   Array<char> workspace;
-   Array<int> a(10);
-
-   for (int use_dev = 0; use_dev < 2; ++use_dev)
-   {
-      CAPTURE(use_dev);
-      a.HostReadWrite();
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         a[i] = i;
-      }
-      auto dptr = a.ReadWrite(use_dev);
-      ExclusiveScan(use_dev, dptr, dptr, a.Size(), 5, workspace);
-      a.HostRead();
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         int expected = (i + 1) * i / 2 - i + 5;
-         CAPTURE(i);
-         REQUIRE(a[i] == expected);
-      }
-      a.HostReadWrite();
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         a[i] = i + 1;
-      }
-      a.ReadWrite(use_dev);
-      ExclusiveScan(use_dev, dptr, dptr, a.Size(), 5, workspace,
-                    std::multiplies<> {});
-      a.HostRead();
-      int expected = 5;
-      for (int i = 0; i < a.Size(); ++i)
-      {
-         CAPTURE(i);
-         REQUIRE(a[i] == expected);
-         expected *= i + 1;
-      }
-   }
-}

tests/unit/linalg/test_vector.cpp

@@ -247,21 +247,3 @@ TEST_CASE("Vector Sum", "[Vector],[GPU]")
 
    REQUIRE(sum_1 == MFEM_Approx(sum_2));
 }
-
-TEST_CASE("Vector delete at indices", "[Vector][GPU]")
-{
-   Vector           test({0,1,2,3,4,5,6,7,8});
-   Array<int> rm_indices({0,    3,4,  6,  8});
-   Vector         result({  1,2,    5,  7  });
-
-   test.UseDevice(true);
-   test.DeleteAt(rm_indices);
-
-   REQUIRE(test.Size() == result.Size());
-
-   test.HostReadWrite();
-   for (int i = 0; i < test.Size(); i++)
-   {
-      CHECK(test[i] == result[i]);
-   }
-}