Merge branch 'master' into hughcars/nc-internal-bdr-project-fix

.gitlab/lassen-build-and-test.yml

@@ -14,14 +14,14 @@ stages:
   - build_and_test
   - report
 
-opt_mpi_cuda_xl_16_1_1_12:
+opt_mpi_cuda_gcc:
   variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
-opt_mpi_cuda_hypre_cuda_xl:
+opt_mpi_cuda_hypre_cuda_gcc:
   variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 # Jobs report

examples/CMakeLists.txt

@@ -146,10 +146,10 @@ if (MFEM_ENABLE_TESTING)
   # Add CUDA/HIP tests.
   set(DEVICE_EXAMPLES
       # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
+      ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
       # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
-      ex34p ex35p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p ex22p ex24p ex25p
+      ex26p ex34p ex35p)
   set(MFEM_TEST_DEVICE)
   if (MFEM_USE_CUDA)
     set(MFEM_TEST_DEVICE "cuda")
@@ -159,6 +159,11 @@ if (MFEM_ENABLE_TESTING)
   if (MFEM_TEST_DEVICE)
     foreach(TEST_NAME ${DEVICE_EXAMPLES})
       set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (${TEST_NAME} MATCHES "ex14p")
+        list(APPEND THIS_TEST_OPTIONS "-rs" "2" "-rp" "0" "-pa")
+      elseif (${TEST_NAME} MATCHES "ex14")
+        list(APPEND THIS_TEST_OPTIONS "-r" "2" "-pa")
+      endif()
       if (NOT (${TEST_NAME} MATCHES ".*p$"))
         add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
           COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})

examples/makefile

@@ -28,9 +28,9 @@ PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
    ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
    ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
    ex37p ex39p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-   ex24p ex25p ex26p ex34p ex35p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
+   ex22p ex24p ex25p ex26p ex34p ex35p
 
 ifeq ($(MFEM_USE_LAPACK),YES)
    SEQ_EXAMPLES += ex38
@@ -138,6 +138,14 @@ ex10-test-seq: ex10
 	@$(call mfem-test,$<,, Serial example,-tf 5)
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-tf 5)
+ex14-test-seq-cuda: ex14
+	@$(call mfem-test,$<,, Serial CUDA example,-r 2 -pa -d cuda)
+ex14p-test-par-cuda: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-rs 2 -rp 0 -pa -d cuda)
+ex14-test-seq-hip: ex14
+	@$(call mfem-test,$<,, Serial HIP example,-r 2 -pa -d hip)
+ex14p-test-par-hip: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-rs 2 -rp 0 -pa -d hip)
 ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p

fem/coefficient.cpp

@@ -807,6 +807,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 
    QuadratureSpaceBase &qspace = *qf.GetSpace();
    const int ne = qspace.GetNE();
+   qf.HostWrite();
    DenseMatrix values;
    DenseSymmetricMatrix matrix;
    for (int iel = 0; iel < ne; ++iel)
@@ -818,7 +819,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
       {
          const IntegrationPoint &ip = ir[iq];
          T.SetIntPoint(&ip);
-         matrix.UseExternalData(&values(0, iq), vdim);
+         matrix.UseExternalData(&values(0, iq), height);
          Eval(matrix, T, ip);
       }
    }
@@ -828,13 +829,12 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
-   mat.SetSize(height);
-   Eval(mat, T, ip);
+   Eval(mat_aux, T, ip);
    for (int j = 0; j < width; ++j)
    {
       for (int i = 0; i < height; ++ i)
       {
-         K(i, j) = mat(i, j);
+         K(i, j) = mat_aux(i, j);
       }
    }
 }

fem/coefficient.hpp

@@ -1426,12 +1426,13 @@ public:
 class SymmetricMatrixCoefficient : public MatrixCoefficient
 {
 protected:
+
    /// Internal matrix used when evaluating this coefficient as a DenseMatrix.
-   DenseSymmetricMatrix mat;
+   mutable DenseSymmetricMatrix mat_aux;
 public:
    /// Construct a dim x dim matrix coefficient.
    explicit SymmetricMatrixCoefficient(int dimension)
-      : MatrixCoefficient(dimension, true) { }
+      : MatrixCoefficient(dimension, true), mat_aux(height) { }
 
    /// Get the size of the matrix.
    int GetSize() const { return height; }
@@ -1464,8 +1465,9 @@ public:
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip);
 
-   /// Return a reference to the constant matrix.
-   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
+   /// @deprecated Return a reference to the internal matrix used when evaluating this coefficient as a DenseMatrix.
+   MFEM_DEPRECATED const DenseSymmetricMatrix& GetMatrix() { return mat_aux; }
 
    virtual ~SymmetricMatrixCoefficient() { }
 };
@@ -1485,6 +1487,10 @@ public:
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
                      const IntegrationPoint &ip) { M = mat; }
+
+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
 };
 
 

fem/gridfunc.cpp

@@ -1321,9 +1321,9 @@ void GridFunction::ProjectVectorFieldOn(GridFunction &vec_field, int comp)
    }
 }
 
-void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
-                                                      GridFunction &der,
-                                                      Array<int> &zones_per_dof)
+void GridFunction::AccumulateAndCountDerivativeValues(
+   int comp, int der_comp, GridFunction &der,
+   Array<int> &zones_per_dof) const
 {
    FiniteElementSpace * der_fes = der.FESpace();
    ElementTransformation * transf;
@@ -1374,7 +1374,8 @@ void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
    }
 }
 
-void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
+void GridFunction::GetDerivative(int comp, int der_comp,
+                                 GridFunction &der) const
 {
    Array<int> overlap;
    AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);

fem/gridfunc.hpp

@@ -321,7 +321,7 @@ public:
        @param[out] der       The resulting derivative (scalar function). The
                              FiniteElementSpace of this function must be set
                              before the call. */
-   void GetDerivative(int comp, int der_comp, GridFunction &der);
+   void GetDerivative(int comp, int der_comp, GridFunction &der) const;
 
    real_t GetDivergence(ElementTransformation &tr) const;
 
@@ -443,7 +443,7 @@ protected:
        GetDerivative() method; see its documentation. */
    void AccumulateAndCountDerivativeValues(int comp, int der_comp,
                                            GridFunction &der,
-                                           Array<int> &zones_per_dof);
+                                           Array<int> &zones_per_dof) const;
 
    void AccumulateAndCountBdrValues(Coefficient *coeff[],
                                     VectorCoefficient *vcoeff,

fem/gslib.cpp

@@ -1393,8 +1393,10 @@ GSOPGSLIB::~GSOPGSLIB()
 void GSOPGSLIB::UpdateIdentifiers(const Array<long long> &ids)
 {
    long long minval = ids.Min();
+#ifdef MFEM_USE_MPI
    MPI_Allreduce(MPI_IN_PLACE, &minval, 1, MPI_LONG_LONG_INT,
                  MPI_MIN, gsl_comm->c);
+#endif
    MFEM_VERIFY(minval >= 0, "Unique identifier cannot be negative.");
    if (gsl_data != NULL) { gslib_gs_free(gsl_data); }
    num_ids = ids.Size();

fem/pgridfunc.cpp

@@ -520,7 +520,7 @@ void ParGridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
 }
 
 void ParGridFunction::GetDerivative(int comp, int der_comp,
-                                    ParGridFunction &der)
+                                    ParGridFunction &der) const
 {
    Array<int> overlap;
    AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);

fem/pgridfunc.hpp

@@ -231,7 +231,7 @@ public:
    void CountElementsPerVDof(Array<int> &elem_per_vdof) const override;
 
    /// Parallel version of GridFunction::GetDerivative(); see its documentation.
-   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
+   void GetDerivative(int comp, int der_comp, ParGridFunction &der) const;
 
    /** Sets the output vector @a dof_vals to the values of the degrees of
        freedom of element @a el. If @a el is greater than or equal to the number

fem/tmop.cpp

@@ -2949,6 +2949,15 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &s0,
    MFEM_VERIFY(surf_fit_pos == NULL,
                "Using both fitting approaches is not supported.");
 
+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   Mesh *mesh = s0.FESpace()->GetMesh();
+   MFEM_VERIFY(mesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
    surf_fit_gf->CountElementsPerVDof(surf_fit_dof_count);
@@ -2987,12 +2996,24 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &pos,
 void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                            const Array<bool> &smarker,
                                            Coefficient &coeff,
-                                           AdaptivityEvaluator &ae)
+                                           AdaptivityEvaluator &ae,
+                                           AdaptivityEvaluator *aegrad,
+                                           AdaptivityEvaluator *aehess)
 {
    // To have both we must duplicate the markers.
    MFEM_VERIFY(surf_fit_pos == NULL,
                "Using both fitting approaches is not supported.");
 
+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   ParMesh *pmesh = s0.ParFESpace()->GetParMesh();
+   MFEM_VERIFY(pmesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
+
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
    s0.CountElementsPerVDof(surf_fit_dof_count);
@@ -3000,11 +3021,80 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
    surf_fit_coeff = &coeff;
    surf_fit_eval = &ae;
 
-   surf_fit_eval->SetParMetaInfo(*s0.ParFESpace()->GetParMesh(),
-                                 *s0.ParFESpace());
+   surf_fit_eval->SetParMetaInfo(*pmesh, *s0.ParFESpace());
    surf_fit_eval->SetInitialField
    (*surf_fit_gf->FESpace()->GetMesh()->GetNodes(), *surf_fit_gf);
-   surf_fit_gf_bg = false;
+
+   if (!aegrad) { return; }
+
+   MFEM_VERIFY(aehess, "AdaptivityEvaluator for Hessians must be provided too.");
+
+   ParFiniteElementSpace *fes = s0.ParFESpace();
+
+   // FE space for gradients.
+   delete surf_fit_grad;
+   H1_FECollection *fec_grad = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_grad = new ParFiniteElementSpace(pmesh, fec_grad,
+                                                               dim);
+   // Initial gradients.
+   surf_fit_grad = new GridFunction(fes_grad);
+   surf_fit_grad->MakeOwner(fec_grad);
+   for (int d = 0; d < dim; d++)
+   {
+      ParGridFunction surf_fit_grad_comp(fes, surf_fit_grad->GetData()+d*s0.Size());
+      s0.GetDerivative(1, d, surf_fit_grad_comp);
+   }
+   surf_fit_eval_grad = aegrad;
+   surf_fit_eval_grad->SetParMetaInfo(*pmesh, *fes_grad);
+   surf_fit_eval_grad->SetInitialField(*pmesh->GetNodes(), *surf_fit_grad);
+
+   // FE space for Hessians.
+   delete surf_fit_hess;
+   H1_FECollection *fec_hess = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_hess = new ParFiniteElementSpace(pmesh, fec_hess,
+                                                               dim*dim);
+   // Initial Hessians.
+   surf_fit_hess = new GridFunction(fes_hess);
+   surf_fit_hess->MakeOwner(fec_hess);
+   int id = 0;
+   for (int d = 0; d < dim; d++)
+   {
+      for (int idir = 0; idir < dim; idir++)
+      {
+         ParGridFunction surf_fit_grad_comp(fes,
+                                            surf_fit_grad->GetData()+d*s0.Size());
+         ParGridFunction surf_fit_hess_comp(fes,
+                                            surf_fit_hess->GetData()+id*s0.Size());
+         surf_fit_grad_comp.GetDerivative(1, idir, surf_fit_hess_comp);
+         id++;
+      }
+   }
+   surf_fit_eval_hess = aehess;
+   surf_fit_eval_hess->SetParMetaInfo(*pmesh, *fes_hess);
+   surf_fit_eval_hess->SetInitialField(*pmesh->GetNodes(), *surf_fit_hess);
+
+   // Store DOF indices that are marked for fitting. Used to reduce work for
+   // transferring information between source/background and current mesh.
+   surf_fit_marker_dof_index.SetSize(0);
+#ifdef MFEM_USE_GSLIB
+   if (dynamic_cast<InterpolatorFP *>(surf_fit_eval) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_grad) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_hess))
+   {
+      for (int i = 0; i < surf_fit_marker->Size(); i++)
+      {
+         if ((*surf_fit_marker)[i] == true)
+         {
+            surf_fit_marker_dof_index.Append(i);
+         }
+      }
+   }
+#endif
+
+   *surf_fit_grad = 0.0;
+   *surf_fit_hess = 0.0;
 }
 
 void TMOP_Integrator::EnableSurfaceFittingFromSource(
@@ -3022,16 +3112,17 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
    // Setup for level set function
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
-   *surf_fit_gf = 0.0;
    surf_fit_marker = &smarker;
    surf_fit_coeff = &coeff;
    surf_fit_eval = &ae;
 
-   surf_fit_gf_bg = true;
    surf_fit_eval->SetParMetaInfo(*s_bg.ParFESpace()->GetParMesh(),
                                  *s_bg.ParFESpace());
    surf_fit_eval->SetInitialField
    (*s_bg.FESpace()->GetMesh()->GetNodes(), s_bg);
+   GridFunction *nodes = s0.FESpace()->GetMesh()->GetNodes();
+   surf_fit_eval->ComputeAtNewPosition(*nodes, *surf_fit_gf,
+                                       nodes->FESpace()->GetOrdering());
 
    // Setup for gradient on background mesh
    MFEM_VERIFY(s_bg_grad.ParFESpace()->GetOrdering() ==
@@ -3041,11 +3132,11 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
    delete surf_fit_grad;
    surf_fit_grad = new GridFunction(s0_grad);
    *surf_fit_grad = 0.0;
-   surf_fit_eval_bg_grad = &age;
-   surf_fit_eval_bg_hess = &ahe;
-   surf_fit_eval_bg_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
-                                         *s_bg_grad.ParFESpace());
-   surf_fit_eval_bg_grad->SetInitialField
+   surf_fit_eval_grad = &age;
+   surf_fit_eval_hess = &ahe;
+   surf_fit_eval_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
+                                      *s_bg_grad.ParFESpace());
+   surf_fit_eval_grad->SetInitialField
    (*s_bg_grad.FESpace()->GetMesh()->GetNodes(), s_bg_grad);
 
    // Setup for Hessian on background mesh
@@ -3056,9 +3147,9 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
    delete surf_fit_hess;
    surf_fit_hess = new GridFunction(s0_hess);
    *surf_fit_hess = 0.0;
-   surf_fit_eval_bg_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
-                                         *s_bg_hess.ParFESpace());
-   surf_fit_eval_bg_hess->SetInitialField
+   surf_fit_eval_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
+                                      *s_bg_hess.ParFESpace());
+   surf_fit_eval_hess->SetInitialField
    (*s_bg_hess.FESpace()->GetMesh()->GetNodes(), s_bg_hess);
 
    // Count number of zones that share each of the DOFs
@@ -3863,7 +3954,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
 
    Vector sigma_e(dof_s);
    DenseMatrix surf_fit_grad_e(dof_s, dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
    {
       surf_fit_gf->GetSubVector(vdofs, sigma_e);
 
@@ -3871,7 +3962,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
       // The FE coefficients of the gradient go in surf_fit_grad_e.
       Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
       DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
       {
          surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
          surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3945,7 +4036,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
    Vector sigma_e(dof_s);
    DenseMatrix surf_fit_grad_e(dof_s, dim);
    DenseMatrix surf_fit_hess_e(dof_s, dim*dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
    {
       surf_fit_gf->GetSubVector(vdofs, sigma_e);
 
@@ -3953,7 +4044,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
       // The FE coefficients of the gradient go in surf_fit_grad_e.
       Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
       DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
       {
          surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
          surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3967,7 +4058,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
       // Project the Hessian of sigma in the same space.
       // The FE coefficients of the Hessian go in surf_fit_hess_e.
       Vector hess_ptr(surf_fit_hess_e.GetData(), dof_s*dim*dim);
-      if (surf_fit_gf_bg)
+      if (surf_fit_hess)
       {
          surf_fit_hess->FESpace()->GetElementVDofs(el_id, dofs);
          surf_fit_hess->GetSubVector(dofs, hess_ptr);
@@ -3994,7 +4085,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
       Tpr.SetIntPoint(&ip);
       real_t w = surf_fit_normal * surf_fit_coeff->Eval(Tpr, ip);
 
-      if (surf_fit_gf || surf_fit_gf_bg)
+      if (surf_fit_gf)
       {
          Vector gg_ptr(surf_fit_hess_s.GetData(), dim * dim);
          surf_fit_hess_e.GetRow(s, gg_ptr);
@@ -4376,6 +4467,130 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
    dx = detv_avg_min / dxscale;
 }
 
+void TMOP_Integrator::RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                                         int new_x_ordering)
+{
+   if (!surf_fit_gf) { return; }
+
+   if (surf_fit_marker_dof_index.Size())
+   {
+      // Interpolate information only at DOFs marked for fitting.
+      const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
+      const int cnt = surf_fit_marker_dof_index.Size();
+      const int total_cnt = new_x.Size()/dim;
+      Vector new_x_sorted(cnt*dim);
+      if (new_x_ordering == 0)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               new_x_sorted(i + d*cnt) = new_x(dof_index + d*total_cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < dim; d++)
+            {
+               new_x_sorted(d + i*dim) = new_x(d + dof_index*dim);
+            }
+         }
+      }
+
+      // Interpolate values of the LS.
+      Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
+      surf_fit_eval->ComputeAtNewPosition(new_x_sorted, surf_fit_gf_int,
+                                          new_x_ordering);
+      for (int i = 0; i < cnt; i++)
+      {
+         int dof_index = surf_fit_marker_dof_index[i];
+         (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
+      }
+
+      // Interpolate gradients of the LS.
+      surf_fit_eval_grad->ComputeAtNewPosition(new_x_sorted, surf_fit_grad_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_grad and surf_fit_gf share the same space
+      const int grad_dim = surf_fit_grad->VectorDim();
+      const int grad_cnt = surf_fit_grad->Size()/grad_dim;
+      if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < grad_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_grad)[dof_index + d*grad_cnt] =
+                  surf_fit_grad_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < grad_dim; d++)
+            {
+               (*surf_fit_grad)[dof_index*grad_dim + d] =
+                  surf_fit_grad_int(i*grad_dim + d);
+            }
+         }
+      }
+
+      // Interpolate Hessians of the LS.
+      surf_fit_eval_hess->ComputeAtNewPosition(new_x_sorted, surf_fit_hess_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_hess and surf_fit_gf share the same space
+      const int hess_dim = surf_fit_hess->VectorDim();
+      const int hess_cnt = surf_fit_hess->Size()/hess_dim;
+      if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < hess_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_hess)[dof_index + d*hess_cnt] =
+                  surf_fit_hess_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < hess_dim; d++)
+            {
+               (*surf_fit_hess)[dof_index*hess_dim + d] =
+                  surf_fit_hess_int(i*hess_dim + d);
+            }
+         }
+      }
+
+   }
+   else
+   {
+      surf_fit_eval->ComputeAtNewPosition(new_x, *surf_fit_gf, new_x_ordering);
+      if (surf_fit_eval_grad)
+      {
+         surf_fit_eval_grad->ComputeAtNewPosition(new_x, *surf_fit_grad,
+                                                  new_x_ordering);
+      }
+      if (surf_fit_eval_hess)
+      {
+         surf_fit_eval_hess->ComputeAtNewPosition(new_x, *surf_fit_hess,
+                                                  new_x_ordering);
+      }
+   }
+}
+
 void TMOP_Integrator::
 UpdateAfterMeshPositionChange(const Vector &x_new,
                               const FiniteElementSpace &x_fes)
@@ -4406,112 +4621,11 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
       adapt_lim_eval->ComputeAtNewPosition(x_new, *adapt_lim_gf, ordering);
    }
 
-   // Update surf_fit_gf if surface fitting is enabled.
+   // Update surf_fit_gf (and optionally its gradients) if surface
+   // fitting is enabled.
    if (surf_fit_gf)
    {
-      if (surf_fit_gf_bg)
-      {
-         // Interpolate information for only DOFs marked for fitting.
-         const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
-         const int cnt = surf_fit_marker_dof_index.Size();
-         const int total_cnt = x_new.Size()/dim;
-         Vector new_x_sorted(cnt*dim);
-         if (ordering == 0)
-         {
-            for (int d = 0; d < dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  new_x_sorted(i + d*cnt) = x_new(dof_index + d*total_cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < dim; d++)
-               {
-                  new_x_sorted(d + i*dim) = x_new(d + dof_index*dim);
-               }
-            }
-         }
-
-         Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
-         surf_fit_eval->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_gf_int, ordering);
-         for (int i = 0; i < cnt; i++)
-         {
-            int dof_index = surf_fit_marker_dof_index[i];
-            (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
-         }
-
-         surf_fit_eval_bg_grad->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_grad_int, ordering);
-         // Assumes surf_fit_grad and surf_fit_gf share the same space
-         const int grad_dim = surf_fit_grad->VectorDim();
-         const int grad_cnt = surf_fit_grad->Size()/grad_dim;
-         if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < grad_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_grad)[dof_index + d*grad_cnt] =
-                     surf_fit_grad_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < grad_dim; d++)
-               {
-                  (*surf_fit_grad)[dof_index*dim + d] =
-                     surf_fit_grad_int(i*dim + d);
-               }
-            }
-         }
-
-         surf_fit_eval_bg_hess->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_hess_int, ordering);
-         // Assumes surf_fit_hess and surf_fit_gf share the same space
-         const int hess_dim = surf_fit_hess->VectorDim();
-         const int hess_cnt = surf_fit_hess->Size()/hess_dim;
-         if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < hess_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_hess)[dof_index + d*hess_cnt] =
-                     surf_fit_hess_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < hess_dim; d++)
-               {
-                  (*surf_fit_hess)[dof_index*dim + d] =
-                     surf_fit_hess_int(i*dim + d);
-               }
-            }
-         }
-      }
-      else
-      {
-         surf_fit_eval->ComputeAtNewPosition(x_new, *surf_fit_gf, ordering);
-      }
+      RemapSurfaceFittingLevelSetAtNodes(x_new, ordering);
    }
 }
 

fem/tmop.hpp

@@ -1784,12 +1784,11 @@ protected:
    // Fitting to given physical positions.
    TMOP_QuadraticLimiter *surf_fit_limiter; // Owned. Created internally.
    const GridFunction *surf_fit_pos;        // Not owned. Positions to fit.
-   real_t surf_fit_normal;
-   bool surf_fit_gf_bg;
-   GridFunction *surf_fit_grad, *surf_fit_hess;
-   AdaptivityEvaluator *surf_fit_eval_bg_grad, *surf_fit_eval_bg_hess;
-   Array<int> surf_fit_dof_count;
-   Array<int> surf_fit_marker_dof_index;
+   real_t surf_fit_normal;                  // Normalization factor.
+   GridFunction *surf_fit_grad, *surf_fit_hess; // Owned. Created internally.
+   AdaptivityEvaluator *surf_fit_eval_grad, *surf_fit_eval_hess; // Not owned.
+   Array<int> surf_fit_dof_count;            // Number of dofs per node.
+   Array<int> surf_fit_marker_dof_index;     // Indices of nodes to fit.
 
    DiscreteAdaptTC *discr_tc;
 
@@ -1985,6 +1984,10 @@ protected:
    real_t ComputeUntanglerMaxMuBarrier(const Vector &x,
                                        const FiniteElementSpace &fes);
 
+   // Remaps the internal surface fitting gridfunction object at provided
+   // locations.
+   void RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                           int new_x_ordering);
 public:
    /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
        @param[in] tc   Target-matrix construction algorithm to use (not owned).
@@ -2000,9 +2003,8 @@ public:
         surf_fit_marker(NULL), surf_fit_coeff(NULL),
         surf_fit_gf(NULL), surf_fit_eval(NULL),
         surf_fit_limiter(NULL), surf_fit_pos(NULL),
-        surf_fit_normal(1.0),
-        surf_fit_gf_bg(false), surf_fit_grad(NULL), surf_fit_hess(NULL),
-        surf_fit_eval_bg_grad(NULL), surf_fit_eval_bg_hess(NULL),
+        surf_fit_normal(1.0), surf_fit_grad(NULL), surf_fit_hess(NULL),
+        surf_fit_eval_grad(NULL), surf_fit_eval_hess(NULL),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
    { PA.enabled = false; }
@@ -2103,9 +2105,15 @@ public:
 
 #ifdef MFEM_USE_MPI
    /// Parallel support for surface fitting to the zero level set of a function.
+   /// Here, we add two optional inputs: @a aegrad and @a aehess. When provided,
+   /// the first and second derivative of the input level set are computed on
+   /// the initial mesh, and @a aegrad and @a aehess are used to remap grad_s(x)
+   /// from grad_s0(x0) and hess_s(x) from hess_s0(x0), respectively.
    void EnableSurfaceFitting(const ParGridFunction &s0,
                              const Array<bool> &smarker, Coefficient &coeff,
-                             AdaptivityEvaluator &ae);
+                             AdaptivityEvaluator &ae,
+                             AdaptivityEvaluator *aegrad = NULL,
+                             AdaptivityEvaluator *aehess = NULL);
 
    /** @brief Fitting of certain DOFs in the current mesh to the zero level set
        of a function defined on another (finer) source mesh.

fem/tmop_tools.cpp

@@ -429,11 +429,13 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 #endif
 
    real_t scale = 1.0;
-   real_t avg_surf_fit_err, max_surf_fit_err = 0.0;
-   if (surf_fit_max_threshold > 0.0)
+   bool fitting = IsSurfaceFittingEnabled();
+   real_t init_fit_avg_err, init_fit_max_err = 0.0;
+   if (fitting && surf_fit_converge_error)
    {
-      GetSurfaceFittingError(x_out_loc, avg_surf_fit_err, max_surf_fit_err);
-      if (max_surf_fit_err < surf_fit_max_threshold)
+      GetSurfaceFittingError(x_out_loc, init_fit_avg_err, init_fit_max_err);
+      // Check for convergence
+      if (init_fit_max_err < surf_fit_max_err_limit)
       {
          if (print_options.iterations)
          {
@@ -444,11 +446,12 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
          return scale;
       }
    }
-   if (adapt_inc_count >= max_adapt_inc_count)
+
+   if (surf_fit_adapt_count >= surf_fit_adapt_count_limit)
    {
       if (print_options.iterations)
       {
-         mfem::out << "TMOPNewtonSolver converged "
+         mfem::out << "TMOPNewtonSolver terminated "
                    "based on max number of times surface fitting weight can"
                    "be increased. \n";
       }
@@ -467,7 +470,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       // reference to detect deteriorations.
       MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
                   " intermediate mesh is invalid. Contact TMOP Developers.");
-      MFEM_VERIFY(min_detJ_threshold == 0.0,
+      MFEM_VERIFY(min_detJ_limit == 0.0,
                   "This setup is not supported. Contact TMOP Developers.");
       *min_det_ptr = untangle_factor * min_detT_in;
    }
@@ -478,6 +481,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
    bool x_out_ok = false;
    real_t energy_out = 0.0, min_detT_out;
    const real_t norm_in = Norm(r);
+   real_t avg_fit_err, max_fit_err = 0.0;
 
    const real_t detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
    compute_metric_quantile_flag = false;
@@ -488,6 +492,9 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
    // Perform the line search.
    for (int i = 0; i < 12; i++)
    {
+      avg_fit_err = 0.0;
+      max_fit_err = 0.0;
+
       // Update the mesh and get the L-vector in x_out_loc.
       add(x, -scale, c, x_out);
       if (serial)
@@ -502,7 +509,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
       // Check the changes in detJ.
       min_detT_out = ComputeMinDet(x_out_loc, *fes);
-      if (untangling == false && min_detT_out <= min_detJ_threshold)
+      if (untangling == false && min_detT_out <= min_detJ_limit)
       {
          // No untangling, and detJ got negative (or small) -- no good.
          if (print_options.iterations)
@@ -529,18 +536,19 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       // Check the changes in total energy.
       ProcessNewState(x_out);
 
-      real_t avg_fit_err, max_fit_err = 0.0;
-      if (surf_fit_max_threshold > 0.0)
+      // Ensure sufficient decrease in fitting error if we are trying to
+      // converge based on error.
+      if (fitting && surf_fit_converge_error)
       {
          GetSurfaceFittingError(x_out_loc, avg_fit_err, max_fit_err);
-      }
-      if (surf_fit_max_threshold > 0.0 && max_fit_err >= 1.2*max_surf_fit_err)
-      {
-         if (print_options.iterations)
+         if (max_fit_err >= 1.2*init_fit_max_err)
          {
-            mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            if (print_options.iterations)
+            {
+               mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            }
+            scale *= 0.5; continue;
          }
-         scale *= 0.5; continue;
       }
 
       if (serial)
@@ -614,7 +622,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
    if (x_out_ok == false) { scale = 0.0; }
 
-   if (surf_fit_scale_factor > 0.0) { update_surf_fit_coeff = true; }
+   if (surf_fit_scale_factor > 0.0) { surf_fit_coeff_update = true; }
    compute_metric_quantile_flag = true;
 
    return scale;
@@ -657,7 +665,7 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
    for (int i = 0; i < integs.Size(); i++)
    {
       ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
+      if (ti && ti->IsSurfaceFittingEnabled())
       {
          weight = ti->GetSurfaceFittingWeight();
          weights.Append(weight);
@@ -668,8 +676,11 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
          Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
          for (int j = 0; j < ati.Size(); j++)
          {
-            weight = ati[j]->GetSurfaceFittingWeight();
-            weights.Append(weight);
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               weight = ati[j]->GetSurfaceFittingWeight();
+               weights.Append(weight);
+            }
          }
       }
    }
@@ -716,6 +727,39 @@ void TMOPNewtonSolver::GetSurfaceFittingError(const Vector &x_loc,
    }
 }
 
+bool TMOPNewtonSolver::IsSurfaceFittingEnabled() const
+{
+   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
+   const Array<NonlinearFormIntegrator*> &integs = *nlf->GetDNFI();
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         if (ti->IsSurfaceFittingEnabled())
+         {
+            return true;
+         }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               return true;
+            }
+         }
+      }
+   }
+   return false;
+}
+
 void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
 {
    const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
@@ -801,38 +845,46 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
    // adaptive surface fitting is enabled. The idea is to increase the
    // coefficient if the surface fitting error does not sufficiently
    // decrease between subsequent TMOPNewtonSolver iterations.
-   if (update_surf_fit_coeff)
+   if (surf_fit_coeff_update)
    {
       // Get surface fitting errors.
-      GetSurfaceFittingError(x_loc, surf_fit_err_avg, surf_fit_err_max);
+      GetSurfaceFittingError(x_loc, surf_fit_avg_err, surf_fit_max_err);
       // Get array with surface fitting weights.
-      Array<real_t> weights;
-      GetSurfaceFittingWeight(weights);
+      Array<real_t> fitweights;
+      GetSurfaceFittingWeight(fitweights);
 
       if (print_options.iterations)
       {
          mfem::out << "Avg/Max surface fitting error: " <<
-                   surf_fit_err_avg << " " <<
-                   surf_fit_err_max << "\n";
+                   surf_fit_avg_err << " " <<
+                   surf_fit_max_err << "\n";
          mfem::out << "Min/Max surface fitting weight: " <<
-                   weights.Min() << " " << weights.Max() << "\n";
+                   fitweights.Min() << " " << fitweights.Max() << "\n";
       }
 
-      real_t change_surf_fit_err = surf_fit_err_avg_prvs-surf_fit_err_avg;
-      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_err_avg_prvs;
+      real_t change_surf_fit_err = surf_fit_avg_err_prvs-surf_fit_avg_err;
+      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_avg_err_prvs;
+
       // Increase the surface fitting coefficient if the surface fitting error
-      // does not decrease sufficiently.
-      if (rel_change_surf_fit_err < surf_fit_rel_change_threshold)
+      // does not decrease sufficiently. If we are converging based on residual,
+      // also make sure we have not reached the maximum fitting weight and
+      // error threshold.
+      if (rel_change_surf_fit_err < surf_fit_err_rel_change_limit &&
+          (surf_fit_converge_error ||
+           (fitweights.Max() < surf_fit_weight_limit &&
+            surf_fit_max_err > surf_fit_max_err_limit)))
       {
-         UpdateSurfaceFittingWeight(surf_fit_scale_factor);
-         adapt_inc_count += 1;
+         real_t scale_factor = std::min(surf_fit_scale_factor,
+                                        surf_fit_weight_limit/fitweights.Max());
+         UpdateSurfaceFittingWeight(scale_factor);
+         surf_fit_adapt_count += 1;
       }
       else
       {
-         adapt_inc_count = 0;
+         surf_fit_adapt_count = 0;
       }
-      surf_fit_err_avg_prvs = surf_fit_err_avg;
-      update_surf_fit_coeff = false;
+      surf_fit_avg_err_prvs = surf_fit_avg_err;
+      surf_fit_coeff_update = false;
    }
 }
 

fem/tmop_tools.hpp

@@ -134,18 +134,20 @@ protected:
    int solver_type;
    bool parallel;
 
-   // Line search step is rejected if min(detJ) <= min_detJ_threshold.
-   real_t min_detJ_threshold = 0.0;
+   // Line search step is rejected if min(detJ) <= min_detJ_limit.
+   real_t min_detJ_limit = 0.0;
 
    // Surface fitting variables.
-   mutable real_t surf_fit_err_avg_prvs = 10000.0;
-   mutable real_t surf_fit_err_avg, surf_fit_err_max;
-   mutable bool update_surf_fit_coeff = false;
-   real_t surf_fit_max_threshold = -1.0;
-   real_t surf_fit_rel_change_threshold = 0.001;
+   mutable real_t surf_fit_avg_err_prvs = 10000.0;
+   mutable real_t surf_fit_avg_err, surf_fit_max_err;
+   mutable bool surf_fit_coeff_update = false;
+   real_t surf_fit_max_err_limit = -1.0;
+   real_t surf_fit_err_rel_change_limit = 0.001;
    real_t surf_fit_scale_factor = 0.0;
-   mutable int adapt_inc_count = 0;
-   mutable int max_adapt_inc_count = 10;
+   mutable int surf_fit_adapt_count = 0;
+   mutable int surf_fit_adapt_count_limit = 10;
+   mutable real_t surf_fit_weight_limit = 1e10;
+   bool surf_fit_converge_error = false;
 
    // Minimum determinant over the whole mesh. Used for mesh untangling.
    real_t *min_det_ptr = nullptr;
@@ -191,6 +193,9 @@ protected:
    void GetSurfaceFittingWeight(Array<real_t> &weights) const;
    ///@}
 
+   /// Check if surface fitting is enabled.
+   bool IsSurfaceFittingEnabled() const;
+
 public:
 #ifdef MFEM_USE_MPI
    TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
@@ -224,38 +229,94 @@ public:
    /// (ii) surface fitting weight.
    virtual void ProcessNewState(const Vector &x) const;
 
-   /** @name Methods for adaptive surface fitting weight. (Experimental) */
-   /// Enable/Disable adaptive surface fitting weight.
-   /// The weight is modified after each TMOPNewtonSolver iteration as:
-   /// w_{k+1} = w_{k} * @a surf_fit_scale_factor if relative change in
-   /// max surface fitting error < @a surf_fit_rel_change_threshold.
-   /// The solver terminates if the maximum surface fitting error does
-   /// not sufficiently decrease for @a max_adapt_inc_count consecutive
-   /// solver iterations or if the max error falls below @a surf_fit_max_threshold.
-   void EnableAdaptiveSurfaceFitting()
-   {
-      surf_fit_scale_factor = 10.0;
-      surf_fit_rel_change_threshold = 0.001;
-   }
+   /** @name Methods for adaptive surface fitting.
+       \brief These methods control the behavior of the weight and the
+       termination of the solver. (Experimental)
+
+       Adaptive fitting weight: The weight is modified after each
+       TMOPNewtonSolver iteration as:
+       w_{k+1} = w_{k} * \ref surf_fit_scale_factor if the relative
+       change in average fitting error < \ref surf_fit_err_rel_change_limit.
+       When converging based on the residual, we enforce the fitting weight
+       to be at-most \ref surf_fit_weight_limit, and increase it only if the
+       fitting error is below user prescribed threshold
+       (\ref surf_fit_max_err_limit).
+       See \ref SetAdaptiveSurfaceFittingScalingFactor and
+       \ref SetAdaptiveSurfaceFittingRelativeChangeThreshold.
+
+       Note that the solver stops if the maximum surface fitting error
+       does not sufficiently decrease for \ref surf_fit_adapt_count_limit (default 10)
+       consecutive increments of the fitting weight during weight adaptation.
+       This typically occurs when the mesh cannot align with the level-set
+       without degrading element quality.
+       See \ref SetMaxNumberofIncrementsForAdaptiveFitting.
+
+       Convergence criterion: There are two modes, residual- and error-based,
+       which can be toggled using \ref SetSurfaceFittingConvergenceBasedOnError.
+
+       (i) Residual based (default): Stop when the norm of the gradient of the
+       TMOP objective reaches the prescribed tolerance. This method is best used
+       with a reasonable value for \ref surf_fit_weight_limit when the
+       adaptive surface fitting scheme is used. See method
+       \ref SetSurfaceFittingWeightLimit.
+
+       (ii) Error based: Stop when the maximum fitting error
+       reaches the user-prescribed threshold, \ref surf_fit_max_err_limit.
+       In this case, \ref surf_fit_weight_limit is ignored during weight
+       adaptation.
+   */
+   ///@{
    void SetAdaptiveSurfaceFittingScalingFactor(real_t factor)
    {
+      MFEM_VERIFY(factor > 1.0, "Scaling factor must be greater than 1.");
       surf_fit_scale_factor = factor;
    }
    void SetAdaptiveSurfaceFittingRelativeChangeThreshold(real_t threshold)
    {
-      surf_fit_rel_change_threshold = threshold;
+      surf_fit_err_rel_change_limit = threshold;
    }
+   /// Used for stopping based on the number of consecutive failed weight
+   /// adaptation iterations.
+   // TODO: Rename to SetMaxNumberofIncrementsForAdaptiveSurfaceFitting
+   // in future.
    void SetMaxNumberofIncrementsForAdaptiveFitting(int count)
    {
-      max_adapt_inc_count = count;
+      surf_fit_adapt_count_limit = count;
    }
+   /// Used for error-based surface fitting termination.
    void SetTerminationWithMaxSurfaceFittingError(real_t max_error)
    {
-      surf_fit_max_threshold = max_error;
+      surf_fit_max_err_limit = max_error;
+      surf_fit_converge_error = true;
    }
+   /// Could be used with both error-based or residual-based convergence.
+   void SetSurfaceFittingMaxErrorLimit(real_t max_error)
+   {
+      surf_fit_max_err_limit = max_error;
+   }
+   /// Used for residual-based surface fitting termination.
+   void SetSurfaceFittingWeightLimit(real_t weight)
+   {
+      surf_fit_weight_limit = weight;
+   }
+   /// Toggle convergence based on residual or error.
+   void SetSurfaceFittingConvergenceBasedOnError(bool mode)
+   {
+      surf_fit_converge_error = mode;
+      if (surf_fit_converge_error)
+      {
+         MFEM_VERIFY(surf_fit_max_err_limit >= 0,
+                     "Fitting error based convergence requires the user to "
+                     "first set the error threshold."
+                     "See SetTerminationWithMaxSurfaceFittingError");
+      }
+   }
+   ///@}
+
+   /// Set minimum determinant enforced during line-search.
    void SetMinimumDeterminantThreshold(real_t threshold)
    {
-      min_detJ_threshold = threshold;
+      min_detJ_limit = threshold;
    }
 
    virtual void Mult(const Vector &b, Vector &x) const

linalg/hypre.cpp

@@ -410,19 +410,19 @@ HYPRE_Int HypreParVector::Randomize(HYPRE_Int seed)
    return hypre_ParVectorSetRandomValues(x,seed);
 }
 
-void HypreParVector::Print(const char *fname) const
+void HypreParVector::Print(const std::string &fname) const
 {
-   hypre_ParVectorPrint(x,fname);
+   hypre_ParVectorPrint(x, fname.c_str());
 }
 
-void HypreParVector::Read(MPI_Comm comm, const char *fname)
+void HypreParVector::Read(MPI_Comm comm, const std::string &fname)
 {
    if (own_ParVector)
    {
       hypre_ParVectorDestroy(x);
    }
    data.Delete();
-   x = hypre_ParVectorRead(comm, fname);
+   x = hypre_ParVectorRead(comm, fname.c_str());
    own_ParVector = true;
    _SetDataAndSize_();
 }
@@ -792,6 +792,44 @@ static void SyncBackBoolCSR(Table *bool_csr, MemoryIJData &mem_csr)
    }
 }
 
+/// @brief Return the size of the partitioning arrays, see @ref
+/// hypre_partitioning_descr.
+static int GetPartitioningArraySize(MPI_Comm comm)
+{
+   if (HYPRE_AssumedPartitionCheck())
+   {
+      return 2;
+   }
+   else
+   {
+      int comm_size;
+      MPI_Comm_size(comm, &comm_size);
+      return comm_size + 1;
+   }
+}
+
+/// @brief Returns true if the row and col arrays are equal (across all MPI
+/// ranks).
+///
+/// Both @a row and @a col are partitioning arrays, whose length is returned by
+/// GetPartitioningArraySize(), see @ref hypre_partitioning_descr.
+static bool RowAndColStartsAreEqual(MPI_Comm comm, HYPRE_BigInt *rows,
+                                    HYPRE_BigInt *cols)
+{
+   const int part_size = GetPartitioningArraySize(comm);
+   bool are_equal = true;
+   for (int i = 0; i < part_size; ++i)
+   {
+      if (rows[i] != cols[i])
+      {
+         are_equal = false;
+         break;
+      }
+   }
+   MPI_Allreduce(MPI_IN_PLACE, &are_equal, 1,  MPI_C_BOOL, MPI_LAND, comm);
+   return are_equal;
+}
+
 // static method
 signed char HypreParMatrix::HypreCsrToMem(hypre_CSRMatrix *h_mat,
                                           MemoryType h_mat_mt,
@@ -924,7 +962,7 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
    hypre_ParCSRMatrixSetNumNonzeros(A);
 
    /* Make sure that the first entry in each row is the diagonal one. */
-   if (row_starts == col_starts)
+   if (RowAndColStartsAreEqual(comm, row_starts, col_starts))
    {
       HypreReadWrite();
       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
@@ -974,11 +1012,12 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
    hypre_ParCSRMatrixSetNumNonzeros(A);
 
    /* Make sure that the first entry in each row is the diagonal one. */
-   if (row_starts == col_starts)
+   if (RowAndColStartsAreEqual(comm, row_starts, col_starts))
    {
       HypreReadWrite();
       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
-      SyncBackCSR(diag, mem_diag); // update diag, if needed
+      // update diag, if needed
+      if (!own_diag_offd) { SyncBackCSR(diag, mem_diag); }
    }
 
    hypre_MatvecCommPkgCreate(A);
@@ -1031,7 +1070,7 @@ HypreParMatrix::HypreParMatrix(
    hypre_ParCSRMatrixSetNumNonzeros(A);
 
    /* Make sure that the first entry in each row is the diagonal one. */
-   if (row_starts == col_starts)
+   if (RowAndColStartsAreEqual(comm, row_starts, col_starts))
    {
       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
    }
@@ -1093,7 +1132,7 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
    hypre_CSRMatrixDestroy(csr_a);
 
    /* Make sure that the first entry in each row is the diagonal one. */
-   if (row_starts == col_starts)
+   if (RowAndColStartsAreEqual(comm, row_starts, col_starts))
    {
       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(new_A));
    }
@@ -1132,7 +1171,7 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
    hypre_ParCSRMatrixSetNumNonzeros(A);
 
    /* Make sure that the first entry in each row is the diagonal one. */
-   if (row_starts == col_starts)
+   if (RowAndColStartsAreEqual(comm, row_starts, col_starts))
    {
       HypreReadWrite();
       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
@@ -1247,11 +1286,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
    Init();
 
    // Determine partitioning size, and my column start and end
-   int part_size;
+   const int part_size = GetPartitioningArraySize(comm);
    HYPRE_BigInt my_col_start, my_col_end; // my range: [my_col_start, my_col_end)
    if (HYPRE_AssumedPartitionCheck())
    {
-      part_size = 2;
       my_col_start = cols[0];
       my_col_end = cols[1];
    }
@@ -1259,15 +1297,14 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
    {
       int myid;
       MPI_Comm_rank(comm, &myid);
-      MPI_Comm_size(comm, &part_size);
-      part_size++;
       my_col_start = cols[myid];
       my_col_end = cols[myid+1];
    }
 
    // Copy in the row and column partitionings
+   const bool rows_eq_cols = RowAndColStartsAreEqual(comm, rows, cols);
    HYPRE_BigInt *row_starts, *col_starts;
-   if (rows == cols)
+   if (rows_eq_cols)
    {
       row_starts = col_starts = mfem_hypre_TAlloc_host(HYPRE_BigInt, part_size);
       for (int i = 0; i < part_size; i++)
@@ -1360,14 +1397,14 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
    }
 
    hypre_ParCSRMatrixSetNumNonzeros(A);
-   /* Make sure that the first entry in each row is the diagonal one. */
-   if (row_starts == col_starts)
+   // Make sure that the first entry in each row is the diagonal one.
+   if (rows_eq_cols)
    {
       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
    }
 #if MFEM_HYPRE_VERSION > 22200
    mfem_hypre_TFree_host(row_starts);
-   if (rows != cols)
+   if (!rows_eq_cols)
    {
       mfem_hypre_TFree_host(col_starts);
    }
@@ -1480,16 +1517,7 @@ void HypreParMatrix::CopyRowStarts()
       return;
    }
 
-   int row_starts_size;
-   if (HYPRE_AssumedPartitionCheck())
-   {
-      row_starts_size = 2;
-   }
-   else
-   {
-      MPI_Comm_size(hypre_ParCSRMatrixComm(A), &row_starts_size);
-      row_starts_size++; // num_proc + 1
-   }
+   const int row_starts_size = GetPartitioningArraySize(hypre_ParCSRMatrixComm(A));
 
    HYPRE_BigInt *old_row_starts = hypre_ParCSRMatrixRowStarts(A);
    HYPRE_BigInt *new_row_starts = mfem_hypre_CTAlloc_host(HYPRE_BigInt,
@@ -1520,16 +1548,7 @@ void HypreParMatrix::CopyColStarts()
       return;
    }
 
-   int col_starts_size;
-   if (HYPRE_AssumedPartitionCheck())
-   {
-      col_starts_size = 2;
-   }
-   else
-   {
-      MPI_Comm_size(hypre_ParCSRMatrixComm(A), &col_starts_size);
-      col_starts_size++; // num_proc + 1
-   }
+   const int col_starts_size = GetPartitioningArraySize(hypre_ParCSRMatrixComm(A));
 
    HYPRE_BigInt *old_col_starts = hypre_ParCSRMatrixColStarts(A);
    HYPRE_BigInt *new_col_starts = mfem_hypre_CTAlloc_host(HYPRE_BigInt,
@@ -2291,13 +2310,8 @@ void HypreParMatrix::Threshold(real_t threshold)
    A = parcsr_A_ptr;
 
    hypre_ParCSRMatrixSetNumNonzeros(A);
-   /* Make sure that the first entry in each row is the diagonal one. */
-#if MFEM_HYPRE_VERSION <= 22200
-   if (row_starts == col_starts)
-#else
-   if ((row_starts[0] == col_starts[0]) &&
-       (row_starts[1] == col_starts[1]))
-#endif
+   // Make sure that the first entry in each row is the diagonal one.
+   if (RowAndColStartsAreEqual(comm, row_starts, col_starts))
    {
       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
    }
@@ -2623,48 +2637,38 @@ void HypreParMatrix::EliminateBC(const Array<int> &ess_dofs,
    mfem_hypre_TFree(eliminate_col);
 }
 
-void HypreParMatrix::Print(const char *fname, HYPRE_Int offi,
+void HypreParMatrix::Print(const std::string &fname, HYPRE_Int offi,
                            HYPRE_Int offj) const
 {
    HostRead();
-   hypre_ParCSRMatrixPrintIJ(A,offi,offj,fname);
+   hypre_ParCSRMatrixPrintIJ(A, offi, offj, fname.c_str());
    HypreRead();
 }
 
-void HypreParMatrix::Read(MPI_Comm comm, const char *fname)
+void HypreParMatrix::Read(MPI_Comm comm, const std::string &fname)
 {
-   Destroy();
-   Init();
-
+   HYPRE_ParCSRMatrix A_parcsr;
    HYPRE_Int base_i, base_j;
-   hypre_ParCSRMatrixReadIJ(comm, fname, &base_i, &base_j, &A);
+   hypre_ParCSRMatrixReadIJ(comm, fname.c_str(), &base_i, &base_j, &A_parcsr);
+
+   WrapHypreParCSRMatrix(A_parcsr, true);
+
    hypre_ParCSRMatrixSetNumNonzeros(A);
-
    if (!hypre_ParCSRMatrixCommPkg(A)) { hypre_MatvecCommPkgCreate(A); }
-
-   height = GetNumRows();
-   width = GetNumCols();
 }
 
-void HypreParMatrix::Read_IJMatrix(MPI_Comm comm, const char *fname)
+void HypreParMatrix::Read_IJMatrix(MPI_Comm comm, const std::string &fname)
 {
-   Destroy();
-   Init();
-
    HYPRE_IJMatrix A_ij;
-   HYPRE_IJMatrixRead(fname, comm, 5555, &A_ij); // HYPRE_PARCSR = 5555
+   HYPRE_IJMatrixRead(fname.c_str(), comm, 5555, &A_ij); // HYPRE_PARCSR = 5555
 
    HYPRE_ParCSRMatrix A_parcsr;
    HYPRE_IJMatrixGetObject(A_ij, (void**) &A_parcsr);
 
-   A = (hypre_ParCSRMatrix*)A_parcsr;
+   WrapHypreParCSRMatrix(A_parcsr, true);
 
    hypre_ParCSRMatrixSetNumNonzeros(A);
-
    if (!hypre_ParCSRMatrixCommPkg(A)) { hypre_MatvecCommPkgCreate(A); }
-
-   height = GetNumRows();
-   width = GetNumCols();
 }
 
 void HypreParMatrix::PrintCommPkg(std::ostream &os) const

linalg/hypre.hpp

@@ -364,10 +364,10 @@ public:
    HYPRE_Int Randomize(HYPRE_Int seed);
 
    /// Prints the locally owned rows in parallel
-   void Print(const char *fname) const;
+   void Print(const std::string &fname) const;
 
    /// Reads a HypreParVector from files saved with HypreParVector::Print
-   void Read(MPI_Comm comm, const char *fname);
+   void Read(MPI_Comm comm, const std::string &fname);
 
    /// Calls hypre's destroy function
    ~HypreParVector();
@@ -919,12 +919,14 @@ public:
    const Memory<HYPRE_Int> &GetDiagMemoryJ() const { return mem_diag.J; }
    const Memory<real_t> &GetDiagMemoryData() const { return mem_diag.data; }
 
-   /// Prints the locally owned rows in parallel
-   void Print(const char *fname, HYPRE_Int offi = 0, HYPRE_Int offj = 0) const;
+   /// @brief Prints the locally owned rows in parallel. The resulting files can
+   /// be read with Read_IJMatrix().
+   void Print(const std::string &fname, HYPRE_Int offi = 0,
+              HYPRE_Int offj = 0) const;
    /// Reads the matrix from a file
-   void Read(MPI_Comm comm, const char *fname);
+   void Read(MPI_Comm comm, const std::string &fname);
    /// Read a matrix saved as a HYPRE_IJMatrix
-   void Read_IJMatrix(MPI_Comm comm, const char *fname);
+   void Read_IJMatrix(MPI_Comm comm, const std::string &fname);
 
    /// Print information about the hypre_ParCSRCommPkg of the HypreParMatrix.
    void PrintCommPkg(std::ostream &out = mfem::out) const;

linalg/operator.hpp

@@ -309,18 +309,51 @@ public:
 
 
 /// Base abstract class for first order time dependent operators.
-/** Operator of the form: (x,t) -> f(x,t), where k = f(x,t) generally solves the
-    algebraic equation F(x,k,t) = G(x,t). The functions F and G represent the
-    _implicit_ and _explicit_ parts of the operator, respectively. For explicit
-    operators, F(x,k,t) = k, so f(x,t) = G(x,t). */
+/** Operator of the form: (u,t) -> k(u,t), where k generally solves the
+    algebraic equation F(u,k,t) = G(u,t). The functions F and G represent the
+    _implicit_ and _explicit_ parts of the operator, respectively.
+
+    A common use for this class is representing a differential algebraic
+    equation of the form $ F(y,\frac{dy}{dt},t) = G(y,t) $.
+
+    For example, consider an ordinary differential equation of the form
+    $ M \frac{dy}{dt} = g(y,t) $. There are various ways of expressing this ODE
+    as a TimeDependentOperator depending on the choices for F and G. Here are
+    some common choices:
+
+      1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t),
+      2. F(u,k,t) = M k and G(u,t) = g(u,t),
+      3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.
+
+    Note that depending on the ODE solver, some of the above choices may be
+    preferable to the others.
+*/
 class TimeDependentOperator : public Operator
 {
 public:
+   /// Enum used to describe the form of the time-dependent operator.
+   /** The type should be set by classes derived from TimeDependentOperator to
+       describe the form, in terms of the functions F and G, used by the
+       specific derived class. This information can be queried by classes or
+       functions (like time stepping algorithms) to make choices about the
+       algorithm to use, or to ensure that the TimeDependentOperator uses the
+       form expected by the class/function.
+
+       For example, assume that a derived class is implementing the ODE
+       $M \frac{dy}{dt} = g(y,t)$ and chooses to define $F(u,k,t) = M k$ and
+       $G(u,t) = g(u,t)$. Then it cannot use type EXPLICIT, unless $M = I$, or
+       type HOMOGENEOUS, unless $g(u,t) = 0$. If, on the other hand, the derived
+       class chooses to define $F(u,k,t) = k$ and $G(u,t) = M^{-1} g(y,t)$, then
+       the natural choice is to set the type to EXPLICIT, even though setting it
+       to IMPLICIT is also not wrong -- doing so will simply fail to inform
+       methods that query this information that it uses a more specific
+       implementation, EXPLICIT, that may allow the use of algorithms that
+       support only the EXPLICIT type. */
    enum Type
    {
-      EXPLICIT,   ///< This type assumes F(x,k,t) = k, i.e. k = f(x,t) = G(x,t).
+      EXPLICIT,   ///< This type assumes F(u,k,t) = k.
       IMPLICIT,   ///< This is the most general type, no assumptions on F and G.
-      HOMOGENEOUS ///< This type assumes that G(x,t) = 0.
+      HOMOGENEOUS ///< This type assumes that G(u,t) = 0.
    };
 
    /// Evaluation mode. See SetEvalMode() for details.
@@ -328,29 +361,30 @@ public:
    {
       /** Normal evaluation. */
       NORMAL,
-      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
-          first term, f1. */
+      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
+          first term, k1. */
       ADDITIVE_TERM_1,
-      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
-          second term, f2. */
+      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
+          second term, k2. */
       ADDITIVE_TERM_2
    };
 
 protected:
    real_t t;  ///< Current time.
-   Type type; ///< Describes the form of the TimeDependentOperator.
+   Type type; /**< @brief Describes the form of the TimeDependentOperator, see
+                   the documentation of #Type. */
    EvalMode eval_mode; ///< Current evaluation mode.
 
 public:
-   /** @brief Construct a "square" TimeDependentOperator y = f(x,t), where x and
-       y have the same dimension @a n. */
+   /** @brief Construct a "square" TimeDependentOperator (u,t) -> k(u,t), where
+       u and k have the same dimension @a n. */
    explicit TimeDependentOperator(int n = 0, real_t t_ = 0.0,
                                   Type type_ = EXPLICIT)
       : Operator(n) { t = t_; type = type_; eval_mode = NORMAL; }
 
-   /** @brief Construct a TimeDependentOperator y = f(x,t), where x and y have
-       dimensions @a w and @a h, respectively. */
-   TimeDependentOperator(int h, int w, real_t t_ = 0.0, Type type_ = EXPLICIT)
+   /** @brief Construct a TimeDependentOperator (u,t) -> k(u,t), where u and k
+       have dimensions @a w and @a h, respectively. */
+   TimeDependentOperator(int h, int w, double t_ = 0.0, Type type_ = EXPLICIT)
       : Operator(h, w) { t = t_; type = type_; eval_mode = NORMAL; }
 
    /// Read the currently set time.
@@ -373,7 +407,7 @@ public:
    /** The evaluation mode is a switch that allows time-stepping methods to
        request evaluation of separate components/terms of the time-dependent
        operator. For example, IMEX methods typically assume additive split of
-       the operator: f(x,t) = f1(x,t) + f2(x,t) and they rely on the ability to
+       the operator: k(u,t) = k1(u,t) + k2(u,t) and they rely on the ability to
        evaluate the two terms separately.
 
        Generally, setting the evaluation mode should affect the behavior of all
@@ -384,62 +418,104 @@ public:
    { eval_mode = new_eval_mode; }
 
    /** @brief Perform the action of the explicit part of the operator, G:
-       @a y = G(@a x, t) where t is the current time.
+       @a v = G(@a u, t) where t is the current time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual. */
-   virtual void ExplicitMult(const Vector &x, Vector &y) const;
+   virtual void ExplicitMult(const Vector &u, Vector &v) const;
 
    /** @brief Perform the action of the implicit part of the operator, F:
-       @a y = F(@a x, @a k, t) where t is the current time.
+       @a v = F(@a u, @a k, t) where t is the current time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual.*/
-   virtual void ImplicitMult(const Vector &x, const Vector &k, Vector &y) const;
+   virtual void ImplicitMult(const Vector &u, const Vector &k, Vector &v) const;
 
-   /** @brief Perform the action of the operator: @a y = k = f(@a x, t), where
-       k solves the algebraic equation F(@a x, k, t) = G(@a x, t) and t is the
-       current time. */
-   virtual void Mult(const Vector &x, Vector &y) const;
+   /** @brief Perform the action of the operator (u,t) -> k(u,t) where t is the
+       current time set by SetTime() and @a k satisfies
+       F(@a u, @a k, t) = G(@a u, t).
 
-   /** @brief Solve the equation: @a k = f(@a x + @a dt @a k, t), for the
-       unknown @a k at the current time t.
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
+       various ways, e.g.:
 
-       For general F and G, the equation for @a k becomes:
-       F(@a x + @a dt @a k, @a k, t) = G(@a x + @a dt @a k, t).
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.
 
-       The input vector @a x corresponds to time index (or cycle) n, while the
-       currently set time, #t, and the result vector @a k correspond to time
-       index n+1. The time step @a dt corresponds to the time interval between
-       cycles n and n+1.
+       Regardless of the choice of F and G, this function should always compute
+       @a k = inv(M) g(@a u, t). */
+   virtual void Mult(const Vector &u, Vector &v) const override;
 
-       This method allows for the abstract implementation of some time
-       integration methods, including diagonal implicit Runge-Kutta (DIRK)
-       methods and the backward Euler method in particular.
+   /** @brief Solve for the unknown @a k, at the current time t, the following
+       equation:
+       F(@a u + @a gamma @a k, @a k, t) = G(@a u + @a gamma @a k, t).
+
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
+       various ways, e.g.:
+
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
+
+       Regardless of the choice of F and G, this function should solve for @a k
+       in M @a k = g(@a u + @a gamma @a k, t).
+
+       To see how @a k can be useful, consider the backward Euler method defined
+       by $ y(t + \Delta t) = y(t) + \Delta t k_0 $ where
+       $ M k_0 = g \big( y(t) + \Delta t k_0, t + \Delta t \big) $. A backward
+       Euler integrator can use @a k from this function for $k_0$, with the call
+       using @a u set to $ y(t) $, @a gamma set to $ \Delta t$, and time set to
+       $t + \Delta t$. See class BackwardEulerSolver.
+
+       Generalizing further, consider a diagonally implicit Runge-Kutta (DIRK)
+       method defined by
+       $ y(t + \Delta t) = y(t) + \Delta t \sum_{i=1}^s b_i k_i $ where
+       $ M k_i = g \big( y(t) + \Delta t \sum_{j=1}^i a_{ij} k_j,
+                         t + c_i \Delta t \big) $.
+       A DIRK integrator can use @a k from this function, with @a u set to
+       $ y(t) + \Delta t \sum_{j=1}^{i-1} a_{ij} k_j $ and @a gamma set to
+       $ a_{ii} \Delta t $, for $ k_i $. For example, see class SDIRK33Solver.
 
        If not re-implemented, this method simply generates an error. */
-   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
+   virtual void ImplicitSolve(const real_t gamma, const Vector &u, Vector &k);
 
-   /** @brief Return an Operator representing (dF/dk @a shift + dF/dx) at the
-       given @a x, @a k, and the currently set time.
+   /** @brief Return an Operator representing (dF/dk @a shift + dF/du) at the
+       given @a u, @a k, and the currently set time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual. */
-   virtual Operator& GetImplicitGradient(const Vector &x, const Vector &k,
+   virtual Operator& GetImplicitGradient(const Vector &u, const Vector &k,
                                          real_t shift) const;
 
-   /** @brief Return an Operator representing dG/dx at the given point @a x and
+   /** @brief Return an Operator representing dG/du at the given point @a u and
        the currently set time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual. */
-   virtual Operator& GetExplicitGradient(const Vector &x) const;
+   virtual Operator& GetExplicitGradient(const Vector &u) const;
 
-   /** @brief Setup the ODE linear system $ A(x,t) = (I - gamma J) $ or
-       $ A = (M - gamma J) $, where $ J(x,t) = \frac{df}{dt(x,t)} $.
+   /** @brief Setup a linear system as needed by some SUNDIALS ODE solvers.
 
-       @param[in]  x     The state at which $A(x,t)$ should be evaluated.
-       @param[in]  fx    The current value of the ODE rhs function, $f(x,t)$.
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined as one
+       of the following:
+
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
+
+       This function performs setup to solve $ A x = b $ where A is either
+
+         1. A(@a y,t) = I - @a gamma inv(M) J(@a y,t)
+         2. A(@a y,t) = M - @a gamma J(@a y,t)
+         3. A(@a y,t) = M - @a gamma J(@a y,t)
+
+       with J = dg/dy (or a reasonable approximation thereof).
+
+       @param[in]  y     The state at which A(@a y,t) should be evaluated.
+       @param[in]  v     The value of inv(M) g(y,t) for 1 or g(y,t) for 2 & 3.
        @param[in]  jok   Flag indicating if the Jacobian should be updated.
        @param[out] jcur  Flag to signal if the Jacobian was updated.
        @param[in]  gamma The scaled time step value.
@@ -448,10 +524,10 @@ public:
 
        Presently, this method is used by SUNDIALS ODE solvers, for more
        details, see the SUNDIALS User Guides. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+   virtual int SUNImplicitSetup(const Vector &y, const Vector &v,
                                 int jok, int *jcur, real_t gamma);
 
-   /** @brief Solve the ODE linear system $ A x = b $ as setup by
+   /** @brief Solve the ODE linear system A @a x = @a b, where A is defined by
        the method SUNImplicitSetup().
 
        @param[in]      b   The linear system right-hand side.
@@ -464,7 +540,8 @@ public:
        details, see the SUNDIALS User Guides. */
    virtual int SUNImplicitSolve(const Vector &b, Vector &x, real_t tol);
 
-   /** @brief Setup the mass matrix in the ODE system $ M y' = f(y,t) $ .
+   /** @brief Setup the mass matrix in the ODE system
+       $ M \frac{dy}{dt} = g(y,t) $ .
 
        If not re-implemented, this method simply generates an error.
 
@@ -472,8 +549,8 @@ public:
        details, see the ARKode User Guide. */
    virtual int SUNMassSetup();
 
-   /** @brief Solve the mass matrix linear system $ M x = b $
-       as setup by the method SUNMassSetup().
+   /** @brief Solve the mass matrix linear system  M @a x = @a b, where M is
+       defined by the method SUNMassSetup().
 
        @param[in]      b   The linear system right-hand side.
        @param[in,out]  x   On input, the initial guess. On output, the solution.
@@ -485,7 +562,8 @@ public:
        details, see the ARKode User Guide. */
    virtual int SUNMassSolve(const Vector &b, Vector &x, real_t tol);
 
-   /** @brief Compute the mass matrix-vector product $ v = M x $ .
+   /** @brief Compute the mass matrix-vector product @a v = M @a x, where M is
+       defined by the method SUNMassSetup().
 
        @param[in]   x The vector to multiply.
        @param[out]  v The result of the matrix-vector product.

linalg/symmat.cpp

@@ -58,6 +58,20 @@ DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(real_t c)
    return *this;
 }
 
+DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(const DenseSymmetricMatrix
+                                                      &m)
+{
+   SetSize(m.height);
+
+   const int hw = m.GetStoredSize();
+   for (int i = 0; i < hw; i++)
+   {
+      data[i] = m.data[i];
+   }
+
+   return *this;
+}
+
 real_t &DenseSymmetricMatrix::Elem(int i, int j)
 {
    return (*this)(i,j);
@@ -89,11 +103,6 @@ MatrixInverse *DenseSymmetricMatrix::Inverse() const
    return nullptr;
 }
 
-void DenseSymmetricMatrix::Print (std::ostream & os, int width_) const
-{
-   mfem_error("DenseSymmetricMatrix::Print() not implemented!");
-}
-
 DenseSymmetricMatrix::~DenseSymmetricMatrix()
 {
    data.Delete();

linalg/symmat.hpp

@@ -102,6 +102,9 @@ public:
 
    DenseSymmetricMatrix &operator*=(real_t c);
 
+   /// Sets the matrix size and elements equal to those of m
+   DenseSymmetricMatrix &operator=(const DenseSymmetricMatrix &m);
+
    std::size_t MemoryUsage() const { return data.Capacity() * sizeof(real_t); }
 
    /// Shortcut for mfem::Read( GetMemory(), TotalSize(), on_dev).
@@ -134,9 +137,6 @@ public:
    /// Returns a pointer to (an approximation) of the matrix inverse.
    virtual MatrixInverse *Inverse() const;
 
-   /// Prints matrix to stream out.
-   virtual void Print (std::ostream & out = mfem::out, int width_ = 4) const;
-
    /// Destroys the symmetric matrix.
    virtual ~DenseSymmetricMatrix();
 };

mesh/mesh.hpp

@@ -857,7 +857,9 @@ public:
        vectors using Mesh::CreatePeriodicVertexMapping.
        @note MFEM requires that each edge of the resulting mesh be uniquely
        identifiable by a pair of distinct vertices. As a consequence, periodic
-       boundaries must be connected by at least three edges. */
+       boundaries must be separated by at least two interior vertices.
+       @note The resulting mesh uses a discontinuous nodal function, see
+       SetCurvature() for further details. */
    static Mesh MakePeriodic(const Mesh &orig_mesh, const std::vector<int> &v2v);
 
    ///@}

mesh/mesh_readers.cpp

@@ -1151,15 +1151,24 @@ void Mesh::ReadXML_VTKMesh(std::istream &input, int &curved, int &read_gf,
    }
    if (cells_xml == NULL) { MFEM_ABORT(erstr); }
 
-   // Read the element attributes, which are stored as CellData named "material"
+   // Read the element attributes, which are stored as CellData named either
+   // "material" or "attribute". We prioritize "material" over "attribute" for
+   // backwards compatibility.
    Array<int> cell_attributes;
+   bool found_attributes = false;
    for (const XMLElement *cell_data_xml = piece->FirstChildElement();
         cell_data_xml != NULL;
         cell_data_xml = cell_data_xml->NextSiblingElement())
    {
-      if (StringCompare(cell_data_xml->Name(), "CellData")
-          && StringCompare(cell_data_xml->Attribute("Scalars"), "material"))
+      const bool is_cell_data =
+         StringCompare(cell_data_xml->Name(), "CellData");
+      const bool is_material =
+         StringCompare(cell_data_xml->Attribute("Scalars"), "material");
+      const bool is_attribute =
+         StringCompare(cell_data_xml->Attribute("Scalars"), "attribute");
+      if (is_cell_data && (is_material || (is_attribute && !found_attributes)))
       {
+         found_attributes = true;
          const XMLElement *data_xml = cell_data_xml->FirstChildElement();
          if (data_xml != NULL && StringCompare(data_xml->Name(), "DataArray"))
          {
@@ -1274,6 +1283,7 @@ void Mesh::ReadVTKMesh(std::istream &input, int &curved, int &read_gf,
    // Read the cell materials
    // bool found_material = false;
    Array<int> cell_attributes;
+   bool found_attributes = false;
    while ((input.good()))
    {
       getline(input, buff);
@@ -1281,8 +1291,10 @@ void Mesh::ReadVTKMesh(std::istream &input, int &curved, int &read_gf,
       {
          break; // We have entered the POINT_DATA block. Quit.
       }
-      else if (buff.rfind("SCALARS material") == 0)
+      else if (buff.rfind("SCALARS material") == 0 ||
+               (buff.rfind("SCALARS attribute") == 0 && !found_attributes))
       {
+         found_attributes = true;
          getline(input, buff); // LOOKUP_TABLE default
          if (buff.rfind("LOOKUP_TABLE default") != 0)
          {

miniapps/meshing/fit-node-position.cpp

@@ -206,7 +206,7 @@ int main (int argc, char *argv[])
    solver.SetMaxIter(200);
    solver.SetRelTol(1e-10);
    solver.SetAbsTol(0.0);
-   solver.EnableAdaptiveSurfaceFitting();
+   solver.SetAdaptiveSurfaceFittingScalingFactor(10);
    solver.SetTerminationWithMaxSurfaceFittingError(1e-3);
 
    // Solve.

miniapps/meshing/mesh-fitting.hpp

@@ -34,6 +34,22 @@ real_t circle_level_set(const Vector &x)
    }
 }
 
+real_t squircle_level_set(const Vector &x)
+{
+   const int dim = x.Size();
+   if (dim == 2)
+   {
+      const real_t xc = x(0) - 0.5, yc = x(1) - 0.5;
+      return std::pow(xc, 4.0) + std::pow(yc, 4.0) - std::pow(0.24, 4.0);
+   }
+   else
+   {
+      const real_t xc = x(0) - 0.5, yc = x(1) - 0.5, zc = x(2) - 0.5;
+      return std::pow(xc, 4.0) + std::pow(yc, 4.0) +
+             std::pow(zc, 4.0) - std::pow(0.24, 4.0);
+   }
+}
+
 real_t in_circle(const Vector &x, const Vector &x_center, real_t radius)
 {
    Vector x_current = x;

miniapps/meshing/pmesh-fitting.cpp

@@ -33,13 +33,15 @@
 // Compile with: make pmesh-fitting
 //
 // Sample runs:
-//  Interface fitting:
-//    mpirun -np 4 pmesh-fitting -o 3 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 5e4 -rtol 1e-5
-//    mpirun -np 4 pmesh-fitting -m square01-tri.mesh -o 3 -rs 0 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 1e4 -rtol 1e-5
+//  Surface fitting:
+//    mpirun -np 4 pmesh-fitting -o 3 -mid 58 -tid 1 -vl 1 -sfc 5e4 -rtol 1e-5
+//    mpirun -np 4 pmesh-fitting -m square01-tri.mesh -o 3 -rs 0 -mid 58 -tid 1 -vl 1 -sfc 1e4 -rtol 1e-5
 //  Surface fitting with weight adaptation and termination based on fitting error:
-//    mpirun -np 4 pmesh-fitting -o 2 -mid 2 -tid 1 -ni 100 -vl 2 -sfc 10 -rtol 1e-20 -st 0 -sfa 10.0 -sft 1e-5
-//  Fitting to Fischer-Tropsch reactor like domain (requires GSLIB):
-//  * mpirun -np 6 pmesh-fitting -m ../../data/inline-tri.mesh -o 2 -rs 4 -mid 2 -tid 1 -vl 2 -sfc 100 -rtol 1e-12 -ni 100 -li 40 -ae 1 -bnd -sbgmesh -slstype 2 -smtype 0 -sfa 10.0 -sft 1e-4 -amriter 5 -dist -mod-bndr-attr
+//    mpirun -np 4 pmesh-fitting -o 2 -mid 2 -tid 1 -vl 2 -sfc 10 -rtol 1e-20 -sfa 10.0 -sft 1e-5 -no-resid
+//  Surface fitting with weight adaptation, limit on max weight, and convergence based on residual.
+//  * mpirun -np 4 pmesh-fitting -m ../../data/inline-tri.mesh -o 2 -mid 2 -tid 4 -vl 2 -sfc 10 -rtol 1e-10 -sfa 10.0 -sft 1e-5 -bgamriter 3 -sbgmesh -ae 1 -marking -slstype 3 -resid -sfcmax 10000 -mod-bndr-attr
+//  Surface fitting to Fischer-Tropsch reactor like domain (requires GSLIB):
+//  * mpirun -np 6 pmesh-fitting -m ../../data/inline-tri.mesh -o 2 -rs 4 -mid 2 -tid 1 -vl 2 -sfc 100 -rtol 1e-12 -li 20 -ae 1 -bnd -sbgmesh -slstype 2 -smtype 0 -sfa 10.0 -sft 1e-4 -no-resid -bgamriter 5 -dist -mod-bndr-attr
 
 #include "mesh-fitting.hpp"
 
@@ -48,12 +50,16 @@ using namespace std;
 
 int main (int argc, char *argv[])
 {
-   // 0. Initialize MPI and HYPRE.
+#ifdef HYPRE_USING_GPU
+   cout << "\nThis miniapp is NOT supported with the GPU version of hypre.\n\n";
+   return MFEM_SKIP_RETURN_VALUE;
+#endif
+
    Mpi::Init(argc, argv);
    int myid = Mpi::WorldRank();
    Hypre::Init();
 
-   // 1. Set the method's default parameters.
+   // Set the method's default parameters.
    const char *mesh_file = "square01.mesh";
    int mesh_poly_deg     = 1;
    int rs_levels         = 1;
@@ -61,7 +67,6 @@ int main (int argc, char *argv[])
    int metric_id         = 2;
    int target_id         = 1;
    real_t surface_fit_const = 100.0;
-   int quad_type         = 1;
    int quad_order        = 8;
    int solver_type       = 0;
    int solver_iter       = 20;
@@ -70,27 +75,28 @@ int main (int argc, char *argv[])
 #else
    real_t solver_rtol    = 1e-10;
 #endif
-   int solver_art_type   = 0;
    int lin_solver        = 2;
    int max_lin_iter      = 100;
    bool move_bnd         = true;
-   bool visualization    = true;
+   bool visualization    = false;
    int verbosity_level   = 0;
    int adapt_eval        = 0;
    const char *devopt    = "cpu";
    real_t surface_fit_adapt = 0.0;
    real_t surface_fit_threshold = -10;
+   real_t surf_fit_const_max    = 1e20;
    bool adapt_marking     = false;
-   bool surf_bg_mesh     = false;
-   bool comp_dist     = false;
-   int surf_ls_type      = 1;
-   int marking_type      = 0;
-   bool mod_bndr_attr    = false;
-   bool material         = false;
+   bool surf_bg_mesh      = false;
+   bool comp_dist         = false;
+   int surf_ls_type       = 1;
+   int marking_type       = 0;
+   bool mod_bndr_attr     = false;
+   bool material          = false;
    int mesh_node_ordering = 0;
-   int amr_iters         = 0;
+   int bg_amr_iters       = 0;
+   bool conv_residual     = true;
 
-   // 2. Parse command-line options.
+   // Parse command-line options.
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
@@ -111,11 +117,6 @@ int main (int argc, char *argv[])
                   "5: Ideal shape, given size (in physical space)");
    args.AddOption(&surface_fit_const, "-sfc", "--surface-fit-const",
                   "Surface preservation constant.");
-   args.AddOption(&quad_type, "-qt", "--quad-type",
-                  "Quadrature rule type:\n\t"
-                  "1: Gauss-Lobatto\n\t"
-                  "2: Gauss-Legendre\n\t"
-                  "3: Closed uniform points");
    args.AddOption(&quad_order, "-qo", "--quad_order",
                   "Order of the quadrature rule.");
    args.AddOption(&solver_type, "-st", "--solver-type",
@@ -124,11 +125,6 @@ int main (int argc, char *argv[])
                   "Maximum number of Newton iterations.");
    args.AddOption(&solver_rtol, "-rtol", "--newton-rel-tolerance",
                   "Relative tolerance for the Newton solver.");
-   args.AddOption(&solver_art_type, "-art", "--adaptive-rel-tol",
-                  "Type of adaptive relative linear solver tolerance:\n\t"
-                  "0: None (default)\n\t"
-                  "1: Eisenstat-Walker type 1\n\t"
-                  "2: Eisenstat-Walker type 2");
    args.AddOption(&lin_solver, "-ls", "--lin-solver",
                   "Linear solver:\n\t"
                   "0: l1-Jacobi\n\t"
@@ -151,10 +147,12 @@ int main (int argc, char *argv[])
    args.AddOption(&devopt, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&surface_fit_adapt, "-sfa", "--adaptive-surface-fit",
-                  "Enable or disable adaptive surface fitting.");
+                  "Scaling factor for surface fitting weight.");
    args.AddOption(&surface_fit_threshold, "-sft", "--surf-fit-threshold",
                   "Set threshold for surface fitting. TMOP solver will"
                   "terminate when max surface fitting error is below this limit");
+   args.AddOption(&surf_fit_const_max, "-sfcmax", "--surf-fit-const-max",
+                  "Max surface fitting weight allowed");
    args.AddOption(&adapt_marking, "-marking", "--adaptive-marking", "-no-amarking",
                   "--no-adaptive-marking",
                   "Enable or disable adaptive marking surface fitting.");
@@ -165,9 +163,9 @@ int main (int argc, char *argv[])
                   "-no-dist","--no-comp-dist",
                   "Compute distance from 0 level set or not.");
    args.AddOption(&surf_ls_type, "-slstype", "--surf-ls-type",
-                  "1 - Circle (DEFAULT), 2 - Squircle, 3 - Butterfly.");
+                  "1 - Circle (DEFAULT), 2 - reactor level-set, 3 - squircle.");
    args.AddOption(&marking_type, "-smtype", "--surf-marking-type",
-                  "1 - Interface (DEFAULT), 2 - Boundary attribute.");
+                  "0 - Interface (DEFAULT), otherwise Boundary attribute.");
    args.AddOption(&mod_bndr_attr, "-mod-bndr-attr", "--modify-boundary-attribute",
                   "-fix-bndr-attr", "--fix-boundary-attribute",
                   "Change boundary attribute based on alignment with Cartesian axes.");
@@ -176,8 +174,11 @@ int main (int argc, char *argv[])
    args.AddOption(&mesh_node_ordering, "-mno", "--mesh_node_ordering",
                   "Ordering of mesh nodes."
                   "0 (default): byNodes, 1: byVDIM");
-   args.AddOption(&amr_iters, "-amriter", "--amr-iter",
+   args.AddOption(&bg_amr_iters, "-bgamriter", "--amr-iter",
                   "Number of amr iterations on background mesh");
+   args.AddOption(&conv_residual, "-resid", "--resid", "-no-resid",
+                  "--no-resid",
+                  "Enable residual based convergence.");
    args.Parse();
    if (!args.Good())
    {
@@ -189,7 +190,11 @@ int main (int argc, char *argv[])
    Device device(devopt);
    if (myid == 0) { device.Print();}
 
-   // 3. Initialize and refine the starting mesh.
+   MFEM_VERIFY(surface_fit_const > 0.0,
+               "This miniapp is for surface fitting only. See (p)mesh-optimizer"
+               "miniapps for general high-order mesh optimization.");
+
+   // Initialize and refine the starting mesh.
    Mesh *mesh = new Mesh(mesh_file, 1, 1, false);
    for (int lev = 0; lev < rs_levels; lev++)
    {
@@ -207,6 +212,10 @@ int main (int argc, char *argv[])
    {
       ls_coeff = new FunctionCoefficient(reactor);
    }
+   else if (surf_ls_type == 3) //squircle
+   {
+      ls_coeff = new FunctionCoefficient(squircle_level_set);
+   }
    else if (surf_ls_type == 6) // 3D shape
    {
       ls_coeff = new FunctionCoefficient(csg_cubecylsph);
@@ -220,7 +229,7 @@ int main (int argc, char *argv[])
    delete mesh;
    for (int lev = 0; lev < rp_levels; lev++) { pmesh->UniformRefinement(); }
 
-   // 4. Setup background mesh for surface fitting
+   // Setup background mesh for surface fitting
    ParMesh *pmesh_surf_fit_bg = NULL;
    if (surf_bg_mesh)
    {
@@ -240,10 +249,10 @@ int main (int argc, char *argv[])
       delete mesh_surf_fit_bg;
    }
 
-   // 5. Define a finite element space on the mesh. Here we use vector finite
-   //    elements which are tensor products of quadratic finite elements. The
-   //    number of components in the vector finite element space is specified by
-   //    the last parameter of the FiniteElementSpace constructor.
+   // Define a finite element space on the mesh. Here we use vector finite
+   // elements which are tensor products of quadratic finite elements. The
+   // number of components in the vector finite element space is specified by
+   // the last parameter of the FiniteElementSpace constructor.
    FiniteElementCollection *fec;
    if (mesh_poly_deg <= 0)
    {
@@ -254,21 +263,21 @@ int main (int argc, char *argv[])
    ParFiniteElementSpace *pfespace =
       new ParFiniteElementSpace(pmesh, fec, dim, mesh_node_ordering);
 
-   // 6. Make the mesh curved based on the above finite element space. This
-   //    means that we define the mesh elements through a fespace-based
-   //    transformation of the reference element.
+   // Make the mesh curved based on the above finite element space. This
+   // means that we define the mesh elements through a fespace-based
+   // transformation of the reference element.
    pmesh->SetNodalFESpace(pfespace);
 
-   // 7. Get the mesh nodes (vertices and other degrees of freedom in the finite
-   //    element space) as a finite element grid function in fespace. Note that
-   //    changing x automatically changes the shapes of the mesh elements.
+   // Get the mesh nodes (vertices and other degrees of freedom in the finite
+   // element space) as a finite element grid function in fespace. Note that
+   // changing x automatically changes the shapes of the mesh elements.
    ParGridFunction x(pfespace);
    pmesh->SetNodalGridFunction(&x);
    x.SetTrueVector();
 
-   // 10. Save the starting (prior to the optimization) mesh to a file. This
-   //     output can be viewed later using GLVis: "glvis -m perturbed -np
-   //     num_mpi_tasks".
+   // Save the starting (prior to the optimization) mesh to a file. This
+   // output can be viewed later using GLVis: "glvis -m perturbed -np
+   // num_mpi_tasks".
    {
       ostringstream mesh_name;
       mesh_name << "perturbed.mesh";
@@ -326,16 +335,7 @@ int main (int argc, char *argv[])
    TMOP_Integrator *tmop_integ = new TMOP_Integrator(metric, target_c);
 
    // Setup the quadrature rules for the TMOP integrator.
-   IntegrationRules *irules = NULL;
-   switch (quad_type)
-   {
-      case 1: irules = &IntRulesLo; break;
-      case 2: irules = &IntRules; break;
-      case 3: irules = &IntRulesCU; break;
-      default:
-         if (myid == 0) { cout << "Unknown quad_type: " << quad_type << endl; }
-         return 3;
-   }
+   IntegrationRules *irules = &IntRulesLo;
    tmop_integ->SetIntegrationRules(*irules, quad_order);
    if (myid == 0 && dim == 2)
    {
@@ -423,7 +423,7 @@ int main (int argc, char *argv[])
       if (surf_bg_mesh)
       {
          OptimizeMeshWithAMRAroundZeroLevelSet(*pmesh_surf_fit_bg, *ls_coeff,
-                                               amr_iters, *surf_fit_bg_gf0);
+                                               bg_amr_iters, *surf_fit_bg_gf0);
          pmesh_surf_fit_bg->Rebalance();
          surf_fit_bg_fes->Update();
          surf_fit_bg_gf0->Update();
@@ -505,6 +505,7 @@ int main (int argc, char *argv[])
          ModifyAttributeForMarkingDOFS(pmesh, mat, 0);
          ModifyAttributeForMarkingDOFS(pmesh, mat, 1);
       }
+      pmesh->SetAttributes();
 
       GridFunctionCoefficient coeff_mat(&mat);
       surf_fit_mat_gf.ProjectDiscCoefficient(coeff_mat,
@@ -579,6 +580,22 @@ int main (int argc, char *argv[])
          }
       }
 
+      // Unify marker across processor boundary
+      surf_fit_mat_gf.ExchangeFaceNbrData();
+      {
+         GroupCommunicator &gcomm = surf_fit_mat_gf.ParFESpace()->GroupComm();
+         Array<real_t> gf_array(surf_fit_mat_gf.GetData(),
+                                surf_fit_mat_gf.Size());
+         gcomm.Reduce<real_t>(gf_array, GroupCommunicator::Max);
+         gcomm.Bcast(gf_array);
+      }
+      surf_fit_mat_gf.ExchangeFaceNbrData();
+
+      for (int i = 0; i < surf_fit_mat_gf.Size(); i++)
+      {
+         surf_fit_marker[i] = surf_fit_mat_gf(i) == 1.0;
+      }
+
       // Set AdaptivityEvaluators for transferring information from initial
       // mesh to current mesh as it moves during adaptivity.
       if (adapt_eval == 0)
@@ -590,11 +607,8 @@ int main (int argc, char *argv[])
       {
 #ifdef MFEM_USE_GSLIB
          adapt_surface = new InterpolatorFP;
-         if (surf_bg_mesh)
-         {
-            adapt_grad_surface = new InterpolatorFP;
-            adapt_hess_surface = new InterpolatorFP;
-         }
+         adapt_grad_surface = new InterpolatorFP;
+         adapt_hess_surface = new InterpolatorFP;
 #else
          MFEM_ABORT("MFEM is not built with GSLIB support!");
 #endif
@@ -604,7 +618,9 @@ int main (int argc, char *argv[])
       if (!surf_bg_mesh)
       {
          tmop_integ->EnableSurfaceFitting(surf_fit_gf0, surf_fit_marker,
-                                          surf_fit_coeff, *adapt_surface);
+                                          surf_fit_coeff, *adapt_surface,
+                                          adapt_grad_surface,
+                                          adapt_hess_surface);
       }
       else
       {
@@ -632,16 +648,9 @@ int main (int argc, char *argv[])
          }
       }
    }
-   pmesh->SetAttributes();
 
-   // 13. Setup the final NonlinearForm (which defines the integral of interest,
-   //     its first and second derivatives). Here we can use a combination of
-   //     metrics, i.e., optimize the sum of two integrals, where both are
-   //     scaled by used-defined space-dependent weights.  Note that there are
-   //     no command-line options for the weights and the type of the second
-   //     metric; one should update those in the code.
+   // Setup the final NonlinearForm.
    ParNonlinearForm a(pfespace);
-   ConstantCoefficient *metric_coeff1 = NULL;
    a.AddDomainIntegrator(tmop_integ);
 
    // Compute the minimum det(J) of the starting mesh.
@@ -674,10 +683,10 @@ int main (int argc, char *argv[])
       surf_fit_coeff.constant  = surface_fit_const;
    }
 
-   // 14. Fix all boundary nodes, or fix only a given component depending on the
-   //     boundary attributes of the given mesh.  Attributes 1/2/3 correspond to
-   //     fixed x/y/z components of the node.  Attribute dim+1 corresponds to
-   //     an entirely fixed node.
+   // Fix all boundary nodes, or fix only a given component depending on the
+   // boundary attributes of the given mesh.  Attributes 1/2/3 correspond to
+   // fixed x/y/z components of the node.  Attribute dim+1 corresponds to
+   // an entirely fixed node.
    if (move_bnd == false)
    {
       Array<int> ess_bdr(pmesh->bdr_attributes.Max());
@@ -733,8 +742,7 @@ int main (int argc, char *argv[])
       a.SetEssentialVDofs(ess_vdofs);
    }
 
-   // 15. As we use the Newton method to solve the resulting nonlinear system,
-   //     here we setup the linear solver for the system's Jacobian.
+   // Setup the linear solver for the system's Jacobian.
    Solver *S = NULL, *S_prec = NULL;
 #ifdef MFEM_USE_SINGLE
    const real_t linsol_rtol = 1e-5;
@@ -784,8 +792,14 @@ int main (int argc, char *argv[])
    }
    if (surface_fit_threshold > 0)
    {
-      solver.SetTerminationWithMaxSurfaceFittingError(surface_fit_threshold);
+      solver.SetSurfaceFittingMaxErrorLimit(surface_fit_threshold);
    }
+   solver.SetSurfaceFittingConvergenceBasedOnError(!conv_residual);
+   if (conv_residual)
+   {
+      solver.SetSurfaceFittingWeightLimit(surf_fit_const_max);
+   }
+
    // Provide all integration rules in case of a mixed mesh.
    solver.SetIntegrationRules(*irules, quad_order);
    if (solver_type == 0)
@@ -797,18 +811,14 @@ int main (int argc, char *argv[])
    solver.SetRelTol(solver_rtol);
    solver.SetAbsTol(0.0);
    solver.SetMinimumDeterminantThreshold(0.001*min_detJ);
-   if (solver_art_type > 0)
-   {
-      solver.SetAdaptiveLinRtol(solver_art_type, 0.5, 0.9);
-   }
    solver.SetPrintLevel(verbosity_level >= 1 ? 1 : -1);
    solver.SetOperator(a);
    Vector b(0);
    solver.Mult(b, x.GetTrueVector());
    x.SetFromTrueVector();
 
-   // 16. Save the optimized mesh to a file. This output can be viewed later
-   //     using GLVis: "glvis -m optimized -np num_mpi_tasks".
+   // Save the optimized mesh to a file. This output can be viewed later
+   // using GLVis: "glvis -m optimized -np num_mpi_tasks".
    {
       ostringstream mesh_name;
       mesh_name << "optimized.mesh";
@@ -842,9 +852,13 @@ int main (int argc, char *argv[])
 
    if (surface_fit_const > 0.0)
    {
+      adapt_surface->ComputeAtNewPosition(x, surf_fit_gf0,
+                                          x.FESpace()->GetOrdering());
       if (visualization)
       {
-         socketstream vis2, vis3;
+         socketstream vis1, vis2, vis3;
+         common::VisualizeField(vis1, "localhost", 19916, surf_fit_gf0,
+                                "Level Set", 000, 400, 300, 300);
          common::VisualizeField(vis2, "localhost", 19916, mat,
                                 "Materials", 300, 400, 300, 300);
          common::VisualizeField(vis3, "localhost", 19916, surf_fit_mat_gf,
@@ -859,7 +873,7 @@ int main (int argc, char *argv[])
       }
    }
 
-   // 18. Visualize the mesh displacement.
+   // Visualize the mesh displacement.
    if (visualization)
    {
       x0 -= x;
@@ -870,7 +884,6 @@ int main (int argc, char *argv[])
 
    delete S;
    delete S_prec;
-   delete metric_coeff1;
    delete adapt_surface;
    delete adapt_grad_surface;
    delete adapt_hess_surface;

miniapps/multidomain/makefile

@@ -55,6 +55,9 @@ MESH_FILES = $(notdir $(wildcard $(SRC)*.mesh))
 $(MESH_FILES): %: $(SRC)%
 	ln -sf $(<) .
 multidomain: | $(MESH_FILES)
+# The target 'copy-data' is used by the makefile in ../../tests/unit
+.PHONY: copy-data
+copy-data: | $(MESH_FILES)
 endif
 
 MFEM_TESTS = MINIAPPS

tests/gitlab/get_mfem_uberenv

@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 
 uberenv_url="https://github.com/mfem/mfem-uberenv.git"
-uberenv_ref="67fab1adaf2d095ffa70dca97381abedbea15c89"
+uberenv_ref="fe5fa88876b29ff03177d44a5bd3e09c84ccdcbf"
 
 [[ ! -d tests/uberenv ]] && git clone ${uberenv_url} tests/uberenv
 cd tests/uberenv

tests/unit/data/quad_attribute.vtu

@@ -0,0 +1,58 @@
+<VTKFile type="UnstructuredGrid" version="1.0" byte_order="LittleEndian" header_type="UInt64">
+  <UnstructuredGrid>
+    <Piece NumberOfPoints="16" NumberOfCells="9">
+      <Points>
+        <DataArray type="Float32" Name="Points" NumberOfComponents="3" format="ascii" RangeMin="0" RangeMax="4.242640687119285">
+          0 0 0 1 0 0
+          2 0 0 3 0 0
+          0 1 0 1 1 0
+          2 1 0 3 1 0
+          0 2 0 1 2 0
+          2 2 0 3 2 0
+          0 3 0 1 3 0
+          2 3 0 3 3 0
+          <InformationKey name="L2_NORM_FINITE_RANGE" location="vtkDataArray" length="2">
+            <Value index="0">
+              0
+            </Value>
+            <Value index="1">
+              4.2426406871
+            </Value>
+          </InformationKey>
+          <InformationKey name="L2_NORM_RANGE" location="vtkDataArray" length="2">
+            <Value index="0">
+              0
+            </Value>
+            <Value index="1">
+              4.2426406871
+            </Value>
+          </InformationKey>
+        </DataArray>
+      </Points>
+      <Cells>
+        <DataArray type="Int64" Name="connectivity" format="ascii" RangeMin="0" RangeMax="15">
+          0 1 5 4 1 2
+          6 5 2 3 7 6
+          4 5 9 8 5 6
+          10 9 6 7 11 10
+          8 9 13 12 9 10
+          14 13 10 11 15 14
+        </DataArray>
+        <DataArray type="Int64" Name="offsets" format="ascii" RangeMin="4" RangeMax="36">
+          4 8 12 16 20 24
+          28 32 36
+        </DataArray>
+        <DataArray type="UInt8" Name="types" format="ascii" RangeMin="9" RangeMax="9">
+          9 9 9 9 9 9
+          9 9 9
+        </DataArray>
+      </Cells>
+      <CellData Scalars="attribute">
+        <DataArray type="UInt8" Name="attribute" format="ascii">
+          1 2 3 4 5 6
+          7 8 9
+        </DataArray>
+      </CellData>
+    </Piece>
+  </UnstructuredGrid>
+</VTKFile>

tests/unit/data/quad_material_attribute.vtu

@@ -0,0 +1,64 @@
+<VTKFile type="UnstructuredGrid" version="1.0" byte_order="LittleEndian" header_type="UInt64">
+  <UnstructuredGrid>
+    <Piece NumberOfPoints="16" NumberOfCells="9">
+      <Points>
+        <DataArray type="Float32" Name="Points" NumberOfComponents="3" format="ascii" RangeMin="0" RangeMax="4.242640687119285">
+          0 0 0 1 0 0
+          2 0 0 3 0 0
+          0 1 0 1 1 0
+          2 1 0 3 1 0
+          0 2 0 1 2 0
+          2 2 0 3 2 0
+          0 3 0 1 3 0
+          2 3 0 3 3 0
+          <InformationKey name="L2_NORM_FINITE_RANGE" location="vtkDataArray" length="2">
+            <Value index="0">
+              0
+            </Value>
+            <Value index="1">
+              4.2426406871
+            </Value>
+          </InformationKey>
+          <InformationKey name="L2_NORM_RANGE" location="vtkDataArray" length="2">
+            <Value index="0">
+              0
+            </Value>
+            <Value index="1">
+              4.2426406871
+            </Value>
+          </InformationKey>
+        </DataArray>
+      </Points>
+      <Cells>
+        <DataArray type="Int64" Name="connectivity" format="ascii" RangeMin="0" RangeMax="15">
+          0 1 5 4 1 2
+          6 5 2 3 7 6
+          4 5 9 8 5 6
+          10 9 6 7 11 10
+          8 9 13 12 9 10
+          14 13 10 11 15 14
+        </DataArray>
+        <DataArray type="Int64" Name="offsets" format="ascii" RangeMin="4" RangeMax="36">
+          4 8 12 16 20 24
+          28 32 36
+        </DataArray>
+        <DataArray type="UInt8" Name="types" format="ascii" RangeMin="9" RangeMax="9">
+          9 9 9 9 9 9
+          9 9 9
+        </DataArray>
+      </Cells>
+      <CellData Scalars="attribute">
+        <DataArray type="UInt8" Name="attribute" format="ascii">
+          10 11 12 13 14 15
+          16 17 18
+        </DataArray>
+      </CellData>
+      <CellData Scalars="material">
+        <DataArray type="UInt8" Name="material" format="ascii">
+          1 2 3 4 5 6
+          7 8 9
+        </DataArray>
+      </CellData>
+    </Piece>
+  </UnstructuredGrid>
+</VTKFile>

tests/unit/fem/test_coefficient.cpp

@@ -303,3 +303,31 @@ TEST_CASE("Piecewise Matrix Coefficient", "[Coefficient]")
       REQUIRE(m.FNorm() == MFEM_Approx(twoNorm));
    }
 }
+
+TEST_CASE("Symmetric Matrix Coefficient", "[Coefficient]")
+{
+   int d = 3;
+   int qfdim = d*(d+1)/2;
+
+   Vector values(qfdim);
+   values.Randomize();
+
+   // Create symmetric matrix initialized w/ values
+   DenseSymmetricMatrix symMat(values.GetData(), d);
+
+   SymmetricMatrixConstantCoefficient symCoeff(symMat);
+
+   // Make mesh of size 1
+   Mesh m = Mesh::MakeCartesian1D(1);
+
+   // Define qspace on mesh w/ 1 integration point
+   QuadratureSpace qspace(&m, 1);
+
+   // Define qf
+   QuadratureFunction qf(qspace, qfdim);
+
+   symCoeff.ProjectSymmetric(qf);
+
+   // Require equality
+   REQUIRE(qf.DistanceTo(values) == MFEM_Approx(0.0));
+}

tests/unit/makefile

@@ -179,6 +179,15 @@ $(eval $(call psedov_tests,debug,DEBUG,debug))
 $(eval $(call psedov_tests,cuda,CUDA,cuda))
 $(eval $(call psedov_tests,cuda_uvm,CUDA_UVM,cuda:uvm))
 
+# For out-of-source builds, copy the meshes in ../../miniapps/multidomain from
+# the source location; these are used by 'punit_tests'.
+ifneq ($(SRC),)
+.PHONY: copy-miniapps-multidomain-data
+copy-miniapps-multidomain-data:
+	$(MAKE) -C ../../miniapps/multidomain copy-data
+punit_tests: | copy-miniapps-multidomain-data
+endif
+
 # For out-of-source builds, copy the meshes in ../../miniapps/meshing from the
 # source location; these are used by the TMOP tests.
 .PHONY: copy-miniapps-meshing-data

tests/unit/mesh/test_vtu.cpp

@@ -29,16 +29,35 @@ TEST_CASE("VTU XML Reader", "[Mesh][VTU][XML]")
       "quad_binary_compress.vtu"
    });
 #endif
-   for (const std::string &fname : mesh_filenames)
+
+   const auto fname = GENERATE_COPY(from_range(mesh_filenames));
+
+   Mesh mesh = Mesh::LoadFromFile("data/" + fname);
+   REQUIRE(mesh.Dimension() == 2);
+   REQUIRE(mesh.GetNE() == 9);
+   REQUIRE(mesh.GetNV() == 16);
+   REQUIRE(mesh.HasGeometry(Geometry::POINT));
+   REQUIRE(mesh.HasGeometry(Geometry::SEGMENT));
+   REQUIRE(mesh.HasGeometry(Geometry::SQUARE));
+   REQUIRE(mesh.GetNumGeometries(2) == 1);
+}
+
+TEST_CASE("VTU Attributes", "[VTU][XML]")
+{
+   // quad_attribute.vtu contains the attributes in a cell data array named
+   // "attribute"
+   Mesh mesh_1 = Mesh::LoadFromFile("data/quad_attribute.vtu");
+   // quad_material_attribute.vtu has cell data arrays named "material" and
+   // "attribute". The one named "material" should take precedence.
+   Mesh mesh_2 = Mesh::LoadFromFile("data/quad_material_attribute.vtu");
+
+   REQUIRE(mesh_1.GetNE() == 9);
+   REQUIRE(mesh_2.GetNE() == 9);
+
+   for (int i = 0; i < mesh_1.GetNE(); ++i)
    {
-      Mesh mesh = Mesh::LoadFromFile(("data/" + fname).c_str());
-      REQUIRE(mesh.Dimension() == 2);
-      REQUIRE(mesh.GetNE() == 9);
-      REQUIRE(mesh.GetNV() == 16);
-      REQUIRE(mesh.HasGeometry(Geometry::POINT));
-      REQUIRE(mesh.HasGeometry(Geometry::SEGMENT));
-      REQUIRE(mesh.HasGeometry(Geometry::SQUARE));
-      REQUIRE(mesh.GetNumGeometries(2) == 1);
+      REQUIRE(mesh_1.GetAttribute(i) == i+1);
+      REQUIRE(mesh_2.GetAttribute(i) == i+1);
    }
 }
 
@@ -52,8 +71,7 @@ TEST_CASE("VTU XML Compressed Blocks", "[VTU][XML][MFEMData]")
                       "bracket_inline_compressed.vtu"
                    );
 
-   std::string mesh_path = mfem_data_dir + "/vtk/" + filename;
-   Mesh mesh = Mesh::LoadFromFile(mesh_path.c_str());
+   Mesh mesh = Mesh::LoadFromFile(mfem_data_dir + "/vtk/" + filename);
 
    REQUIRE(mesh.Dimension() == 3);
    REQUIRE(mesh.GetNE() == 206208);