Removed blobs, added .gitignore for this directory.

CHANGELOG

@@ -10,48 +10,19 @@
 
 Version 4.0.1 (development)
 ===========================
+- Improved RAJA backend
+- Improved multi-GPU MPI communication.
 
-Improved GPU support
---------------------
+GPU support
+-----------
 - Added initial support for AMD GPUs based on HIP: a C++ runtime API and kernel
-  language that can run on both AMD and NVIDIA hardware. With this change, the
-  list of backends is: "occa-cuda", "raja-cuda", "cuda", "hip", "occa-omp",
-  "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".
-
-- Improved RAJA backend and multi-GPU MPI communications.
-
-Discretization improvements
----------------------------
-- Added support for non-conforming prism AMR, including coarsening and parallel
-  load balancing. Anisotropic prism refinement is only available in the serial
-  version at the moment.
-
-Meshing improvements
---------------------
-- The TMOP mesh optimization algorithms were extended to support r-adaptivity.
-  Target matrices can now be constructed either via a given analytical function
-  (e.g. spatial dependence of size, aspect ratio, etc., for each element) or via
-  a (Par)GridFunction specified on the original mesh.
-
-- The TMOP mesh optimization algorithms have been improved to support AMR meshes.
-
-- Added support for creating refined versions of periodic meshes, making use of
-  the new L2ElementRestriction class. This class also allows for computing
-  geometric factors on periodic meshes using partial assembly.
-
-- Improved element numbering after uniform mesh refinement.
-
-New and updated examples and miniapps
--------------------------------------
-- The mesh-optimizer and pmesh-optimizer miniapps have been updated to
-  demonstrate the new r-adaptivity capabilities of TMOP.
-
-- The (p)mesh-optimizer miniapp has been updated to demonstrate mesh
-  optimization for an AMR mesh.
+  language that can run on both AMD and NVIDIA hardware. The list of current
+  backends is: "occa-cuda", "raja-cuda", "cuda", "hip", "occa-omp", "raja-omp",
+  "omp", "occa-cpu", "raja-cpu", and "cpu".
 
 Miscellaneous
 -------------
-- Upgraded the SUNDIALS interface to utilize SUNDIALS version 5.0.  This
+- Upgraded the SUNDIALS interface to utilize SUNDIALS version 5.0.  This 
   necessitated a complete rework of the interface and requires changes at
   the application level.  Example usage of this new interface can be found
   in the examples/sundials directory.
@@ -98,6 +69,7 @@ GPU support
 
 Discretization improvements
 ---------------------------
+
 - Partial assembled finite element operators are now available in the core
   library, based on the new classes PABilinearFormExtension, ElementRestriction,
   DofToQuad and GeometricFactors (associated with the classes BilinearForm,

CMakeLists.txt

@@ -254,7 +254,7 @@ endif()
 
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
-  find_package(Axom REQUIRED Axom)
+  find_package(Axom REQUIRED Sidre SLIC axom_utils)
 endif()
 
 # PUMI

INSTALL

@@ -383,11 +383,11 @@ MFEM_USE_MPFR = YES/NO
    see below.
 
 MFEM_USE_SIDRE = YES/NO
-   Sidre is a component of LLNL's axom project, https://github.com/LLNL/axom,
-   that provides an HDF5-based file format for visualization or restart
-   capability following the Conduit (https://github.com/LLNL/conduit) mesh
-   blueprint specification. When enabled, this option requires installation of
-   HDF5 (see also MFEM_USE_NETCDF), Conduit and LLNL's axom project.
+   Sidre is a component of LLNL's axom project, http://goo.gl/cZyJdn, that
+   provides an HDF5-based file format for visualization or restart capability
+   following the Conduit (https://github.com/LLNL/conduit) mesh blueprint
+   specification. When enabled, this option requires installation of HDF5 (see
+   also MFEM_USE_NETCDF), Conduit and LLNL's axom project.
 
 MFEM_USE_CONDUIT = YES/NO
    Enables support for converting MFEM Mesh and Grid Function objects to and
@@ -543,8 +543,7 @@ The specific libraries and their options are:
   Options: PETSC_OPT, PETSC_LIB.
 
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
-  Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
-  URL: https://github.com/LLNL/axom
+  URL: http://goo.gl/cZyJdn (axom, to be released)
        https://github.com/LLNL/conduit (Conduit)
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: SIDRE_OPT, SIDRE_LIB.
@@ -572,7 +571,6 @@ The specific libraries and their options are:
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
 
 - RAJA, used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
   URL: https://github.com/LLNL/RAJA
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
 
@@ -715,7 +713,6 @@ MFEM_USE_PUMI
 MFEM_USE_CUDA
 MFEM_USE_OCCA
 MFEM_USE_RAJA
-MFEM_USE_SIDRE
 
 The following options are CMake specific:
 
@@ -764,7 +761,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - PUMI
  - OCCA
  - RAJA
- - AXOM - Used when MFEM_USE_SIDRE is enabled
 
 The following built-in CMake packages are also used:
 

config/cmake/modules/FindAxom.cmake

@@ -18,4 +18,6 @@ include(MfemCmakeUtilities)
 # Note: components are enabled based on the find_package() parameters.
 mfem_find_package(Axom AXOM AXOM_DIR "include" "" "lib" ""
   "Paths to headers required by Axom." "Libraries required by Axom."
-  ADD_COMPONENT Axom "include" axom/config.hpp "lib" axom)
+  ADD_COMPONENT Sidre "include" sidre/sidre.hpp "lib" sidre
+  ADD_COMPONENT SLIC "include" slic/slic.hpp "lib" slic
+  ADD_COMPONENT axom_utils "include" axom_utils/Utilities.hpp "lib" axom_utils)

config/defaults.cmake

@@ -154,7 +154,7 @@ set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
 
 set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 # May need to add "Boost" as requirement.
-set(Axom_REQUIRED_PACKAGES "Conduit/relay/blueprint" CACHE STRING
+set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
     "Additional packages required by Axom.")
 
 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING

config/defaults.mk

@@ -299,7 +299,7 @@ SIDRE_LIB = \
    -Wl,-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
    -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
-   -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl
+   -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
 # PUMI
 # Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h

examples/ex12p.cpp

@@ -5,11 +5,11 @@
 // Sample runs:
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh
-//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
-//    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
-//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
+//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 79 -n 10 -o 2 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3876
+//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 79
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3876 -o 2 -sys
-//    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4526 -n 6 -o 3 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-hex-nurbs.mesh
 //

examples/ex17.cpp

@@ -252,7 +252,7 @@ int main(int argc, char *argv[])
    }
    else
    {
-      GMRES(A, M, B, X, 3, 5000, 100, rtol*rtol, 0.0);
+      GMRES(A, M, B, X, 3, 5000, 50, rtol*rtol, 0.0);
    }
 #else
    // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.

examples/ex3p.cpp

@@ -102,7 +102,8 @@ int main(int argc, char *argv[])
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
    {
-      int ref_levels = (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels =
+         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();

examples/makefile

@@ -22,7 +22,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
-  ex18 ex19 ex20 ex21
+  ex18 ex19 ex20 ex21 drl4amr
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
  ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p
 
@@ -71,6 +71,10 @@ ifeq ($(MFEM_USE_MPI),YES)
 ex18p: $(SRC)ex18.hpp
 endif
 
+drl4amr:
+	python drl4amr.py build
+	g++ -pthread -shared -Wl,-z,relro build/temp.linux-x86_64-2.7/drl4amr.o -L/usr/lib64 -lpython2.7 -o build/lib.linux-x86_64-2.7/drl4amr.so -L.. -lmfem
+
 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)
 test: $(SUBDIRS_TEST)

fem/CMakeLists.txt

@@ -32,7 +32,6 @@ set(SRCS
   nonlininteg.cpp
   staticcond.cpp
   tmop.cpp
-  tmop_tools.cpp
   )
 
 set(HDRS
@@ -65,7 +64,6 @@ set(HDRS
   tfespace.hpp
   tintrules.hpp
   tmop.hpp
-  tmop_tools.hpp
   )
 
 if (MFEM_USE_SIDRE)

fem/bilininteg_diffusion.cpp

@@ -27,7 +27,7 @@ static void OccaPADiffusionSetup2D(const int D1D,
                                    const int NE,
                                    const Array<double> &W,
                                    const Vector &J,
-                                   const Vector &C,
+                                   const double COEFF,
                                    Vector &op)
 {
    occa::properties props;
@@ -35,9 +35,7 @@ static void OccaPADiffusionSetup2D(const int D1D,
    props["defines/Q1D"] = Q1D;
    const occa::memory o_W = OccaMemoryRead(W.GetMemory(), W.Size());
    const occa::memory o_J = OccaMemoryRead(J.GetMemory(), J.Size());
-   const occa::memory o_C = OccaMemoryRead(C.GetMemory(), C.Size());
    occa::memory o_op = OccaMemoryWrite(op.GetMemory(), op.Size());
-   const bool const_c = C.Size() == 1;
    const occa_id_t id = std::make_pair(D1D,Q1D);
    static occa_kernel_t OccaDiffSetup2D_ker;
    if (OccaDiffSetup2D_ker.find(id) == OccaDiffSetup2D_ker.end())
@@ -47,7 +45,7 @@ static void OccaPADiffusionSetup2D(const int D1D,
                                      "DiffusionSetup2D", props);
       OccaDiffSetup2D_ker.emplace(id, DiffusionSetup2D);
    }
-   OccaDiffSetup2D_ker.at(id)(NE, o_W, o_J, o_C, o_op, const_c);
+   OccaDiffSetup2D_ker.at(id)(NE, o_W, o_J, COEFF, o_op);
 }
 
 static void OccaPADiffusionSetup3D(const int D1D,
@@ -55,7 +53,7 @@ static void OccaPADiffusionSetup3D(const int D1D,
                                    const int NE,
                                    const Array<double> &W,
                                    const Vector &J,
-                                   const Vector &C,
+                                   const double COEFF,
                                    Vector &op)
 {
    occa::properties props;
@@ -63,9 +61,7 @@ static void OccaPADiffusionSetup3D(const int D1D,
    props["defines/Q1D"] = Q1D;
    const occa::memory o_W = OccaMemoryRead(W.GetMemory(), W.Size());
    const occa::memory o_J = OccaMemoryRead(J.GetMemory(), J.Size());
-   const occa::memory o_C = OccaMemoryRead(C.GetMemory(), C.Size());
    occa::memory o_op = OccaMemoryWrite(op.GetMemory(), op.Size());
-   const bool const_c = C.Size() == 1;
    const occa_id_t id = std::make_pair(D1D,Q1D);
    static occa_kernel_t OccaDiffSetup3D_ker;
    if (OccaDiffSetup3D_ker.find(id) == OccaDiffSetup3D_ker.end())
@@ -75,7 +71,7 @@ static void OccaPADiffusionSetup3D(const int D1D,
                                      "DiffusionSetup3D", props);
       OccaDiffSetup3D_ker.emplace(id, DiffusionSetup3D);
    }
-   OccaDiffSetup3D_ker.at(id)(NE, o_W, o_J, o_C, o_op, const_c);
+   OccaDiffSetup3D_ker.at(id)(NE, o_W, o_J, COEFF, o_op);
 }
 #endif // MFEM_USE_OCCA
 
@@ -84,16 +80,14 @@ static void PADiffusionSetup2D(const int Q1D,
                                const int NE,
                                const Array<double> &w,
                                const Vector &j,
-                               const Vector &c,
-                               Vector &d)
+                               const double COEFF,
+                               Vector &op)
 {
    const int NQ = Q1D*Q1D;
-   const bool const_c = c.Size() == 1;
    auto W = w.Read();
 
    auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
-   auto D = Reshape(d.Write(), NQ, 3, NE);
+   auto y = Reshape(op.Write(), NQ, 3, NE);
 
    MFEM_FORALL(e, NE,
    {
@@ -103,11 +97,10 @@ static void PADiffusionSetup2D(const int Q1D,
          const double J21 = J(q,1,0,e);
          const double J12 = J(q,0,1,e);
          const double J22 = J(q,1,1,e);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double c_detJ = W[q] * coeff / ((J11*J22)-(J21*J12));
-         D(q,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
-         D(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
-         D(q,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
+         const double c_detJ = W[q] * COEFF / ((J11*J22)-(J21*J12));
+         y(q,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
+         y(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
+         y(q,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
       }
    });
 }
@@ -117,15 +110,13 @@ static void PADiffusionSetup3D(const int Q1D,
                                const int NE,
                                const Array<double> &w,
                                const Vector &j,
-                               const Vector &c,
-                               Vector &d)
+                               const double COEFF,
+                               Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
-   const bool const_c = c.Size() == 1;
    auto W = w.Read();
    auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
-   auto D = Reshape(d.Write(), NQ, 6, NE);
+   auto y = Reshape(op.Write(), NQ, 6, NE);
    MFEM_FORALL(e, NE,
    {
       for (int q = 0; q < NQ; ++q)
@@ -142,8 +133,7 @@ static void PADiffusionSetup3D(const int Q1D,
          const double detJ = J11 * (J22 * J33 - J32 * J23) -
          /* */               J21 * (J12 * J33 - J32 * J13) +
          /* */               J31 * (J12 * J23 - J22 * J13);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double c_detJ = W[q] * coeff / detJ;
+         const double c_detJ = W[q] * COEFF / detJ;
          // adj(J)
          const double A11 = (J22 * J33) - (J23 * J32);
          const double A12 = (J32 * J13) - (J12 * J33);
@@ -155,12 +145,12 @@ static void PADiffusionSetup3D(const int Q1D,
          const double A32 = (J31 * J12) - (J11 * J32);
          const double A33 = (J11 * J22) - (J12 * J21);
          // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-         D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-         D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-         D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-         D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-         D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-         D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+         y(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+         y(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+         y(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+         y(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+         y(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+         y(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
       }
    });
 }
@@ -171,8 +161,8 @@ static void PADiffusionSetup(const int dim,
                              const int NE,
                              const Array<double> &W,
                              const Vector &J,
-                             const Vector &C,
-                             Vector &D)
+                             const double COEFF,
+                             Vector &op)
 {
    if (dim == 1) { MFEM_ABORT("dim==1 not supported in PADiffusionSetup"); }
    if (dim == 2)
@@ -180,22 +170,22 @@ static void PADiffusionSetup(const int dim,
 #ifdef MFEM_USE_OCCA
       if (DeviceCanUseOcca())
       {
-         OccaPADiffusionSetup2D(D1D, Q1D, NE, W, J, C, D);
+         OccaPADiffusionSetup2D(D1D, Q1D, NE, W, J, COEFF, op);
          return;
       }
 #endif // MFEM_USE_OCCA
-      PADiffusionSetup2D(Q1D, NE, W, J, C, D);
+      PADiffusionSetup2D(Q1D, NE, W, J, COEFF, op);
    }
    if (dim == 3)
    {
 #ifdef MFEM_USE_OCCA
       if (DeviceCanUseOcca())
       {
-         OccaPADiffusionSetup3D(D1D, Q1D, NE, W, J, C, D);
+         OccaPADiffusionSetup3D(D1D, Q1D, NE, W, J, COEFF, op);
          return;
       }
 #endif // MFEM_USE_OCCA
-      PADiffusionSetup3D(Q1D, NE, W, J, C, D);
+      PADiffusionSetup3D(Q1D, NE, W, J, COEFF, op);
    }
 }
 
@@ -215,32 +205,11 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.Write(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-   PADiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J, coeff,
-                    pa_data);
+   ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
+   MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+   const double coeff = cQ->constant;
+   PADiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
+                    coeff, pa_data);
 }
 
 #ifdef MFEM_USE_OCCA

fem/bilininteg_mass.cpp

@@ -38,40 +38,24 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(ne*nq, Device::GetMemoryType());
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.Write(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   ConstantCoefficient *const_coeff = dynamic_cast<ConstantCoefficient*>(Q);
+   // TODO: other types of coefficients ...
    if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
    if (dim==2)
    {
+      double constant = 0.0;
+      if (const_coeff)
+      {
+         constant = const_coeff->constant;
+      }
+      else
+      {
+         MFEM_ABORT("Coefficient type not supported");
+      }
       const int NE = ne;
       const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
       auto w = ir->GetWeights().Read();
       auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
-      auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
       auto v = Reshape(pa_data.Write(), NQ, NE);
       MFEM_FORALL(e, NE,
       {
@@ -82,20 +66,25 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
             const double J21 = J(q,0,1,e);
             const double J22 = J(q,1,1,e);
             const double detJ = (J11*J22)-(J21*J12);
-            const double coeff = const_c ? C(0,0) : C(q,e);
-            v(q,e) =  w[q] * coeff * detJ;
+            v(q,e) =  w[q] * constant * detJ;
          }
       });
    }
    if (dim==3)
    {
+      double constant = 0.0;
+      if (const_coeff)
+      {
+         constant = const_coeff->constant;
+      }
+      else
+      {
+         MFEM_ABORT("Coefficient type not supported");
+      }
       const int NE = ne;
       const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
       auto W = ir->GetWeights().Read();
       auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
-      auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
       auto v = Reshape(pa_data.Write(), NQ,NE);
       MFEM_FORALL(e, NE,
       {
@@ -107,8 +96,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
             const double detJ = J11 * (J22 * J33 - J32 * J23) -
             /* */               J21 * (J12 * J33 - J32 * J13) +
             /* */               J31 * (J12 * J23 - J22 * J13);
-            const double coeff = const_c ? C(0,0) : C(q,e);
-            v(q,e) = W[q] * coeff * detJ;
+            v(q,e) = W[q] * constant * detJ;
          }
       });
    }
@@ -763,7 +751,7 @@ static void PAMassApply(const int dim,
 #endif // MFEM_USE_OCCA
    if (dim == 2)
    {
-      switch ((D1D << 4) | Q1D)
+      switch ((D1D << 4 ) | Q1D)
       {
          case 0x22: return SmemPAMassApply2D<2,2,16>(NE, B, Bt, op, x, y);
          case 0x33: return SmemPAMassApply2D<3,3,16>(NE, B, Bt, op, x, y);
@@ -778,7 +766,7 @@ static void PAMassApply(const int dim,
    }
    else if (dim == 3)
    {
-      switch ((D1D << 4) | Q1D)
+      switch ((D1D << 4 ) | Q1D)
       {
          case 0x23: return SmemPAMassApply3D<2,3>(NE, B, Bt, op, x, y);
          case 0x34: return SmemPAMassApply3D<3,4>(NE, B, Bt, op, x, y);

fem/fe.cpp

@@ -9635,7 +9635,6 @@ void L2_TetrahedronElement::ProjectDelta(int vertex, Vector &dofs) const
             const IntegrationPoint &ip = Nodes.IntPoint(i);
             dofs[i] = pow(ip.y, Order);
          }
-         break;
       case 3:
          for (int i = 0; i < Dof; i++)
          {
@@ -11966,10 +11965,6 @@ Linear3DFiniteElement TetrahedronFE;
 // Object declared in mesh/wedge.hpp.
 // Defined here to ensure it is constructed after 'poly1d' and before
 // 'Geometries'.
-// TODO: define as thread_local to prevent race conditions in GLVis, because
-// there is no "LinearWedgeFiniteElement" and WedgeFE is in turn used from two
-// different threads for different things in GLVis. We also don't want to turn
-// MFEM_THREAD_SAFE on globally. (See PR #731)
 H1_WedgeElement WedgeFE(1);
 
 // Object declared in geom.hpp.

fem/fem.hpp

@@ -31,7 +31,6 @@
 #include "estimators.hpp"
 #include "staticcond.hpp"
 #include "tmop.hpp"
-#include "tmop_tools.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/fespace.cpp

@@ -567,40 +567,6 @@ bool FiniteElementSpace::DofFinalizable(int dof, const Array<bool>& finalized,
    return true;
 }
 
-void FiniteElementSpace::GetDegenerateFaceDofs(int index,
-                                               Array<int> &dofs) const
-{
-   // In NC meshes with prisms, a special constraint occurs where a prism edge
-   // is slave to a quadrilateral face. Rather than introduce a new edge-face
-   // constraint type, we handle such cases as degenerate face-face constraints,
-   // where the point-matrix rectangle has zero height. This method returns
-   // DOFs for the first edge of the rectangle, duplicated in the orthogonal
-   // direction, to resemble DOFs for a quadrilateral face. The extra DOFs are
-   // ignored by FiniteElementSpace::AddDependencies.
-
-   Array<int> edof;
-   GetEdgeDofs(-1 - index, edof);
-
-   int nv = fec->DofForGeometry(Geometry::POINT);
-   int ne = fec->DofForGeometry(Geometry::SEGMENT);
-   int nn = 2*nv + ne;
-
-   dofs.SetSize(nn*nn);
-   dofs = edof[0];
-
-   // copy first two vertex DOFs
-   for (int i = 0; i < nv; i++)
-   {
-      dofs[i] = edof[i];
-      dofs[nv+i] = edof[nv+i];
-   }
-   // copy first edge DOFs
-   for (int i = 0; i < ne; i++)
-   {
-      dofs[4*nv + i] = edof[2*nv + i];
-   }
-}
-
 void
 FiniteElementSpace::GetEntityDofs(int entity, int index, Array<int> &dofs) const
 {
@@ -608,8 +574,7 @@ FiniteElementSpace::GetEntityDofs(int entity, int index, Array<int> &dofs) const
    {
       case 0: GetVertexDofs(index, dofs); break;
       case 1: GetEdgeDofs(index, dofs); break;
-      case 2: (index >= 0) ? GetFaceDofs(index, dofs)
-         /*             */ : GetDegenerateFaceDofs(index, dofs);
+      case 2: GetFaceDofs(index, dofs); break;
    }
 }
 
@@ -632,33 +597,28 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    // collect local edge/face dependencies
    for (int entity = 1; entity <= 2; entity++)
    {
-      const NCMesh::NCList &list = mesh->ncmesh->GetNCList(entity);
+      const NCMesh::NCList &list = (entity > 1) ? mesh->ncmesh->GetFaceList()
+                                   /*        */ : mesh->ncmesh->GetEdgeList();
       if (!list.masters.size()) { continue; }
 
-      Array<int> master_dofs, slave_dofs;
-
       IsoparametricTransformation T;
-      DenseMatrix I;
+      if (entity > 1) { T.SetFE(&QuadrilateralFE); }
+      else { T.SetFE(&SegmentFE); }
+
+      Geometry::Type geom = (entity > 1) ? Geometry::SQUARE : Geometry::SEGMENT;
+      const FiniteElement* fe = fec->FiniteElementForGeometry(geom);
+      if (!fe) { continue; }
+
+      Array<int> master_dofs, slave_dofs;
+      DenseMatrix I(fe->GetDof());
 
       // loop through all master edges/faces, constrain their slave edges/faces
       for (unsigned mi = 0; mi < list.masters.size(); mi++)
       {
          const NCMesh::Master &master = list.masters[mi];
-
          GetEntityDofs(entity, master.index, master_dofs);
          if (!master_dofs.Size()) { continue; }
 
-         const FiniteElement* fe = fec->FiniteElementForGeometry(master.Geom());
-         if (!fe) { continue; }
-
-         switch (master.geom)
-         {
-            case Geometry::SQUARE:   T.SetFE(&QuadrilateralFE); break;
-            case Geometry::TRIANGLE: T.SetFE(&TriangleFE); break;
-            case Geometry::SEGMENT:  T.SetFE(&SegmentFE); break;
-            default: MFEM_ABORT("unsupported geometry");
-         }
-
          for (int si = master.slaves_begin; si < master.slaves_end; si++)
          {
             const NCMesh::Slave &slave = list.slaves[si];
@@ -831,14 +791,8 @@ const Operator *FiniteElementSpace::GetElementRestriction(
 {
    // Check if we have a discontinuous space using the FE collection:
    const L2_FECollection *dg_space = dynamic_cast<const L2_FECollection*>(fec);
-   if (dg_space)
-   {
-      if (L2E_nat.Ptr() == NULL)
-      {
-         L2E_nat.Reset(new L2ElementRestriction(*this));
-      }
-      return L2E_nat.Ptr();
-   }
+   if (dg_space) { return NULL; }
+   // TODO: support other DG collections.
    if (e_ordering == ElementDofOrdering::LEXICOGRAPHIC)
    {
       if (L2E_lex.Ptr() == NULL)
@@ -951,7 +905,7 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
    const FiniteElement *fe = fec->FiniteElementForGeometry(geom);
 
    const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();
-   const DenseTensor &pmats = rtrans.point_matrices[geom];
+   const DenseTensor &pmats = rtrans.GetPointMatrices(geom);
 
    int nmat = pmats.SizeK();
    int ldof = fe->GetDof(); // assuming the same FE everywhere
@@ -990,9 +944,7 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
    : fespace(fespace)
    , old_elem_dof(old_elem_dof)
 {
-   const Mesh* mesh = fespace->GetMesh();
-   MFEM_VERIFY(mesh->ReduceInt(fespace->GetNDofs()) >=
-               mesh->ReduceInt(old_ndofs),
+   MFEM_VERIFY(fespace->GetNDofs() >= old_ndofs,
                "Previous space is not coarser.");
 
    width = old_ndofs * fespace->GetVDim();
@@ -1102,7 +1054,7 @@ FiniteElementSpace::DerefinementOperator::DerefinementOperator(
          f_fes->fec->FiniteElementForGeometry(geom);
       const FiniteElement *coarse_fe =
          c_fes->fec->FiniteElementForGeometry(geom);
-      const DenseTensor &pmats = rtrans.point_matrices[geom];
+      const DenseTensor &pmats = rtrans.GetPointMatrices(geom);
 
       lP.SetSize(fine_fe->GetDof(), coarse_fe->GetDof(), pmats.SizeK());
       lM.SetSize(fine_fe->GetDof(),   fine_fe->GetDof(), pmats.SizeK());
@@ -1218,7 +1170,7 @@ void FiniteElementSpace::GetLocalDerefinementMatrices(Geometry::Type geom,
 
    const CoarseFineTransformations &dtrans =
       mesh->ncmesh->GetDerefinementTransforms();
-   const DenseTensor &pmats = dtrans.point_matrices[geom];
+   const DenseTensor &pmats = dtrans.GetPointMatrices(geom);
 
    const int nmat = pmats.SizeK();
    const int ldof = fe->GetDof();
@@ -1323,7 +1275,7 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
       coarse_fes.fec->FiniteElementForGeometry(geom);
 
    const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();
-   const DenseTensor &pmats = rtrans.point_matrices[geom];
+   const DenseTensor &pmats = rtrans.GetPointMatrices(geom);
 
    int nmat = pmats.SizeK();
 
@@ -1416,26 +1368,31 @@ void FiniteElementSpace::UpdateNURBS()
 void FiniteElementSpace::Construct()
 {
    // This method should be used only for non-NURBS spaces.
-   MFEM_VERIFY(!NURBSext, "internal error");
+   MFEM_ASSERT(!NURBSext, "internal error");
 
    elem_dof = NULL;
    bdrElem_dof = NULL;
 
+   nvdofs = mesh->GetNV() * fec->DofForGeometry(Geometry::POINT);
+
+   if ( mesh->Dimension() > 1 )
+   {
+      nedofs = mesh->GetNEdges() * fec->DofForGeometry(Geometry::SEGMENT);
+   }
+   else
+   {
+      nedofs = 0;
+   }
+
    ndofs = 0;
-   nedofs = nfdofs = nbdofs = 0;
+   nfdofs = 0;
+   nbdofs = 0;
    bdofs = NULL;
    fdofs = NULL;
    cP = NULL;
    cR = NULL;
    cP_is_set = false;
-   // 'Th' is initialized/destroyed before this method is called.
-
-   nvdofs = mesh->GetNV() * fec->DofForGeometry(Geometry::POINT);
-
-   if (mesh->Dimension() > 1)
-   {
-      nedofs = mesh->GetNEdges() * fec->DofForGeometry(Geometry::SEGMENT);
-   }
+   // Th is initialized/destroyed before this method is called.
 
    if (mesh->GetNFaces() > 0)
    {
@@ -1467,7 +1424,8 @@ void FiniteElementSpace::Construct()
       bdofs[0] = 0;
       for (int i = 0; i < mesh->GetNE(); i++)
       {
-         nbdofs += fec->DofForGeometry(mesh->GetElementBaseGeometry(i));
+         Geometry::Type geom = mesh->GetElementBaseGeometry(i);
+         nbdofs += fec->DofForGeometry(geom);
          bdofs[i+1] = nbdofs;
       }
    }
@@ -1478,7 +1436,7 @@ void FiniteElementSpace::Construct()
    // later.
 }
 
-void FiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
+void FiniteElementSpace::GetElementDofs (int i, Array<int> &dofs) const
 {
    if (elem_dof)
    {
@@ -2555,7 +2513,7 @@ L2ProjectionGridTransfer::L2Projection::L2Projection(
    Vector shape_lor(ndof_lor);
 
    const Geometry::Type geom = fe_ho->GetGeomType();
-   const DenseTensor &pmats = cf_tr.point_matrices[geom];
+   const DenseTensor &pmats = cf_tr.GetPointMatrices(geom);
    emb_tr.SetIdentityTransformation(geom);
 
    for (int iho=0; iho<nel_ho; ++iho)
@@ -2578,7 +2536,7 @@ L2ProjectionGridTransfer::L2Projection::L2Projection(
 
          // Create the transformation that embeds the fine low-order element
          // within the coarse high-order element in reference space
-         emb_tr.GetPointMat() = pmats(cf_tr.embeddings[ilor].matrix);
+         emb_tr.GetPointMat() = pmats(iref);
          emb_tr.FinalizeTransformation();
 
          int order = fe_lor->GetOrder() + fe_ho->GetOrder() + el_tr->OrderW();
@@ -2679,70 +2637,6 @@ const Operator &L2ProjectionGridTransfer::BackwardOperator()
    return *B;
 }
 
-L2ElementRestriction::L2ElementRestriction(const FiniteElementSpace &fes)
-   : ne(fes.GetNE()),
-     vdim(fes.GetVDim()),
-     byvdim(fes.GetOrdering() == Ordering::byVDIM),
-     ndof(ne > 0 ? fes.GetFE(0)->GetDof() : 0)
-{
-   height = vdim*ne*ndof;
-   width = vdim*ne*ndof;
-}
-
-void L2ElementRestriction::Mult(const Vector &x, Vector &y) const
-{
-   for (int iel=0; iel<ne; ++iel)
-   {
-      for (int vd=0; vd<vdim; ++vd)
-      {
-         for (int idof=0; idof<ndof; ++idof)
-         {
-            // E-vector dimensions (dofs, vdim, elements)
-            // L-vector dimensions: byVDIM:  (vdim, dofs, element)
-            //                      byNODES: (dofs, elements, vdim)
-            int yidx = iel*vdim*ndof + vd*ndof + idof;
-            int xidx;
-            if (byvdim)
-            {
-               xidx = iel*ndof*vdim + idof*vdim + vd;
-            }
-            else
-            {
-               xidx = vd*ne*ndof + iel*ndof + idof;
-            }
-            y[yidx] = x[xidx];
-         }
-      }
-   }
-}
-
-void L2ElementRestriction::MultTranspose(const Vector &x, Vector &y) const
-{
-   // Since this restriction is a permutation, the transpose is the inverse
-   for (int iel=0; iel<ne; ++iel)
-   {
-      for (int vd=0; vd<vdim; ++vd)
-      {
-         for (int idof=0; idof<ndof; ++idof)
-         {
-            // E-vector dimensions (dofs, vdim, elements)
-            // L-vector dimensions: byVDIM:  (vdim, dofs, element)
-            //                      byNODES: (dofs, elements, vdim)
-            int xidx = iel*vdim*ndof + vd*ndof + idof;
-            int yidx;
-            if (byvdim)
-            {
-               yidx = iel*ndof*vdim + idof*vdim + vd;
-            }
-            else
-            {
-               yidx = vd*ne*ndof + iel*ndof + idof;
-            }
-            y[yidx] = x[xidx];
-         }
-      }
-   }
-}
 
 ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
                                        ElementDofOrdering e_ordering)

fem/fespace.hpp

@@ -146,8 +146,6 @@ protected:
 
    /// Helper to get vertex, edge or face DOFs (entity=0,1,2 resp.).
    void GetEntityDofs(int entity, int index, Array<int> &dofs) const;
-   // Get degenerate face DOFs: see explanation in method implementation.
-   void GetDegenerateFaceDofs(int index, Array<int> &dofs) const;
 
    /// Calculate the cP and cR matrices for a nonconforming mesh.
    void BuildConformingInterpolation() const;
@@ -158,7 +156,6 @@ protected:
    static bool DofFinalizable(int dof, const Array<bool>& finalized,
                               const SparseMatrix& deps);
 
-   /// Replicate 'mat' in the vector dimension, according to vdim ordering mode.
    void MakeVDimMatrix(SparseMatrix &mat) const;
 
    /// GridFunction interpolation operator applicable after mesh refinement.
@@ -307,9 +304,8 @@ public:
        The parameter @a e_ordering describes how the local DOFs in each element
        should be ordered, see ElementDofOrdering.
 
-       For discontinuous spaces, the element restriction corresponds to a
-       permutation of the degrees of freedom, implemented by the
-       L2ElementRestriction class.
+       For discontinuous spaces, where the element-restriction is the identity,
+       this method will return NULL.
 
        The returned Operator is owned by the FiniteElementSpace. */
    const Operator *GetElementRestriction(ElementDofOrdering e_ordering) const;
@@ -900,22 +896,6 @@ public:
    void MultTranspose(const Vector &x, Vector &y) const;
 };
 
-/// Operator that converts L2 FiniteElementSpace L-vectors to E-vectors.
-/** Objects of this type are typically created and owned by FiniteElementSpace
-    objects, see FiniteElementSpace::GetElementRestriction(). L-vectors
-    corresponding to grid functions in L2 finite element spaces differ from
-    E-vectors only in the ordering of the degrees of freedom. */
-class L2ElementRestriction : public Operator
-{
-   const int ne;
-   const int vdim;
-   const bool byvdim;
-   const int ndof;
-public:
-   L2ElementRestriction(const FiniteElementSpace&);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
 
 /** @brief A class that performs interpolation from an E-vector to quadrature
     point values and/or derivatives (Q-vectors). */

fem/gridfunc.cpp

@@ -1716,7 +1716,6 @@ void GridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff,
    Array<int> vdofs;
    Vector vals;
 
-   HostWrite();
    // maximal element attribute for each dof
    dof_attr.SetSize(fes->GetVSize());
    dof_attr = -1;

fem/gridfunc.hpp

@@ -434,8 +434,6 @@ public:
    /** The GridFunction is resized using the SetSize() method. */
    virtual void SetSpace(FiniteElementSpace *f);
 
-   using Vector::MakeRef;
-
    /** @brief Make the GridFunction reference external data on a new
        FiniteElementSpace. */
    /** This method changes the FiniteElementSpace associated with the

fem/linearform.cpp

@@ -138,11 +138,7 @@ void LinearForm::Assemble()
          eltrans = fes -> GetBdrElementTransformation (i);
          for (int k=0; k < blfi.Size(); k++)
          {
-            if (blfi_marker[k] &&
-                (*blfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
             blfi[k]->AssembleRHSElementVect(*fes->GetBE(i), *eltrans, elemvect);
-
             AddElementVector (vdofs, elemvect);
          }
       }

fem/lininteg.cpp

@@ -350,6 +350,7 @@ void VectorFEDomainLFIntegrator::AssembleDeltaElementVect(
    vshape.Mult(vec, elvect);
 }
 
+
 void VectorBoundaryFluxLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -396,26 +397,19 @@ void VectorFEBoundaryFluxLFIntegrator::AssembleRHSElementVect(
    if (ir == NULL)
    {
       int intorder = 2*el.GetOrder();  // <----------
-      if (F == NULL)
-      {
-         intorder -= el.GetOrder() + 1;
-      }
       ir = &IntRules.Get(el.GetGeomType(), intorder);
    }
 
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetIntPoint (&ip);
+      double val = ip.weight*F.Eval(Tr, ip);
+
       el.CalcShape(ip, shape);
 
-      double val = ip.weight;
-      if (F)
-      {
-         Tr.SetIntPoint (&ip);
-         val *= F->Eval(Tr, ip);
-      }
-
-      elvect.Add(val, shape);
+      add(elvect, val, shape, elvect);
    }
 }
 

fem/lininteg.hpp

@@ -279,12 +279,11 @@ public:
 class VectorFEBoundaryFluxLFIntegrator : public LinearFormIntegrator
 {
 private:
-   Coefficient *F;
+   Coefficient &F;
    Vector shape;
 
 public:
-   VectorFEBoundaryFluxLFIntegrator() : F(NULL) { }
-   VectorFEBoundaryFluxLFIntegrator(Coefficient &f) : F(&f) { }
+   VectorFEBoundaryFluxLFIntegrator(Coefficient &f) : F(f) { }
 
    virtual void AssembleRHSElementVect(const FiniteElement &el,
                                        ElementTransformation &Tr,

fem/nonlinearform.cpp

@@ -65,8 +65,6 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
    Vector el_x;
    const FiniteElement *fe;
    ElementTransformation *T;
-   Mesh *mesh = fes->GetMesh();
-
    double energy = 0.0;
 
    if (dnfi.Size())
@@ -86,81 +84,14 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
 
    if (fnfi.Size())
    {
-      FaceElementTransformations *tr;
-      const FiniteElement *fe1, *fe2;
-      Array<int> vdofs2;
-
-      for (int i = 0; i < mesh->GetNumFaces(); i++)
-      {
-         tr = mesh->GetInteriorFaceTransformations(i);
-         if (tr != NULL)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fes->GetElementVDofs(tr->Elem2No, vdofs2);
-            vdofs.Append (vdofs2);
-            x.GetSubVector(vdofs, el_x);
-            fe1 = fes->GetFE(tr->Elem1No);
-            fe2 = fes->GetFE(tr->Elem2No);
-            for (int k = 0; k < fnfi.Size(); k++)
-            {
-               energy += fnfi[k]->GetFaceEnergy(*fe1, *fe2, *tr, el_x);
-            }
-         }
-      }
+      MFEM_ABORT("TODO: add energy contribution from interior face terms");
    }
 
    if (bfnfi.Size())
    {
-      FaceElementTransformations *tr;
-      const FiniteElement *fe1, *fe2;
-
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < bfnfi.Size(); k++)
-      {
-         if (bfnfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *bfnfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary face integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         tr = mesh->GetBdrFaceTransformations (i);
-         if (tr != NULL)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            x.GetSubVector(vdofs, el_x);
-
-            fe1 = fes->GetFE(tr->Elem1No);
-            // The fe2 object is really a dummy and not used on the boundaries,
-            // but we can't dereference a NULL pointer, and we don't want to
-            // actually make a fake element.
-            fe2 = fe1;
-            for (int k = 0; k < bfnfi.Size(); k++)
-            {
-               if (bfnfi_marker[k] &&
-                   (*bfnfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-               energy += bfnfi[k]->GetFaceEnergy(*fe1, *fe2, *tr, el_x);
-            }
-         }
-      }
+      MFEM_ABORT("TODO: add energy contribution from boundary face terms");
    }
+
    return energy;
 }
 

fem/nonlinearform.hpp

@@ -111,7 +111,7 @@ public:
        be fes->GetVSize(). */
    double GetGridFunctionEnergy(const Vector &x) const;
 
-   /// Compute the energy corresponding to the state @a x.
+   /// Compute the enery corresponding to the state @a x.
    /** In general, @a x may have non-homogeneous essential boundary values.
 
        The state @a x must be a true-dof vector. */

fem/nonlininteg.cpp

@@ -55,14 +55,6 @@ double NonlinearFormIntegrator::GetElementEnergy(
    return 0.0;
 }
 
-double NonlinearFormIntegrator::GetFaceEnergy(
-   const FiniteElement &el1, const FiniteElement &el2,
-   FaceElementTransformations &Tr, const Vector &elfun)
-{
-   mfem_error("NonlinearFormIntegrator::GetFaceEnergy"
-              " is not overloaded!");
-   return 0.0;
-}
 
 void BlockNonlinearFormIntegrator::AssembleElementVector(
    const Array<const FiniteElement *> &el,

fem/nonlininteg.hpp

@@ -63,17 +63,11 @@ public:
                                  FaceElementTransformations &Tr,
                                  const Vector &elfun, DenseMatrix &elmat);
 
-   /// Compute the local energy/functional
+   /// Compute the local energy
    virtual double GetElementEnergy(const FiniteElement &el,
                                    ElementTransformation &Tr,
                                    const Vector &elfun);
 
-   /// Compute the face(s) contribution to the energy/functional
-   virtual double GetFaceEnergy(const FiniteElement &el1,
-                                const FiniteElement &el2,
-                                FaceElementTransformations &Tr,
-                                const Vector &elfun);
-
    virtual ~NonlinearFormIntegrator() { }
 };
 

fem/occa.okl

@@ -38,24 +38,19 @@ typedef double* QLocal3D_t @dim(Q1D, Q1D, Q1D, NE);
 typedef double* Jacobian2D_t @dim(Q2D, 2, 2, NE);
 typedef double* Jacobian3D_t @dim(Q3D, 3, 3, NE);
 
-typedef double* Coeff2D_t @dim(Q2D, NE);
-typedef double* Coeff3D_t @dim(Q3D, NE);
-
 typedef double* SymmOperator2D_t @dim(Q2D, 3, NE);
 typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
 
 @kernel void DiffusionSetup2D(const int NE,
                               @restrict const double *W,
                               @restrict const Jacobian2D_t J,
-                              @restrict const Coeff2D_t C,
-                              @restrict SymmOperator2D_t op,
-                              const bool const_c) {
+                              const double COEFF,
+                              @restrict SymmOperator2D_t op) {
   for (int e = 0; e < NE; ++e; @outer) {
     for (int q = 0; q < Q2D; ++q; @inner) {
       const double J11 = J(q, 0, 0, e), J12 = J(q, 1, 0, e);
       const double J21 = J(q, 0, 1, e), J22 = J(q, 1, 1, e);
-      const double coeff = const_c ? C(0,0) : C(q,e);
-      const double c_detJ = W[q] * coeff / ((J11 * J22) - (J21 * J12));
+      const double c_detJ = W[q] * COEFF / ((J11 * J22) - (J21 * J12));
       op(q, 0, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
       op(q, 1, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
       op(q, 2, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
@@ -66,9 +61,8 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
 @kernel void DiffusionSetup3D(const int NE,
                               @restrict const double *W,
                               @restrict const Jacobian3D_t J,
-                              @restrict const Coeff3D_t C,
-                              @restrict SymmOperator3D_t op,
-                              const bool const_c) {
+                              const double COEFF,
+                              @restrict SymmOperator3D_t op) {
   for (int e = 0; e < NE; ++e; @outer) {
     for (int q = 0; q < Q3D; ++q; @inner) {
       const double J11 = J(q, 0, 0, e), J12 = J(q, 1, 0, e), J13 = J(q, 2, 0, e);
@@ -78,8 +72,7 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
       const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
                            (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
 
-      const double coeff = const_c ? C(0,0) : C(q,e);
-      const double c_detJ = W[q] * coeff / detJ;
+      const double c_detJ = W[q] * COEFF / detJ;
 
       // adj(J)
       const double A11 = (J22 * J33) - (J23 * J32);

fem/pfespace.cpp

@@ -98,8 +98,6 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
 
    gcomm = NULL;
 
-   gfdofs = NULL;
-
    P = NULL;
    Pconf = NULL;
    R = NULL;
@@ -150,37 +148,20 @@ void ParFiniteElementSpace::Construct()
       // cut space.
       ConstructTrueDofs();
 
-      ngedofs = ngfdofs = 0;
-      gfdofs = NULL;
-
       // calculate number of ghost DOFs
       ngvdofs = pncmesh->GetNGhostVertices()
                 * fec->DofForGeometry(Geometry::POINT);
 
+      ngedofs = ngfdofs = 0;
       if (pmesh->Dimension() > 1)
       {
          ngedofs = pncmesh->GetNGhostEdges()
                    * fec->DofForGeometry(Geometry::SEGMENT);
       }
-
       if (pmesh->Dimension() > 2)
       {
-         if (fdofs != NULL) // have mixed faces
-         {
-            gfdofs = new int[pncmesh->GetNGhostFaces()+1];
-            gfdofs[0] = 0;
-            for (int i = 0; i < pncmesh->GetNGhostFaces(); i++)
-            {
-               int ghost = pncmesh->GetNFaces() + i;
-               ngfdofs += fec->DofForGeometry(pncmesh->GetFaceGeometry(ghost));
-               gfdofs[i+1] = ngfdofs;
-            }
-         }
-         else
-         {
-            ngfdofs = pncmesh->GetNGhostFaces()
-                      * fec->DofForGeometry(pncmesh->GetFaceGeometry(0));
-         }
+         ngfdofs = pncmesh->GetNGhostFaces()
+                   * fec->DofForGeometry(pncmesh->GetGhostFaceGeometry(0));
       }
 
       // total number of ghost DOFs. Ghost DOFs start at index 'ndofs', i.e.,
@@ -1343,18 +1324,20 @@ void ParFiniteElementSpace::GetGhostEdgeDofs(const MeshId &edge_id,
 void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
                                              Array<int> &dofs) const
 {
-   int nfv, V[4], E[4], Eo[4];
-   nfv = pmesh->pncmesh->GetFaceVerticesEdges(face_id, V, E, Eo);
+   const int ghost_face_index = face_id.index - pncmesh->GetNFaces();
+   MFEM_ASSERT(pncmesh->GetGhostFaceGeometry(ghost_face_index)
+               == Geometry::SQUARE, "");
 
    int nv = fec->DofForGeometry(Geometry::POINT);
    int ne = fec->DofForGeometry(Geometry::SEGMENT);
-   int nf = fec->DofForGeometry((nfv == 3) ?
-                                Geometry::TRIANGLE : Geometry::SQUARE);
+   int nf = fec->DofForGeometry(Geometry::SQUARE);
+   dofs.SetSize(4*nv + 4*ne + nf);
 
-   dofs.SetSize(nfv*(nv + ne) + nf);
+   int V[4], E[4], Eo[4];
+   pmesh->pncmesh->GetFaceVerticesEdges(face_id, V, E, Eo);
 
    int offset = 0;
-   for (int i = 0; i < nfv; i++)
+   for (int i = 0; i < 4; i++)
    {
       int ghost = pncmesh->GetNVertices();
       int first = (V[i] < ghost) ? V[i]*nv : (ndofs + (V[i] - ghost)*nv);
@@ -1364,7 +1347,7 @@ void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
       }
    }
 
-   for (int i = 0; i < nfv; i++)
+   for (int i = 0; i < 4; i++)
    {
       int ghost = pncmesh->GetNEdges();
       int first = (E[i] < ghost) ? nvdofs + E[i]*ne
@@ -1377,10 +1360,8 @@ void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
       }
    }
 
-   const int ghost_face_index = face_id.index - pncmesh->GetNFaces();
-   int first = ndofs + ngvdofs + ngedofs;
-   first += gfdofs ? gfdofs[ghost_face_index] : nf*ghost_face_index;
-
+   // Assuming all ghost faces have the same number of dofs:
+   int first = ndofs + ngvdofs + ngedofs + ghost_face_index*nf;
    for (int j = 0; j < nf; j++)
    {
       dofs[offset++] = first + j;
@@ -1422,19 +1403,12 @@ void ParFiniteElementSpace::GetBareDofs(int entity, int index,
          break;
 
       default:
-         ned = fec->DofForGeometry(pncmesh->GetFaceGeometry(index));
+         MFEM_ASSERT(!pmesh->HasGeometry(Geometry::TRIANGLE), "");
+         ned = fec->DofForGeometry(Geometry::SQUARE);
          ghost = pncmesh->GetNFaces();
-
-         if (index < ghost) // regular face
-         {
-            first = nvdofs + nedofs + (fdofs ? fdofs[index] : index*ned);
-         }
-         else // ghost face
-         {
-            index -= ghost;
-            first = ndofs + ngvdofs + ngedofs +
-                    (gfdofs ? gfdofs[index] : index*ned);
-         }
+         first = (index < ghost)
+                 ? nvdofs + nedofs + index*ned // regular face
+                 : ndofs + ngvdofs + ngedofs + (index - ghost)*ned; // ghost
          break;
    }
 
@@ -1470,30 +1444,16 @@ int ParFiniteElementSpace::PackDof(int entity, int index, int edof) const
                 : ndofs + ngvdofs + (index - ghost)*ned + edof; // ghost edge
 
       default:
+         MFEM_ASSERT(!pmesh->HasGeometry(Geometry::TRIANGLE), "");
          ghost = pncmesh->GetNFaces();
-         ned = fec->DofForGeometry(pncmesh->GetFaceGeometry(index));
+         ned = fec->DofForGeometry(Geometry::SQUARE);
 
-         if (index < ghost) // regular face
-         {
-            return nvdofs + nedofs + (fdofs ? fdofs[index] : index*ned) + edof;
-         }
-         else // ghost face
-         {
-            index -= ghost;
-            return ndofs + ngvdofs + ngedofs +
-                   (gfdofs ? gfdofs[index] : index*ned) + edof;
-         }
+         return (index < ghost)
+                ? nvdofs + nedofs + index*ned + edof // regular face
+                : ndofs + ngvdofs + ngedofs + (index - ghost)*ned + edof; //ghost
    }
 }
 
-static int bisect(int* array, int size, int value)
-{
-   int* end = array + size;
-   int* pos = std::upper_bound(array, end, value);
-   MFEM_VERIFY(pos != end, "value not found");
-   return pos - array;
-}
-
 /** Dissect a DOF number to obtain the entity type (0=vertex, 1=edge, 2=face),
  *  entity index and the DOF number within the entity.
  */
@@ -1519,17 +1479,9 @@ void ParFiniteElementSpace::UnpackDof(int dof,
       dof -= nedofs;
       if (dof < nfdofs) // regular face
       {
-         if (fdofs) // have mixed faces
-         {
-            index = bisect(fdofs+1, mesh->GetNFaces(), dof);
-            edof = dof - fdofs[index];
-         }
-         else // uniform faces
-         {
-            int nf = fec->DofForGeometry(pncmesh->GetFaceGeometry(0));
-            index = dof / nf, edof = dof % nf;
-         }
-         entity = 2;
+         MFEM_ASSERT(!pmesh->HasGeometry(Geometry::TRIANGLE), "");
+         int nf = fec->DofForGeometry(Geometry::SQUARE);
+         entity = 2, index = dof / nf, edof = dof % nf;
          return;
       }
       MFEM_ABORT("Cannot unpack internal DOF");
@@ -1553,17 +1505,8 @@ void ParFiniteElementSpace::UnpackDof(int dof,
       dof -= ngedofs;
       if (dof < ngfdofs) // ghost face
       {
-         if (gfdofs) // have mixed faces
-         {
-            index = bisect(gfdofs+1, pncmesh->GetNGhostFaces(), dof);
-            edof = dof - gfdofs[index];
-         }
-         else // uniform faces
-         {
-            int nf = fec->DofForGeometry(pncmesh->GetFaceGeometry(0));
-            index = pncmesh->GetNFaces() + dof / nf, edof = dof % nf;
-         }
-         entity = 2;
+         int nf = fec->DofForGeometry(pncmesh->GetGhostFaceGeometry(0));
+         entity = 2, index = pncmesh->GetNFaces() + dof / nf, edof = dof % nf;
          return;
       }
       MFEM_ABORT("Out of range DOF.");
@@ -1746,7 +1689,7 @@ void NeighborRowMessage::Encode(int rank)
          mfem::out << "Rank " << pncmesh->MyRank << " sending to " << rank
                    << ": ent " << ri.entity << ", index " << ri.index
                    << ", edof " << ri.edof << " (id " << id.element << "/"
-                   << int(id.local) << ")" << std::endl;
+                   << id.local << ")" << std::endl;
 #endif
 
          // handle orientation and sign change
@@ -1789,6 +1732,8 @@ void NeighborRowMessage::Decode(int rank)
    rows.clear();
    rows.reserve(nrows);
 
+   Geometry::Type fgeom = pncmesh->GetFaceGeometry();
+
    // read rows
    for (int ent = 0, gi = 0; ent < 3; ent++)
    {
@@ -1807,9 +1752,8 @@ void NeighborRowMessage::Decode(int rank)
          }
          else if (ent == 2)
          {
-            Geometry::Type geom = pncmesh->GetFaceGeometry(id.index);
             int fo = pncmesh->GetFaceOrientation(id.index);
-            ind = fec->DofOrderForOrientation(geom, fo);
+            ind = fec->DofOrderForOrientation(fgeom, fo);
          }
 
          double s = 1.0;
@@ -1898,7 +1842,7 @@ void ParFiniteElementSpace
    for (int i = 0; i < dof_group.Size(); i++)
    {
       os << i << ": ";
-      if (i < (nvdofs + nedofs + nfdofs) || i >= ndofs)
+      if (i < (nvdofs + nedofs + nfdofs) || i > ndofs)
       {
          int ent, idx, edof;
          UnpackDof(i, ent, idx, edof);
@@ -1980,7 +1924,15 @@ int ParFiniteElementSpace
          if (!list.masters.size()) { continue; }
 
          IsoparametricTransformation T;
-         DenseMatrix I;
+         if (entity > 1) { T.SetFE(&QuadrilateralFE); }
+         else { T.SetFE(&SegmentFE); }
+
+         Geometry::Type geom = (entity > 1) ?
+                               Geometry::SQUARE : Geometry::SEGMENT;
+         const FiniteElement* fe = fec->FiniteElementForGeometry(geom);
+         if (!fe) { continue; }
+
+         DenseMatrix I(fe->GetDof());
 
          // process masters that we own or that affect our edges/faces
          for (unsigned mi = 0; mi < list.masters.size(); mi++)
@@ -1994,17 +1946,6 @@ int ParFiniteElementSpace
 
             if (!master_dofs.Size()) { continue; }
 
-            const FiniteElement* fe = fec->FiniteElementForGeometry(mf.Geom());
-            if (!fe) { continue; }
-
-            switch (mf.Geom())
-            {
-               case Geometry::SQUARE:   T.SetFE(&QuadrilateralFE); break;
-               case Geometry::TRIANGLE: T.SetFE(&TriangleFE); break;
-               case Geometry::SEGMENT:  T.SetFE(&SegmentFE); break;
-               default: MFEM_ABORT("unsupported geometry");
-            }
-
             // constrain slaves that exist in our mesh
             for (int si = mf.slaves_begin; si < mf.slaves_end; si++)
             {
@@ -2055,8 +1996,6 @@ int ParFiniteElementSpace
                   (l == 1) ? (const MeshId&) list.masters[i]
                   /*    */ : (const MeshId&) list.slaves[i];
 
-               if (id.index < 0) { continue; }
-
                GroupId owner = pncmesh->GetEntityOwnerId(entity, id.index);
                GroupId group = pncmesh->GetEntityGroupId(entity, id.index);
 
@@ -2553,9 +2492,6 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    int nrk = HYPRE_AssumedPartitionCheck() ? 2 : NRanks;
 
    MFEM_VERIFY(Nonconforming(), "Not implemented for conforming meshes.");
-   MFEM_VERIFY(pmesh->GetNumGeometries(pmesh->Dimension()) == 1,
-               "Not implemented for mixed meshes.");
-
    MFEM_VERIFY(old_dof_offsets[nrk], "Missing previous (finer) space.");
    MFEM_VERIFY(dof_offsets[nrk] <= old_dof_offsets[nrk],
                "Previous space is not finer.");
@@ -2569,7 +2505,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    Vector row;
 
    ParNCMesh* pncmesh = pmesh->pncmesh;
-   Geometry::Type geom = pncmesh->GetElementGeometry(0); // TODO mixed meshes
+   Geometry::Type geom = pncmesh->GetElementGeometry();
    int ldof = fec->FiniteElementForGeometry(geom)->GetDof();
 
    const CoarseFineTransformations &dtrans = pncmesh->GetDerefinementTransforms();
@@ -2793,8 +2729,6 @@ void ParFiniteElementSpace::Destroy()
    delete Pconf; Pconf = NULL;
    delete R; R = NULL;
 
-   delete [] gfdofs; gfdofs = NULL;
-
    delete gcomm; gcomm = NULL;
 
    num_face_nbr_dofs = -1;

fem/pfespace.hpp

@@ -46,7 +46,6 @@ private:
 
    /// Number of vertex/edge/face/total ghost DOFs (nonconforming case).
    int ngvdofs, ngedofs, ngfdofs, ngdofs;
-   int* gfdofs;
 
    /// The group of each local dof.
    Array<int> ldof_group;
@@ -114,7 +113,7 @@ private:
    void GetGhostFaceDofs(const MeshId &face_id, Array<int> &dofs) const;
 
    void GetGhostDofs(int entity, const MeshId &id, Array<int> &dofs) const;
-   /// Return the dofs associated with the interior of the given mesh entity.
+   // Return the dofs associated with the interior of the given mesh entity.
    void GetBareDofs(int entity, int index, Array<int> &dofs) const;
 
    int  PackDof(int entity, int index, int edof) const;

fem/pgridfunc.cpp

@@ -225,13 +225,11 @@ void ParGridFunction::ExchangeFaceNbrData()
    MPI_Request *recv_requests = requests + num_face_nbrs;
    MPI_Status  *statuses = new MPI_Status[num_face_nbrs];
 
-   const double *h_data = this->HostRead();
    for (int i = 0; i < send_data.Size(); i++)
    {
-      send_data[i] = h_data[send_ldof[i]];
+      send_data[i] = data[send_ldof[i]];
    }
 
-   double *h_face_nbr_data = face_nbr_data.HostWrite();
    for (int fn = 0; fn < num_face_nbrs; fn++)
    {
       int nbr_rank = pmesh->GetFaceNbrRank(fn);
@@ -241,7 +239,7 @@ void ParGridFunction::ExchangeFaceNbrData()
                 send_offset[fn+1] - send_offset[fn],
                 MPI_DOUBLE, nbr_rank, tag, MyComm, &send_requests[fn]);
 
-      MPI_Irecv(&h_face_nbr_data[recv_offset[fn]],
+      MPI_Irecv(&face_nbr_data(recv_offset[fn]),
                 recv_offset[fn+1] - recv_offset[fn],
                 MPI_DOUBLE, nbr_rank, tag, MyComm, &recv_requests[fn]);
    }

fem/pgridfunc.hpp

@@ -112,8 +112,6 @@ public:
    /// Associate a new parallel space with the ParGridFunction.
    void SetSpace(ParFiniteElementSpace *f);
 
-   using GridFunction::MakeRef;
-
    /** @brief Make the ParGridFunction reference external data on a new
        FiniteElementSpace. */
    /** This method changes the FiniteElementSpace associated with the

fem/pnonlinearform.cpp

@@ -46,7 +46,7 @@ double ParNonlinearForm::GetParGridFunctionEnergy(const Vector &x) const
 void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
    NonlinearForm::Mult(x, y); // x --(P)--> aux1 --(A_local)--> aux2
-   Y.MakeRef(aux2, 0); // aux2 contains A_local.P.x
+   Y.SetData(aux2.GetData()); // aux2 contains A_local.P.x
 
    if (fnfi.Size())
    {
@@ -58,7 +58,7 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
       Array<int> vdofs1, vdofs2;
       Vector el_x, el_y;
 
-      X.MakeRef(aux1, 0); // aux1 contains P.x
+      X.SetData(aux1.GetData()); // aux1 contains P.x
       X.ExchangeFaceNbrData();
       const int n_shared_faces = pmesh->GetNSharedFaces();
       for (int i = 0; i < n_shared_faces; i++)

fem/sidredatacollection.cpp

@@ -16,7 +16,9 @@
 
 #include "fem.hpp"
 
-#include <axom/sidre.hpp>
+#ifdef MFEM_USE_MPI
+#include <sidre/IOManager.hpp>
+#endif
 
 #include <string>
 #include <iomanip>      // for setw, setfill
@@ -202,10 +204,10 @@ SidreDataCollection::get_file_path(const std::string &filename) const
 
 axom::sidre::View *
 SidreDataCollection::AllocNamedBuffer(const std::string& buffer_name,
-                                      axom::sidre::IndexType sz,
+                                      axom::sidre::SidreLength sz,
                                       axom::sidre::TypeID type)
 {
-   sz = std::max(sz, sidre::IndexType(0));
+   sz = std::max(sz, sidre::SidreLength(0));
    sidre::Group *f = named_buffers_grp();
    sidre::View  *v = NULL;
 
@@ -823,7 +825,7 @@ void SidreDataCollection::Save(const std::string& filename,
 void SidreDataCollection::
 addScalarBasedGridFunction(const std::string &field_name, GridFunction *gf,
                            const std::string &buffer_name,
-                           axom::sidre::IndexType offset)
+                           axom::sidre::SidreLength offset)
 {
    sidre::Group* grp = m_bp_grp->getGroup("fields/" + field_name);
    MFEM_ASSERT(grp != NULL, "field " << field_name << " does not exist");
@@ -886,7 +888,7 @@ addScalarBasedGridFunction(const std::string &field_name, GridFunction *gf,
 void SidreDataCollection::
 addVectorBasedGridFunction(const std::string& field_name, GridFunction *gf,
                            const std::string &buffer_name,
-                           axom::sidre::IndexType offset)
+                           axom::sidre::SidreLength offset)
 {
    sidre::Group* grp = m_bp_grp->getGroup("fields/" + field_name);
    MFEM_ASSERT(grp != NULL, "field " << field_name << " does not exist");
@@ -1011,7 +1013,7 @@ DeregisterFieldInBPIndex(const std::string& field_name)
 void SidreDataCollection::RegisterField(const std::string &field_name,
                                         GridFunction *gf,
                                         const std::string &buffer_name,
-                                        axom::sidre::IndexType offset)
+                                        axom::sidre::SidreLength offset)
 {
    if ( field_name.empty() || buffer_name.empty() ||
         gf == NULL || gf->FESpace() == NULL )

fem/sidredatacollection.hpp

@@ -25,7 +25,7 @@
 #  pragma GCC diagnostic ignored "-Wpedantic"
 # endif
 #endif
-#include <axom/sidre.hpp>
+#include <sidre/sidre.hpp>
 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
 # pragma GCC diagnostic pop
 #endif
@@ -246,7 +246,7 @@ public:
     */
    void RegisterField(const std::string &field_name, GridFunction *gf,
                       const std::string &buffer_name,
-                      axom::sidre::IndexType offset);
+                      axom::sidre::SidreLength offset);
 
    /// Registers an attribute field in the Sidre DataStore
    /** The registration process is similar to that of RegisterField()
@@ -385,7 +385,7 @@ public:
     */
    axom::sidre::View *
    AllocNamedBuffer(const std::string& buffer_name,
-                    axom::sidre::IndexType sz,
+                    axom::sidre::SidreLength sz,
                     axom::sidre::TypeID type =
                        axom::sidre::DOUBLE_ID);
 
@@ -469,7 +469,7 @@ private:
    void addScalarBasedGridFunction(const std::string& field_name,
                                    GridFunction* gf,
                                    const std::string &buffer_name,
-                                   axom::sidre::IndexType offset);
+                                   axom::sidre::SidreLength offset);
 
    /**
     * \brief A private helper function to set up the views associated with the
@@ -483,7 +483,7 @@ private:
    void addVectorBasedGridFunction(const std::string& field_name,
                                    GridFunction* gf,
                                    const std::string &buffer_name,
-                                   axom::sidre::IndexType offset);
+                                   axom::sidre::SidreLength offset);
 
    /** @brief A private helper function to set up the Views associated with
        attribute field named @a field_name */

fem/tmop.cpp

@@ -12,7 +12,6 @@
 #include "tmop.hpp"
 #include "linearform.hpp"
 #include "pgridfunc.hpp"
-#include "tmop_tools.hpp"
 
 namespace mfem
 {
@@ -769,7 +768,7 @@ void TMOP_Metric_352::AssembleH(const DenseMatrix &Jpt,
 void TargetConstructor::ComputeAvgVolume() const
 {
    MFEM_VERIFY(nodes, "Nodes are not given!");
-   MFEM_ASSERT(avg_volume == 0.0, "The average volume is already computed!");
+   MFEM_ASSERT(avg_volume == 0.0, "the average volume is already computed!");
 
    Mesh *mesh = nodes->FESpace()->GetMesh();
    const int NE = mesh->GetNE();
@@ -788,13 +787,9 @@ void TargetConstructor::ComputeAvgVolume() const
          volume += ip.weight * Tr.Weight();
       }
    }
-
-   NCMesh *ncmesh = mesh->ncmesh;
-   if (Parallel() == false)
+   if (!Parallel())
    {
-      avg_volume = (ncmesh == NULL) ?
-                   volume / NE : volume / ncmesh->GetNumRootElements();
-
+      avg_volume = volume / NE;
    }
 #ifdef MFEM_USE_MPI
    else
@@ -802,8 +797,7 @@ void TargetConstructor::ComputeAvgVolume() const
       double area_NE[4];
       area_NE[0] = volume; area_NE[1] = NE;
       MPI_Allreduce(area_NE, area_NE + 2, 2, MPI_DOUBLE, MPI_SUM, comm);
-      avg_volume = (ncmesh == NULL) ?
-                   area_NE[2] / area_NE[3] : area_NE[2] / ncmesh->GetNumRootElements();
+      avg_volume = area_NE[2] / area_NE[3];
    }
 #endif
 }
@@ -811,7 +805,6 @@ void TargetConstructor::ComputeAvgVolume() const
 // virtual method
 void TargetConstructor::ComputeElementTargets(int e_id, const FiniteElement &fe,
                                               const IntegrationRule &ir,
-                                              const Vector &elfun,
                                               DenseTensor &Jtr) const
 {
    MFEM_ASSERT(target_type == IDEAL_SHAPE_UNIT_SIZE || nodes != NULL, "");
@@ -834,15 +827,7 @@ void TargetConstructor::ComputeElementTargets(int e_id, const FiniteElement &fe,
       {
          if (avg_volume == 0.0) { ComputeAvgVolume(); }
          DenseMatrix W(Wideal.Height());
-
-         NCMesh *ncmesh = nodes->FESpace()->GetMesh()->ncmesh;
-         double el_volume = avg_volume;
-         if (ncmesh)
-         {
-            el_volume = avg_volume / ncmesh->GetElementSizeReduction(e_id);
-         }
-
-         W.Set(std::pow(volume_scale * el_volume / Wideal.Det(),
+         W.Set(std::pow(volume_scale * avg_volume / Wideal.Det(),
                         1./W.Height()), Wideal);
          for (int i = 0; i < ir.GetNPoints(); i++) { Jtr(i) = W; }
          break;
@@ -868,7 +853,7 @@ void TargetConstructor::ComputeElementTargets(int e_id, const FiniteElement &fe,
             if (target_type == IDEAL_SHAPE_GIVEN_SIZE)
             {
                const double det = Jtr(i).Det();
-               MFEM_VERIFY(det > 0.0, "The given mesh is inverted!");
+               MFEM_VERIFY(det > 0.0, "Initial mesh is inverted!");
                Jtr(i).Set(std::pow(det / detW, 1./dim), Wideal);
             }
          }
@@ -879,162 +864,6 @@ void TargetConstructor::ComputeElementTargets(int e_id, const FiniteElement &fe,
    }
 }
 
-void AnalyticAdaptTC::SetAnalyticTargetSpec(Coefficient *sspec,
-                                            VectorCoefficient *vspec,
-                                            MatrixCoefficient *mspec)
-{
-   scalar_tspec = sspec;
-   vector_tspec = vspec;
-   matrix_tspec = mspec;
-}
-
-void AnalyticAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
-                                            const IntegrationRule &ir,
-                                            const Vector &elfun,
-                                            DenseTensor &Jtr) const
-{
-   DenseMatrix point_mat;
-   point_mat.UseExternalData(elfun.GetData(), fe.GetDof(), fe.GetDim());
-
-   switch (target_type)
-   {
-      case GIVEN_FULL:
-      {
-         MFEM_VERIFY(matrix_tspec != NULL,
-                     "Target type GIVEN_FULL requires a MatrixCoefficient.");
-
-         IsoparametricTransformation Tpr;
-         Tpr.SetFE(&fe);
-         Tpr.ElementNo = e_id;
-         Tpr.GetPointMat().Transpose(point_mat);
-
-         for (int i = 0; i < ir.GetNPoints(); i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            Tpr.SetIntPoint(&ip);
-            matrix_tspec->Eval(Jtr(i), Tpr, ip);
-         }
-         break;
-      }
-      default:
-         MFEM_ABORT("Incompatible target type for analytic adaptation!");
-   }
-}
-
-#ifdef MFEM_USE_MPI
-void DiscreteAdaptTC::SetParDiscreteTargetSpec(ParGridFunction &tspec)
-{
-   target_spec.SetSize(tspec.Size());
-   target_spec = tspec;
-   tspec_fes   = tspec.FESpace();
-
-   // Default evaluator is based on CG advection.
-   if (adapt_eval == NULL) { adapt_eval = new AdvectorCG; }
-
-   adapt_eval->SetParMetaInfo(*tspec.ParFESpace()->GetParMesh(),
-                              *tspec.FESpace()->FEColl(),
-                              tspec.FESpace()->GetVDim());
-
-   adapt_eval->SetInitialField
-   (*tspec.FESpace()->GetMesh()->GetNodes(), target_spec);
-}
-#endif
-
-void DiscreteAdaptTC::SetSerialDiscreteTargetSpec(GridFunction &tspec)
-{
-   target_spec.SetSize(tspec.Size());
-   target_spec = tspec;
-   tspec_fes   = tspec.FESpace();
-
-   // Default evaluator is based on CG advection.
-   if (adapt_eval == NULL) { adapt_eval = new AdvectorCG; }
-
-   adapt_eval->SetSerialMetaInfo(*tspec.FESpace()->GetMesh(),
-                                 *tspec.FESpace()->FEColl(),
-                                 tspec.FESpace()->GetVDim());
-
-   adapt_eval->SetInitialField
-   (*tspec.FESpace()->GetMesh()->GetNodes(), target_spec);
-}
-
-void DiscreteAdaptTC::UpdateTargetSpecification(const Vector &new_x)
-{
-   MFEM_VERIFY(target_spec.Size() > 0, "Target specification is not set!");
-
-   adapt_eval->ComputeAtNewPosition(new_x, target_spec);
-}
-
-void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
-                                            const IntegrationRule &ir,
-                                            const Vector &elfun,
-                                            DenseTensor &Jtr) const
-{
-   MFEM_VERIFY(tspec_fes, "A call to SetDiscreteTargerSpec() is needed.");
-
-   switch (target_type)
-   {
-      case IDEAL_SHAPE_GIVEN_SIZE:
-      {
-         const DenseMatrix &Wideal =
-            Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
-         const int dim = Wideal.Height(),
-                   ntspec_dofs = tspec_fes->GetFE(0)->GetDof();
-
-         Vector shape(ntspec_dofs), tspec_vals(ntspec_dofs);
-         Array<int> dofs;
-         tspec_fes->GetElementDofs(e_id, dofs);
-         target_spec.GetSubVector(dofs, tspec_vals);
-
-         const double min_size = tspec_vals.Min();
-         MFEM_ASSERT(min_size > 0.0,
-                     "Non-positive size propagated in the target definition.");
-
-         for (int i = 0; i < ir.GetNPoints(); i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-            const double size = std::max(shape * tspec_vals, min_size);
-            Jtr(i).Set(std::pow(size / Wideal.Det(), 1.0/dim), Wideal);
-         }
-         break;
-      }
-      default:
-         MFEM_ABORT("Incompatible target type for analytic adaptation!");
-   }
-}
-
-void AdaptivityEvaluator::SetSerialMetaInfo(const Mesh &m,
-                                            const FiniteElementCollection &fec,
-                                            int num_comp)
-{
-   delete fes;
-   delete mesh;
-   mesh = new Mesh(m, true);
-   fes = new FiniteElementSpace(mesh, &fec, num_comp);
-}
-
-#ifdef MFEM_USE_MPI
-void AdaptivityEvaluator::SetParMetaInfo(const ParMesh &m,
-                                         const FiniteElementCollection &fec,
-                                         int num_comp)
-{
-   delete pfes;
-   delete pmesh;
-   pmesh = new ParMesh(m, true);
-   pfes  = new ParFiniteElementSpace(pmesh, &fec, num_comp);
-}
-#endif
-
-AdaptivityEvaluator::~AdaptivityEvaluator()
-{
-   delete fes;
-   delete mesh;
-#ifdef MFEM_USE_MPI
-   delete pfes;
-   delete pmesh;
-#endif
-}
-
 void TMOP_Integrator::EnableLimiting(const GridFunction &n0,
                                      const GridFunction &dist, Coefficient &w0,
                                      TMOP_LimiterFunction *lfunc)
@@ -1092,7 +921,7 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
 
    energy = 0.0;
    DenseTensor Jtr(dim, dim, ir->GetNPoints());
-   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+   targetC->ComputeElementTargets(T.ElementNo, el, *ir, Jtr);
 
    // Limited case.
    Vector shape, p, p0, d_vals;
@@ -1161,7 +990,6 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
       energy += weight * val;
    }
    delete Tpr;
-
    return energy;
 }
 
@@ -1188,7 +1016,7 @@ void TMOP_Integrator::AssembleElementVector(const FiniteElement &el,
 
    elvect = 0.0;
    DenseTensor Jtr(dim, dim, ir->GetNPoints());
-   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+   targetC->ComputeElementTargets(T.ElementNo, el, *ir, Jtr);
 
    // Limited case.
    DenseMatrix pos0;
@@ -1244,8 +1072,6 @@ void TMOP_Integrator::AssembleElementVector(const FiniteElement &el,
       P *= weight_m;
       AddMultABt(DS, P, PMatO);
 
-      // TODO: derivatives of adaptivity-based targets.
-
       if (coeff0)
       {
          el.CalcShape(ip, shape);
@@ -1281,7 +1107,7 @@ void TMOP_Integrator::AssembleElementGrad(const FiniteElement &el,
 
    elmat = 0.0;
    DenseTensor Jtr(dim, dim, ir->GetNPoints());
-   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+   targetC->ComputeElementTargets(T.ElementNo, el, *ir, Jtr);
 
    // Limited case.
    DenseMatrix pos0, grad_grad;
@@ -1334,8 +1160,6 @@ void TMOP_Integrator::AssembleElementGrad(const FiniteElement &el,
 
       metric->AssembleH(Jpt, DS, weight_m, elmat);
 
-      // TODO: derivatives of adaptivity-based targets.
-
       if (coeff0)
       {
          el.CalcShape(ip, shape);
@@ -1410,12 +1234,11 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
    for (int i = 0; i < fes->GetNE(); i++)
    {
       fe = fes->GetFE(i);
+      targetC->ComputeElementTargets(i, *fe, *ir, Jtr);
       fes->GetElementVDofs(i, vdofs);
       x.GetSubVector(vdofs, x_vals);
       PMatI.UseExternalData(x_vals.GetData(), dof, dim);
 
-      targetC->ComputeElementTargets(i, *fe, *ir, x_vals, Jtr);
-
       for (int i = 0; i < ir->GetNPoints(); i++)
       {
          const IntegrationPoint &ip = ir->IntPoint(i);
@@ -1451,6 +1274,9 @@ void InterpolateTMOP_QualityMetric(TMOP_QualityMetric &metric,
       const IntegrationRule &ir = metric_gf.FESpace()->GetFE(i)->GetNodes();
       const int nsp = ir.GetNPoints(), dof = fe_pos.GetDof();
 
+      W.SetSize(dim, dim, nsp);
+      tc.ComputeElementTargets(i, fe_pos, ir, W);
+
       dshape.SetSize(dof, dim);
       pos.SetSize(dof, dim);
       posV.SetDataAndSize(pos.Data(), dof * dim);
@@ -1459,9 +1285,6 @@ void InterpolateTMOP_QualityMetric(TMOP_QualityMetric &metric,
       nodes.FESpace()->GetElementVDofs(i, pos_dofs);
       nodes.GetSubVector(pos_dofs, posV);
 
-      W.SetSize(dim, dim, nsp);
-      tc.ComputeElementTargets(i, fe_pos, ir, posV, W);
-
       for (int j = 0; j < nsp; j++)
       {
          const DenseMatrix &Wj = W(j);

fem/tmop.hpp

@@ -12,6 +12,7 @@
 #ifndef MFEM_TMOP_HPP
 #define MFEM_TMOP_HPP
 
+#include "../config/config.hpp"
 #include "../linalg/invariants.hpp"
 #include "nonlininteg.hpp"
 
@@ -513,51 +514,6 @@ public:
    virtual ~TMOP_QuadraticLimiter() { }
 };
 
-class FiniteElementCollection;
-class FiniteElementSpace;
-class ParFiniteElementSpace;
-
-class AdaptivityEvaluator
-{
-protected:
-   // Owned.
-   Mesh *mesh;
-   FiniteElementSpace *fes;
-
-#ifdef MFEM_USE_MPI
-   // Owned.
-   ParMesh *pmesh;
-   ParFiniteElementSpace *pfes;
-#endif
-
-public:
-   AdaptivityEvaluator() : mesh(NULL), fes(NULL)
-   {
-#ifdef MFEM_USE_MPI
-      pmesh = NULL;
-      pfes = NULL;
-#endif
-   }
-   virtual ~AdaptivityEvaluator();
-
-   /** Specifies the Mesh and FiniteElementCollection of the solution that will
-       be evaluated. The given mesh will be copied into the internal object. */
-   void SetSerialMetaInfo(const Mesh &m,
-                          const FiniteElementCollection &fec, int num_comp);
-
-#ifdef MFEM_USE_MPI
-   /// Parallel version of SetSerialMetaInfo.
-   void SetParMetaInfo(const ParMesh &m,
-                       const FiniteElementCollection &fec, int num_comp);
-#endif
-
-   // TODO use GridFunctions to make clear it's on the ldofs?
-   virtual void SetInitialField(const Vector &init_nodes,
-                                const Vector &init_field) = 0;
-
-   virtual void ComputeAtNewPosition(const Vector &new_nodes,
-                                     Vector &new_field) = 0;
-};
 
 /** @brief Base class representing target-matrix construction algorithms for
     mesh optimization via the target-matrix optimization paradigm (TMOP). */
@@ -582,11 +538,9 @@ public:
       IDEAL_SHAPE_GIVEN_SIZE, /**<
          Ideal shape, given size/volume; the given nodes define the target
          volume at all quadrature points. */
-      GIVEN_SHAPE_AND_SIZE, /**<
+      GIVEN_SHAPE_AND_SIZE /**<
          Given shape, given size/volume; the given nodes define the exact target
          Jacobian matrix at all quadrature points. */
-      GIVEN_FULL /**<
-         Full target tensor is specified at every quadrature point. */
    };
 
 protected:
@@ -635,89 +589,14 @@ public:
    void SetVolumeScale(double vol_scale) { volume_scale = vol_scale; }
 
    /** @brief Given an element and quadrature rule, computes ref->target
-       transformation Jacobians for each quadrature point in the element.
-       The physical positions of the element's nodes are given by @a elfun. */
+       transformation Jacobians for each quadrature point in the element. */
    virtual void ComputeElementTargets(int e_id, const FiniteElement &fe,
                                       const IntegrationRule &ir,
-                                      const Vector &elfun,
-                                      DenseTensor &Jtr) const;
-};
-
-class AnalyticAdaptTC : public TargetConstructor
-{
-protected:
-   // Analytic target specification.
-   Coefficient *scalar_tspec;
-   VectorCoefficient *vector_tspec;
-   MatrixCoefficient *matrix_tspec;
-
-public:
-   AnalyticAdaptTC(TargetType ttype)
-      : TargetConstructor(ttype),
-        scalar_tspec(NULL), vector_tspec(NULL), matrix_tspec(NULL) { }
-
-   virtual void SetAnalyticTargetSpec(Coefficient *sspec,
-                                      VectorCoefficient *vspec,
-                                      MatrixCoefficient *mspec);
-
-   /** @brief Given an element and quadrature rule, computes ref->target
-       transformation Jacobians for each quadrature point in the element.
-       The physical positions of the element's nodes are given by @a elfun. */
-   virtual void ComputeElementTargets(int e_id, const FiniteElement &fe,
-                                      const IntegrationRule &ir,
-                                      const Vector &elfun,
                                       DenseTensor &Jtr) const;
 };
 
 class ParGridFunction;
 
-class DiscreteAdaptTC : public TargetConstructor
-{
-protected:
-   // Discrete target specification.
-   // Data is owned, updated by UpdateTargetSpecification.
-   Vector target_spec;
-   // Note: do not use the Nodes of this space as they may not be on the
-   // positions corresponding to the values of tspec.
-   const FiniteElementSpace *tspec_fes;
-
-   // Evaluation of the discrete target specification on different meshes.
-   // Owned.
-   AdaptivityEvaluator *adapt_eval;
-
-public:
-   DiscreteAdaptTC(TargetType ttype)
-      : TargetConstructor(ttype),
-        target_spec(), tspec_fes(NULL), adapt_eval(NULL) { }
-
-   virtual ~DiscreteAdaptTC() { delete adapt_eval; }
-
-   virtual void SetSerialDiscreteTargetSpec(GridFunction &tspec);
-#ifdef MFEM_USE_MPI
-   virtual void SetParDiscreteTargetSpec(ParGridFunction &tspec);
-#endif
-
-   /** Used to update the target specification after the mesh has changed. The
-       new mesh positions are given by new_x. */
-   void UpdateTargetSpecification(const Vector &new_x);
-
-   void SetAdaptivityEvaluator(AdaptivityEvaluator *ae)
-   {
-      if (adapt_eval) { delete adapt_eval; }
-      adapt_eval = ae;
-   }
-
-   /** @brief Given an element and quadrature rule, computes ref->target
-       transformation Jacobians for each quadrature point in the element.
-       The physical positions of the element's nodes are given by @a elfun.
-       Note that this function assumes that UpdateTargetSpecification() has
-       been called with the position vector corresponding to @a elfun. */
-   virtual void ComputeElementTargets(int e_id, const FiniteElement &fe,
-                                      const IntegrationRule &ir,
-                                      const Vector &elfun,
-                                      DenseTensor &Jtr) const;
-};
-
 /** @brief A TMOP integrator class based on any given TMOP_QualityMetric and
     TargetConstructor.
 

fem/tmop_tools.cpp

@@ -1,518 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "tmop_tools.hpp"
-#include "nonlinearform.hpp"
-#include "pnonlinearform.hpp"
-#include "../general/osockstream.hpp"
-
-namespace mfem
-{
-
-using namespace mfem;
-
-void AdvectorCG::SetInitialField(const Vector &init_nodes,
-                                 const Vector &init_field)
-{
-   nodes0 = init_nodes;
-   field0 = init_field;
-}
-
-void AdvectorCG::ComputeAtNewPosition(const Vector &new_nodes,
-                                      Vector &new_field)
-{
-   int myid = 0;
-   Mesh *m = mesh;
-
-#ifdef MFEM_USE_MPI
-   if (pfes) { MPI_Comm_rank(pfes->GetComm(), &myid); }
-   if (pmesh) { m = pmesh; }
-#endif
-
-   MFEM_VERIFY(m != NULL, "No mesh has been given to the AdaptivityEvaluator.");
-
-   // This will be used to move the positions.
-   GridFunction *mesh_nodes = m->GetNodes();
-   *mesh_nodes = nodes0;
-   new_field = field0;
-
-   // Velocity of the positions.
-   GridFunction u(mesh_nodes->FESpace());
-   subtract(new_nodes, nodes0, u);
-
-   TimeDependentOperator *oper = NULL;
-   // This must be the fes of the ind, associated with the object's mesh.
-   if (fes)  { oper = new SerialAdvectorCGOper(nodes0, u, *fes); }
-#ifdef MFEM_USE_MPI
-   else if (pfes) { oper = new ParAdvectorCGOper(nodes0, u, *pfes); }
-#endif
-   MFEM_VERIFY(oper != NULL,
-               "No FE space has been given to the AdaptivityEvaluator.");
-   ode_solver.Init(*oper);
-
-   // Compute some time step [mesh_size / speed].
-   double min_h = std::numeric_limits<double>::infinity();
-   for (int i = 0; i < m->GetNE(); i++)
-   {
-      min_h = std::min(min_h, m->GetElementSize(i));
-   }
-   double v_max = 0.0;
-   const int s = u.FESpace()->GetVSize() / 2;
-   for (int i = 0; i < s; i++)
-   {
-      const double vel = u(i) * u(i) + u(i+s) * u(i+s);
-      v_max = std::max(v_max, vel);
-   }
-   if (v_max == 0.0)
-   {
-      // No need to change the field.
-      return;
-   }
-   v_max = std::sqrt(v_max);
-   double dt = 0.5 * min_h / v_max;
-   double glob_dt = dt;
-#ifdef MFEM_USE_MPI
-   if (pfes)
-   {
-      MPI_Allreduce(&dt, &glob_dt, 1, MPI_DOUBLE, MPI_MIN, pfes->GetComm());
-   }
-#endif
-
-   double t = 0.0;
-   bool last_step = false;
-   for (int ti = 1; !last_step; ti++)
-   {
-      if (t + glob_dt >= 1.0)
-      {
-#ifdef MFEM_DEBUG
-         if (myid == 0)
-         {
-            mfem::out << "Remap took " << ti << " steps." << std::endl;
-         }
-#endif
-         glob_dt = 1.0 - t;
-         last_step = true;
-      }
-      ode_solver.Step(new_field, t, glob_dt);
-   }
-
-   // Trim the overshoots and undershoots.
-   const double minv = field0.Min(), maxv = field0.Max();
-   for (int i = 0; i < new_field.Size(); i++)
-   {
-      if (new_field(i) < minv) { new_field(i) = minv; }
-      if (new_field(i) > maxv) { new_field(i) = maxv; }
-   }
-
-   nodes0 = new_nodes;
-   field0 = new_field;
-
-   delete oper;
-}
-
-SerialAdvectorCGOper::SerialAdvectorCGOper(const Vector &x_start,
-                                           GridFunction &vel,
-                                           FiniteElementSpace &fes)
-   : TimeDependentOperator(fes.GetVSize()),
-     x0(x_start), x_now(*fes.GetMesh()->GetNodes()),
-     u(vel), u_coeff(&u), M(&fes), K(&fes)
-{
-   ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
-   K.AddDomainIntegrator(Kinteg);
-   K.Assemble(0);
-   K.Finalize(0);
-
-   MassIntegrator *Minteg = new MassIntegrator;
-   M.AddDomainIntegrator(Minteg);
-   M.Assemble();
-   M.Finalize();
-}
-
-void SerialAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
-{
-   // Move the mesh.
-   const double t = GetTime();
-   add(x0, t, u, x_now);
-
-   // Assemble on the new mesh.
-   K.BilinearForm::operator=(0.0);
-   K.Assemble();
-   Vector rhs(K.Size());
-   K.Mult(ind, rhs);
-   M.BilinearForm::operator=(0.0);
-   M.Assemble();
-
-   di_dt = 0.0;
-   CGSolver lin_solver;
-   DSmoother prec;
-   lin_solver.SetPreconditioner(prec);
-   lin_solver.SetOperator(M.SpMat());
-   lin_solver.SetRelTol(1e-12); lin_solver.SetAbsTol(0.0);
-   lin_solver.SetMaxIter(100);
-   lin_solver.SetPrintLevel(0);
-   lin_solver.Mult(rhs, di_dt);
-}
-
-#ifdef MFEM_USE_MPI
-ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
-                                     GridFunction &vel,
-                                     ParFiniteElementSpace &pfes)
-   : TimeDependentOperator(pfes.GetVSize()),
-     x0(x_start), x_now(*pfes.GetMesh()->GetNodes()),
-     u(vel), u_coeff(&u), M(&pfes), K(&pfes)
-{
-   ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
-   K.AddDomainIntegrator(Kinteg);
-   K.Assemble(0);
-   K.Finalize(0);
-
-   MassIntegrator *Minteg = new MassIntegrator;
-   M.AddDomainIntegrator(Minteg);
-   M.Assemble();
-   M.Finalize();
-}
-
-void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
-{
-   // Move the mesh.
-   const double t = GetTime();
-   add(x0, t, u, x_now);
-
-   // Assemble on the new mesh.
-   K.BilinearForm::operator=(0.0);
-   K.Assemble();
-   ParGridFunction rhs(K.ParFESpace());
-   K.Mult(ind, rhs);
-   M.BilinearForm::operator=(0.0);
-   M.Assemble();
-
-   HypreParVector *RHS = rhs.ParallelAssemble();
-   HypreParVector X(K.ParFESpace());
-   X = 0.0;
-   HypreParMatrix *Mh  = M.ParallelAssemble();
-
-   CGSolver lin_solver(M.ParFESpace()->GetParMesh()->GetComm());
-   HypreSmoother prec;
-   prec.SetType(HypreSmoother::Jacobi, 1);
-   lin_solver.SetPreconditioner(prec);
-   lin_solver.SetOperator(*Mh);
-   lin_solver.SetRelTol(1e-8);
-   lin_solver.SetAbsTol(0.0);
-   lin_solver.SetMaxIter(100);
-   lin_solver.SetPrintLevel(0);
-   lin_solver.Mult(*RHS, X);
-   K.ParFESpace()->GetProlongationMatrix()->Mult(X, di_dt);
-
-   delete Mh;
-   delete RHS;
-}
-#endif
-
-double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
-                                              const Vector &b) const
-{
-   const FiniteElementSpace *fes = NULL;
-   double energy_in = 0.0;
-#ifdef MFEM_USE_MPI
-   const ParNonlinearForm *p_nlf = dynamic_cast<const ParNonlinearForm *>(oper);
-   MFEM_VERIFY(!(parallel && p_nlf == NULL), "Invalid Operator subclass.");
-   if (parallel)
-   {
-      fes = p_nlf->FESpace();
-      energy_in = p_nlf->GetEnergy(x);
-   }
-#endif
-   const bool serial = !parallel;
-   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
-   MFEM_VERIFY(!(serial && nlf == NULL), "Invalid Operator subclass.");
-   if (serial)
-   {
-      fes = nlf->FESpace();
-      energy_in = nlf->GetEnergy(x);
-   }
-
-   const bool have_b = (b.Size() == Height());
-
-   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
-             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
-   Array<int> xdofs(dof * dim);
-   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
-   Vector posV(pos.Data(), dof * dim);
-
-   Vector x_out(x.Size()), x_out_loc(fes->GetVSize());
-   bool x_out_ok = false;
-   double scale = 1.0, energy_out;
-   double norm0 = Norm(r);
-
-   // Decreases the scaling of the update until the new mesh is valid.
-   for (int i = 0; i < 12; i++)
-   {
-      add(x, -scale, c, x_out);
-
-      if (serial)
-      {
-         const SparseMatrix *cP = fes->GetConformingProlongation();
-         if (!cP) {x_out_loc.SetData(x_out.GetData());}
-         else {cP->Mult(x_out,x_out_loc);}
-         energy_out = nlf->GetGridFunctionEnergy(x_out_loc);
-      }
-#ifdef MFEM_USE_MPI
-      else
-      {
-         fes->GetProlongationMatrix()->Mult(x_out, x_out_loc);
-         energy_out = p_nlf->GetParGridFunctionEnergy(x_out_loc);
-      }
-#endif
-
-      if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
-      {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
-         scale *= 0.5; continue;
-      }
-
-      int jac_ok = 1;
-      for (int i = 0; i < NE; i++)
-      {
-         fes->GetElementVDofs(i, xdofs);
-         x_out_loc.GetSubVector(xdofs, posV);
-         for (int j = 0; j < nsp; j++)
-         {
-            fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-            MultAtB(pos, dshape, Jpr);
-            if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
-         }
-      }
-   break2:
-      int jac_ok_all = jac_ok;
-#ifdef MFEM_USE_MPI
-      if (parallel)
-      {
-         MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
-                       p_nlf->ParFESpace()->GetComm());
-      }
-#endif
-
-      if (jac_ok_all == 0)
-      {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
-         scale *= 0.5; continue;
-      }
-
-      oper->Mult(x_out, r);
-      if (have_b) { r -= b; }
-      double norm = Norm(r);
-
-      if (norm > 1.2*norm0)
-      {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
-         scale *= 0.5; continue;
-      }
-      else { x_out_ok = true; break; }
-   }
-
-   if (print_level >= 0)
-   {
-      mfem::out << "Energy decrease: "
-                << (energy_in - energy_out) / energy_in * 100.0
-                << "% with " << scale << " scaling.\n";
-   }
-
-   if (x_out_ok == false) { scale = 0.0; }
-   return scale;
-}
-
-void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
-{
-   if (discr_tc)
-   {
-      if (parallel)
-      {
-#ifdef MFEM_USE_MPI
-         const ParNonlinearForm *nlf =
-            dynamic_cast<const ParNonlinearForm *>(oper);
-         Vector x_loc(nlf->ParFESpace()->GetVSize());
-         nlf->ParFESpace()->GetProlongationMatrix()->Mult(x, x_loc);
-         discr_tc->UpdateTargetSpecification(x_loc);
-#endif
-      }
-      else { discr_tc->UpdateTargetSpecification(x); }
-   }
-}
-
-double TMOPDescentNewtonSolver::ComputeScalingFactor(const Vector &x,
-                                                     const Vector &b) const
-{
-   const FiniteElementSpace *fes = NULL;
-   double energy_in = 0.0;
-#ifdef MFEM_USE_MPI
-   const ParNonlinearForm *p_nlf = dynamic_cast<const ParNonlinearForm *>(oper);
-   MFEM_VERIFY(!(parallel && p_nlf == NULL), "Invalid Operator subclass.");
-   if (parallel)
-   {
-      fes = p_nlf->FESpace();
-      energy_in = p_nlf->GetEnergy(x);
-   }
-#endif
-   const bool serial = !parallel;
-   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
-   MFEM_VERIFY(!(serial && nlf == NULL), "Invalid Operator subclass.");
-   if (serial)
-   {
-      fes = nlf->FESpace();
-      energy_in = nlf->GetEnergy(x);
-   }
-
-   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
-             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
-   Array<int> xdofs(dof * dim);
-   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
-   Vector posV(pos.Data(), dof * dim);
-   Vector x_loc(fes->GetVSize());
-
-   double min_detJ = infinity();
-   for (int i = 0; i < NE; i++)
-   {
-      fes->GetElementVDofs(i, xdofs);
-      x_loc.GetSubVector(xdofs, posV);
-
-      for (int j = 0; j < nsp; j++)
-      {
-         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-         MultAtB(pos, dshape, Jpr);
-         min_detJ = std::min(min_detJ, Jpr.Det());
-      }
-   }
-   double min_detJ_all = min_detJ;
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
-                    p_nlf->ParFESpace()->GetComm());
-   }
-#endif
-   if (print_level >= 0)
-   {
-      mfem::out << "Minimum det(J) = " << min_detJ_all << '\n';
-   }
-
-   Vector x_out(x.Size());
-   bool x_out_ok = false;
-   double scale = 1.0, energy_out;
-
-   for (int i = 0; i < 7; i++)
-   {
-      add(x, -scale, c, x_out);
-      if (serial)
-      {
-         const SparseMatrix *cP = fes->GetConformingProlongation();
-         if (!cP) {x_loc.SetData(x_out.GetData());}
-         else {cP->Mult(x_out,x_loc);}
-         energy_out = nlf->GetGridFunctionEnergy(x_loc);
-      }
-#ifdef MFEM_USE_MPI
-      else
-      {
-         fes->GetProlongationMatrix()->Mult(x_out, x_loc);
-         energy_out = p_nlf->GetParGridFunctionEnergy(x_loc);
-      }
-#endif
-
-      if (energy_out > energy_in || std::isnan(energy_out) != 0)
-      {
-         scale *= 0.5;
-      }
-      else { x_out_ok = true; break; }
-   }
-
-   if (print_level >= 0)
-   {
-      mfem::out << "Energy decrease: "
-                << (energy_in - energy_out) / energy_in * 100.0
-                << "% with " << scale << " scaling.\n";
-   }
-
-   if (x_out_ok == false) { return 0.0; }
-
-   return scale;
-}
-
-void TMOPDescentNewtonSolver::ProcessNewState(const Vector &x) const
-{
-   if (discr_tc)
-   {
-      if (parallel)
-      {
-#ifdef MFEM_USE_MPI
-         const ParNonlinearForm *nlf =
-            dynamic_cast<const ParNonlinearForm *>(oper);
-         Vector x_loc(nlf->ParFESpace()->GetVSize());
-         nlf->ParFESpace()->GetProlongationMatrix()->Mult(x, x_loc);
-         discr_tc->UpdateTargetSpecification(x_loc);
-#endif
-      }
-      else { discr_tc->UpdateTargetSpecification(x); }
-   }
-}
-
-#ifdef MFEM_USE_MPI
-// Metric values are visualized by creating an L2 finite element functions and
-// computing the metric values at the nodes.
-void vis_tmop_metric_p(int order, TMOP_QualityMetric &qm,
-                       const TargetConstructor &tc, ParMesh &pmesh,
-                       char *title, int position)
-{
-   L2_FECollection fec(order, pmesh.Dimension(), BasisType::GaussLobatto);
-   ParFiniteElementSpace fes(&pmesh, &fec, 1);
-   ParGridFunction metric(&fes);
-   InterpolateTMOP_QualityMetric(qm, tc, pmesh, metric);
-   socketstream sock;
-   if (pmesh.GetMyRank() == 0)
-   {
-      sock.open("localhost", 19916);
-      sock << "solution\n";
-   }
-   pmesh.PrintAsOne(sock);
-   metric.SaveAsOne(sock);
-   if (pmesh.GetMyRank() == 0)
-   {
-      sock << "window_title '"<< title << "'\n"
-           << "window_geometry "
-           << position << " " << 0 << " " << 600 << " " << 600 << "\n"
-           << "keys jRmclA\n";
-   }
-}
-#endif
-
-// Metric values are visualized by creating an L2 finite element functions and
-// computing the metric values at the nodes.
-void vis_tmop_metric_s(int order, TMOP_QualityMetric &qm,
-                       const TargetConstructor &tc, Mesh &mesh,
-                       char *title, int position)
-{
-   L2_FECollection fec(order, mesh.Dimension(), BasisType::GaussLobatto);
-   FiniteElementSpace fes(&mesh, &fec, 1);
-   GridFunction metric(&fes);
-   InterpolateTMOP_QualityMetric(qm, tc, mesh, metric);
-   osockstream sock(19916, "localhost");
-   sock << "solution\n";
-   mesh.Print(sock);
-   metric.Save(sock);
-   sock.send();
-   sock << "window_title '"<< title << "'\n"
-        << "window_geometry "
-        << position << " " << 0 << " " << 600 << " " << 600 << "\n"
-        << "keys jRmclA\n";
-}
-
-}

fem/tmop_tools.hpp

@@ -1,140 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_TMOP_TOOLS_HPP
-#define MFEM_TMOP_TOOLS_HPP
-
-#include "bilinearform.hpp"
-#include "pbilinearform.hpp"
-#include "tmop.hpp"
-
-namespace mfem
-{
-
-// Performs the full remap advection loop.
-class AdvectorCG : public AdaptivityEvaluator
-{
-private:
-   RK4Solver ode_solver;
-   Vector nodes0;
-   Vector field0;
-
-public:
-   AdvectorCG() : AdaptivityEvaluator(), ode_solver(), nodes0(), field0() { }
-
-   virtual void SetInitialField(const Vector &init_nodes,
-                                const Vector &init_field);
-
-   virtual void ComputeAtNewPosition(const Vector &new_nodes,
-                                     Vector &new_field);
-};
-
-/// Performs a single remap advection step in serial.
-class SerialAdvectorCGOper : public TimeDependentOperator
-{
-protected:
-   const Vector &x0;
-   Vector &x_now;
-   GridFunction &u;
-   VectorGridFunctionCoefficient u_coeff;
-   mutable BilinearForm M, K;
-
-public:
-   /** Here @a fes is the FESpace of the function that will be moved. Note
-       that Mult() moves the nodes of the mesh corresponding to @a fes. */
-   SerialAdvectorCGOper(const Vector &x_start, GridFunction &vel,
-                        FiniteElementSpace &fes);
-
-   virtual void Mult(const Vector &ind, Vector &di_dt) const;
-};
-
-#ifdef MFEM_USE_MPI
-/// Performs a single remap advection step in parallel.
-class ParAdvectorCGOper : public TimeDependentOperator
-{
-protected:
-   const Vector &x0;
-   Vector &x_now;
-   GridFunction &u;
-   VectorGridFunctionCoefficient u_coeff;
-   mutable ParBilinearForm M, K;
-
-public:
-   /** Here @a pfes is the ParFESpace of the function that will be moved. Note
-       that Mult() moves the nodes of the mesh corresponding to @a pfes. */
-   ParAdvectorCGOper(const Vector &x_start, GridFunction &vel,
-                     ParFiniteElementSpace &pfes);
-
-   virtual void Mult(const Vector &ind, Vector &di_dt) const;
-};
-#endif
-
-class TMOPNewtonSolver : public NewtonSolver
-{
-private:
-   bool parallel;
-
-   // Quadrature points that are checked for negative Jacobians etc.
-   const IntegrationRule &ir;
-
-   mutable DiscreteAdaptTC *discr_tc;
-
-public:
-#ifdef MFEM_USE_MPI
-   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule)
-      : NewtonSolver(comm), parallel(true), ir(irule), discr_tc(NULL) { }
-#endif
-   TMOPNewtonSolver(const IntegrationRule &irule)
-      : NewtonSolver(), parallel(false), ir(irule), discr_tc(NULL) { }
-
-   void SetDiscreteAdaptTC(DiscreteAdaptTC *tc) { discr_tc = tc; }
-
-   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
-
-   virtual void ProcessNewState(const Vector &x) const;
-};
-
-/// Allows negative Jacobians. Used for untangling.
-class TMOPDescentNewtonSolver : public NewtonSolver
-{
-private:
-   bool parallel;
-
-   // Quadrature points that are checked for negative Jacobians etc.
-   const IntegrationRule &ir;
-
-   mutable DiscreteAdaptTC *discr_tc;
-
-public:
-#ifdef MFEM_USE_MPI
-   TMOPDescentNewtonSolver(MPI_Comm comm, const IntegrationRule &irule)
-      : NewtonSolver(comm), parallel(true), ir(irule), discr_tc(NULL) { }
-#endif
-   TMOPDescentNewtonSolver(const IntegrationRule &irule)
-      : NewtonSolver(), parallel(false), ir(irule), discr_tc(NULL) { }
-
-   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
-
-   virtual void ProcessNewState(const Vector &x) const;
-};
-
-void vis_tmop_metric_s(int order, TMOP_QualityMetric &qm,
-                       const TargetConstructor &tc, Mesh &pmesh,
-                       char *title, int position);
-#ifdef MFEM_USE_MPI
-void vis_tmop_metric_p(int order, TMOP_QualityMetric &qm,
-                       const TargetConstructor &tc, ParMesh &pmesh,
-                       char *title, int position);
-#endif
-
-}
-
-#endif

general/array.hpp

@@ -433,7 +433,7 @@ class BlockArray
 public:
    BlockArray(int block_size = 16*1024);
    BlockArray(const BlockArray<T> &other); // deep copy
-   ~BlockArray() { Destroy(); }
+   ~BlockArray();
 
    /// Allocate and construct a new item in the array, return its index.
    int Append();
@@ -463,9 +463,6 @@ public:
    /// Return the current capacity of the BlockArray.
    int Capacity() const { return blocks.Size()*(mask+1); }
 
-   /// Destroy all items, set size to zero.
-   void DeleteAll() { Destroy(); blocks.DeleteAll(); size = 0; }
-
    void Swap(BlockArray<T> &other);
 
    long MemoryUsage() const;
@@ -570,8 +567,6 @@ protected:
       MFEM_ASSERT(index >= 0 && index < size,
                   "Out of bounds access: " << index << ", size = " << size);
    }
-
-   void Destroy();
 };
 
 
@@ -1003,7 +998,7 @@ long BlockArray<T>::MemoryUsage() const
 }
 
 template<typename T>
-void BlockArray<T>::Destroy()
+BlockArray<T>::~BlockArray()
 {
    int bsize = size & mask;
    for (int i = blocks.Size(); i != 0; )

general/communication.hpp

@@ -322,9 +322,7 @@ struct VarMessage
    std::string data;
    MPI_Request send_request;
 
-   /** Non-blocking send to processor 'rank'. Returns immediately. Completion
-       (as tested by MPI_Wait/Test) does not mean the message was received --
-       it may be on its way or just buffered locally. */
+   /// Non-blocking send to processor 'rank'.
    void Isend(int rank, MPI_Comm comm)
    {
       Encode(rank);
@@ -332,20 +330,12 @@ struct VarMessage
                 &send_request);
    }
 
-   /** Non-blocking synchronous send to processor 'rank'. Returns immediately.
-       Completion (MPI_Wait/Test) means that the message was received. */
-   void Issend(int rank, MPI_Comm comm)
-   {
-      Encode(rank);
-      MPI_Issend((void*) data.data(), data.length(), MPI_BYTE, rank, Tag, comm,
-                 &send_request);
-   }
-
    /// Helper to send all messages in a rank-to-message map container.
    template<typename MapT>
    static void IsendAll(MapT& rank_msg, MPI_Comm comm)
    {
-      for (auto it = rank_msg.begin(); it != rank_msg.end(); ++it)
+      typename MapT::iterator it;
+      for (it = rank_msg.begin(); it != rank_msg.end(); ++it)
       {
          it->second.Isend(it->first, comm);
       }
@@ -355,32 +345,14 @@ struct VarMessage
    template<typename MapT>
    static void WaitAllSent(MapT& rank_msg)
    {
-      for (auto it = rank_msg.begin(); it != rank_msg.end(); ++it)
+      typename MapT::iterator it;
+      for (it = rank_msg.begin(); it != rank_msg.end(); ++it)
       {
          MPI_Wait(&it->second.send_request, MPI_STATUS_IGNORE);
          it->second.Clear();
       }
    }
 
-   /** Return true if all messages in the map container were sent, otherwise
-       return false, without waiting. */
-   template<typename MapT>
-   static bool TestAllSent(MapT& rank_msg)
-   {
-      for (auto it = rank_msg.begin(); it != rank_msg.end(); ++it)
-      {
-         VarMessage &msg = it->second;
-         if (msg.send_request != MPI_REQUEST_NULL)
-         {
-            int sent;
-            MPI_Test(&msg.send_request, &sent, MPI_STATUS_IGNORE);
-            if (!sent) { return false; }
-            msg.Clear();
-         }
-      }
-      return true;
-   }
-
    /** Blocking probe for incoming message of this type from any rank.
        Returns the rank and message size. */
    static void Probe(int &rank, int &size, MPI_Comm comm)

general/device.hpp

@@ -276,7 +276,7 @@ inline T *Write(Memory<T> &mem, int size, bool on_dev = true)
 
 /** @brief Shortcut to Write(const Memory<T> &mem, int size, false) */
 template <typename T>
-inline T *HostWrite(Memory<T> &mem, int size)
+inline const T *HostWrite(const Memory<T> &mem, int size)
 {
    return mfem::Write(mem, size, false);
 }
@@ -298,9 +298,9 @@ inline T *ReadWrite(Memory<T> &mem, int size, bool on_dev = true)
    }
 }
 
-/** @brief Shortcut to ReadWrite(Memory<T> &mem, int size, false) */
+/** @brief Shortcut to ReadWrite(const Memory<T> &mem, int size, false) */
 template <typename T>
-inline T *HostReadWrite(Memory<T> &mem, int size)
+inline const T *HostReadWrite(const Memory<T> &mem, int size)
 {
    return mfem::ReadWrite(mem, size, false);
 }

general/error.hpp

@@ -138,10 +138,4 @@ void mfem_warning(const char *msg = NULL);
 // Generate a warning message - always generated, regardless of MFEM_DEBUG.
 #define MFEM_WARNING(msg) _MFEM_MESSAGE("MFEM Warning: " << msg, 1)
 
-// Macro that checks (in MFEM_DEBUG mode) that i is in the range [imin,imax).
-#define MFEM_ASSERT_INDEX_IN_RANGE(i,imin,imax) \
-   MFEM_ASSERT((imin) <= (i) && (i) < (imax), \
-   "invalid index " #i << " = " << (i) << \
-   ", valid range is [" << (imin) << ',' << (imax) << ')')
-
 #endif

general/forall.hpp

@@ -91,8 +91,7 @@ using RAJA::statement::Segs;
 template <const int BLOCKS = MFEM_CUDA_BLOCKS, typename DBODY>
 void RajaCudaWrap1D(const int N, DBODY &&d_body)
 {
-   //true denotes asynchronous kernel
-   RAJA::forall<RAJA::cuda_exec<BLOCKS,true>>(RAJA::RangeSegment(0,N),d_body);
+   RAJA::forall<RAJA::cuda_exec<BLOCKS>>(RAJA::RangeSegment(0,N),d_body);
 }
 
 template <typename DBODY>
@@ -103,8 +102,8 @@ void RajaCudaWrap2D(const int N, DBODY &&d_body,
    MFEM_VERIFY(BZ>0, "");
    const int G = (N+BZ-1)/BZ;
    RAJA::kernel<RAJA::KernelPolicy<
-   RAJA::statement::CudaKernelAsync<
-   RAJA::statement::For<0, RAJA::cuda_block_x_direct,
+   RAJA::statement::CudaKernel<
+   RAJA::statement::For<0, RAJA::cuda_block_x_loop,
         RAJA::statement::For<1, RAJA::cuda_thread_x_direct,
         RAJA::statement::For<2, RAJA::cuda_thread_y_direct,
         RAJA::statement::For<3, RAJA::cuda_thread_z_direct,
@@ -116,6 +115,7 @@ void RajaCudaWrap2D(const int N, DBODY &&d_body,
       const int k = n*BZ + threadIdx.z;
       if (k >= N) { return; }
       d_body(k);
+      MFEM_SYNC_THREAD;
    });
    MFEM_GPU_CHECK(cudaGetLastError());
 }
@@ -126,15 +126,15 @@ void RajaCudaWrap3D(const int N, DBODY &&d_body,
 {
    MFEM_VERIFY(N>0, "");
    RAJA::kernel<RAJA::KernelPolicy<
-   RAJA::statement::CudaKernelAsync<
-   RAJA::statement::For<0, RAJA::cuda_block_x_direct,
+   RAJA::statement::CudaKernel<
+   RAJA::statement::For<0, RAJA::cuda_block_x_loop,
         RAJA::statement::For<1, RAJA::cuda_thread_x_direct,
         RAJA::statement::For<2, RAJA::cuda_thread_y_direct,
         RAJA::statement::For<3, RAJA::cuda_thread_z_direct,
         RAJA::statement::Lambda<0, Segs<0>>>>>>>>>
         (RAJA::make_tuple(RAJA::RangeSegment(0,N), RAJA::RangeSegment(0,X),
                           RAJA::RangeSegment(0,Y), RAJA::RangeSegment(0,Z)),
-   [=] RAJA_DEVICE (const int k) { d_body(k); });
+   [=] RAJA_DEVICE (const int k) { d_body(k); MFEM_SYNC_THREAD; });
    MFEM_GPU_CHECK(cudaGetLastError());
 }
 
@@ -147,11 +147,35 @@ void RajaCudaWrap3D(const int N, DBODY &&d_body,
 using RAJA::statement::Segs;
 
 template <typename HBODY>
-void RajaOmpWrap(const int N, HBODY &&h_body)
+void RajaOmpWrap1D(const int N, HBODY &&h_body)
 {
    RAJA::forall<RAJA::omp_parallel_for_exec>(RAJA::RangeSegment(0,N), h_body);
 }
 
+template <typename HBODY>
+void RajaOmpWrap2D(const int N, HBODY &&h_body,
+                   const int X, const int Y, const int BZ)
+{
+   RAJA::kernel<RAJA::KernelPolicy<
+   RAJA::statement::For<0, RAJA::omp_parallel_for_exec,
+        RAJA::statement::Lambda<0, Segs<0>>>>>
+        (RAJA::make_tuple(RAJA::RangeSegment(0,N), RAJA::RangeSegment(0,X),
+                          RAJA::RangeSegment(0,Y), RAJA::RangeSegment(0,BZ)),
+   [=] (int k) { h_body(k); });
+}
+
+template <typename HBODY>
+void RajaOmpWrap3D(const int N, HBODY &&h_body,
+                   const int X, const int Y, const int Z)
+{
+   RAJA::kernel<RAJA::KernelPolicy<
+   RAJA::statement::For<0, RAJA::omp_parallel_for_exec,
+        RAJA::statement::Lambda<0, Segs<0>>>>>
+        (RAJA::make_tuple(RAJA::RangeSegment(0,N), RAJA::RangeSegment(0,X),
+                          RAJA::RangeSegment(0,Y), RAJA::RangeSegment(0,Z)),
+   [=] (int k) { h_body(k); });
+}
+
 #endif
 
 
@@ -336,8 +360,14 @@ inline void ForallWrap(const bool use_dev, const int N,
 
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
    // Handle all allowed OpenMP backends except Backend::OMP
-   if (Device::Allows(Backend::OMP_MASK & ~Backend::OMP))
-   { return RajaOmpWrap(N, h_body); }
+   if (DIM == 1 && Device::Allows(Backend::OMP_MASK & ~Backend::OMP))
+   { return RajaOmpWrap1D(N, h_body); }
+
+   if (DIM == 2 && Device::Allows(Backend::OMP_MASK & ~Backend::OMP))
+   { return RajaOmpWrap2D(N, h_body, X, Y, Z); }
+
+   if (DIM == 3 && Device::Allows(Backend::OMP_MASK & ~Backend::OMP))
+   { return RajaOmpWrap3D(N, h_body, X, Y, Z); }
 #endif
 
 #ifdef MFEM_USE_OPENMP

general/globals.hpp

@@ -107,9 +107,4 @@ void SetGlobalMPI_Comm(MPI_Comm comm);
 
 } // namespace mfem
 
-
-// Request a global object to be instantiated for each thread in its TLS.
-#define MFEM_THREAD_LOCAL thread_local
-
-
 #endif

general/hash.hpp

@@ -79,22 +79,22 @@ public:
 
    /// Get item whose parents are p1, p2... Create it if it doesn't exist.
    T* Get(int p1, int p2);
-   T* Get(int p1, int p2, int p3, int p4 = -1 /* p4 optional */);
+   T* Get(int p1, int p2, int p3, int p4);
 
    /// Get id of item whose parents are p1, p2... Create it if it doesn't exist.
    int GetId(int p1, int p2);
-   int GetId(int p1, int p2, int p3, int p4 = -1);
+   int GetId(int p1, int p2, int p3, int p4);
 
    /// Find item whose parents are p1, p2... Return NULL if it doesn't exist.
    T* Find(int p1, int p2);
-   T* Find(int p1, int p2, int p3, int p4 = -1);
+   T* Find(int p1, int p2, int p3, int p4);
 
    const T* Find(int p1, int p2) const;
-   const T* Find(int p1, int p2, int p3, int p4 = -1) const;
+   const T* Find(int p1, int p2, int p3, int p4) const;
 
    /// Find id of item whose parents are p1, p2... Return -1 if it doesn't exist.
    int FindId(int p1, int p2) const;
-   int FindId(int p1, int p2, int p3, int p4 = -1) const;
+   int FindId(int p1, int p2, int p3, int p4) const;
 
    /// Return the number of elements currently stored in the HashTable.
    int Size() const { return Base::Size() - unused.Size(); }
@@ -113,12 +113,9 @@ public:
    /** Its id will be reused by newly added items. */
    void Delete(int id);
 
-   /// Remove all items.
-   void DeleteAll();
-
    /// Make an item hashed under different parent IDs.
    void Reparent(int id, int new_p1, int new_p2);
-   void Reparent(int id, int new_p1, int new_p2, int new_p3, int new_p4 = -1);
+   void Reparent(int id, int new_p1, int new_p2, int new_p3, int new_p4);
 
    /// Return total size of allocated memory (tables plus items), in bytes.
    long MemoryUsage() const;
@@ -249,18 +246,6 @@ inline void sort4(int &a, int &b, int &c, int &d)
    sort3(b, c, d);
 }
 
-inline void sort4_ext(int &a, int &b, int &c, int &d)
-{
-   if (d < 0) // support optional last index
-   {
-      sort3(a, b, c);
-   }
-   else
-   {
-      sort4(a, b, c, d);
-   }
-}
-
 } // internal
 
 template<typename T>
@@ -310,7 +295,7 @@ template<typename T>
 int HashTable<T>::GetId(int p1, int p2, int p3, int p4)
 {
    // search for the item in the hashtable
-   internal::sort4_ext(p1, p2, p3, p4);
+   internal::sort4(p1, p2, p3, p4);
    int idx = Hash(p1, p2, p3);
    int id = SearchList(table[idx], p1, p2, p3);
    if (id >= 0) { return id; }
@@ -376,7 +361,7 @@ int HashTable<T>::FindId(int p1, int p2) const
 template<typename T>
 int HashTable<T>::FindId(int p1, int p2, int p3, int p4) const
 {
-   internal::sort4_ext(p1, p2, p3, p4);
+   internal::sort4(p1, p2, p3, p4);
    return SearchList(table[Hash(p1, p2, p3)], p1, p2, p3);
 }
 
@@ -474,14 +459,6 @@ void HashTable<T>::Delete(int id)
    unused.Append(id); // add its id to the unused ids
 }
 
-template<typename T>
-void HashTable<T>::DeleteAll()
-{
-   Base::DeleteAll();
-   for (int i = 0; i <= mask; i++) { table[i] = -1; }
-   unused.DeleteAll();
-}
-
 template<typename T>
 void HashTable<T>::Reparent(int id, int new_p1, int new_p2)
 {
@@ -504,7 +481,7 @@ void HashTable<T>::Reparent(int id,
    T& item = Base::At(id);
    Unlink(Hash(item), id);
 
-   internal::sort4_ext(new_p1, new_p2, new_p3, new_p4);
+   internal::sort4(new_p1, new_p2, new_p3, new_p4);
    item.p1 = new_p1;
    item.p2 = new_p2;
    item.p3 = new_p3;

general/mem_manager.cpp

@@ -11,7 +11,7 @@
 
 #include "../general/forall.hpp"
 
-#include <cstring> // std::memcpy, std::memcmp
+#include <cstring> // std::memcpy
 
 #include <list>
 #include <unordered_map>
@@ -57,15 +57,6 @@ MemoryClass operator*(MemoryClass mc1, MemoryClass mc2)
 }
 
 
-// Instantiate Memory<T>::PrintFlags for T = int and T = double.
-template void Memory<int>::PrintFlags() const;
-template void Memory<double>::PrintFlags() const;
-
-// Instantiate Memory<T>::CompareHostAndDevice for T = int and T = double.
-template int Memory<int>::CompareHostAndDevice(int size) const;
-template int Memory<double>::CompareHostAndDevice(int size) const;
-
-
 namespace internal
 {
 
@@ -704,19 +695,6 @@ void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
                 ~(dest_on_host ? Mem::VALID_DEVICE : Mem::VALID_HOST);
 }
 
-int MemoryManager::CompareHostAndDevice_(void *h_ptr, size_t size,
-                                         unsigned flags)
-{
-   void *d_ptr = (flags & Mem::ALIAS) ?
-                 mm.GetAliasDevicePtr(h_ptr, size, false) :
-                 mm.GetDevicePtr(h_ptr, size, false);
-   char *h_buf = new char[size];
-   CuMemcpyDtoH(h_buf, d_ptr, size);
-   int res = std::memcmp(h_ptr, h_buf, size);
-   delete [] h_buf;
-   return res;
-}
-
 
 void MemoryPrintFlags(unsigned flags)
 {

general/mem_manager.hpp

@@ -377,16 +377,6 @@ public:
    /// Copy @a size entries from @a *this to the host pointer @a dest.
    /** The given @a size should not exceed the Capacity() of @a *this. */
    inline void CopyToHost(T *dest, int size) const;
-
-   /// Print the internal flags.
-   /** This method can be useful for debugging. It is explicitly instantiated
-       for Memory<T> with T = int and T = double. */
-   inline void PrintFlags() const;
-
-   /// If both the host and the device data are valid, compare their contents.
-   /** This method can be useful for debugging. It is explicitly instantiated
-       for Memory<T> with T = int and T = double. */
-   inline int CompareHostAndDevice(int size) const;
 };
 
 
@@ -457,11 +447,6 @@ private:
    static void CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
                              std::size_t size, unsigned &dest_flags);
 
-   // Compare the contents of the host and the device memory - useful for
-   // debugging.
-   static int CompareHostAndDevice_(void *h_ptr, size_t size, unsigned flags);
-
-
    /// Adds an address in the map
    void *Insert(void *ptr, const std::size_t bytes);
 
@@ -742,24 +727,10 @@ inline void Memory<T>::CopyToHost(T *dest, int size) const
 
 
 /** @brief Print the state of a Memory object based on its internal flags.
-    Useful in a debugger. See also Memory<T>::PrintFlags(). */
+    Useful in a debugger. */
 extern void MemoryPrintFlags(unsigned flags);
 
 
-template <typename T>
-inline void Memory<T>::PrintFlags() const
-{
-   MemoryPrintFlags(flags);
-}
-
-template <typename T>
-inline int Memory<T>::CompareHostAndDevice(int size) const
-{
-   if (!(flags & VALID_HOST) || !(flags & VALID_DEVICE)) { return 0; }
-   return MemoryManager::CompareHostAndDevice_(h_ptr, size*sizeof(T), flags);
-}
-
-
 /// The (single) global memory manager object
 extern MemoryManager mm;
 

linalg/densemat.cpp

@@ -4456,8 +4456,7 @@ const
    int n = SizeI(), ne = SizeK();
    const int *I = elem_dof.GetI(), *J = elem_dof.GetJ(), *dofs;
    const double *d_col = tdata;
-   double *yp = y.HostReadWrite();
-   double x_col;
+   double *yp = y, x_col;
    const double *xp = x;
    // the '4' here can be tuned for given platform and compiler
    if (n <= 4)

linalg/densemat.hpp

@@ -721,26 +721,11 @@ public:
    { return const_cast<DenseTensor&>(*this)(k); }
 
    double &operator()(int i, int j, int k)
-   {
-      MFEM_ASSERT_INDEX_IN_RANGE(i, 0, SizeI());
-      MFEM_ASSERT_INDEX_IN_RANGE(j, 0, SizeJ());
-      MFEM_ASSERT_INDEX_IN_RANGE(k, 0, SizeK());
-      return tdata[i+SizeI()*(j+SizeJ()*k)];
-   }
-
+   { return tdata[i+SizeI()*(j+SizeJ()*k)]; }
    const double &operator()(int i, int j, int k) const
-   {
-      MFEM_ASSERT_INDEX_IN_RANGE(i, 0, SizeI());
-      MFEM_ASSERT_INDEX_IN_RANGE(j, 0, SizeJ());
-      MFEM_ASSERT_INDEX_IN_RANGE(k, 0, SizeK());
-      return tdata[i+SizeI()*(j+SizeJ()*k)];
-   }
+   { return tdata[i+SizeI()*(j+SizeJ()*k)]; }
 
-   double *GetData(int k)
-   {
-      MFEM_ASSERT_INDEX_IN_RANGE(k, 0, SizeK());
-      return tdata+k*Mk.Height()*Mk.Width();
-   }
+   double *GetData(int k) { return tdata+k*Mk.Height()*Mk.Width(); }
 
    double *Data() { return tdata; }
 

linalg/hypre.cpp

@@ -185,7 +185,7 @@ Vector * HypreParVector::GlobalVector() const
 
 HypreParVector& HypreParVector::operator=(double d)
 {
-   Vector::operator=(d);
+   hypre_ParVectorSetConstantValues(x,d);
    return *this;
 }
 
@@ -198,7 +198,10 @@ HypreParVector& HypreParVector::operator=(const HypreParVector &y)
    }
 #endif
 
-   Vector::operator=(y);
+   for (int i = 0; i < size; i++)
+   {
+      data[i] = y.data[i];
+   }
    return *this;
 }
 
@@ -1021,8 +1024,6 @@ HypreParMatrix * HypreParMatrix::Transpose() const
 HYPRE_Int HypreParMatrix::Mult(HypreParVector &x, HypreParVector &y,
                                double a, double b)
 {
-   x.HostRead();
-   (b == 0.0) ? y.HostWrite() : y.HostReadWrite();
    return hypre_ParCSRMatrixMatvec(a, A, x, b, y);
 }
 
@@ -1034,7 +1035,7 @@ void HypreParMatrix::Mult(double a, const Vector &x, double b, Vector &y) const
                << ", expected size = " << Height());
 
    auto x_data = x.HostRead();
-   auto y_data = (b == 0.0) ? y.HostWrite() : y.HostReadWrite();
+   auto y_data = y.HostWrite();
    if (X == NULL)
    {
       X = new HypreParVector(A->comm,
@@ -1066,7 +1067,7 @@ void HypreParMatrix::MultTranspose(double a, const Vector &x,
    // Note: x has the dimensions of Y (height), and
    //       y has the dimensions of X (width)
    auto x_data = x.HostRead();
-   auto y_data = (b == 0.0) ? y.HostWrite() : y.HostReadWrite();
+   auto y_data = y.HostWrite();
    if (X == NULL)
    {
       X = new HypreParVector(A->comm,
@@ -2008,12 +2009,10 @@ void HypreSmoother::Mult(const HypreParVector &b, HypreParVector &x) const
       return;
    }
 
-   b.HostRead();
    if (!iterative_mode)
    {
       if (type == 0 && relax_times == 1)
       {
-         x.HostWrite();
          HYPRE_ParCSRDiagScale(NULL, *A, b, x);
          if (relax_weight != 1.0)
          {
@@ -2023,7 +2022,6 @@ void HypreSmoother::Mult(const HypreParVector &b, HypreParVector &x) const
       }
       x = 0.0;
    }
-   x.HostReadWrite();
 
    if (V == NULL)
    {
@@ -2073,25 +2071,21 @@ void HypreSmoother::Mult(const Vector &b, Vector &x) const
       mfem_error("HypreSmoother::Mult (...) : HypreParMatrix A is missing");
       return;
    }
-
-   auto b_data = b.HostRead();
-   auto x_data = iterative_mode ? x.HostReadWrite() : x.HostWrite();
-
    if (B == NULL)
    {
       B = new HypreParVector(A->GetComm(),
                              A -> GetGlobalNumRows(),
-                             const_cast<double*>(b_data),
+                             b.GetData(),
                              A -> GetRowStarts());
       X = new HypreParVector(A->GetComm(),
                              A -> GetGlobalNumCols(),
-                             x_data,
+                             x.GetData(),
                              A -> GetColStarts());
    }
    else
    {
-      B -> SetData(const_cast<double*>(b_data));
-      X -> SetData(x_data);
+      B -> SetData(b.GetData());
+      X -> SetData(x.GetData());
    }
 
    Mult(*B, *X);
@@ -2208,14 +2202,7 @@ HypreSolver::~HypreSolver()
 }
 
 
-HyprePCG::HyprePCG(MPI_Comm comm) : precond(NULL)
-{
-   iterative_mode = true;
-
-   HYPRE_ParCSRPCGCreate(comm, &pcg_solver);
-}
-
-HyprePCG::HyprePCG(HypreParMatrix &_A) : HypreSolver(&_A), precond(NULL)
+HyprePCG::HyprePCG(HypreParMatrix &_A) : HypreSolver(&_A)
 {
    MPI_Comm comm;
 
@@ -2226,26 +2213,6 @@ HyprePCG::HyprePCG(HypreParMatrix &_A) : HypreSolver(&_A), precond(NULL)
    HYPRE_ParCSRPCGCreate(comm, &pcg_solver);
 }
 
-void HyprePCG::SetOperator(const Operator &op)
-{
-   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
-   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
-
-   // update base classes: Operator, Solver, HypreSolver
-   height = new_A->Height();
-   width  = new_A->Width();
-   A = const_cast<HypreParMatrix *>(new_A);
-   if (precond)
-   {
-      precond->SetOperator(*A);
-      this->SetPreconditioner(*precond);
-   }
-   setup_called = 0;
-   delete X;
-   delete B;
-   B = X = NULL;
-}
-
 void HyprePCG::SetTol(double tol)
 {
    HYPRE_PCGSetTol(pcg_solver, tol);
@@ -2266,14 +2233,12 @@ void HyprePCG::SetPrintLevel(int print_lvl)
    HYPRE_ParCSRPCGSetPrintLevel(pcg_solver, print_lvl);
 }
 
-void HyprePCG::SetPreconditioner(HypreSolver &_precond)
+void HyprePCG::SetPreconditioner(HypreSolver &precond)
 {
-   precond = &_precond;
-
    HYPRE_ParCSRPCGSetPrecond(pcg_solver,
-                             _precond.SolveFcn(),
-                             _precond.SetupFcn(),
-                             _precond);
+                             precond.SolveFcn(),
+                             precond.SetupFcn(),
+                             precond);
 }
 
 void HyprePCG::SetResidualConvergenceOptions(int res_frequency, double rtol)
@@ -2371,57 +2336,24 @@ HyprePCG::~HyprePCG()
 }
 
 
-HypreGMRES::HypreGMRES(MPI_Comm comm) : precond(NULL)
-{
-   iterative_mode = true;
-
-   HYPRE_ParCSRGMRESCreate(comm, &gmres_solver);
-   SetDefaultOptions();
-}
-
 HypreGMRES::HypreGMRES(HypreParMatrix &_A) : HypreSolver(&_A)
 {
    MPI_Comm comm;
 
+   int k_dim    = 50;
+   int max_iter = 100;
+   double tol   = 1e-6;
+
    iterative_mode = true;
 
    HYPRE_ParCSRMatrixGetComm(*A, &comm);
 
    HYPRE_ParCSRGMRESCreate(comm, &gmres_solver);
-   SetDefaultOptions();
-}
-
-void HypreGMRES::SetDefaultOptions()
-{
-   int k_dim    = 50;
-   int max_iter = 100;
-   double tol   = 1e-6;
-
    HYPRE_ParCSRGMRESSetKDim(gmres_solver, k_dim);
    HYPRE_ParCSRGMRESSetMaxIter(gmres_solver, max_iter);
    HYPRE_ParCSRGMRESSetTol(gmres_solver, tol);
 }
 
-void HypreGMRES::SetOperator(const Operator &op)
-{
-   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
-   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
-
-   // update base classes: Operator, Solver, HypreSolver
-   height = new_A->Height();
-   width  = new_A->Width();
-   A = const_cast<HypreParMatrix *>(new_A);
-   if (precond)
-   {
-      precond->SetOperator(*A);
-      this->SetPreconditioner(*precond);
-   }
-   setup_called = 0;
-   delete X;
-   delete B;
-   B = X = NULL;
-}
-
 void HypreGMRES::SetTol(double tol)
 {
    HYPRE_GMRESSetTol(gmres_solver, tol);
@@ -2447,14 +2379,12 @@ void HypreGMRES::SetPrintLevel(int print_lvl)
    HYPRE_GMRESSetPrintLevel(gmres_solver, print_lvl);
 }
 
-void HypreGMRES::SetPreconditioner(HypreSolver &_precond)
+void HypreGMRES::SetPreconditioner(HypreSolver &precond)
 {
-   precond = &_precond;
-
    HYPRE_ParCSRGMRESSetPrecond(gmres_solver,
-                               _precond.SolveFcn(),
-                               _precond.SetupFcn(),
-                               _precond);
+                               precond.SolveFcn(),
+                               precond.SetupFcn(),
+                               precond);
 }
 
 void HypreGMRES::Mult(const HypreParVector &b, HypreParVector &x) const
@@ -2531,40 +2461,10 @@ HypreGMRES::~HypreGMRES()
 }
 
 
-void HypreDiagScale::SetOperator(const Operator &op)
-{
-   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
-   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
-
-   // update base classes: Operator, Solver, HypreSolver
-   height = new_A->Height();
-   width  = new_A->Width();
-   A = const_cast<HypreParMatrix *>(new_A);
-   setup_called = 0;
-   delete X;
-   delete B;
-   B = X = NULL;
-}
-
-
-HypreParaSails::HypreParaSails(MPI_Comm comm)
-{
-   HYPRE_ParaSailsCreate(comm, &sai_precond);
-   SetDefaultOptions();
-}
-
 HypreParaSails::HypreParaSails(HypreParMatrix &A) : HypreSolver(&A)
 {
    MPI_Comm comm;
 
-   HYPRE_ParCSRMatrixGetComm(A, &comm);
-
-   HYPRE_ParaSailsCreate(comm, &sai_precond);
-   SetDefaultOptions();
-}
-
-void HypreParaSails::SetDefaultOptions()
-{
    int    sai_max_levels = 1;
    double sai_threshold  = 0.1;
    double sai_filter     = 0.1;
@@ -2573,36 +2473,9 @@ void HypreParaSails::SetDefaultOptions()
    int    sai_reuse      = 0;
    int    sai_logging    = 1;
 
-   HYPRE_ParaSailsSetParams(sai_precond, sai_threshold, sai_max_levels);
-   HYPRE_ParaSailsSetFilter(sai_precond, sai_filter);
-   HYPRE_ParaSailsSetSym(sai_precond, sai_sym);
-   HYPRE_ParaSailsSetLoadbal(sai_precond, sai_loadbal);
-   HYPRE_ParaSailsSetReuse(sai_precond, sai_reuse);
-   HYPRE_ParaSailsSetLogging(sai_precond, sai_logging);
-}
+   HYPRE_ParCSRMatrixGetComm(A, &comm);
 
-void HypreParaSails::ResetSAIPrecond(MPI_Comm comm)
-{
-   HYPRE_Int  sai_max_levels;
-   HYPRE_Real sai_threshold;
-   HYPRE_Real sai_filter;
-   HYPRE_Int  sai_sym;
-   HYPRE_Real sai_loadbal;
-   HYPRE_Int  sai_reuse;
-   HYPRE_Int  sai_logging;
-
-   // hypre_ParAMGData *amg_data = (hypre_ParAMGData *)sai_precond;
-   HYPRE_ParaSailsGetNlevels(sai_precond, &sai_max_levels);
-   HYPRE_ParaSailsGetThresh(sai_precond, &sai_threshold);
-   HYPRE_ParaSailsGetFilter(sai_precond, &sai_filter);
-   HYPRE_ParaSailsGetSym(sai_precond, &sai_sym);
-   HYPRE_ParaSailsGetLoadbal(sai_precond, &sai_loadbal);
-   HYPRE_ParaSailsGetReuse(sai_precond, &sai_reuse);
-   HYPRE_ParaSailsGetLogging(sai_precond, &sai_logging);
-
-   HYPRE_ParaSailsDestroy(sai_precond);
    HYPRE_ParaSailsCreate(comm, &sai_precond);
-
    HYPRE_ParaSailsSetParams(sai_precond, sai_threshold, sai_max_levels);
    HYPRE_ParaSailsSetFilter(sai_precond, sai_filter);
    HYPRE_ParaSailsSetSym(sai_precond, sai_sym);
@@ -2611,28 +2484,6 @@ void HypreParaSails::ResetSAIPrecond(MPI_Comm comm)
    HYPRE_ParaSailsSetLogging(sai_precond, sai_logging);
 }
 
-void HypreParaSails::SetOperator(const Operator &op)
-{
-   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
-   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
-
-   if (A)
-   {
-      MPI_Comm comm;
-      HYPRE_ParCSRMatrixGetComm(*A, &comm);
-      ResetSAIPrecond(comm);
-   }
-
-   // update base classes: Operator, Solver, HypreSolver
-   height = new_A->Height();
-   width  = new_A->Width();
-   A = const_cast<HypreParMatrix *>(new_A);
-   setup_called = 0;
-   delete X;
-   delete B;
-   B = X = NULL;
-}
-
 void HypreParaSails::SetSymmetry(int sym)
 {
    HYPRE_ParaSailsSetSym(sai_precond, sym);
@@ -2644,30 +2495,19 @@ HypreParaSails::~HypreParaSails()
 }
 
 
-HypreEuclid::HypreEuclid(MPI_Comm comm)
-{
-   HYPRE_EuclidCreate(comm, &euc_precond);
-   SetDefaultOptions();
-}
-
 HypreEuclid::HypreEuclid(HypreParMatrix &A) : HypreSolver(&A)
 {
    MPI_Comm comm;
 
-   HYPRE_ParCSRMatrixGetComm(A, &comm);
-
-   HYPRE_EuclidCreate(comm, &euc_precond);
-   SetDefaultOptions();
-}
-
-void HypreEuclid::SetDefaultOptions()
-{
    int    euc_level = 1; // We use ILU(1)
    int    euc_stats = 0; // No logging
    int    euc_mem   = 0; // No memory logging
    int    euc_bj    = 0; // 1: Use Block Jacobi
    int    euc_ro_sc = 0; // 1: Use Row scaling
 
+   HYPRE_ParCSRMatrixGetComm(A, &comm);
+
+   HYPRE_EuclidCreate(comm, &euc_precond);
    HYPRE_EuclidSetLevel(euc_precond, euc_level);
    HYPRE_EuclidSetStats(euc_precond, euc_stats);
    HYPRE_EuclidSetMem(euc_precond, euc_mem);
@@ -2675,38 +2515,6 @@ void HypreEuclid::SetDefaultOptions()
    HYPRE_EuclidSetRowScale(euc_precond, euc_ro_sc);
 }
 
-void HypreEuclid::ResetEuclidPrecond(MPI_Comm comm)
-{
-   // Euclid does not seem to offer access to its current configuration, so we
-   // simply reset it to its default options.
-   HYPRE_EuclidDestroy(euc_precond);
-   HYPRE_EuclidCreate(comm, &euc_precond);
-
-   SetDefaultOptions();
-}
-
-void HypreEuclid::SetOperator(const Operator &op)
-{
-   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
-   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
-
-   if (A)
-   {
-      MPI_Comm comm;
-      HYPRE_ParCSRMatrixGetComm(*new_A, &comm);
-      ResetEuclidPrecond(comm);
-   }
-
-   // update base classes: Operator, Solver, HypreSolver
-   height = new_A->Height();
-   width  = new_A->Width();
-   A = const_cast<HypreParMatrix *>(new_A);
-   setup_called = 0;
-   delete X;
-   delete B;
-   B = X = NULL;
-}
-
 HypreEuclid::~HypreEuclid()
 {
    HYPRE_EuclidDestroy(euc_precond);
@@ -2982,18 +2790,9 @@ HypreBoomerAMG::~HypreBoomerAMG()
    HYPRE_BoomerAMGDestroy(amg_precond);
 }
 
-HypreAMS::HypreAMS(ParFiniteElementSpace *edge_fespace)
-{
-   Init(edge_fespace);
-}
 
 HypreAMS::HypreAMS(HypreParMatrix &A, ParFiniteElementSpace *edge_fespace)
    : HypreSolver(&A)
-{
-   Init(edge_fespace);
-}
-
-void HypreAMS::Init(ParFiniteElementSpace *edge_fespace)
 {
    int cycle_type       = 13;
    int rlx_type         = 2;
@@ -3177,22 +2976,6 @@ void HypreAMS::Init(ParFiniteElementSpace *edge_fespace)
    error_mode = IGNORE_HYPRE_ERRORS;
 }
 
-void HypreAMS::SetOperator(const Operator &op)
-{
-   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
-   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
-
-   // update base classes: Operator, Solver, HypreSolver
-   height = new_A->Height();
-   width  = new_A->Width();
-   A = const_cast<HypreParMatrix *>(new_A);
-
-   setup_called = 0;
-   delete X;
-   delete B;
-   B = X = NULL;
-}
-
 HypreAMS::~HypreAMS()
 {
    HYPRE_AMSDestroy(ams);
@@ -3213,18 +2996,8 @@ void HypreAMS::SetPrintLevel(int print_lvl)
    HYPRE_AMSSetPrintLevel(ams, print_lvl);
 }
 
-HypreADS::HypreADS(ParFiniteElementSpace *face_fespace)
-{
-   Init(face_fespace);
-}
-
 HypreADS::HypreADS(HypreParMatrix &A, ParFiniteElementSpace *face_fespace)
    : HypreSolver(&A)
-{
-   Init(face_fespace);
-}
-
-void HypreADS::Init(ParFiniteElementSpace *face_fespace)
 {
    int cycle_type       = 11;
    int rlx_type         = 2;
@@ -3447,22 +3220,6 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace)
    error_mode = IGNORE_HYPRE_ERRORS;
 }
 
-void HypreADS::SetOperator(const Operator &op)
-{
-   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
-   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
-
-   // update base classes: Operator, Solver, HypreSolver
-   height = new_A->Height();
-   width  = new_A->Width();
-   A = const_cast<HypreParMatrix *>(new_A);
-
-   setup_called = 0;
-   delete X;
-   delete B;
-   B = X = NULL;
-}
-
 HypreADS::~HypreADS()
 {
    HYPRE_ADSDestroy(ads);

linalg/hypre.hpp

@@ -735,15 +735,9 @@ class HyprePCG : public HypreSolver
 private:
    HYPRE_Solver pcg_solver;
 
-   HypreSolver * precond;
-
 public:
-   HyprePCG(MPI_Comm comm);
-
    HyprePCG(HypreParMatrix &_A);
 
-   virtual void SetOperator(const Operator &op);
-
    void SetTol(double tol);
    void SetMaxIter(int max_iter);
    void SetLogging(int logging);
@@ -790,18 +784,9 @@ class HypreGMRES : public HypreSolver
 private:
    HYPRE_Solver gmres_solver;
 
-   HypreSolver * precond;
-
-   /// Default, generally robust, GMRES options
-   void SetDefaultOptions();
-
 public:
-   HypreGMRES(MPI_Comm comm);
-
    HypreGMRES(HypreParMatrix &_A);
 
-   virtual void SetOperator(const Operator &op);
-
    void SetTol(double tol);
    void SetMaxIter(int max_iter);
    void SetKDim(int dim);
@@ -853,8 +838,6 @@ public:
    explicit HypreDiagScale(HypreParMatrix &A) : HypreSolver(&A) { }
    virtual operator HYPRE_Solver() const { return NULL; }
 
-   virtual void SetOperator(const Operator &op);
-
    virtual HYPRE_PtrToParSolverFcn SetupFcn() const
    { return (HYPRE_PtrToParSolverFcn) HYPRE_ParCSRDiagScaleSetup; }
    virtual HYPRE_PtrToParSolverFcn SolveFcn() const
@@ -870,21 +853,9 @@ class HypreParaSails : public HypreSolver
 private:
    HYPRE_Solver sai_precond;
 
-   /// Default, generally robust, ParaSails options
-   void SetDefaultOptions();
-
-   // If sai_precond is NULL, this method allocates it and sets default options.
-   // Otherwise the method saves the options from sai_precond, destroys it,
-   // allocates a new object, and sets its options to the saved values.
-   void ResetSAIPrecond(MPI_Comm comm);
-
 public:
-   HypreParaSails(MPI_Comm comm);
-
    HypreParaSails(HypreParMatrix &A);
 
-   virtual void SetOperator(const Operator &op);
-
    void SetSymmetry(int sym);
 
    /// The typecast to HYPRE_Solver returns the internal sai_precond
@@ -911,21 +882,9 @@ class HypreEuclid : public HypreSolver
 private:
    HYPRE_Solver euc_precond;
 
-   /// Default, generally robust, Euclid options
-   void SetDefaultOptions();
-
-   // If euc_precond is NULL, this method allocates it and sets default options.
-   // Otherwise the method saves the options from euc_precond, destroys it,
-   // allocates a new object, and sets its options to the saved values.
-   void ResetEuclidPrecond(MPI_Comm comm);
-
 public:
-   HypreEuclid(MPI_Comm comm);
-
    HypreEuclid(HypreParMatrix &A);
 
-   virtual void SetOperator(const Operator &op);
-
    /// The typecast to HYPRE_Solver returns the internal euc_precond
    virtual operator HYPRE_Solver() const { return euc_precond; }
 
@@ -1004,9 +963,6 @@ HypreParMatrix* DiscreteCurl(ParFiniteElementSpace *face_fespace,
 class HypreAMS : public HypreSolver
 {
 private:
-   /// Constuct AMS solver from finite element space
-   void Init(ParFiniteElementSpace *edge_space);
-
    HYPRE_Solver ams;
 
    /// Vertex coordinates
@@ -1017,12 +973,8 @@ private:
    HypreParMatrix *Pi, *Pix, *Piy, *Piz;
 
 public:
-   HypreAMS(ParFiniteElementSpace *edge_fespace);
-
    HypreAMS(HypreParMatrix &A, ParFiniteElementSpace *edge_fespace);
 
-   virtual void SetOperator(const Operator &op);
-
    void SetPrintLevel(int print_lvl);
 
    /// Set this option when solving a curl-curl problem with zero mass term
@@ -1043,9 +995,6 @@ public:
 class HypreADS : public HypreSolver
 {
 private:
-   /// Constuct ADS solver from finite element space
-   void Init(ParFiniteElementSpace *face_fespace);
-
    HYPRE_Solver ads;
 
    /// Vertex coordinates
@@ -1060,12 +1009,8 @@ private:
    HypreParMatrix *RT_Pi, *RT_Pix, *RT_Piy, *RT_Piz;
 
 public:
-   HypreADS(ParFiniteElementSpace *face_fespace);
-
    HypreADS(HypreParMatrix &A, ParFiniteElementSpace *face_fespace);
 
-   virtual void SetOperator(const Operator &op);
-
    void SetPrintLevel(int print_lvl);
 
    /// The typecast to HYPRE_Solver returns the internal ads object

linalg/petsc.cpp

@@ -422,12 +422,6 @@ PetscParVector& PetscParVector::operator*=(PetscScalar s)
    return *this;
 }
 
-PetscParVector& PetscParVector::operator+=(PetscScalar s)
-{
-   ierr = VecShift(x,s); PCHKERRQ(x,ierr);
-   return *this;
-}
-
 void PetscParVector::PlaceArray(PetscScalar *temp_data)
 {
    ierr = VecPlaceArray(x,temp_data); PCHKERRQ(x,ierr);
@@ -2309,6 +2303,17 @@ void PetscLinearSolver::SetOperator(const Operator &op)
                               (dynamic_cast<const PetscParMatrix *>(&op));
    const Operator       *oA = dynamic_cast<const Operator *>(&op);
 
+   // Preserve Pmat if already set
+   KSP ksp = (KSP)obj;
+   Mat P = NULL;
+   PetscBool pmat;
+   ierr = KSPGetOperatorsSet(ksp,NULL,&pmat); PCHKERRQ(ksp,ierr);
+   if (pmat)
+   {
+      ierr = KSPGetOperators(ksp,NULL,&P); PCHKERRQ(ksp,ierr);
+      ierr = PetscObjectReference((PetscObject)P); PCHKERRQ(ksp,ierr);
+   }
+
    // update base classes: Operator, Solver, PetscLinearSolver
    bool delete_pA = false;
    if (!pA)
@@ -2331,7 +2336,6 @@ void PetscLinearSolver::SetOperator(const Operator &op)
    MFEM_VERIFY(pA, "Unsupported operation!");
 
    // Set operators into PETSc KSP
-   KSP ksp = (KSP)obj;
    Mat A = pA->A;
    if (operatorset)
    {
@@ -2352,7 +2356,15 @@ void PetscLinearSolver::SetOperator(const Operator &op)
          wrap = false;
       }
    }
-   ierr = KSPSetOperators(ksp,A,A); PCHKERRQ(ksp,ierr);
+   if (P)
+   {
+      ierr = KSPSetOperators(ksp,A,P); PCHKERRQ(ksp,ierr);
+      ierr = MatDestroy(&P); PCHKERRQ(ksp,ierr);
+   }
+   else
+   {
+      ierr = KSPSetOperators(ksp,A,A); PCHKERRQ(ksp,ierr);
+   }
 
    // Update PetscSolver
    operatorset = true;
@@ -2488,7 +2500,7 @@ void PetscLinearSolver::SetPreconditioner(Solver &precond)
    }
 }
 
-void PetscLinearSolver::MultKernel(const Vector &b, Vector &x, bool trans) const
+void PetscLinearSolver::Mult(const Vector &b, Vector &x) const
 {
    KSP ksp = (KSP)obj;
 
@@ -2516,28 +2528,11 @@ void PetscLinearSolver::MultKernel(const Vector &b, Vector &x, bool trans) const
    PCHKERRQ(ksp, ierr);
 
    // Solve the system.
-   if (trans)
-   {
-      ierr = KSPSolveTranspose(ksp, B->x, X->x); PCHKERRQ(ksp,ierr);
-   }
-   else
-   {
-      ierr = KSPSolve(ksp, B->x, X->x); PCHKERRQ(ksp,ierr);
-   }
+   ierr = KSPSolve(ksp, B->x, X->x); PCHKERRQ(ksp,ierr);
    B->ResetArray();
    X->ResetArray();
 }
 
-void PetscLinearSolver::Mult(const Vector &b, Vector &x) const
-{
-   (*this).MultKernel(b,x,false);
-}
-
-void PetscLinearSolver::MultTranspose(const Vector &b, Vector &x) const
-{
-   (*this).MultKernel(b,x,true);
-}
-
 PetscLinearSolver::~PetscLinearSolver()
 {
    MPI_Comm comm;
@@ -2659,8 +2654,7 @@ void PetscPreconditioner::SetOperator(const Operator &op)
    if (delete_pA) { delete pA; };
 }
 
-void PetscPreconditioner::MultKernel(const Vector &b, Vector &x,
-                                     bool trans) const
+void PetscPreconditioner::Mult(const Vector &b, Vector &x) const
 {
    PC pc = (PC)obj;
 
@@ -2685,28 +2679,11 @@ void PetscPreconditioner::MultKernel(const Vector &b, Vector &x,
    Customize();
 
    // Apply the preconditioner.
-   if (trans)
-   {
-      ierr = PCApplyTranspose(pc, B->x, X->x); PCHKERRQ(pc, ierr);
-   }
-   else
-   {
-      ierr = PCApply(pc, B->x, X->x); PCHKERRQ(pc, ierr);
-   }
+   ierr = PCApply(pc, B->x, X->x); PCHKERRQ(pc, ierr);
    B->ResetArray();
    X->ResetArray();
 }
 
-void PetscPreconditioner::Mult(const Vector &b, Vector &x) const
-{
-   (*this).MultKernel(b,x,false);
-}
-
-void PetscPreconditioner::MultTranspose(const Vector &b, Vector &x) const
-{
-   (*this).MultKernel(b,x,true);
-}
-
 PetscPreconditioner::~PetscPreconditioner()
 {
    MPI_Comm comm;
@@ -3211,27 +3188,26 @@ PetscFieldSplitSolver::PetscFieldSplitSolver(MPI_Comm comm, Operator &op,
    : PetscPreconditioner(comm,op,prefix)
 {
    PC pc = (PC)obj;
-   ierr = PCSetType(pc,PCFIELDSPLIT); PCHKERRQ(pc,ierr);
 
    Mat pA;
    ierr = PCGetOperators(pc,&pA,NULL); PCHKERRQ(pc,ierr);
 
    // Check if pA is of type MATNEST
+   // (this requirement can be removed when we can pass fields).
    PetscBool isnest;
    ierr = PetscObjectTypeCompare((PetscObject)pA,MATNEST,&isnest);
+   PCHKERRQ(pA,ierr);
+   MFEM_VERIFY(isnest,
+               "PetscFieldSplitSolver needs the matrix in nested format.");
 
-   PetscInt nr = 0;
-   IS  *isrow = NULL;
-   if (isnest) // we now the fields
-   {
-      ierr = MatNestGetSize(pA,&nr,NULL); PCHKERRQ(pc,ierr);
-      ierr = PetscCalloc1(nr,&isrow); CCHKERRQ(PETSC_COMM_SELF,ierr);
-      ierr = MatNestGetISs(pA,isrow,NULL); PCHKERRQ(pc,ierr);
-   }
+   PetscInt nr;
+   IS  *isrow;
+   ierr = PCSetType(pc,PCFIELDSPLIT); PCHKERRQ(pc,ierr);
+   ierr = MatNestGetSize(pA,&nr,NULL); PCHKERRQ(pc,ierr);
+   ierr = PetscCalloc1(nr,&isrow); CCHKERRQ(PETSC_COMM_SELF,ierr);
+   ierr = MatNestGetISs(pA,isrow,NULL); PCHKERRQ(pc,ierr);
 
    // We need to customize here, before setting the index sets.
-   // This is because PCFieldSplitSetType customizes the function
-   // pointers. SubSolver options will be processed during PCApply
    Customize();
 
    for (PetscInt i=0; i<nr; i++)
@@ -3599,7 +3575,6 @@ void PetscODESolver::Run(Vector &x, double &t, double &dt, double t_final)
 }  // namespace mfem
 
 #include "petsc/private/petscimpl.h"
-#include "petsc/private/matimpl.h"
 
 // auxiliary functions
 static PetscErrorCode __mfem_ts_monitor(TS ts, PetscInt it, PetscReal t, Vec x,
@@ -3755,37 +3730,17 @@ static PetscErrorCode __mfem_ts_ijacobian(TS ts, PetscReal t, Vec x,
       pA->EliminateRowsCols(bchandler->GetTDofs(),dummy,dummy);
    }
 
-   // Get nonzerostate
-   PetscObjectState nonzerostate;
-   ierr = MatGetNonzeroState(P,&nonzerostate); CHKERRQ(ierr);
-
    // Avoid unneeded copy of the matrix by hacking
    Mat B;
    B = pA->ReleaseMat(false);
    ierr = MatHeaderReplace(P,&B); CHKERRQ(ierr);
    if (delete_pA) { delete pA; }
-
-   // Matrix-free case
    if (A && A != P)
    {
       ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
       ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
    }
 
-   // When using MATNEST and PCFIELDSPLIT, the second setup of the
-   // preconditioner fails because MatCreateSubMatrix_Nest does not
-   // actually return a matrix. Instead, for efficiency reasons,
-   // it returns a reference to the submatrix. The second time it
-   // is called, MAT_REUSE_MATRIX is used and MatCreateSubMatrix_Nest
-   // aborts since the two submatrices are actually different.
-   // We circumvent this issue by incrementing the nonzero state
-   // (i.e. PETSc thinks the operator sparsity pattern has changed)
-   // This does not impact performances in the case of MATNEST
-   PetscBool isnest;
-   ierr = PetscObjectTypeCompare((PetscObject)P,MATNEST,&isnest);
-   CHKERRQ(ierr);
-   if (isnest) { P->nonzerostate = nonzerostate + 1; }
-
    // Jacobian reusage
    ierr = PetscObjectStateGet((PetscObject)P,&ts_ctx->cached_ijacstate);
    CHKERRQ(ierr);
@@ -3942,7 +3897,6 @@ static PetscErrorCode __mfem_ts_computesplits(TS ts,PetscReal t,Vec x,Vec xp,
       ierr = MatAXPY(*pJxp,-1.0,*pJx,SAME_NONZERO_PATTERN); PCHKERRQ(ts,ierr);
    }
 
-   // Matrix-free cases
    if (Ax && Ax != Jx)
    {
       ierr = MatAssemblyBegin(Ax,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
@@ -4029,31 +3983,12 @@ static PetscErrorCode __mfem_ts_rhsjacobian(TS ts, PetscReal t, Vec x,
       pA->EliminateRowsCols(bchandler->GetTDofs(),dummy,dummy);
    }
 
-   // Get nonzerostate
-   PetscObjectState nonzerostate;
-   ierr = MatGetNonzeroState(P,&nonzerostate); CHKERRQ(ierr);
-
    // Avoid unneeded copy of the matrix by hacking
    Mat B;
    B = pA->ReleaseMat(false);
    ierr = MatHeaderReplace(P,&B); CHKERRQ(ierr);
    if (delete_pA) { delete pA; }
 
-   // When using MATNEST and PCFIELDSPLIT, the second setup of the
-   // preconditioner fails because MatCreateSubMatrix_Nest does not
-   // actually return a matrix. Instead, for efficiency reasons,
-   // it returns a reference to the submatrix. The second time it
-   // is called, MAT_REUSE_MATRIX is used and MatCreateSubMatrix_Nest
-   // aborts since the two submatrices are actually different.
-   // We circumvent this issue by incrementing the nonzero state
-   // (i.e. PETSc thinks the operator sparsity pattern has changed)
-   // This does not impact performances in the case of MATNEST
-   PetscBool isnest;
-   ierr = PetscObjectTypeCompare((PetscObject)P,MATNEST,&isnest);
-   CHKERRQ(ierr);
-   if (isnest) { P->nonzerostate = nonzerostate + 1; }
-
-   // Matrix-free case
    if (A && A != P)
    {
       ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
@@ -4157,30 +4092,10 @@ static PetscErrorCode __mfem_snes_jacobian(SNES snes, Vec x, Mat A, Mat P,
       pA->EliminateRowsCols(bchandler->GetTDofs(),dummy,dummy);
    }
 
-   // Get nonzerostate
-   PetscObjectState nonzerostate;
-   ierr = MatGetNonzeroState(P,&nonzerostate); CHKERRQ(ierr);
-
    // Avoid unneeded copy of the matrix by hacking
    Mat B = pA->ReleaseMat(false);
    ierr = MatHeaderReplace(P,&B); CHKERRQ(ierr);
    if (delete_pA) { delete pA; }
-
-   // When using MATNEST and PCFIELDSPLIT, the second setup of the
-   // preconditioner fails because MatCreateSubMatrix_Nest does not
-   // actually return a matrix. Instead, for efficiency reasons,
-   // it returns a reference to the submatrix. The second time it
-   // is called, MAT_REUSE_MATRIX is used and MatCreateSubMatrix_Nest
-   // aborts since the two submatrices are actually different.
-   // We circumvent this issue by incrementing the nonzero state
-   // (i.e. PETSc thinks the operator sparsity pattern has changed)
-   // This does not impact performances in the case of MATNEST
-   PetscBool isnest;
-   ierr = PetscObjectTypeCompare((PetscObject)P,MATNEST,&isnest);
-   CHKERRQ(ierr);
-   if (isnest) { P->nonzerostate = nonzerostate + 1; }
-
-   // Matrix-free case
    if (A && A != P)
    {
       ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
@@ -4528,11 +4443,6 @@ PetscErrorCode MakeShellPC(PC pc, mfem::Solver &precond, bool ownsop)
    ctx->factory = NULL;
    ctx->numprec = 0;
 
-   // In case the PC was already of type SHELL, this will destroy any
-   // previous user-defined data structure
-   // We cannot call PCReset as it will wipe out any operator already set
-   ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
-
    ierr = PCSetType(pc,PCSHELL); CHKERRQ(ierr);
    ierr = PCShellSetName(pc,"MFEM Solver (unknown Pmat)"); CHKERRQ(ierr);
    ierr = PCShellSetContext(pc,(void *)ctx); CHKERRQ(ierr);
@@ -4557,11 +4467,6 @@ PetscErrorCode MakeShellPCWithFactory(PC pc,
    ctx->factory = factory;
    ctx->numprec = 0;
 
-   // In case the PC was already of type SHELL, this will destroy any
-   // previous user-defined data structure
-   // We cannot call PCReset as it will wipe out any operator already set
-   ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
-
    ierr = PCSetType(pc,PCSHELL); CHKERRQ(ierr);
    ierr = PCShellSetName(pc,factory->GetName()); CHKERRQ(ierr);
    ierr = PCShellSetContext(pc,(void *)ctx); CHKERRQ(ierr);

linalg/petsc.hpp

@@ -170,7 +170,6 @@ public:
    PetscParVector& operator+= (const PetscParVector &y);
    PetscParVector& operator-= (const PetscParVector &y);
    PetscParVector& operator*= (PetscScalar d);
-   PetscParVector& operator+= (PetscScalar d);
 
    /** @brief Temporarily replace the data of the PETSc Vec object. To return to
        the original data array, call ResetArray().
@@ -602,7 +601,6 @@ class PetscLinearSolver : public PetscSolver, public Solver
 private:
    /// Internal flag to handle HypreParMatrix conversion or not.
    bool wrap;
-   void MultKernel(const Vector &b, Vector &x, bool trans) const;
 
 public:
    PetscLinearSolver(MPI_Comm comm, const std::string &prefix = std::string(),
@@ -618,8 +616,6 @@ public:
                      const std::string &prefix = std::string());
    virtual ~PetscLinearSolver();
 
-   /// Sets the operator to be used for mat-vec operations and
-   /// for the construction of the preconditioner
    virtual void SetOperator(const Operator &op);
 
    /// Allows to prescribe a different operator (@a pop) to construct
@@ -627,12 +623,10 @@ public:
    void SetOperator(const Operator &op, const Operator &pop);
 
    /// Sets the solver to perform preconditioning
-   /// preserves the linear operator for the mat-vec
    void SetPreconditioner(Solver &precond);
 
    /// Application of the solver.
    virtual void Mult(const Vector &b, Vector &x) const;
-   virtual void MultTranspose(const Vector &b, Vector &x) const;
 
    /// Conversion function to PETSc's KSP type.
    operator KSP() const { return (KSP)obj; }
@@ -652,9 +646,6 @@ public:
 /// Abstract class for PETSc's preconditioners.
 class PetscPreconditioner : public PetscSolver, public Solver
 {
-private:
-   void MultKernel(const Vector &b, Vector &x, bool trans) const;
-
 public:
    PetscPreconditioner(MPI_Comm comm,
                        const std::string &prefix = std::string());
@@ -668,7 +659,6 @@ public:
 
    /// Application of the preconditioner.
    virtual void Mult(const Vector &b, Vector &x) const;
-   virtual void MultTranspose(const Vector &b, Vector &x) const;
 
    /// Conversion function to PETSc's PC type.
    operator PC() const { return (PC)obj; }

linalg/solvers.cpp

@@ -1311,8 +1311,6 @@ void NewtonSolver::Mult(const Vector &b, Vector &x) const
       }
       add(x, -c_scale, c, x);
 
-      ProcessNewState(x);
-
       oper->Mult(x, r);
       if (have_b)
       {

linalg/solvers.hpp

@@ -283,10 +283,6 @@ public:
        value of 0 indicates a failure, interrupting the Newton iteration. */
    virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const
    { return 1.0; }
-
-   /** @brief This method can be overloaded in derived classes to perform
-       computations that need knowledge of the newest Newton state. */
-   virtual void ProcessNewState(const Vector &x) const { }
 };
 
 /** Adaptive restarted GMRES.

linalg/sparsemat.cpp

@@ -2640,11 +2640,9 @@ SparseMatrix &SparseMatrix::operator=(double a)
 {
    if (Rows == NULL)
    {
-      const int nnz = J.Capacity();
-      double *h_A = HostWrite(A, nnz);
-      for (int i = 0; i < nnz; i++)
+      for (int i = 0, nnz = I[height]; i < nnz; i++)
       {
-         h_A[i] = a;
+         A[i] = a;
       }
    }
    else

linalg/sparsemat.hpp

@@ -152,54 +152,6 @@ public:
    /// Return the element data, i.e. the array #A, const version.
    inline const double *GetData() const { return A; }
 
-   // Memory access methods for the #I array.
-   Memory<int> &GetMemoryI() { return I; }
-   const Memory<int> &GetMemoryI() const { return I; }
-   const int *ReadI(bool on_dev = true) const
-   { return mfem::Read(I, Height()+1, on_dev); }
-   int *WriteI(bool on_dev = true)
-   { return mfem::Write(I, Height()+1, on_dev); }
-   int *ReadWriteI(bool on_dev = true)
-   { return mfem::ReadWrite(I, Height()+1, on_dev); }
-   const int *HostReadI() const
-   { return mfem::Read(I, Height()+1, false); }
-   int *HostWriteI()
-   { return mfem::Write(I, Height()+1, false); }
-   int *HostReadWriteI()
-   { return mfem::ReadWrite(I, Height()+1, false); }
-
-   // Memory access methods for the #J array.
-   Memory<int> &GetMemoryJ() { return J; }
-   const Memory<int> &GetMemoryJ() const { return J; }
-   const int *ReadJ(bool on_dev = true) const
-   { return mfem::Read(J, J.Capacity(), on_dev); }
-   int *WriteJ(bool on_dev = true)
-   { return mfem::Write(J, J.Capacity(), on_dev); }
-   int *ReadWriteJ(bool on_dev = true)
-   { return mfem::ReadWrite(J, J.Capacity(), on_dev); }
-   const int *HostReadJ() const
-   { return mfem::Read(J, J.Capacity(), false); }
-   int *HostWriteJ()
-   { return mfem::Write(J, J.Capacity(), false); }
-   int *HostReadWriteJ()
-   { return mfem::ReadWrite(J, J.Capacity(), false); }
-
-   // Memory access methods for the #A array.
-   Memory<double> &GetMemoryData() { return A; }
-   const Memory<double> &GetMemoryData() const { return A; }
-   const double *ReadData(bool on_dev = true) const
-   { return mfem::Read(A, A.Capacity(), on_dev); }
-   double *WriteData(bool on_dev = true)
-   { return mfem::Write(A, A.Capacity(), on_dev); }
-   double *ReadWriteData(bool on_dev = true)
-   { return mfem::ReadWrite(A, A.Capacity(), on_dev); }
-   const double *HostReadData() const
-   { return mfem::Read(A, A.Capacity(), false); }
-   double *HostWriteData()
-   { return mfem::Write(A, A.Capacity(), false); }
-   double *HostReadWriteData()
-   { return mfem::ReadWrite(A, A.Capacity(), false); }
-
    /// Returns the number of elements in row @a i.
    int RowSize(const int i) const;
 

linalg/sundials.cpp

@@ -155,16 +155,12 @@ void CVODESolver::Init(TimeDependentOperator &f_)
 
    // Get the vector length
    long local_size = f_.Height();
-#ifdef MFEM_USE_MPI
    long global_size;
-#endif
 
    if (Parallel())
    {
-#ifdef MFEM_USE_MPI
       MPI_Allreduce(&local_size, &global_size, 1, MPI_LONG, MPI_SUM,
                     NV_COMM_P(y));
-#endif
    }
 
    // Get current time
@@ -173,7 +169,7 @@ void CVODESolver::Init(TimeDependentOperator &f_)
    if (sundials_mem)
    {
       // Check if the problem size has changed since the last Init() call
-      int resize = 0;
+      int resize;
       if (!Parallel())
       {
          resize = (NV_LENGTH_S(y) != local_size);
@@ -569,16 +565,12 @@ void ARKStepSolver::Init(TimeDependentOperator &f_)
 
    // Get the vector length
    long local_size = f_.Height();
-#ifdef MFEM_USE_MPI
    long global_size;
-#endif
 
    if (Parallel())
    {
-#ifdef MFEM_USE_MPI
       MPI_Allreduce(&local_size, &global_size, 1, MPI_LONG, MPI_SUM,
                     NV_COMM_P(y));
-#endif
    }
 
    // Get current time
@@ -587,7 +579,7 @@ void ARKStepSolver::Init(TimeDependentOperator &f_)
    if (sundials_mem)
    {
       // Check if the problem size has changed since the last Init() call
-      int resize = 0;
+      int resize;
       if (!Parallel())
       {
          resize = (NV_LENGTH_S(y) != local_size);
@@ -1057,22 +1049,18 @@ void KINSolver::SetOperator(const Operator &op)
 
    // Get the vector length
    long local_size = height;
-#ifdef MFEM_USE_MPI
    long global_size;
-#endif
 
    if (Parallel())
    {
-#ifdef MFEM_USE_MPI
       MPI_Allreduce(&local_size, &global_size, 1, MPI_LONG, MPI_SUM,
                     NV_COMM_P(y));
-#endif
    }
 
    if (sundials_mem)
    {
       // Check if the problem size has changed since the last SetOperator call
-      int resize = 0;
+      int resize;
       if (!Parallel())
       {
          resize = (NV_LENGTH_S(y) != local_size);

linalg/superlu.cpp

@@ -20,10 +20,6 @@
 #include "superlu_defs.h"
 #include "superlu_ddefs.h"
 
-#if XSDK_INDEX_SIZE == 64
-#error "SuperLUDist has been built with 64bit integers. This is not supported"
-#endif
-
 using namespace std;
 
 namespace mfem
@@ -134,11 +130,6 @@ SuperLURowLocMatrix::SuperLURowLocMatrix( const HypreParMatrix & hypParMat )
    // hypre_CSRMatrix.
    hypre_CSRMatrix * csr_op = hypre_MergeDiagAndOffd(parcsr_op);
    hypre_CSRMatrixSetDataOwner(csr_op,0);
-#if MFEM_HYPRE_VERSION >= 21600
-   MFEM_VERIFY(csr_op->num_rows < INT_MAX,"SuperLU: number of local rows "
-               "is too large to store as an integer.");
-   hypre_CSRMatrixBigJtoJ(csr_op);
-#endif
 
    int m         = parcsr_op->global_num_rows;
    int n         = parcsr_op->global_num_cols;

linalg/vector.cpp

@@ -834,10 +834,9 @@ double Vector::Sum() const
 {
    double sum = 0.0;
 
-   const double *h_data = this->HostRead();
    for (int i = 0; i < size; i++)
    {
-      sum += h_data[i];
+      sum += data[i];
    }
 
    return sum;

linalg/vector.hpp

@@ -140,13 +140,6 @@ public:
        @sa NewDataAndSize(). */
    inline void NewMemoryAndSize(const Memory<double> &mem, int s, bool own_mem);
 
-   /// Reset the Vector to be a reference to a sub-vector of @a base.
-   inline void MakeRef(Vector &base, int offset, int size);
-
-   /** @brief Reset the Vector to be a reference to a sub-vector of @a base
-       without changing its current size. */
-   inline void MakeRef(Vector &base, int offset);
-
    /// Set the Vector data (host pointer) ownership flag.
    void MakeDataOwner() const { data.SetHostPtrOwner(true); }
 
@@ -462,19 +455,6 @@ inline void Vector::NewMemoryAndSize(const Memory<double> &mem, int s,
    if (!own_mem) { data.ClearOwnerFlags(); }
 }
 
-inline void Vector::MakeRef(Vector &base, int offset, int s)
-{
-   data.Delete();
-   size = s;
-   data.MakeAlias(base.GetMemory(), offset, s);
-}
-
-inline void Vector::MakeRef(Vector &base, int offset)
-{
-   data.Delete();
-   data.MakeAlias(base.GetMemory(), offset, size);
-}
-
 inline void Vector::Destroy()
 {
    const bool use_dev = data.UseDevice();

mesh/element.hpp

@@ -74,13 +74,8 @@ public:
 
    virtual const int *GetEdgeVertices(int) const = 0;
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const = 0;
 
-   virtual int GetNFaces() const = 0;
-
-   virtual int GetNFaceVertices(int fi) const = 0;
-
    virtual const int *GetFaceVertices(int fi) const = 0;
 
    /// Mark the longest edge by assuming/changing the order of the vertices.

mesh/hexahedron.hpp

@@ -51,14 +51,9 @@ public:
    virtual const int *GetEdgeVertices(int ei) const
    { return geom_t::Edges[ei]; }
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const
    { nFaceVertices = 4; return 6; }
 
-   virtual int GetNFaces() const { return 6; }
-
-   virtual int GetNFaceVertices(int) const { return 4; }
-
    virtual const int *GetFaceVertices(int fi) const
    { return geom_t::FaceVert[fi]; }
 

mesh/mesh.cpp

@@ -430,8 +430,6 @@ void Mesh::GetBdrElementTransformation(int i, IsoparametricTransformation* ElTr)
    else
    {
       const FiniteElement *bdr_el = Nodes->FESpace()->GetBE(i);
-      Nodes->HostRead();
-      const GridFunction &nodes = *Nodes;
       if (bdr_el)
       {
          Array<int> vdofs;
@@ -442,7 +440,7 @@ void Mesh::GetBdrElementTransformation(int i, IsoparametricTransformation* ElTr)
          {
             for (int j = 0; j < n; j++)
             {
-               pm(k,j) = nodes(vdofs[n*k+j]);
+               pm(k,j) = (*Nodes)(vdofs[n*k+j]);
             }
          }
          ElTr->SetFE(bdr_el);
@@ -494,8 +492,6 @@ void Mesh::GetFaceTransformation(int FaceNo, IsoparametricTransformation *FTr)
    else // curved mesh
    {
       const FiniteElement *face_el = Nodes->FESpace()->GetFaceElement(FaceNo);
-      Nodes->HostRead();
-      const GridFunction &nodes = *Nodes;
       if (face_el)
       {
          Array<int> vdofs;
@@ -506,7 +502,7 @@ void Mesh::GetFaceTransformation(int FaceNo, IsoparametricTransformation *FTr)
          {
             for (int j = 0; j < n; j++)
             {
-               pm(i, j) = nodes(vdofs[n*i+j]);
+               pm(i, j) = (*Nodes)(vdofs[n*i+j]);
             }
          }
          FTr->SetFE(face_el);
@@ -3302,10 +3298,6 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)
    Array<int> rdofs;
    DenseMatrix phys_pts;
    int max_nv = 0;
-
-   DenseMatrix node_coordinates(spaceDim*pow(2, Dim), r_num_elem);
-   H1_FECollection vertex_fec(1, Dim);
-
    for (int el = 0; el < orig_mesh->GetNE(); el++)
    {
       Geometry::Type geom = orig_mesh->GetElementBaseGeometry(el);
@@ -3320,7 +3312,6 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)
       orig_mesh->GetElementTransformation(el)->Transform(rfe->GetNodes(),
                                                          phys_pts);
       const int *c2h_map = rfec.GetDofMap(geom);
-      const int *vertex_map = vertex_fec.GetDofMap(geom);
       for (int i = 0; i < phys_pts.Width(); i++)
       {
          vertices[rdofs[i]].SetCoords(spaceDim, phys_pts.GetColumn(i));
@@ -3335,24 +3326,9 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)
             int cid = RG.RefGeoms[k+nvert*j]; // local Cartesian index
             v[k] = rdofs[c2h_map[cid]];
          }
-         for (int k = 0; k < nvert; k++)
-         {
-            for (int j = 0; j < spaceDim; ++j)
-            {
-               node_coordinates(k*spaceDim + j, NumOfElements)
-                  = vertices[v[vertex_map[k]]](j);
-            }
-         }
          AddElement(elem);
       }
    }
-
-   SetCurvature(1, true, spaceDim);
-   Vector node_coordinates_vec(
-      node_coordinates.Data(),
-      node_coordinates.Width()*node_coordinates.Height());
-   SetNodes(node_coordinates_vec);
-
    // Add refined boundary elements
    for (int el = 0; el < orig_mesh->GetNBE(); el++)
    {
@@ -3453,28 +3429,6 @@ void Mesh::KnotInsert(Array<KnotVector *> &kv)
    UpdateNURBS();
 }
 
-void Mesh::KnotInsert(Array<Vector *> &kv)
-{
-   if (NURBSext == NULL)
-   {
-      mfem_error("Mesh::KnotInsert : Not a NURBS mesh!");
-   }
-
-   if (kv.Size() != NURBSext->GetNKV())
-   {
-      mfem_error("Mesh::KnotInsert : KnotVector array size mismatch!");
-   }
-
-   NURBSext->ConvertToPatches(*Nodes);
-
-   NURBSext->KnotInsert(kv);
-
-   last_operation = Mesh::NONE; // FiniteElementSpace::Update is not supported
-   sequence++;
-
-   UpdateNURBS();
-}
-
 void Mesh::NURBSUniformRefinement()
 {
    // do not check for NURBSext since this method is protected
@@ -4805,13 +4759,7 @@ void Mesh::GenerateNCFaceInfo()
    for (unsigned i = 0; i < list.slaves.size(); i++)
    {
       const NCMesh::Slave &slave = list.slaves[i];
-
-      if (slave.index < 0 || // degenerate slave face
-          slave.index >= nfaces || // ghost slave
-          slave.master >= nfaces) // has ghost master
-      {
-         continue;
-      }
+      if (slave.index >= nfaces || slave.master >= nfaces) { continue; }
 
       FaceInfo &slave_fi = faces_info[slave.index];
       FaceInfo &master_fi = faces_info[slave.master];
@@ -4964,39 +4912,15 @@ STable3D *Mesh::GetElementToFaceTable(int ret_ftbl)
    return NULL;
 }
 
-// shift cyclically 3 integers so that the smallest is first
-static inline
-void Rotate3(int &a, int &b, int &c)
-{
-   if (a < b)
-   {
-      if (a > c)
-      {
-         ShiftRight(a, b, c);
-      }
-   }
-   else
-   {
-      if (b < c)
-      {
-         ShiftRight(c, b, a);
-      }
-      else
-      {
-         ShiftRight(a, b, c);
-      }
-   }
-}
-
 void Mesh::ReorientTetMesh()
 {
+   int *v;
+
    if (Dim != 3 || !(meshgen & 1))
    {
       return;
    }
 
-   DeleteLazyTables();
-
    DSTable *old_v_to_v = NULL;
    Table *old_elem_vert = NULL;
 
@@ -5009,7 +4933,7 @@ void Mesh::ReorientTetMesh()
    {
       if (GetElementType(i) == Element::TETRAHEDRON)
       {
-         int *v = elements[i]->GetVertices();
+         v = elements[i]->GetVertices();
 
          Rotate3(v[0], v[1], v[2]);
          if (v[0] < v[3])
@@ -5018,7 +4942,7 @@ void Mesh::ReorientTetMesh()
          }
          else
          {
-            ShiftRight(v[0], v[1], v[3]);
+            ShiftL2R(v[0], v[1], v[3]);
          }
       }
    }
@@ -5027,7 +4951,7 @@ void Mesh::ReorientTetMesh()
    {
       if (GetBdrElementType(i) == Element::TRIANGLE)
       {
-         int *v = boundary[i]->GetVertices();
+         v = boundary[i]->GetVertices();
 
          Rotate3(v[0], v[1], v[2]);
       }
@@ -5859,7 +5783,7 @@ void Mesh::SetVertices(const Vector &vert_coord)
       }
 }
 
-void Mesh::GetNode(int i, double *coord) const
+void Mesh::GetNode(int i, double *coord)
 {
    if (Nodes)
    {
@@ -6010,19 +5934,16 @@ void Mesh::UniformRefinement2D()
    const int oedge = NumOfVertices;
    const int oelem = oedge + NumOfEdges;
 
-   Array<Element*> new_elements;
-   Array<Element*> new_boundary;
-
    vertices.SetSize(oelem + quad_counter);
-   new_elements.SetSize(4 * NumOfElements);
+   elements.SetSize(4 * NumOfElements);
    quad_counter = 0;
-
-   for (int i = 0, j = 0; i < NumOfElements; i++)
+   for (int i = 0; i < NumOfElements; i++)
    {
       const Element::Type el_type = elements[i]->GetType();
       const int attr = elements[i]->GetAttribute();
       int *v = elements[i]->GetVertices();
       const int *e = el_to_edge->GetRow(i);
+      const int j = NumOfElements + 3 * i;
       int vv[2];
 
       if (el_type == Element::TRIANGLE)
@@ -6036,14 +5957,12 @@ void Mesh::UniformRefinement2D()
             AverageVertices(vv, 2, oedge+e[ei]);
          }
 
-         new_elements[j++] =
-            new Triangle(v[0], oedge+e[0], oedge+e[2], attr);
-         new_elements[j++] =
-            new Triangle(oedge+e[1], oedge+e[2], oedge+e[0], attr);
-         new_elements[j++] =
-            new Triangle(oedge+e[0], v[1], oedge+e[1], attr);
-         new_elements[j++] =
-            new Triangle(oedge+e[2], oedge+e[1], v[2], attr);
+         elements[j+0] = new Triangle(oedge+e[1], oedge+e[2], oedge+e[0], attr);
+         elements[j+1] = new Triangle(oedge+e[0], v[1], oedge+e[1], attr);
+         elements[j+2] = new Triangle(oedge+e[2], oedge+e[1], v[2], attr);
+
+         v[1] = oedge+e[0];
+         v[2] = oedge+e[2];
       }
       else if (el_type == Element::QUADRILATERAL)
       {
@@ -6060,36 +5979,34 @@ void Mesh::UniformRefinement2D()
             AverageVertices(vv, 2, oedge+e[ei]);
          }
 
-         new_elements[j++] =
-            new Quadrilateral(v[0], oedge+e[0], oelem+qe, oedge+e[3], attr);
-         new_elements[j++] =
-            new Quadrilateral(oedge+e[0], v[1], oedge+e[1], oelem+qe, attr);
-         new_elements[j++] =
-            new Quadrilateral(oelem+qe, oedge+e[1], v[2], oedge+e[2], attr);
-         new_elements[j++] =
-            new Quadrilateral(oedge+e[3], oelem+qe, oedge+e[2], v[3], attr);
+         elements[j+0] = new Quadrilateral(oedge+e[0], v[1], oedge+e[1],
+                                           oelem+qe, attr);
+         elements[j+1] = new Quadrilateral(oelem+qe, oedge+e[1],
+                                           v[2], oedge+e[2], attr);
+         elements[j+2] = new Quadrilateral(oedge+e[3], oelem+qe,
+                                           oedge+e[2], v[3], attr);
+
+         v[1] = oedge+e[0];
+         v[2] = oelem+qe;
+         v[3] = oedge+e[3];
       }
       else
       {
          MFEM_ABORT("unknown element type: " << el_type);
       }
-      FreeElement(elements[i]);
    }
-   mfem::Swap(elements, new_elements);
 
-   // refine boundary elements
-   new_boundary.SetSize(2 * NumOfBdrElements);
-   for (int i = 0, j = 0; i < NumOfBdrElements; i++)
+   boundary.SetSize(2 * NumOfBdrElements);
+   for (int i = 0; i < NumOfBdrElements; i++)
    {
       const int attr = boundary[i]->GetAttribute();
       int *v = boundary[i]->GetVertices();
+      const int j = NumOfBdrElements + i;
 
-      new_boundary[j++] = new Segment(v[0], oedge+be_to_edge[i], attr);
-      new_boundary[j++] = new Segment(oedge+be_to_edge[i], v[1], attr);
+      boundary[j] = new Segment(oedge+be_to_edge[i], v[1], attr);
 
-      FreeElement(boundary[i]);
+      v[1] = oedge+be_to_edge[i];
    }
-   mfem::Swap(boundary, new_boundary);
 
    static const double A = 0.0, B = 0.5, C = 1.0;
    static double tri_children[2*3*4] =
@@ -6107,17 +6024,17 @@ void Mesh::UniformRefinement2D()
       A,B, B,B, B,C, A,C  // upper-left
    };
 
-   CoarseFineTr.point_matrices[Geometry::TRIANGLE]
-   .UseExternalData(tri_children, 2, 3, 4);
-   CoarseFineTr.point_matrices[Geometry::SQUARE]
-   .UseExternalData(quad_children, 2, 4, 4);
+   CoarseFineTr.point_matrices[Geometry::TRIANGLE].
+   UseExternalData(tri_children, 2, 3, 4);
+   CoarseFineTr.point_matrices[Geometry::SQUARE].
+   UseExternalData(quad_children, 2, 4, 4);
    CoarseFineTr.embeddings.SetSize(elements.Size());
 
    for (int i = 0; i < elements.Size(); i++)
    {
       Embedding &emb = CoarseFineTr.embeddings[i];
-      emb.parent = i / 4;
-      emb.matrix = i % 4;
+      emb.parent = (i < NumOfElements) ? i : (i - NumOfElements) / 3;
+      emb.matrix = (i < NumOfElements) ? 0 : (i - NumOfElements) % 3 + 1;
    }
 
    NumOfVertices    = vertices.Size();
@@ -6249,20 +6166,17 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p)
    const int oface = oedge + NumOfEdges;
    const int oelem = oface + NumOfQuadFaces;
 
-   Array<Element*> new_elements;
-   Array<Element*> new_boundary;
-
    vertices.SetSize(oelem + hex_counter);
-   new_elements.SetSize(8 * NumOfElements);
-   CoarseFineTr.embeddings.SetSize(new_elements.Size());
-
+   elements.SetSize(8 * NumOfElements);
+   CoarseFineTr.embeddings.SetSize(elements.Size());
    hex_counter = 0;
-   for (int i = 0, j = 0; i < NumOfElements; i++)
+   for (int i = 0; i < NumOfElements; i++)
    {
       const Element::Type el_type = elements[i]->GetType();
       const int attr = elements[i]->GetAttribute();
       int *v = elements[i]->GetVertices();
       const int *e = el_to_edge->GetRow(i);
+      const int j = NumOfElements + 7 * i;
       int vv[4], ev[12];
 
       if (e2v.Size())
@@ -6408,54 +6322,51 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p)
             const int (&mv)[4][4] = mv_all[rt];
 
 #ifndef MFEM_USE_MEMALLOC
-            new_elements[j+0] =
-               new Tetrahedron(v[0], oedge+e[0], oedge+e[1], oedge+e[2], attr);
-            new_elements[j+1] =
-               new Tetrahedron(oedge+e[0], v[1], oedge+e[3], oedge+e[4], attr);
-            new_elements[j+2] =
-               new Tetrahedron(oedge+e[1], oedge+e[3], v[2], oedge+e[5], attr);
-            new_elements[j+3] =
-               new Tetrahedron(oedge+e[2], oedge+e[4], oedge+e[5], v[3], attr);
-
+            elements[j+0] = new Tetrahedron(oedge+e[0], v[1],
+                                            oedge+e[3], oedge+e[4], attr);
+            elements[j+1] = new Tetrahedron(oedge+e[1], oedge+e[3],
+                                            v[2], oedge+e[5], attr);
+            elements[j+2] = new Tetrahedron(oedge+e[2], oedge+e[4],
+                                            oedge+e[5], v[3], attr);
             for (int k = 0; k < 4; k++)
             {
-               new_elements[j+4+k] =
+               elements[j+k+3] =
                   new Tetrahedron(oedge+e[mv[k][0]], oedge+e[mv[k][1]],
                                   oedge+e[mv[k][2]], oedge+e[mv[k][3]], attr);
             }
 #else
             Tetrahedron *tet;
-            new_elements[j+0] = tet = TetMemory.Alloc();
-            tet->Init(v[0], oedge+e[0], oedge+e[1], oedge+e[2], attr);
-
-            new_elements[j+1] = tet = TetMemory.Alloc();
+            elements[j+0] = tet = TetMemory.Alloc();
             tet->Init(oedge+e[0], v[1], oedge+e[3], oedge+e[4], attr);
-
-            new_elements[j+2] = tet = TetMemory.Alloc();
+            elements[j+1] = tet = TetMemory.Alloc();
             tet->Init(oedge+e[1], oedge+e[3], v[2], oedge+e[5], attr);
-
-            new_elements[j+3] = tet = TetMemory.Alloc();
+            elements[j+2] = tet = TetMemory.Alloc();
             tet->Init(oedge+e[2], oedge+e[4], oedge+e[5], v[3], attr);
-
             for (int k = 0; k < 4; k++)
             {
-               new_elements[j+4+k] = tet = TetMemory.Alloc();
+               elements[j+k+3] = tet = TetMemory.Alloc();
                tet->Init(oedge+e[mv[k][0]], oedge+e[mv[k][1]],
                          oedge+e[mv[k][2]], oedge+e[mv[k][3]], attr);
             }
 #endif
-            for (int k = 0; k < 4; k++)
+
+            v[1] = oedge+e[0];
+            v[2] = oedge+e[1];
+            v[3] = oedge+e[2];
+            ((Tetrahedron*)elements[i])->SetRefinementFlag(0);
+
+            CoarseFineTr.embeddings[i].parent = i;
+            CoarseFineTr.embeddings[i].matrix = 0;
+            for (int k = 0; k < 3; k++)
             {
                CoarseFineTr.embeddings[j+k].parent = i;
-               CoarseFineTr.embeddings[j+k].matrix = k;
+               CoarseFineTr.embeddings[j+k].matrix = k+1;
             }
             for (int k = 0; k < 4; k++)
             {
-               CoarseFineTr.embeddings[j+4+k].parent = i;
-               CoarseFineTr.embeddings[j+4+k].matrix = 4*(rt+1)+k;
+               CoarseFineTr.embeddings[j+k+3].parent = i;
+               CoarseFineTr.embeddings[j+k+3].matrix = 4*(rt+1)+k;
             }
-
-            j += 8;
          }
          break;
 
@@ -6485,37 +6396,33 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p)
             const int qf3 = f2qf[f[3]];
             const int qf4 = f2qf[f[4]];
 
-            new_elements[j++] =
-               new Wedge(v[0], oedge+e[0], oedge+e[2],
-                         oedge+e[6], oface+qf2, oface+qf4, attr);
+            elements[j+0] = new Wedge(oedge+e[1], oedge+e[2], oedge+e[0],
+                                      oface+qf3, oface+qf4, oface+qf2,
+                                      attr);
+            elements[j+1] = new Wedge(oedge+e[0], v[1], oedge+e[1],
+                                      oface+qf2, oedge+e[7], oface+qf3,
+                                      attr);
+            elements[j+2] = new Wedge(oedge+e[2], oedge+e[1], v[2],
+                                      oface+qf4, oface+qf3, oedge+e[8],
+                                      attr);
+            elements[j+3] = new Wedge(oedge+e[6], oface+qf2, oface+qf4,
+                                      v[3], oedge+e[3], oedge+e[5],
+                                      attr);
+            elements[j+4] = new Wedge(oface+qf3, oface+qf4, oface+qf2,
+                                      oedge+e[4], oedge+e[5], oedge+e[3],
+                                      attr);
+            elements[j+5] = new Wedge(oface+qf2, oedge+e[7], oface+qf3,
+                                      oedge+e[3], v[4], oedge+e[4],
+                                      attr);
+            elements[j+6] = new Wedge(oface+qf4, oface+qf3, oedge+e[8],
+                                      oedge+e[5], oedge+e[4], v[5],
+                                      attr);
 
-            new_elements[j++] =
-               new Wedge(oedge+e[1], oedge+e[2], oedge+e[0],
-                         oface+qf3, oface+qf4, oface+qf2, attr);
-
-            new_elements[j++] =
-               new Wedge(oedge+e[0], v[1], oedge+e[1],
-                         oface+qf2, oedge+e[7], oface+qf3, attr);
-
-            new_elements[j++] =
-               new Wedge(oedge+e[2], oedge+e[1], v[2],
-                         oface+qf4, oface+qf3, oedge+e[8], attr);
-
-            new_elements[j++] =
-               new Wedge(oedge+e[6], oface+qf2, oface+qf4,
-                         v[3], oedge+e[3], oedge+e[5], attr);
-
-            new_elements[j++] =
-               new Wedge(oface+qf3, oface+qf4, oface+qf2,
-                         oedge+e[4], oedge+e[5], oedge+e[3], attr);
-
-            new_elements[j++] =
-               new Wedge(oface+qf2, oedge+e[7], oface+qf3,
-                         oedge+e[3], v[4], oedge+e[4], attr);
-
-            new_elements[j++] =
-               new Wedge(oface+qf4, oface+qf3, oedge+e[8],
-                         oedge+e[5], oedge+e[4], v[5], attr);
+            v[1] = oedge+e[0];
+            v[2] = oedge+e[2];
+            v[3] = oedge+e[6];
+            v[4] = oface+qf2;
+            v[5] = oface+qf4;
          }
          break;
 
@@ -6557,38 +6464,35 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p)
                AverageVertices(vv, 2, oedge+e[ei]);
             }
 
-            new_elements[j++] =
-               new Hexahedron(v[0], oedge+e[0], oface+qf[0],
-                              oedge+e[3], oedge+e[8], oface+qf[1],
-                              oelem+he, oface+qf[4], attr);
-            new_elements[j++] =
-               new Hexahedron(oedge+e[0], v[1], oedge+e[1],
-                              oface+qf[0], oface+qf[1], oedge+e[9],
-                              oface+qf[2], oelem+he, attr);
-            new_elements[j++] =
-               new Hexahedron(oface+qf[0], oedge+e[1], v[2],
-                              oedge+e[2], oelem+he, oface+qf[2],
-                              oedge+e[10], oface+qf[3], attr);
-            new_elements[j++] =
-               new Hexahedron(oedge+e[3], oface+qf[0], oedge+e[2],
-                              v[3], oface+qf[4], oelem+he,
-                              oface+qf[3], oedge+e[11], attr);
-            new_elements[j++] =
-               new Hexahedron(oedge+e[8], oface+qf[1], oelem+he,
-                              oface+qf[4], v[4], oedge+e[4],
-                              oface+qf[5], oedge+e[7], attr);
-            new_elements[j++] =
-               new Hexahedron(oface+qf[1], oedge+e[9], oface+qf[2],
-                              oelem+he, oedge+e[4], v[5],
-                              oedge+e[5], oface+qf[5], attr);
-            new_elements[j++] =
-               new Hexahedron(oelem+he, oface+qf[2], oedge+e[10],
-                              oface+qf[3], oface+qf[5], oedge+e[5],
-                              v[6], oedge+e[6], attr);
-            new_elements[j++] =
-               new Hexahedron(oface+qf[4], oelem+he, oface+qf[3],
-                              oedge+e[11], oedge+e[7], oface+qf[5],
-                              oedge+e[6], v[7], attr);
+            elements[j+0] = new Hexahedron(oedge+e[0], v[1], oedge+e[1],
+                                           oface+qf[0], oface+qf[1], oedge+e[9],
+                                           oface+qf[2], oelem+he, attr);
+            elements[j+1] = new Hexahedron(oface+qf[0], oedge+e[1], v[2],
+                                           oedge+e[2], oelem+he, oface+qf[2],
+                                           oedge+e[10], oface+qf[3], attr);
+            elements[j+2] = new Hexahedron(oedge+e[3], oface+qf[0], oedge+e[2],
+                                           v[3], oface+qf[4], oelem+he,
+                                           oface+qf[3], oedge+e[11], attr);
+            elements[j+3] = new Hexahedron(oedge+e[8], oface+qf[1], oelem+he,
+                                           oface+qf[4], v[4], oedge+e[4],
+                                           oface+qf[5], oedge+e[7], attr);
+            elements[j+4] = new Hexahedron(oface+qf[1], oedge+e[9], oface+qf[2],
+                                           oelem+he, oedge+e[4], v[5],
+                                           oedge+e[5], oface+qf[5], attr);
+            elements[j+5] = new Hexahedron(oelem+he, oface+qf[2], oedge+e[10],
+                                           oface+qf[3], oface+qf[5], oedge+e[5],
+                                           v[6], oedge+e[6], attr);
+            elements[j+6] = new Hexahedron(oface+qf[4], oelem+he, oface+qf[3],
+                                           oedge+e[11], oedge+e[7], oface+qf[5],
+                                           oedge+e[6], v[7], attr);
+
+            v[1] = oedge+e[0];
+            v[2] = oface+qf[0];
+            v[3] = oedge+e[3];
+            v[4] = oedge+e[8];
+            v[5] = oface+qf[1];
+            v[6] = oelem+he;
+            v[7] = oface+qf[4];
          }
          break;
 
@@ -6596,18 +6500,16 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p)
             MFEM_ABORT("Unknown 3D element type \"" << el_type << "\"");
             break;
       }
-      FreeElement(elements[i]);
    }
-   mfem::Swap(elements, new_elements);
 
-   // refine boundary elements
-   new_boundary.SetSize(4 * NumOfBdrElements);
-   for (int i = 0, j = 0; i < NumOfBdrElements; i++)
+   boundary.SetSize(4 * NumOfBdrElements);
+   for (int i = 0; i < NumOfBdrElements; i++)
    {
       const Element::Type bdr_el_type = boundary[i]->GetType();
       const int attr = boundary[i]->GetAttribute();
       int *v = boundary[i]->GetVertices();
       const int *e = bel_to_edge->GetRow(i);
+      const int j = NumOfBdrElements + 3 * i;
       int ev[4];
 
       if (e2v.Size())
@@ -6619,36 +6521,34 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p)
 
       if (bdr_el_type == Element::TRIANGLE)
       {
-         new_boundary[j++] =
-            new Triangle(v[0], oedge+e[0], oedge+e[2], attr);
-         new_boundary[j++] =
-            new Triangle(oedge+e[1], oedge+e[2], oedge+e[0], attr);
-         new_boundary[j++] =
-            new Triangle(oedge+e[0], v[1], oedge+e[1], attr);
-         new_boundary[j++] =
-            new Triangle(oedge+e[2], oedge+e[1], v[2], attr);
+         boundary[j+0] = new Triangle(oedge+e[1], oedge+e[2], oedge+e[0], attr);
+         boundary[j+1] = new Triangle(oedge+e[0], v[1], oedge+e[1], attr);
+         boundary[j+2] = new Triangle(oedge+e[2], oedge+e[1], v[2], attr);
+
+         v[1] = oedge+e[0];
+         v[2] = oedge+e[2];
       }
       else if (bdr_el_type == Element::QUADRILATERAL)
       {
          const int qf =
             (f2qf.Size() == 0) ? be_to_face[i] : f2qf[be_to_face[i]];
 
-         new_boundary[j++] =
-            new Quadrilateral(v[0], oedge+e[0], oface+qf, oedge+e[3], attr);
-         new_boundary[j++] =
-            new Quadrilateral(oedge+e[0], v[1], oedge+e[1], oface+qf, attr);
-         new_boundary[j++] =
-            new Quadrilateral(oface+qf, oedge+e[1], v[2], oedge+e[2], attr);
-         new_boundary[j++] =
-            new Quadrilateral(oedge+e[3], oface+qf, oedge+e[2], v[3], attr);
+         boundary[j+0] = new Quadrilateral(oedge+e[0], v[1], oedge+e[1],
+                                           oface+qf, attr);
+         boundary[j+1] = new Quadrilateral(oface+qf, oedge+e[1], v[2],
+                                           oedge+e[2], attr);
+         boundary[j+2] = new Quadrilateral(oedge+e[3], oface+qf,
+                                           oedge+e[2], v[3], attr);
+
+         v[1] = oedge+e[0];
+         v[2] = oface+qf;
+         v[3] = oedge+e[3];
       }
       else
       {
          MFEM_ABORT("boundary Element is not a triangle or a quad!");
       }
-      FreeElement(boundary[i]);
    }
-   mfem::Swap(boundary, new_boundary);
 
    static const double A = 0.0, B = 0.5, C = 1.0;
    static double tet_children[3*4*16] =
@@ -6699,21 +6599,20 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p)
       A,B,B, B,B,B, B,C,B, A,C,B, A,B,C, B,B,C, B,C,C, A,C,C
    };
 
-   CoarseFineTr.point_matrices[Geometry::TETRAHEDRON]
-   .UseExternalData(tet_children, 3, 4, 16);
-   CoarseFineTr.point_matrices[Geometry::PRISM]
-   .UseExternalData(pri_children, 3, 6, 8);
-   CoarseFineTr.point_matrices[Geometry::CUBE]
-   .UseExternalData(hex_children, 3, 8, 8);
+   CoarseFineTr.point_matrices[Geometry::TETRAHEDRON].
+   UseExternalData(tet_children, 3, 4, 16);
+   CoarseFineTr.point_matrices[Geometry::PRISM].
+   UseExternalData(pri_children, 3, 6, 8);
+   CoarseFineTr.point_matrices[Geometry::CUBE].
+   UseExternalData(hex_children, 3, 8, 8);
 
    for (int i = 0; i < elements.Size(); i++)
    {
-      // tetrahedron elements are handled above:
+      // Tetrahedron elements are handled above:
       if (elements[i]->GetType() == Element::TETRAHEDRON) { continue; }
-
       Embedding &emb = CoarseFineTr.embeddings[i];
-      emb.parent = i / 8;
-      emb.matrix = i % 8;
+      emb.parent = (i < NumOfElements) ? i : (i - NumOfElements) / 7;
+      emb.matrix = (i < NumOfElements) ? 0 : (i - NumOfElements) % 7 + 1;
    }
 
    NumOfVertices    = vertices.Size();
@@ -7297,13 +7196,13 @@ void Mesh::GeneralRefinement(const Array<Refinement> &refinements,
    else if (nonconforming < 0)
    {
       // determine if nonconforming refinement is suitable
-      if ((meshgen & 2) || (meshgen & 4))
+      if (meshgen & 2)
       {
-         nonconforming = 1; // tensor product elements and wedges
+         nonconforming = 1;
       }
       else
       {
-         nonconforming = 0; // simplices
+         nonconforming = 0;
       }
    }
 
@@ -7358,9 +7257,8 @@ void Mesh::EnsureNCMesh(bool triangles_nonconforming)
 
    if (!ncmesh)
    {
-      if ((meshgen & 0x2) /* quads/hexes */ ||
-          (meshgen & 0x4) /* wedges */ ||
-          (triangles_nonconforming && Dim == 2 && (meshgen & 0x1)))
+      if ((meshgen & 2) /* quads/hexes */ ||
+          (triangles_nonconforming && Dim == 2 && (meshgen & 1)))
       {
          MFEM_VERIFY(GetNumGeometries(Dim) <= 1,
                      "mixed meshes are not supported");
@@ -7783,7 +7681,12 @@ void Mesh::UniformRefinement(int i, const DSTable &v_to_v,
 void Mesh::InitRefinementTransforms()
 {
    // initialize CoarseFineTr
-   CoarseFineTr.Clear();
+   map<Geometry::Type,DenseTensor> &pms = CoarseFineTr.point_matrices;
+   map<Geometry::Type,DenseTensor>::iterator pms_iter;
+   for (pms_iter = pms.begin(); pms_iter != pms.end(); ++pms_iter)
+   {
+      pms_iter->second.SetSize(0, 0, 0);
+   }
    CoarseFineTr.embeddings.SetSize(NumOfElements);
    for (int i = 0; i < NumOfElements; i++)
    {
@@ -10113,51 +10016,4 @@ Mesh *Extrude2D(Mesh *mesh, const int nz, const double sz)
    return mesh3d;
 }
 
-#ifdef MFEM_DEBUG
-void Mesh::DebugDump(std::ostream &out) const
-{
-   // dump vertices and edges (NCMesh "nodes")
-   out << NumOfVertices + NumOfEdges << "\n";
-   for (int i = 0; i < NumOfVertices; i++)
-   {
-      const double *v = GetVertex(i);
-      out << i << " " << v[0] << " " << v[1] << " " << v[2]
-          << " 0 0 " << i << " -1 0\n";
-   }
-
-   Array<int> ev;
-   for (int i = 0; i < NumOfEdges; i++)
-   {
-      GetEdgeVertices(i, ev);
-      double mid[3] = {0, 0, 0};
-      for (int j = 0; j < 2; j++)
-      {
-         for (int k = 0; k < spaceDim; k++)
-         {
-            mid[k] += GetVertex(ev[j])[k];
-         }
-      }
-      out << NumOfVertices+i << " "
-          << mid[0]/2 << " " << mid[1]/2 << " " << mid[2]/2 << " "
-          << ev[0] << " " << ev[1] << " -1 " << i << " 0\n";
-   }
-
-   // dump elements
-   out << NumOfElements << "\n";
-   for (int i = 0; i < NumOfElements; i++)
-   {
-      const Element* e = elements[i];
-      out << e->GetNVertices() << " ";
-      for (int j = 0; j < e->GetNVertices(); j++)
-      {
-         out << e->GetVertices()[j] << " ";
-      }
-      out << e->GetAttribute() << " 0 " << i << "\n";
-   }
-
-   // dump faces
-   out << "0\n";
-}
-#endif
-
 }

mesh/mesh.hpp

@@ -386,6 +386,11 @@ protected:
       return FaceIsInterior(FaceNo) || (faces_info[FaceNo].Elem2Inf >= 0);
    }
 
+   // shift cyclically 3 integers left-to-right
+   inline static void ShiftL2R(int &, int &, int &);
+   // shift cyclically 3 integers so that the smallest is first
+   inline static void Rotate3(int &, int &, int &);
+
    void FreeElement(Element *E);
 
    void GenerateFaces();
@@ -1009,7 +1014,7 @@ public:
    // Nodes are only active for higher order meshes, and share locations with
    // the vertices, plus all the higher- order control points within the element
    // and along the edges and on the faces.
-   void GetNode(int i, double *coord) const;
+   void GetNode(int i, double *coord);
    void SetNode(int i, const double *coord);
 
    // Node operations for curved mesh.
@@ -1117,7 +1122,6 @@ public:
 
    ///@{ @name NURBS mesh refinement methods
    void KnotInsert(Array<KnotVector *> &kv);
-   void KnotInsert(Array<Vector *> &kv);
    /* For each knot vector:
          new_degree = max(old_degree, min(old_degree + rel_degree, degree)). */
    void DegreeElevate(int rel_degree, int degree = 16);
@@ -1266,11 +1270,6 @@ public:
 
    /// Destroys Mesh.
    virtual ~Mesh() { DestroyPointers(); }
-
-#ifdef MFEM_DEBUG
-   /// Output an NCMesh-compatible debug dump.
-   void DebugDump(std::ostream &out) const;
-#endif
 };
 
 /** Overload operator<< for std::ostream and Mesh; valid also for the derived
@@ -1359,13 +1358,35 @@ public:
 };
 
 
-// shift cyclically 3 integers left-to-right
-inline void ShiftRight(int &a, int &b, int &c)
+// inline functions
+inline void Mesh::ShiftL2R(int &a, int &b, int &c)
 {
    int t = a;
    a = c;  c = b;  b = t;
 }
 
+inline void Mesh::Rotate3(int &a, int &b, int &c)
+{
+   if (a < b)
+   {
+      if (a > c)
+      {
+         ShiftL2R(a, b, c);
+      }
+   }
+   else
+   {
+      if (b < c)
+      {
+         ShiftL2R(c, b, a);
+      }
+      else
+      {
+         ShiftL2R(a, b, c);
+      }
+   }
+}
+
 }
 
 #endif

mesh/ncmesh.hpp

@@ -15,7 +15,6 @@
 #include "../config/config.hpp"
 #include "../general/hash.hpp"
 #include "../general/globals.hpp"
-#include "../general/sort_pairs.hpp"
 #include "../linalg/densemat.hpp"
 #include "element.hpp"
 #include "vertex.hpp"
@@ -60,23 +59,24 @@ struct Embedding
 struct CoarseFineTransformations
 {
    /// Matrices for IsoparametricTransformation organized by Geometry::Type
-   DenseTensor point_matrices[Geometry::NumGeom];
+   std::map<Geometry::Type, DenseTensor> point_matrices;
    /// Fine element positions in their parents.
    Array<Embedding> embeddings;
 
+   const DenseTensor &GetPointMatrices(Geometry::Type geom) const;
+
    void GetCoarseToFineMap(const Mesh &fine_mesh,
                            Table &coarse_to_fine,
                            Array<int> &coarse_to_ref_type,
                            Table &ref_type_to_matrix,
                            Array<Geometry::Type> &ref_type_to_geom) const;
 
-   void Clear();
-   bool IsInitialized() const;
+   void Clear() { point_matrices.clear(); embeddings.DeleteAll(); }
    long MemoryUsage() const;
 };
 
 
-/** \brief A class for non-conforming AMR on higher-order hexahedral, prismatic,
+/** \brief A class for non-conforming AMR on higher-order hexahedral,
  *  quadrilateral or triangular meshes.
  *
  *  The class is used as follows:
@@ -152,13 +152,10 @@ public:
    {
       int index;   ///< Mesh number
       int element; ///< NCMesh::Element containing this vertex/edge/face
-      char local;  ///< local number within 'element'
-      char geom;   ///< Geometry::Type (faces only) (char storage to save RAM)
+      int local;   ///< local number within 'element'
 
-      MeshId(int index = -1, int element = -1, char local = -1, char geom = -1)
-         : index(index), element(element), local(local), geom(geom) {}
-
-      Geometry::Type Geom() const { return Geometry::Type(geom); }
+      MeshId(int index = -1, int element = -1, int local = -1)
+         : index(index), element(element), local(local) {}
    };
 
    /** Nonconforming edge/face that has more than one neighbor. The neighbors
@@ -167,9 +164,8 @@ public:
    {
       int slaves_begin, slaves_end; ///< slave faces
 
-      Master(int index, int element, char local, char geom, int sb, int se)
-         : MeshId(index, element, local, geom)
-         , slaves_begin(sb), slaves_end(se) {}
+      Master(int index, int element, int local, int sb, int se)
+         : MeshId(index, element, local), slaves_begin(sb), slaves_end(se) {}
    };
 
    /// Nonconforming edge/face within a bigger edge/face.
@@ -179,9 +175,8 @@ public:
       int edge_flags; ///< edge orientation flags
       DenseMatrix point_matrix; ///< position within the master edge/face
 
-      Slave(int index, int element, char local, char geom)
-         : MeshId(index, element, local, geom)
-         , master(-1), edge_flags(0) {}
+      Slave(int index, int element, int local)
+         : MeshId(index, element, local), master(-1), edge_flags(0) {}
 
       /// Return the point matrix oriented according to the master and slave edges
       void OrientedPointMatrix(DenseMatrix &oriented_matrix) const;
@@ -291,11 +286,10 @@ public:
        processor independent. TODO: this seems only partially true? */
    int GetEdgeNCOrientation(const MeshId &edge_id) const;
 
-   /** Return Mesh vertex and edge indices of a face identified by 'face_id'.
-       The return value is the number of face vertices. */
-   int GetFaceVerticesEdges(const MeshId &face_id,
-                            int vert_index[4], int edge_index[4],
-                            int edge_orientation[4]) const;
+   /// Return Mesh vertex and edge indices of a face identified by 'face_id'.
+   void GetFaceVerticesEdges(const MeshId &face_id,
+                             int vert_index[4], int edge_index[4],
+                             int edge_orientation[4]) const;
 
    /** Given an edge (by its vertex indices v1 and v2) return the first
        (geometric) parent edge that exists in the Mesh or -1 if there is no such
@@ -311,29 +305,14 @@ public:
                                    Array<int> &bdr_vertices,
                                    Array<int> &bdr_edges);
 
-   /// Return element geometry type. @a index is the Mesh element number.
-   Geometry::Type GetElementGeometry(int index) const
-   { return elements[leaf_elements[index]].Geom(); }
+   /// Return the type of elements in the mesh.
+   Geometry::Type GetElementGeometry() const { return elements[0].geom; }
 
-   /// Return face geometry type. @a index is the Mesh face number.
-   Geometry::Type GetFaceGeometry(int index) const
-   { return Geometry::Type(face_geom[index]); }
-
-   /// Return the number of root elements.
-   int GetNumRootElements() { return root_state.Size(); }
+   Geometry::Type GetFaceGeometry() const { return Geometry::SQUARE; }
 
    /// Return the distance of leaf 'i' from the root.
    int GetElementDepth(int i) const;
 
-   /** Return the size reduction compared to the root element (ignoring local
-       stretching and curvature). */
-   int GetElementSizeReduction(int i) const;
-
-   /// Return the faces and face attributes of leaf element 'i'.
-   void GetElementFacesAttributes(int i, Array<int> &faces,
-                                  Array<int> &fattr) const;
-
-
    /// I/O: Print the "vertex_parents" section of the mesh file (ver. >= 1.1).
    void PrintVertexParents(std::ostream &out) const;
 
@@ -360,17 +339,15 @@ public:
 
    void PrintStats(std::ostream &out = mfem::out) const;
 
-   typedef int64_t RefCoord;
-
 
 protected: // interface for Mesh to be able to construct itself from NCMesh
 
    friend class Mesh;
 
    /// Return the basic Mesh arrays for the current finest level.
-   void GetMeshComponents(Array<mfem::Vertex> &mvertices,
-                          Array<mfem::Element*> &melements,
-                          Array<mfem::Element*> &mboundary) const;
+   void GetMeshComponents(Array<mfem::Vertex>& mvertices,
+                          Array<mfem::Element*>& melements,
+                          Array<mfem::Element*>& mboundary) const;
 
    /** Get edge and face numbering from 'mesh' (i.e., set all Edge::index and
        Face::index) after a new mesh was created from us. */
@@ -381,7 +358,6 @@ protected: // implementation
 
    int Dim, spaceDim; ///< dimensions of the elements and the vertex coordinates
    bool Iso; ///< true if the mesh only contains isotropic refinements
-   int Geoms; ///< bit mask of element geometries present, see InitGeomFlags()
 
    /** A Node can hold a vertex, an edge, or both. Elements directly point to
        their corner nodes, but edge nodes also exist and can be accessed using
@@ -435,7 +411,7 @@ protected: // implementation
        to its vertex nodes. */
    struct Element
    {
-      char geom;     ///< Geometry::Type of the element (char for storage only)
+      Geometry::Type geom; ///< Geometry::Type of the element
       char ref_type; ///< bit mask of X,Y,Z refinements (bits 0,1,2 respectively)
       char flag;     ///< generic flag/marker, can be used by algorithms
       int index;     ///< element number in the Mesh, -1 if refined
@@ -449,8 +425,6 @@ protected: // implementation
       int parent; ///< parent element, -1 if this is a root element, -2 if free
 
       Element(Geometry::Type geom, int attr);
-
-      Geometry::Type Geom() const { return Geometry::Type(geom); }
    };
 
    // primary data
@@ -496,7 +470,6 @@ protected: // implementation
    NCList vertex_list; ///< lazy-initialized list of vertices, see GetVertexList
 
    Array<int> boundary_faces; ///< subset of all faces, set by BuildFaceList
-   Array<char> face_geom; ///< face geometry by face index, set by OnMeshUpdated
 
    Table element_vertex; ///< leaf-element to vertex table, see FindSetNeighbors
 
@@ -518,15 +491,10 @@ protected: // implementation
    virtual int GetNumGhostElements() const { return 0; }
    virtual int GetNumGhostVertices() const { return 0; }
 
-   void InitGeomFlags();
-   bool HavePrisms() const { return Geoms & (1 << Geometry::PRISM); }
-
 
    // refinement/derefinement
 
    Array<Refinement> ref_stack; ///< stack of scheduled refinements (temporary)
-   HashTable<Node> shadow; ///< temporary storage for reparented nodes
-   Array<Triple<int, int, int> > reparents; ///< scheduled node reparents (tmp)
 
    Table derefinements; ///< possible derefinements, see GetDerefinementTable
 
@@ -551,16 +519,13 @@ protected: // implementation
    }
 
    int NewHexahedron(int n0, int n1, int n2, int n3,
-                     int n4, int n5, int n6, int n7, int attr,
+                     int n4, int n5, int n6, int n7,
+                     int attr,
                      int fattr0, int fattr1, int fattr2,
                      int fattr3, int fattr4, int fattr5);
 
-   int NewWedge(int n0, int n1, int n2,
-                int n3, int n4, int n5, int attr,
-                int fattr0, int fattr1,
-                int fattr2, int fattr3, int fattr4);
-
-   int NewQuadrilateral(int n0, int n1, int n2, int n3, int attr,
+   int NewQuadrilateral(int n0, int n1, int n2, int n3,
+                        int attr,
                         int eattr0, int eattr1, int eattr2, int eattr3);
 
    int NewTriangle(int n0, int n1, int n2,
@@ -568,62 +533,57 @@ protected: // implementation
 
    mfem::Element* NewMeshElement(int geom) const;
 
-   int QuadFaceSplitType(int v1, int v2, int v3, int v4, int mid[5]
-                         = NULL /*optional output of mid-edge nodes*/) const;
+   int GetMidEdgeNode(int vn1, int vn2);
+   int GetMidFaceNode(int en1, int en2, int en3, int en4);
 
-   bool TriFaceSplit(int v1, int v2, int v3, int mid[3] = NULL) const;
+   int FaceSplitType(int v1, int v2, int v3, int v4, int mid[4]
+                     = NULL /*optional output of mid-edge nodes*/) const;
 
    void ForceRefinement(int vn1, int vn2, int vn3, int vn4);
 
-   void FindEdgeElements(int vn1, int vn2, int vn3, int vn4,
-                         Array<MeshId> &prisms) const;
-
-   void CheckAnisoPrism(int vn1, int vn2, int vn3, int vn4,
-                        const Refinement *refs, int nref);
-
    void CheckAnisoFace(int vn1, int vn2, int vn3, int vn4,
                        int mid12, int mid34, int level = 0);
 
    void CheckIsoFace(int vn1, int vn2, int vn3, int vn4,
                      int en1, int en2, int en3, int en4, int midf);
 
-   void ReparentNode(int node, int new_p1, int new_p2);
-
-   int FindMidEdgeNode(int node1, int node2) const;
-   int GetMidEdgeNode(int node1, int node2);
-
-   int GetMidFaceNode(int en1, int en2, int en3, int en4);
-
-   void ReferenceElement(int elem);
-   void UnreferenceElement(int elem, Array<int> &elemFaces);
+   void RefElement(int elem);
+   void UnrefElement(int elem, Array<int> &elemFaces);
 
    Face* GetFace(Element &elem, int face_no);
    void RegisterFaces(int elem, int *fattr = NULL);
    void DeleteUnusedFaces(const Array<int> &elemFaces);
 
+   int FindAltParents(int node1, int node2);
+
+   bool NodeSetX1(int node, int* n);
+   bool NodeSetX2(int node, int* n);
+   bool NodeSetY1(int node, int* n);
+   bool NodeSetY2(int node, int* n);
+   bool NodeSetZ1(int node, int* n);
+   bool NodeSetZ2(int node, int* n);
+
    void CollectDerefinements(int elem, Array<Connection> &list);
 
    /// Return el.node[index] correctly, even if the element is refined.
    int RetrieveNode(const Element &el, int index);
 
-   /// Extended version of find_node: works if 'el' is refined.
-   int FindNodeExt(const Element &el, int node, bool abort = true);
+   /// Extended version of find_node: works if 'el' is refined; optional abort.
+   int FindNodeExt(const Element &el, int node, bool abort = false);
 
 
    // face/edge lists
 
    static int find_node(const Element &el, int node);
-   static int find_element_edge(const Element &el, int vn0, int vn1,
-                                bool abort = true);
-   static int find_local_face(int geom, int a, int b, int c);
+   static int find_element_edge(const Element &el, int vn0, int vn1);
+   static int find_hex_face(int a, int b, int c);
 
    int ReorderFacePointMat(int v0, int v1, int v2, int v3,
                            int elem, DenseMatrix& mat) const;
    struct PointMatrix;
-   void TraverseQuadFace(int vn0, int vn1, int vn2, int vn3,
-                         const PointMatrix& pm, int level, Face* eface[4]);
-   void TraverseTriFace(int vn0, int vn1, int vn2,
-                        const PointMatrix& pm, int level);
+   void TraverseFace(int vn0, int vn1, int vn2, int vn3,
+                     const PointMatrix& pm, int level);
+
    void TraverseEdge(int vn0, int vn1, double t0, double t1, int flags,
                      int level);
 
@@ -666,9 +626,8 @@ protected: // implementation
 
 
    void CollectEdgeVertices(int v0, int v1, Array<int> &indices);
-   void CollectTriFaceVertices(int v0, int v1, int v2, Array<int> &indices);
-   void CollectQuadFaceVertices(int v0, int v1, int v2, int v3,
-                                Array<int> &indices);
+   void CollectFaceVertices(int v0, int v1, int v2, int v3,
+                            Array<int> &indices);
    void BuildElementToVertexTable();
 
    void UpdateElementToVertexTable()
@@ -676,15 +635,6 @@ protected: // implementation
       if (element_vertex.Size() < 0) { BuildElementToVertexTable(); }
    }
 
-   int GetVertexRootCoord(int elem, RefCoord coord[3]) const;
-   void CollectIncidentElements(int elem, const RefCoord coord[3],
-                                Array<int> &list) const;
-
-   /** Return elements neighboring to a local vertex of element 'elem'. Only
-       elements from within the same refinement tree ('cousins') are returned.
-       Complexity is proportional to the depth of elem's refinement tree. */
-   void FindVertexCousins(int elem, int local, Array<int> &cousins) const;
-
 
    // coarse/fine transformations
 
@@ -740,13 +690,6 @@ protected: // implementation
       PointMatrix(const Point& p0, const Point& p1, const Point& p2, const Point& p3)
       { np = 4; points[0] = p0; points[1] = p1; points[2] = p2; points[3] = p3; }
 
-      PointMatrix(const Point& p0, const Point& p1, const Point& p2,
-                  const Point& p3, const Point& p4, const Point& p5)
-      {
-         np = 6;
-         points[0] = p0; points[1] = p1; points[2] = p2;
-         points[3] = p3; points[4] = p4; points[5] = p5;
-      }
       PointMatrix(const Point& p0, const Point& p1, const Point& p2,
                   const Point& p3, const Point& p4, const Point& p5,
                   const Point& p6, const Point& p7)
@@ -764,13 +707,11 @@ protected: // implementation
 
    static PointMatrix pm_tri_identity;
    static PointMatrix pm_quad_identity;
-   static PointMatrix pm_prism_identity;
    static PointMatrix pm_hex_identity;
 
-   static const PointMatrix& GetGeomIdentity(Geometry::Type geom);
+   static const PointMatrix& GetGeomIdentity(int geom);
 
-   void GetPointMatrix(Geometry::Type geom, const char* ref_path,
-                       DenseMatrix& matrix);
+   void GetPointMatrix(int geom, const char* ref_path, DenseMatrix& matrix);
 
    typedef std::map<std::string, int> RefPathMap;
 
@@ -807,10 +748,9 @@ protected: // implementation
 
    void FindFaceNodes(int face, int node[4]);
 
-   int EdgeSplitLevel(int vn1, int vn2) const;
-   int TriFaceSplitLevel(int vn1, int vn2, int vn3) const;
-   void QuadFaceSplitLevel(int vn1, int vn2, int vn3, int vn4,
-                           int& h_level, int& v_level) const;
+   int  EdgeSplitLevel(int vn1, int vn2) const;
+   void FaceSplitLevel(int vn1, int vn2, int vn3, int vn4,
+                       int& h_level, int& v_level) const;
 
    void CountSplits(int elem, int splits[3]) const;
    void GetLimitRefinements(Array<Refinement> &refinements, int max_level);
@@ -825,10 +765,9 @@ protected: // implementation
        (triangles, quads, cubes) */
    struct GeomInfo
    {
-      int nv, ne, nf;   // number of: vertices, edges, faces
-      int edges[12][2]; // edge vertices (up to 12 edges)
-      int faces[6][4];  // face vertices (up to 6 faces)
-      int nfv[6];       // number of face vertices
+      int nv, ne, nf, nfv; // number of: vertices, edges, faces, face vertices
+      int edges[12][2];    // edge vertices (up to 12 edges)
+      int faces[6][4];     // face vertices (up to 6 faces)
 
       bool initialized;
       GeomInfo() : initialized(false) {}
@@ -837,7 +776,7 @@ protected: // implementation
 
    static GeomInfo GI[Geometry::NumGeom];
 
-   static GeomInfo &gi_hex, &gi_wedge, &gi_quad, &gi_tri;
+   static GeomInfo &gi_hex, &gi_quad, &gi_tri;
 
 #ifdef MFEM_DEBUG
 public:

mesh/ncmesh_tables.hpp

@@ -1,469 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-namespace mfem
-{
-
-static int ref_type_num_children[8] = { 0, 2, 2, 4, 2, 4, 4, 8 };
-
-
-// derefinement tables
-
-static int quad_deref_table[3][4 + 4] =
-{
-   { 0, 1, 1, 0, /**/ 1, 1, 0, 0 }, // 1 - X
-   { 0, 0, 1, 1, /**/ 0, 0, 1, 1 }, // 2 - Y
-   { 0, 1, 2, 3, /**/ 1, 1, 3, 3 }  // 3 - iso
-};
-
-static int hex_deref_table[7][8 + 6] =
-{
-   { 0, 1, 1, 0, 0, 1, 1, 0, /**/ 1, 1, 1, 0, 0, 0 }, // 1 - X
-   { 0, 0, 1, 1, 0, 0, 1, 1, /**/ 0, 0, 0, 1, 1, 1 }, // 2 - Y
-   { 0, 1, 2, 3, 0, 1, 2, 3, /**/ 1, 1, 1, 3, 3, 3 }, // 3 - XY
-   { 0, 0, 0, 0, 1, 1, 1, 1, /**/ 0, 0, 0, 1, 1, 1 }, // 4 - Z
-   { 0, 1, 1, 0, 3, 2, 2, 3, /**/ 1, 1, 1, 3, 3, 3 }, // 5 - XZ
-   { 0, 0, 1, 1, 2, 2, 3, 3, /**/ 0, 0, 0, 3, 3, 3 }, // 6 - YZ
-   { 0, 1, 2, 3, 4, 5, 6, 7, /**/ 1, 1, 1, 7, 7, 7 }  // 7 - iso
-};
-
-static int prism_deref_table[7][6 + 5] =
-{
-   {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 1
-   {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 2
-   { 0, 1, 2, 0, 1, 2, /**/  0, 0, 0, 1, 0 }, // 3 - XY
-   { 0, 0, 0, 1, 1, 1, /**/  0, 1, 0, 0, 0 }, // 4 - Z
-   {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 5
-   {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 6
-   { 0, 1, 2, 4, 5, 6, /**/  0, 5, 0, 5, 0 }  // 7 - iso
-};
-
-
-// child ordering tables
-
-static char quad_hilbert_child_order[8][4] =
-{
-   {0,1,2,3}, {0,3,2,1}, {1,2,3,0}, {1,0,3,2},
-   {2,3,0,1}, {2,1,0,3}, {3,0,1,2}, {3,2,1,0}
-};
-
-static char quad_hilbert_child_state[8][4] =
-{
-   {1,0,0,5}, {0,1,1,4}, {3,2,2,7}, {2,3,3,6},
-   {5,4,4,1}, {4,5,5,0}, {7,6,6,3}, {6,7,7,2}
-};
-
-static char hex_hilbert_child_order[24][8] =
-{
-   {0,1,2,3,7,6,5,4}, {0,3,7,4,5,6,2,1}, {0,4,5,1,2,6,7,3},
-   {1,0,3,2,6,7,4,5}, {1,2,6,5,4,7,3,0}, {1,5,4,0,3,7,6,2},
-   {2,1,5,6,7,4,0,3}, {2,3,0,1,5,4,7,6}, {2,6,7,3,0,4,5,1},
-   {3,0,4,7,6,5,1,2}, {3,2,1,0,4,5,6,7}, {3,7,6,2,1,5,4,0},
-   {4,0,1,5,6,2,3,7}, {4,5,6,7,3,2,1,0}, {4,7,3,0,1,2,6,5},
-   {5,1,0,4,7,3,2,6}, {5,4,7,6,2,3,0,1}, {5,6,2,1,0,3,7,4},
-   {6,2,3,7,4,0,1,5}, {6,5,1,2,3,0,4,7}, {6,7,4,5,1,0,3,2},
-   {7,3,2,6,5,1,0,4}, {7,4,0,3,2,1,5,6}, {7,6,5,4,0,1,2,3}
-};
-
-static char hex_hilbert_child_state[24][8] =
-{
-   {1,2,2,7,7,21,21,17},     {2,0,0,22,22,16,16,8},    {0,1,1,15,15,6,6,23},
-   {4,5,5,10,10,18,18,14},   {5,3,3,19,19,13,13,11},   {3,4,4,12,12,9,9,20},
-   {8,7,7,17,17,23,23,2},    {6,8,8,0,0,15,15,22},     {7,6,6,21,21,1,1,16},
-   {11,10,10,14,14,20,20,5}, {9,11,11,3,3,12,12,19},   {10,9,9,18,18,4,4,13},
-   {13,14,14,5,5,19,19,10},  {14,12,12,20,20,11,11,4}, {12,13,13,9,9,3,3,18},
-   {16,17,17,2,2,22,22,7},   {17,15,15,23,23,8,8,1},   {15,16,16,6,6,0,0,21},
-   {20,19,19,11,11,14,14,3}, {18,20,20,4,4,10,10,12},  {19,18,18,13,13,5,5,9},
-   {23,22,22,8,8,17,17,0},   {21,23,23,1,1,7,7,15},    {22,21,21,16,16,2,2,6}
-};
-
-
-// child/parent reference domain transforms
-
-typedef NCMesh::RefCoord RefCoord;
-
-// reference domain coordinates as fixed point numbers
-const RefCoord T_HALF = (1ll << 59);
-const RefCoord T_ONE = (1ll << 60);
-const RefCoord T_TWO = (1ll << 61);
-
-// (scaling factors have a different fixed point multiplier)
-const RefCoord S_HALF = 1;
-const RefCoord S_ONE = 2;
-const RefCoord S_TWO = 4;
-
-static RefCoord tri_corners[3][3] =
-{
-   {    0,     0, 0},
-   {T_ONE,     0, 0},
-   {    0, T_ONE, 0}
-};
-
-static RefCoord quad_corners[4][3] =
-{
-   {    0,     0, 0},
-   {T_ONE,     0, 0},
-   {T_ONE, T_ONE, 0},
-   {    0, T_ONE, 0}
-};
-
-static RefCoord hex_corners[8][3] =
-{
-   {    0,     0,     0},
-   {T_ONE,     0,     0},
-   {T_ONE, T_ONE,     0},
-   {    0, T_ONE,     0},
-   {    0,     0, T_ONE},
-   {T_ONE,     0, T_ONE},
-   {T_ONE, T_ONE, T_ONE},
-   {    0, T_ONE, T_ONE}
-};
-
-static RefCoord prism_corners[6][3] =
-{
-   {    0,     0,     0},
-   {T_ONE,     0,     0},
-   {    0, T_ONE,     0},
-   {    0,     0, T_ONE},
-   {T_ONE,     0, T_ONE},
-   {    0, T_ONE, T_ONE}
-};
-
-typedef RefCoord RefPoint[3];
-static RefPoint* geom_corners[7] =
-{
-   NULL, // point
-   NULL, // segment
-   tri_corners,
-   quad_corners,
-   NULL, // tetrahedron
-   hex_corners,
-   prism_corners
-};
-
-// reference domain transform: 3 scales, 3 translations
-struct RefTrf
-{
-   RefCoord s[3], t[3];
-
-   void Apply(const RefCoord src[3], RefCoord dst[3]) const;
-};
-
-static RefTrf quad_parent_rt1[2] =
-{
-   { {S_HALF, S_ONE, 0}, {     0, 0, 0} },
-   { {S_HALF, S_ONE, 0}, {T_HALF, 0, 0} }
-};
-
-static RefTrf quad_child_rt1[2] =
-{
-   { {S_TWO, S_ONE, 0}, {     0, 0, 0} },
-   { {S_TWO, S_ONE, 0}, {-T_ONE, 0, 0} }
-};
-
-static RefTrf quad_parent_rt2[2] =
-{
-   { {S_ONE, S_HALF, 0}, {0,      0, 0} },
-   { {S_ONE, S_HALF, 0}, {0, T_HALF, 0} }
-};
-
-static RefTrf quad_child_rt2[2] =
-{
-   { {S_ONE, S_TWO, 0}, {0,      0, 0} },
-   { {S_ONE, S_TWO, 0}, {0, -T_ONE, 0} }
-};
-
-static RefTrf quad_parent_rt3[4] =
-{
-   { {S_HALF, S_HALF, 0}, {     0,      0, 0} },
-   { {S_HALF, S_HALF, 0}, {T_HALF,      0, 0} },
-   { {S_HALF, S_HALF, 0}, {T_HALF, T_HALF, 0} },
-   { {S_HALF, S_HALF, 0}, {     0, T_HALF, 0} }
-};
-
-static RefTrf quad_child_rt3[4] =
-{
-   { {S_TWO, S_TWO, 0}, {     0,      0, 0} },
-   { {S_TWO, S_TWO, 0}, {-T_ONE,      0, 0} },
-   { {S_TWO, S_TWO, 0}, {-T_ONE, -T_ONE, 0} },
-   { {S_TWO, S_TWO, 0}, {     0, -T_ONE, 0} }
-};
-
-static RefTrf* quad_parent[4] =
-{
-   NULL,
-   quad_parent_rt1,
-   quad_parent_rt2,
-   quad_parent_rt3
-};
-
-static RefTrf* quad_child[4] =
-{
-   NULL,
-   quad_child_rt1,
-   quad_child_rt2,
-   quad_child_rt3
-};
-
-static RefTrf hex_parent_rt1[2] =
-{
-   { {S_HALF, S_ONE, S_ONE}, {     0, 0, 0} },
-   { {S_HALF, S_ONE, S_ONE}, {T_HALF, 0, 0} }
-};
-
-static RefTrf hex_child_rt1[2] =
-{
-   { {S_TWO, S_ONE, S_ONE}, {     0, 0, 0} },
-   { {S_TWO, S_ONE, S_ONE}, {-T_ONE, 0, 0} }
-};
-
-static RefTrf hex_parent_rt2[2] =
-{
-   { {S_ONE, S_HALF, S_ONE}, {0,      0, 0} },
-   { {S_ONE, S_HALF, S_ONE}, {0, T_HALF, 0} }
-};
-
-static RefTrf hex_child_rt2[2] =
-{
-   { {S_ONE, S_TWO, S_ONE}, {0,      0, 0} },
-   { {S_ONE, S_TWO, S_ONE}, {0, -T_ONE, 0} }
-};
-
-static RefTrf hex_parent_rt3[4] =
-{
-   { {S_HALF, S_HALF, S_ONE}, {     0,      0, 0} },
-   { {S_HALF, S_HALF, S_ONE}, {T_HALF,      0, 0} },
-   { {S_HALF, S_HALF, S_ONE}, {T_HALF, T_HALF, 0} },
-   { {S_HALF, S_HALF, S_ONE}, {     0, T_HALF, 0} }
-};
-
-static RefTrf hex_child_rt3[4] =
-{
-   { {S_TWO, S_TWO, S_ONE}, {     0,      0, 0} },
-   { {S_TWO, S_TWO, S_ONE}, {-T_ONE,      0, 0} },
-   { {S_TWO, S_TWO, S_ONE}, {-T_ONE, -T_ONE, 0} },
-   { {S_TWO, S_TWO, S_ONE}, {     0, -T_ONE, 0} }
-};
-
-static RefTrf hex_parent_rt4[2] =
-{
-   { {S_ONE, S_ONE, S_HALF}, {0, 0,      0} },
-   { {S_ONE, S_ONE, S_HALF}, {0, 0, T_HALF} }
-};
-
-static RefTrf hex_child_rt4[2] =
-{
-   { {S_ONE, S_ONE, S_TWO}, {0, 0,      0} },
-   { {S_ONE, S_ONE, S_TWO}, {0, 0, -T_ONE} }
-};
-
-static RefTrf hex_parent_rt5[4] =
-{
-   { {S_HALF, S_ONE, S_HALF}, {     0, 0,      0} },
-   { {S_HALF, S_ONE, S_HALF}, {T_HALF, 0,      0} },
-   { {S_HALF, S_ONE, S_HALF}, {T_HALF, 0, T_HALF} },
-   { {S_HALF, S_ONE, S_HALF}, {     0, 0, T_HALF} }
-};
-
-static RefTrf hex_child_rt5[4] =
-{
-   { {S_TWO, S_ONE, S_TWO}, {     0, 0,      0} },
-   { {S_TWO, S_ONE, S_TWO}, {-T_ONE, 0,      0} },
-   { {S_TWO, S_ONE, S_TWO}, {-T_ONE, 0, -T_ONE} },
-   { {S_TWO, S_ONE, S_TWO}, {     0, 0, -T_ONE} }
-};
-
-static RefTrf hex_parent_rt6[4] =
-{
-   { {S_ONE, S_HALF, S_HALF}, {0,      0,      0} },
-   { {S_ONE, S_HALF, S_HALF}, {0, T_HALF,      0} },
-   { {S_ONE, S_HALF, S_HALF}, {0,      0, T_HALF} },
-   { {S_ONE, S_HALF, S_HALF}, {0, T_HALF, T_HALF} }
-};
-
-static RefTrf hex_child_rt6[4] =
-{
-   { {S_ONE, S_TWO, S_TWO}, {0,      0,      0} },
-   { {S_ONE, S_TWO, S_TWO}, {0, -T_ONE,      0} },
-   { {S_ONE, S_TWO, S_TWO}, {0,      0, -T_ONE} },
-   { {S_ONE, S_TWO, S_TWO}, {0, -T_ONE, -T_ONE} }
-};
-
-static RefTrf hex_parent_rt7[8] =
-{
-   { {S_HALF, S_HALF, S_HALF}, {     0,      0,      0} },
-   { {S_HALF, S_HALF, S_HALF}, {T_HALF,      0,      0} },
-   { {S_HALF, S_HALF, S_HALF}, {T_HALF, T_HALF,      0} },
-   { {S_HALF, S_HALF, S_HALF}, {     0, T_HALF,      0} },
-   { {S_HALF, S_HALF, S_HALF}, {     0,      0, T_HALF} },
-   { {S_HALF, S_HALF, S_HALF}, {T_HALF,      0, T_HALF} },
-   { {S_HALF, S_HALF, S_HALF}, {T_HALF, T_HALF, T_HALF} },
-   { {S_HALF, S_HALF, S_HALF}, {     0, T_HALF, T_HALF} }
-};
-
-static RefTrf hex_child_rt7[8] =
-{
-   { {S_TWO, S_TWO, S_TWO}, {     0,      0,      0} },
-   { {S_TWO, S_TWO, S_TWO}, {-T_ONE,      0,      0} },
-   { {S_TWO, S_TWO, S_TWO}, {-T_ONE, -T_ONE,      0} },
-   { {S_TWO, S_TWO, S_TWO}, {     0, -T_ONE,      0} },
-   { {S_TWO, S_TWO, S_TWO}, {     0,      0, -T_ONE} },
-   { {S_TWO, S_TWO, S_TWO}, {-T_ONE,      0, -T_ONE} },
-   { {S_TWO, S_TWO, S_TWO}, {-T_ONE, -T_ONE, -T_ONE} },
-   { {S_TWO, S_TWO, S_TWO}, {     0, -T_ONE, -T_ONE} }
-};
-
-static RefTrf* hex_parent[8] =
-{
-   NULL,
-   hex_parent_rt1,
-   hex_parent_rt2,
-   hex_parent_rt3,
-   hex_parent_rt4,
-   hex_parent_rt5,
-   hex_parent_rt6,
-   hex_parent_rt7
-};
-
-static RefTrf* hex_child[8] =
-{
-   NULL,
-   hex_child_rt1,
-   hex_child_rt2,
-   hex_child_rt3,
-   hex_child_rt4,
-   hex_child_rt5,
-   hex_child_rt6,
-   hex_child_rt7
-};
-
-static RefTrf tri_parent_rt3[4] =
-{
-   { { S_HALF,  S_HALF, 0}, {     0,      0, 0} },
-   { { S_HALF,  S_HALF, 0}, {T_HALF,      0, 0} },
-   { { S_HALF,  S_HALF, 0}, {     0, T_HALF, 0} },
-   { {-S_HALF, -S_HALF, 0}, {T_HALF, T_HALF, 0} }
-};
-
-static RefTrf tri_child_rt3[4] =
-{
-   { { S_TWO,  S_TWO, 0}, {     0,      0, 0} },
-   { { S_TWO,  S_TWO, 0}, {-T_ONE,      0, 0} },
-   { { S_TWO,  S_TWO, 0}, {     0, -T_ONE, 0} },
-   { {-S_TWO, -S_TWO, 0}, { T_ONE,  T_ONE, 0} }
-};
-
-static RefTrf* tri_parent[4] =
-{
-   NULL, NULL, NULL,
-   tri_parent_rt3
-};
-
-static RefTrf* tri_child[4] =
-{
-   NULL, NULL, NULL,
-   tri_child_rt3
-};
-
-static RefTrf prism_parent_rt3[4] =
-{
-   { { S_HALF,  S_HALF, S_ONE}, {     0,      0, 0} },
-   { { S_HALF,  S_HALF, S_ONE}, {T_HALF,      0, 0} },
-   { { S_HALF,  S_HALF, S_ONE}, {     0, T_HALF, 0} },
-   { {-S_HALF, -S_HALF, S_ONE}, {T_HALF, T_HALF, 0} }
-};
-
-static RefTrf prism_child_rt3[4] =
-{
-   { { S_TWO,  S_TWO, S_ONE}, {     0,      0, 0} },
-   { { S_TWO,  S_TWO, S_ONE}, {-T_ONE,      0, 0} },
-   { { S_TWO,  S_TWO, S_ONE}, {     0, -T_ONE, 0} },
-   { {-S_TWO, -S_TWO, S_ONE}, { T_ONE,  T_ONE, 0} }
-};
-
-static RefTrf prism_parent_rt4[2] =
-{
-   { {S_ONE, S_ONE, S_HALF}, {0, 0,      0} },
-   { {S_ONE, S_ONE, S_HALF}, {0, 0, T_HALF} }
-};
-
-static RefTrf prism_child_rt4[2] =
-{
-   { {S_ONE, S_ONE, S_TWO}, {0, 0,      0} },
-   { {S_ONE, S_ONE, S_TWO}, {0, 0, -T_ONE} }
-};
-
-static RefTrf prism_parent_rt7[8] =
-{
-   { { S_HALF,  S_HALF, S_HALF}, {     0,      0,      0} },
-   { { S_HALF,  S_HALF, S_HALF}, {T_HALF,      0,      0} },
-   { { S_HALF,  S_HALF, S_HALF}, {     0, T_HALF,      0} },
-   { {-S_HALF, -S_HALF, S_HALF}, {T_HALF, T_HALF,      0} },
-   { { S_HALF,  S_HALF, S_HALF}, {     0,      0, T_HALF} },
-   { { S_HALF,  S_HALF, S_HALF}, {T_HALF,      0, T_HALF} },
-   { { S_HALF,  S_HALF, S_HALF}, {     0, T_HALF, T_HALF} },
-   { {-S_HALF, -S_HALF, S_HALF}, {T_HALF, T_HALF, T_HALF} }
-};
-
-static RefTrf prism_child_rt7[8] =
-{
-   { { S_TWO,  S_TWO, S_TWO}, {     0,      0,      0} },
-   { { S_TWO,  S_TWO, S_TWO}, {-T_ONE,      0,      0} },
-   { { S_TWO,  S_TWO, S_TWO}, {     0, -T_ONE,      0} },
-   { {-S_TWO, -S_TWO, S_TWO}, { T_ONE,  T_ONE,      0} },
-   { { S_TWO,  S_TWO, S_TWO}, {     0,      0, -T_ONE} },
-   { { S_TWO,  S_TWO, S_TWO}, {-T_ONE,      0, -T_ONE} },
-   { { S_TWO,  S_TWO, S_TWO}, {     0, -T_ONE, -T_ONE} },
-   { {-S_TWO, -S_TWO, S_TWO}, { T_ONE,  T_ONE, -T_ONE} }
-};
-
-static RefTrf* prism_parent[8] =
-{
-   NULL, NULL, NULL,
-   prism_parent_rt3,
-   prism_parent_rt4,
-   NULL, NULL,
-   prism_parent_rt7
-};
-
-static RefTrf* prism_child[8] =
-{
-   NULL, NULL, NULL,
-   prism_child_rt3,
-   prism_child_rt4,
-   NULL, NULL,
-   prism_child_rt7
-};
-
-static RefTrf** geom_parent[7] =
-{
-   NULL,
-   NULL,
-   tri_parent,
-   quad_parent,
-   NULL,
-   hex_parent,
-   prism_parent
-};
-
-static RefTrf** geom_child[7] =
-{
-   NULL,
-   NULL,
-   tri_child,
-   quad_child,
-   NULL,
-   hex_child,
-   prism_child
-};
-
-
-} // namespace mfem

mesh/nurbs.cpp

@@ -582,19 +582,9 @@ void NURBSPatch::KnotInsert(int dir, const KnotVector &newkv)
    }
 }
 
-void NURBSPatch::KnotInsert(Array<Vector *> &newkv)
-{
-   for (int dir = 0; dir < kv.Size(); dir++)
-   {
-      KnotInsert(dir, *newkv[dir]);
-   }
-}
-
 // Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
 void NURBSPatch::KnotInsert(int dir, const Vector &knot)
 {
-   if (knot.Size() == 0 ) { return; }
-
    if (dir >= kv.Size() || dir < 0)
    {
       mfem_error("NURBSPatch::KnotInsert : Incorrect direction!");
@@ -2992,34 +2982,6 @@ void NURBSExtension::KnotInsert(Array<KnotVector *> &kv)
    }
 }
 
-void NURBSExtension::KnotInsert(Array<Vector *> &kv)
-{
-   Array<int> edges;
-   Array<int> orient;
-
-   Array<Vector *> pkv(Dimension());
-
-   for (int p = 0; p < patches.Size(); p++)
-   {
-      patchTopo->GetElementEdges(p, edges, orient);
-
-      if (Dimension()==2)
-      {
-         pkv[0] = kv[KnotInd(edges[0])];
-         pkv[1] = kv[KnotInd(edges[1])];
-      }
-      else
-      {
-         pkv[0] = kv[KnotInd(edges[0])];
-         pkv[1] = kv[KnotInd(edges[3])];
-         pkv[2] = kv[KnotInd(edges[8])];
-      }
-
-      patches[p]->KnotInsert(pkv);
-   }
-}
-
-
 void NURBSExtension::GetPatchNets(const Vector &coords, int vdim)
 {
    if (Dimension() == 2)

mesh/nurbs.hpp

@@ -119,9 +119,7 @@ public:
    void KnotInsert   (int dir, const KnotVector &knot);
    void KnotInsert   (int dir, const Vector     &knot);
 
-   void KnotInsert(Array<Vector *> &knot);
    void KnotInsert(Array<KnotVector *> &knot);
-
    void DegreeElevate(int t);
    void UniformRefinement();
 
@@ -391,7 +389,6 @@ public:
    void DegreeElevate(int rel_degree, int degree = 16);
    void UniformRefinement();
    void KnotInsert(Array<KnotVector *> &kv);
-   void KnotInsert(Array<Vector *> &kv);
 };
 
 

mesh/pmesh.cpp

@@ -2399,32 +2399,6 @@ int ParMesh::GetSharedFace(int sface) const
    }
 }
 
-// shift cyclically 3 integers a, b, c, so that the smallest of
-// order[a], order[b], order[c] is first
-static inline
-void Rotate3Indirect(int &a, int &b, int &c,
-                     const Array<std::int64_t> &order)
-{
-   if (order[a] < order[b])
-   {
-      if (order[a] > order[c])
-      {
-         ShiftRight(a, b, c);
-      }
-   }
-   else
-   {
-      if (order[b] < order[c])
-      {
-         ShiftRight(c, b, a);
-      }
-      else
-      {
-         ShiftRight(a, b, c);
-      }
-   }
-}
-
 void ParMesh::ReorientTetMesh()
 {
    if (Dim != 3 || !(meshgen & 1))
@@ -2432,109 +2406,7 @@ void ParMesh::ReorientTetMesh()
       return;
    }
 
-   DeleteLazyTables();
-
-   DSTable *old_v_to_v = NULL;
-   Table *old_elem_vert = NULL;
-
-   if (Nodes)
-   {
-      PrepareNodeReorder(&old_v_to_v, &old_elem_vert);
-   }
-
-   // create a GroupCommunicator over shared vertices
-   GroupCommunicator svert_comm(gtopo);
-   {
-      // initialize svert_comm
-      Table &gr_svert = svert_comm.GroupLDofTable();
-      // gr_svert differs from group_svert - the latter does not store gr. 0
-      gr_svert.SetDims(GetNGroups(), svert_lvert.Size());
-      gr_svert.GetI()[0] = 0;
-      for (int gr = 1; gr <= GetNGroups(); gr++)
-      {
-         gr_svert.GetI()[gr] = group_svert.GetI()[gr-1];
-      }
-      for (int k = 0; k < svert_lvert.Size(); k++)
-      {
-         gr_svert.GetJ()[k] = group_svert.GetJ()[k];
-      }
-      svert_comm.Finalize();
-   }
-
-   // communicate the local index of each shared vertex from the group master to
-   // other ranks in the group
-   Array<int> svert_master_rank(svert_lvert.Size());
-   Array<int> svert_master_index(svert_lvert);
-   {
-      for (int i = 0; i < group_svert.Size(); i++)
-      {
-         int rank = gtopo.GetGroupMasterRank(i+1);
-         for (int j = 0; j < group_svert.RowSize(i); j++)
-         {
-            svert_master_rank[group_svert.GetRow(i)[j]] = rank;
-         }
-      }
-      svert_comm.Bcast(svert_master_index);
-   }
-
-   // the pairs (master rank, master local index) define a globally consistent
-   // vertex ordering
-   Array<std::int64_t> glob_vert_order(vertices.Size());
-   {
-      Array<int> lvert_svert(vertices.Size());
-      lvert_svert = -1;
-      for (int i = 0; i < svert_lvert.Size(); i++)
-      {
-         lvert_svert[svert_lvert[i]] = i;
-      }
-
-      for (int i = 0; i < vertices.Size(); i++)
-      {
-         int s = lvert_svert[i];
-         if (s >= 0)
-         {
-            glob_vert_order[i] =
-               (std::int64_t(svert_master_rank[s]) << 32) + svert_master_index[s];
-         }
-         else
-         {
-            glob_vert_order[i] = (std::int64_t(MyRank) << 32) + i;
-         }
-      }
-   }
-
-   // rotate tetrahedra so that vertex zero is the lowest (global) index vertex,
-   // vertex 1 is the second lowest (global) index and vertices 2 and 3 preserve
-   // positive orientation of the element
-   for (int i = 0; i < NumOfElements; i++)
-   {
-      if (GetElementType(i) == Element::TETRAHEDRON)
-      {
-         int *v = elements[i]->GetVertices();
-
-         Rotate3Indirect(v[0], v[1], v[2], glob_vert_order);
-
-         if (glob_vert_order[v[0]] < glob_vert_order[v[3]])
-         {
-            Rotate3Indirect(v[1], v[2], v[3], glob_vert_order);
-         }
-         else
-         {
-            ShiftRight(v[0], v[1], v[3]);
-         }
-      }
-   }
-
-   // rotate also boundary triangles
-   for (int i = 0; i < NumOfBdrElements; i++)
-   {
-      if (GetBdrElementType(i) == Element::TRIANGLE)
-      {
-         int *v = boundary[i]->GetVertices();
-
-         Rotate3Indirect(v[0], v[1], v[2], glob_vert_order);
-      }
-   }
+   Mesh::ReorientTetMesh();
 
    const bool check_consistency = true;
    if (check_consistency)
@@ -2561,56 +2433,37 @@ void ParMesh::ReorientTetMesh()
       for (int i = 0; i < stria_flag.Size(); i++)
       {
          const int *v = shared_trias[i].v;
-         if (glob_vert_order[v[0]] < glob_vert_order[v[1]])
+         if (v[0] < v[1])
          {
-            stria_flag[i] = (glob_vert_order[v[0]] < glob_vert_order[v[2]]) ? 0 : 2;
+            stria_flag[i] = (v[0] < v[2]) ? 0 : 2;
          }
          else // v[1] < v[0]
          {
-            stria_flag[i] = (glob_vert_order[v[1]] < glob_vert_order[v[2]]) ? 1 : 2;
+            stria_flag[i] = (v[1] < v[2]) ? 1 : 2;
          }
       }
-
       Array<int> stria_master_flag(stria_flag);
       stria_comm.Bcast(stria_master_flag);
       for (int i = 0; i < stria_flag.Size(); i++)
       {
-         const int *v = shared_trias[i].v;
          MFEM_VERIFY(stria_flag[i] == stria_master_flag[i],
-                     "inconsistent vertex ordering found, shared triangle "
-                     << i << ": ("
-                     << v[0] << ", " << v[1] << ", " << v[2] << "), "
-                     << "local flag: " << stria_flag[i]
-                     << ", master flag: " << stria_master_flag[i]);
+                     "inconsistent vertex ordering found");
       }
    }
 
-   // rotate shared triangle faces
+   // Rotate shared triangle faces.
+   // Note that no communication is needed to ensure that the shared
+   // faces are rotated in the same way in both processors. This is
+   // automatic due to various things, e.g. the global to local vertex
+   // mapping preserves the global order; also the way new vertices
+   // are introduced during refinement is essential.
    for (int i = 0; i < shared_trias.Size(); i++)
    {
       int *v = shared_trias[i].v;
-
-      Rotate3Indirect(v[0], v[1], v[2], glob_vert_order);
+      Rotate3(v[0], v[1], v[2]);
    }
 
-   // finalize
-   if (!Nodes)
-   {
-      GetElementToFaceTable();
-      GenerateFaces();
-      if (el_to_edge)
-      {
-         NumOfEdges = GetElementToEdgeTable(*el_to_edge, be_to_edge);
-      }
-   }
-   else
-   {
-      DoNodeReorder(old_v_to_v, old_elem_vert);
-      delete old_elem_vert;
-      delete old_v_to_v;
-   }
-
-   // the local edge and face numbering is changed therefore we need to
+   // The local edge and face numbering is changed therefore we need to
    // update sedge_ledge and sface_lface.
    FinalizeParTopo();
 }
@@ -3202,9 +3055,6 @@ bool ParMesh::NonconformingDerefinement(Array<double> &elem_error,
    long glob_size = ReduceInt(derefs.Size());
    if (!glob_size) { return false; }
 
-   // Destroy face-neighbor data only when actually de-refining.
-   DeleteFaceNbrData();
-
    pncmesh->Derefine(derefs);
 
    ParMesh* mesh2 = new ParMesh(*pncmesh);
@@ -3232,18 +3082,7 @@ bool ParMesh::NonconformingDerefinement(Array<double> &elem_error,
    return true;
 }
 
-
 void ParMesh::Rebalance()
-{
-   RebalanceImpl(NULL); // default SFC-based partition
-}
-
-void ParMesh::Rebalance(const Array<int> &partition)
-{
-   RebalanceImpl(&partition);
-}
-
-void ParMesh::RebalanceImpl(const Array<int> *partition)
 {
    if (Conforming())
    {
@@ -3270,7 +3109,7 @@ void ParMesh::RebalanceImpl(const Array<int> *partition)
 
    DeleteFaceNbrData();
 
-   pncmesh->Rebalance(partition);
+   pncmesh->Rebalance();
 
    ParMesh* pmesh2 = new ParMesh(*pncmesh);
    pncmesh->OnMeshUpdated(pmesh2);
@@ -4171,7 +4010,6 @@ void ParMesh::PrintAsOne(std::ostream &out)
           "# SQUARE      = 3\n"
           "# TETRAHEDRON = 4\n"
           "# CUBE        = 5\n"
-          "# PRISM       = 6\n"
           "#\n";
 
       out << "\ndimension\n" << Dim;
@@ -4270,15 +4108,6 @@ void ParMesh::PrintAsOne(std::ostream &out)
    {
       switch (Dim)
       {
-         case 1:
-            for (i = 0; i < svert_lvert.Size(); i++)
-            {
-               ints.Append(Geometry::POINT);
-               ints.Append(svert_lvert[i]);
-               ne++;
-            }
-            break;
-
          case 2:
             for (i = 0; i < shared_edges.Size(); i++)
             {

mesh/pmesh.hpp

@@ -141,9 +141,6 @@ protected:
    virtual bool NonconformingDerefinement(Array<double> &elem_error,
                                           double threshold, int nc_limit = 0,
                                           int op = 1);
-
-   void RebalanceImpl(const Array<int> *partition);
-
    void DeleteFaceNbrData();
 
    bool WantSkipSharedMaster(const NCMesh::Master &master) const;
@@ -289,15 +286,9 @@ public:
    /// Utility function: sum integers from all processors (Allreduce).
    virtual long ReduceInt(int value) const;
 
-   /** Load balance the mesh by equipartitioning the global space-filling
-       sequence of elements. Works for nonconforming meshes only. */
+   /// Load balance the mesh. NC meshes only.
    void Rebalance();
 
-   /** Load balance a nonconforming mesh using a user-defined partition.
-       Each local element 'i' is migrated to processor rank 'partition[i]',
-       for 0 <= i < GetNE(). */
-   void Rebalance(const Array<int> &partition);
-
    /** Print the part of the mesh in the calling processor adding the interface
        as boundary (for visualization purposes) using the mfem v1.0 format. */
    virtual void Print(std::ostream &out = mfem::out) const;

mesh/pncmesh.cpp

@@ -156,7 +156,7 @@ void ParNCMesh::UpdateVertices()
       Element &el = elements[leaf_elements[i]];
       if (el.rank == MyRank)
       {
-         for (int j = 0; j < GI[el.Geom()].nv; j++)
+         for (int j = 0; j < GI[(int) el.geom].nv; j++)
          {
             int &vindex = nodes[el.node[j]].vert_index;
             if (vindex < 0) { vindex = NVertices++; }
@@ -227,30 +227,6 @@ void ParNCMesh::OnMeshUpdated(Mesh *mesh)
       MFEM_ASSERT(NFaces == NEdges, "");
       MFEM_ASSERT(NGhostFaces == NGhostEdges, "");
    }
-
-   // update face_geom for ghost faces
-   face_geom.SetSize(NFaces + NGhostFaces, Geometry::SQUARE);
-   for (int i = 0; i < NGhostElements; i++)
-   {
-      Element &el = elements[leaf_elements[NElements + i]]; // ghost element
-      GeomInfo &gi = GI[el.Geom()];
-
-      for (int j = 0; j < gi.nf; j++)
-      {
-         const int *fv = gi.faces[j];
-         Face* face = faces.Find(el.node[fv[0]], el.node[fv[1]],
-                                 el.node[fv[2]], el.node[fv[3]]);
-         MFEM_ASSERT(face, "face not found!");
-
-         static const Geometry::Type types[5] =
-         {
-            Geometry::INVALID, Geometry::INVALID,
-            Geometry::SEGMENT, Geometry::TRIANGLE, Geometry::SQUARE
-         };
-
-         face_geom[face->index] = types[gi.nfv[j]];
-      }
-   }
 }
 
 void ParNCMesh::ElementSharesFace(int elem, int local, int face)
@@ -281,9 +257,6 @@ void ParNCMesh::BuildFaceList()
    // This is an extension of NCMesh::BuildFaceList() which also determines
    // face ownership and prepares face processor groups.
 
-   // (special case for prisms: to be able to handle edge-face constraints)
-   if (HavePrisms()) { GetEdgeList(); }
-
    int nfaces = NFaces + NGhostFaces;
 
    tmp_owner.SetSize(nfaces);
@@ -443,25 +416,13 @@ void ParNCMesh::MakeSharedList(const NCList &list, NCList &shared)
    for (unsigned i = 0; i < list.masters.size(); i++)
    {
       const Master &master = list.masters[i];
-      char &master_flag = tmp_shared_flag[master.index];
-      char master_old_flag = master_flag;
+      char master_old_flag = tmp_shared_flag[master.index];
 
       for (int j = master.slaves_begin; j < master.slaves_end; j++)
       {
-         int si = list.slaves[j].index;
-         if (si >= 0)
-         {
-            char &slave_flag = tmp_shared_flag[si];
-            master_flag |= slave_flag;
-            slave_flag |= master_old_flag;
-         }
-         else // special case: prism edge-face constraint
-         {
-            if (entity_owner[1][-1-si] != MyRank)
-            {
-               master_flag |= 0x2;
-            }
-         }
+         char &slave_flag = tmp_shared_flag[list.slaves[j].index];
+         tmp_shared_flag[master.index] |= slave_flag;
+         slave_flag |= master_old_flag;
       }
    }
 
@@ -483,8 +444,7 @@ void ParNCMesh::MakeSharedList(const NCList &list, NCList &shared)
    }
    for (unsigned i = 0; i < list.slaves.size(); i++)
    {
-      int si = list.slaves[i].index;
-      if (si >= 0 && tmp_shared_flag[si] == 0x3)
+      if (tmp_shared_flag[list.slaves[i].index] == 0x3)
       {
          shared.slaves.push_back(list.slaves[i]);
       }
@@ -634,9 +594,7 @@ void ParNCMesh::CalculatePMatrixGroups()
       ranks.SetSize(0);
       for (int j = master_face.slaves_begin; j < master_face.slaves_end; j++)
       {
-         int si = face_list.slaves[j].index;
-         int owner = (si >= 0) ? entity_owner[2][si] // standard face dependency
-                     /*     */ : entity_owner[1][-1 - si]; // prism edge-face dep
+         int owner = entity_owner[2][face_list.slaves[j].index];
          ranks.Append(groups[owner][0]);
       }
       ranks.Sort();
@@ -644,8 +602,8 @@ void ParNCMesh::CalculatePMatrixGroups()
 
       AddConnections(2, master_face.index, ranks);
 
-      int nfv = GetFaceVerticesEdges(master_face, v, e, eo);
-      for (int j = 0; j < nfv; j++)
+      GetFaceVerticesEdges(master_face, v, e, eo);
+      for (int j = 0; j < 4; j++)
       {
          AddConnections(0, v[j], ranks);
          AddConnections(1, e[j], ranks);
@@ -677,23 +635,20 @@ int ParNCMesh::get_face_orientation(Face &face, Element &e1, Element &e2,
    {
       // get local face number (remember that p1, p2, p3 are not in order, and
       // p4 is not stored)
-      int lf = find_local_face(e[i]->Geom(),
-                               find_node(*e[i], face.p1),
-                               find_node(*e[i], face.p2),
-                               find_node(*e[i], face.p3));
+      int lf = find_hex_face(find_node(*e[i], face.p1),
+                             find_node(*e[i], face.p2),
+                             find_node(*e[i], face.p3));
       // optional output
       if (local) { local[i] = lf; }
 
       // get node IDs for the face as seen from e[i]
-      const int* fv = GI[e[i]->Geom()].faces[lf];
+      const int* fv = GI[Geometry::CUBE].faces[lf];
       for (int j = 0; j < 4; j++)
       {
          ids[i][j] = e[i]->node[fv[j]];
       }
    }
-
-   return (ids[0][3] >= 0) ? Mesh::GetQuadOrientation(ids[0], ids[1])
-          /*            */ : Mesh::GetTriOrientation(ids[0], ids[1]);
+   return Mesh::GetQuadOrientation(ids[0], ids[1]);
 }
 
 void ParNCMesh::CalcFaceOrientations()
@@ -838,8 +793,6 @@ void ParNCMesh::NeighborProcessors(Array<int> &neighbors)
 {
    UpdateLayers();
 
-   // TODO: look at groups instead?
-
    std::set<int> ranks;
    for (int i = 0; i < ghost_layer.Size(); i++)
    {
@@ -851,9 +804,30 @@ void ParNCMesh::NeighborProcessors(Array<int> &neighbors)
 
 //// ParMesh compatibility /////////////////////////////////////////////////////
 
+struct CompareShared // TODO: use lambda when C++11 available
+{
+   const Array<int> &elem_local, &leaf_glob_order, &shared_local;
+
+   CompareShared
+   (const Array<int> &el, const Array<int> &lgo, const Array<int> &sl)
+      : elem_local(el), leaf_glob_order(lgo), shared_local(sl) {}
+
+   inline bool operator()(const int a, const int b)
+   {
+      int el_loc_a = elem_local[shared_local[a]];
+      int el_loc_b = elem_local[shared_local[b]];
+
+      int lgo_a = leaf_glob_order[el_loc_a >> 4];
+      int lgo_b = leaf_glob_order[el_loc_b >> 4];
+
+      if (lgo_a != lgo_b) { return lgo_a < lgo_b; }
+
+      return (el_loc_a & 0xf) < (el_loc_b & 0xf);
+   }
+};
+
 void ParNCMesh::MakeSharedTable(int ngroups, int ent, Array<int> &shared_local,
-                                Table &group_shared, Array<char> *entity_geom,
-                                char geom)
+                                Table &group_shared)
 {
    const Array<GroupId> &conf_group = entity_conf_group[ent];
 
@@ -865,8 +839,6 @@ void ParNCMesh::MakeSharedTable(int ngroups, int ent, Array<int> &shared_local,
    {
       if (conf_group[i])
       {
-         if (entity_geom && (*entity_geom)[i] != geom) { continue; }
-
          num_shared++;
          group_shared.AddAColumnInRow(conf_group[i]-1);
       }
@@ -880,8 +852,6 @@ void ParNCMesh::MakeSharedTable(int ngroups, int ent, Array<int> &shared_local,
    {
       if (conf_group[i])
       {
-         if (entity_geom && (*entity_geom)[i] != geom) { continue; }
-
          shared_local[j] = i;
          group_shared.AddConnection(conf_group[i]-1, j);
          j++;
@@ -896,18 +866,8 @@ void ParNCMesh::MakeSharedTable(int ngroups, int ent, Array<int> &shared_local,
       int *row = group_shared.GetRow(i);
 
       Array<int> ref_row(row, size);
-      ref_row.Sort([&](const int a, const int b)
-      {
-         int el_loc_a = entity_elem_local[ent][shared_local[a]];
-         int el_loc_b = entity_elem_local[ent][shared_local[b]];
-
-         int lgo_a = leaf_glob_order[el_loc_a >> 4];
-         int lgo_b = leaf_glob_order[el_loc_b >> 4];
-
-         if (lgo_a != lgo_b) { return lgo_a < lgo_b; }
-
-         return (el_loc_a & 0xf) < (el_loc_b & 0xf);
-      });
+      ref_row.Sort(
+         CompareShared(entity_elem_local[ent], leaf_glob_order, shared_local));
    }
 }
 
@@ -953,16 +913,15 @@ void ParNCMesh::GetConformingSharedStructures(ParMesh &pmesh)
    }
 
    // create shared to local index mappings and group tables
-   int ng = pmesh.gtopo.NGroups();
-   MakeSharedTable(ng, 0, pmesh.svert_lvert, pmesh.group_svert);
-   MakeSharedTable(ng, 1, pmesh.sedge_ledge, pmesh.group_sedge);
+   int ngroups = pmesh.gtopo.NGroups();
+   MakeSharedTable(ngroups, 0, pmesh.svert_lvert, pmesh.group_svert);
+   MakeSharedTable(ngroups, 1, pmesh.sedge_ledge, pmesh.group_sedge);
+   MakeSharedTable(ngroups, 2, pmesh.sface_lface, pmesh.group_squad);
 
-   Array<int> slt, slq;
-   MakeSharedTable(ng, 2, slt, pmesh.group_stria, &face_geom, Geometry::TRIANGLE);
-   MakeSharedTable(ng, 2, slq, pmesh.group_squad, &face_geom, Geometry::SQUARE);
-
-   pmesh.sface_lface = slt;
-   pmesh.sface_lface.Append(slq);
+   // create an empty group_stria (we currently don't have triangle faces)
+   pmesh.group_stria.MakeI(ngroups-1);
+   pmesh.group_stria.MakeJ();
+   pmesh.group_stria.ShiftUpI();
 
    // create shared_edges
    for (int i = 0; i < pmesh.shared_edges.Size(); i++)
@@ -980,23 +939,11 @@ void ParNCMesh::GetConformingSharedStructures(ParMesh &pmesh)
       pmesh.shared_edges[i] = new Segment(v, 1);
    }
 
-   // create shared_trias
-   pmesh.shared_trias.SetSize(slt.Size());
-   for (int i = 0; i < slt.Size(); i++)
+   // create shared_faces
+   pmesh.shared_quads.SetSize(pmesh.sface_lface.Size());
+   for (int i = 0; i < pmesh.shared_quads.Size(); i++)
    {
-      int el_loc = entity_elem_local[2][slt[i]];
-      MeshId face_id(-1, leaf_elements[(el_loc >> 4)], (el_loc & 0xf));
-
-      int v[4], e[4], eo[4];
-      GetFaceVerticesEdges(face_id, v, e, eo);
-      pmesh.shared_trias[i].Set(v);
-   }
-
-   // create shared_quads
-   pmesh.shared_quads.SetSize(slq.Size());
-   for (int i = 0; i < slq.Size(); i++)
-   {
-      int el_loc = entity_elem_local[2][slq[i]];
+      int el_loc = entity_elem_local[2][pmesh.sface_lface[i]];
       MeshId face_id(-1, leaf_elements[(el_loc >> 4)], (el_loc & 0xf));
 
       int e[4], eo[4];
@@ -1012,6 +959,12 @@ void ParNCMesh::GetConformingSharedStructures(ParMesh &pmesh)
    leaf_glob_order.DeleteAll();
 }
 
+bool ParNCMesh::compare_ranks_indices(const Element* a, const Element* b)
+{
+   return (a->rank != b->rank) ? a->rank < b->rank
+          /*                */ : a->index < b->index;
+}
+
 void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
 {
    ClearAuxPM();
@@ -1050,7 +1003,6 @@ void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
       for (int j = mf.slaves_begin; j < mf.slaves_end; j++)
       {
          const Slave &sf = full_list.slaves[j];
-         if (sf.index < 0) { continue; }
 
          MFEM_ASSERT(mf.element >= 0 && sf.element >= 0, "");
          Element* e[2] = { &elements[mf.element], &elements[sf.element] };
@@ -1072,11 +1024,7 @@ void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
    // same on different processors, this is important for ExchangeFaceNbrData)
    fnbr.Sort();
    fnbr.Unique();
-   fnbr.Sort([](const Element* a, const Element* b)
-   {
-      return (a->rank != b->rank) ? a->rank < b->rank
-             /*                */ : a->index < b->index;
-   });
+   fnbr.Sort(compare_ranks_indices);
 
    // put the ranks into 'face_nbr_group'
    for (int i = 0; i < fnbr.Size(); i++)
@@ -1208,7 +1156,6 @@ void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
          for (int j = mf.slaves_begin; j < mf.slaves_end; j++)
          {
             const Slave &sf = full_list.slaves[j];
-            if (sf.index < 0) { continue; }
 
             MFEM_ASSERT(sf.element >= 0 && mf.element >= 0, "");
             Element &sfe = elements[sf.element];
@@ -1356,18 +1303,13 @@ void ParNCMesh::Prune()
    {
       if (PruneTree(i)) { DerefineElement(i); }
    }
+
    Update();
 }
 
 
 void ParNCMesh::Refine(const Array<Refinement> &refinements)
 {
-   if (NRanks == 1)
-   {
-      NCMesh::Refine(refinements);
-      return;
-   }
-
    for (int i = 0; i < refinements.Size(); i++)
    {
       const Refinement &ref = refinements[i];
@@ -1766,7 +1708,7 @@ void ParNCMesh::CheckDerefinementNCLevel(const Table &deref_table,
 
 //// Rebalance /////////////////////////////////////////////////////////////////
 
-void ParNCMesh::Rebalance(const Array<int> *custom_partition)
+void ParNCMesh::Rebalance()
 {
    send_rebalance_dofs.clear();
    recv_rebalance_dofs.clear();
@@ -1774,46 +1716,30 @@ void ParNCMesh::Rebalance(const Array<int> *custom_partition)
    Array<int> old_elements;
    leaf_elements.GetSubArray(0, NElements, old_elements);
 
-   if (!custom_partition) // SFC based partitioning
+   // figure out new assignments for Element::rank
+   long local_elems = NElements, total_elems = 0;
+   MPI_Allreduce(&local_elems, &total_elems, 1, MPI_LONG, MPI_SUM, MyComm);
+
+   long first_elem_global = 0;
+   MPI_Scan(&local_elems, &first_elem_global, 1, MPI_LONG, MPI_SUM, MyComm);
+   first_elem_global -= local_elems;
+
+   Array<int> new_ranks(leaf_elements.Size());
+   new_ranks = -1;
+
+   for (int i = 0, j = 0; i < leaf_elements.Size(); i++)
    {
-      Array<int> new_ranks(leaf_elements.Size());
-      new_ranks = -1;
-
-      // figure out new assignments for Element::rank
-      long local_elems = NElements, total_elems = 0;
-      MPI_Allreduce(&local_elems, &total_elems, 1, MPI_LONG, MPI_SUM, MyComm);
-
-      long first_elem_global = 0;
-      MPI_Scan(&local_elems, &first_elem_global, 1, MPI_LONG, MPI_SUM, MyComm);
-      first_elem_global -= local_elems;
-
-      for (int i = 0, j = 0; i < leaf_elements.Size(); i++)
+      if (elements[leaf_elements[i]].rank == MyRank)
       {
-         if (elements[leaf_elements[i]].rank == MyRank)
-         {
-            new_ranks[i] = Partition(first_elem_global + (j++), total_elems);
-         }
+         new_ranks[i] = Partition(first_elem_global + (j++), total_elems);
       }
-
-      int target_elements = PartitionFirstIndex(MyRank+1, total_elems)
-                            - PartitionFirstIndex(MyRank, total_elems);
-
-      // assign the new ranks and send elements (plus ghosts) to new owners
-      RedistributeElements(new_ranks, target_elements, true);
    }
-   else // whatever partitioning the user has passed
-   {
-      MFEM_VERIFY(custom_partition->Size() == NElements,
-                  "Size of the partition array must match the number "
-                  "of local mesh elements (ParMesh::GetNE()).");
 
-      Array<int> new_ranks;
-      custom_partition->Copy(new_ranks);
+   int target_elements = PartitionFirstIndex(MyRank+1, total_elems)
+                         - PartitionFirstIndex(MyRank, total_elems);
 
-      new_ranks.SetSize(leaf_elements.Size(), -1); // make room for ghosts
-
-      RedistributeElements(new_ranks, -1, true);
-   }
+   // assign the new ranks and send elements (plus ghosts) to new owners
+   RedistributeElements(new_ranks, target_elements, true);
 
    // set up the old index array
    old_index_or_rank.SetSize(NElements);
@@ -1828,22 +1754,28 @@ void ParNCMesh::Rebalance(const Array<int> *custom_partition)
    Prune();
 }
 
+struct CompareRanks // TODO: use lambda when C++11 available
+{
+   typedef BlockArray<NCMesh::Element> ElemArray;
+   const ElemArray &elements;
+   CompareRanks(const ElemArray &elements) : elements(elements) {}
+
+   inline bool operator()(const int a, const int b)
+   {
+      return elements[a].rank < elements[b].rank;
+   }
+};
+
 void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
                                      bool record_comm)
 {
-   bool sfc = (target_elements >= 0);
-
    UpdateLayers();
 
    // *** STEP 1: communicate new rank assignments for the ghost layer ***
 
    NeighborElementRankMessage::Map send_ghost_ranks, recv_ghost_ranks;
 
-   ghost_layer.Sort([&](const int a, const int b)
-   {
-      return elements[a].rank < elements[b].rank;
-   });
-
+   ghost_layer.Sort(CompareRanks(elements));
    {
       Array<int> rank_neighbors;
 
@@ -1921,17 +1853,12 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
       el.rank = new_ranks[i];
    }
 
-   int nsent = 0, nrecv = 0; // for debug check
-
    RebalanceMessage::Map send_elems;
    {
       // sort elements we own by the new rank
       Array<int> owned_elements;
       owned_elements.MakeRef(leaf_elements.GetData(), NElements);
-      owned_elements.Sort([&](const int a, const int b)
-      {
-         return elements[a].rank < elements[b].rank;
-      });
+      owned_elements.Sort(CompareRanks(elements));
 
       Array<int> batch;
       batch.Reserve(1024);
@@ -1973,16 +1900,7 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
                // disrupting the termination mechanism in Step 4.
             }
 
-            if (sfc)
-            {
-               msg.Isend(rank, MyComm);
-            }
-            else
-            {
-               // custom partitioning needs synchronous sends
-               msg.Issend(rank, MyComm);
-            }
-            nsent++;
+            msg.Isend(rank, MyComm);
 
             // also: record what elements we sent (excluding the ghosts)
             // so that SendRebalanceDofs can later send data for them
@@ -1998,108 +1916,41 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
 
    // *** STEP 3: receive elements from others ***
 
+   /* We don't know from whom we're going to receive so we need to probe.
+      Fortunately, we do know how many elements we're going to own eventually
+      so the termination condition is easy. */
+
    RebalanceMessage msg;
    msg.SetNCMesh(this);
 
-   if (sfc)
+   while (received_elements < target_elements)
    {
-      /* We don't know from whom we're going to receive, so we need to probe.
-         However, for the default SFC partitioning, we do know how many elements
-         we're going to own eventually, so the termination condition is easy. */
+      int rank, size;
+      RebalanceMessage::Probe(rank, size, MyComm);
 
-      while (received_elements < target_elements)
+      // receive message; note: elements are created as the message is decoded
+      msg.Recv(rank, size, MyComm);
+
+      for (int i = 0; i < msg.Size(); i++)
       {
-         int rank, size;
-         RebalanceMessage::Probe(rank, size, MyComm);
+         int elem_rank = msg.values[i];
+         elements[msg.elements[i]].rank = elem_rank;
 
-         // receive message; note: elements are created as the message is decoded
-         msg.Recv(rank, size, MyComm);
-         nrecv++;
-
-         for (int i = 0; i < msg.Size(); i++)
-         {
-            int elem_rank = msg.values[i];
-            elements[msg.elements[i]].rank = elem_rank;
-
-            if (elem_rank == MyRank) { received_elements++; }
-         }
-
-         // save the ranks we received from, for later use in RecvRebalanceDofs
-         if (record_comm)
-         {
-            recv_rebalance_dofs[rank].SetNCMesh(this);
-         }
+         if (elem_rank == MyRank) { received_elements++; }
       }
 
-      Update();
-
-      RebalanceMessage::WaitAllSent(send_elems);
-   }
-   else
-   {
-      /* The case (target_elements < 0) is used for custom partitioning.
-         Here we need to employ the "non-blocking consensus" algorithm
-         (https://scorec.rpi.edu/REPORTS/2015-9.pdf) to determine when the
-         element exchange is finished. The algorithm uses a non-blocking
-         barrier. */
-
-      MPI_Request barrier = MPI_REQUEST_NULL;
-      int done = 0;
-
-      while (!done)
+      // save the ranks we received from, for later use in RecvRebalanceDofs
+      if (record_comm)
       {
-         int rank, size;
-         while (RebalanceMessage::IProbe(rank, size, MyComm))
-         {
-            // receive message; note: elements are created as the msg is decoded
-            msg.Recv(rank, size, MyComm);
-            nrecv++;
-
-            for (int i = 0; i < msg.Size(); i++)
-            {
-               elements[msg.elements[i]].rank = msg.values[i];
-            }
-
-            // save the ranks we received from, for later use in RecvRebalanceDofs
-            if (record_comm)
-            {
-               recv_rebalance_dofs[rank].SetNCMesh(this);
-            }
-         }
-
-         if (barrier != MPI_REQUEST_NULL)
-         {
-            MPI_Test(&barrier, &done, MPI_STATUS_IGNORE);
-         }
-         else
-         {
-            if (RebalanceMessage::TestAllSent(send_elems))
-            {
-               int err = MPI_Ibarrier(MyComm, &barrier);
-
-               MFEM_VERIFY(err == MPI_SUCCESS, "");
-               MFEM_VERIFY(barrier != MPI_REQUEST_NULL, "");
-            }
-         }
+         recv_rebalance_dofs[rank].SetNCMesh(this);
       }
-
-      Update();
    }
 
+   Update();
+
+   // make sure we can delete all send buffers
    NeighborElementRankMessage::WaitAllSent(send_ghost_ranks);
-
-#ifdef MFEM_DEBUG
-   int glob_sent, glob_recv;
-   MPI_Reduce(&nsent, &glob_sent, 1, MPI_INT, MPI_SUM, 0, MyComm);
-   MPI_Reduce(&nrecv, &glob_recv, 1, MPI_INT, MPI_SUM, 0, MyComm);
-
-   if (MyRank == 0)
-   {
-      MFEM_ASSERT(glob_sent == glob_recv,
-                  "(glob_sent, glob_recv) = ("
-                  << glob_sent << ", " << glob_recv << ")");
-   }
-#endif
+   NeighborElementRankMessage::WaitAllSent(send_elems);
 }
 
 
@@ -2391,8 +2242,8 @@ void ParNCMesh::AdjustMeshIds(Array<MeshId> ids[], int rank)
       if (contains_rank[entity_pmat_group[2][face_id.index]])
       {
          int v[4], e[4], eo[4], pos, k;
-         int nfv = GetFaceVerticesEdges(face_id, v, e, eo);
-         for (int j = 0; j < nfv; j++)
+         GetFaceVerticesEdges(face_id, v, e, eo);
+         for (int j = 0; j < 4; j++)
          {
             if ((pos = find_v.FindSorted(Pair<int, int>(v[j], 0))) != -1)
             {
@@ -2416,7 +2267,7 @@ void ParNCMesh::ChangeVertexMeshIdElement(NCMesh::MeshId &id, int elem)
    Element &el = elements[elem];
    MFEM_ASSERT(el.ref_type == 0, "");
 
-   GeomInfo& gi = GI[el.Geom()];
+   GeomInfo& gi = GI[(int) el.geom];
    for (int i = 0; i < gi.nv; i++)
    {
       if (nodes[el.node[i]].vert_index == id.index)
@@ -2432,14 +2283,14 @@ void ParNCMesh::ChangeVertexMeshIdElement(NCMesh::MeshId &id, int elem)
 void ParNCMesh::ChangeEdgeMeshIdElement(NCMesh::MeshId &id, int elem)
 {
    Element &old = elements[id.element];
-   const int *ev = GI[old.Geom()].edges[(int) id.local];
+   const int *ev = GI[(int) old.geom].edges[id.local];
    Node* node = nodes.Find(old.node[ev[0]], old.node[ev[1]]);
    MFEM_ASSERT(node != NULL, "Edge not found.");
 
    Element &el = elements[elem];
    MFEM_ASSERT(el.ref_type == 0, "");
 
-   GeomInfo& gi = GI[el.Geom()];
+   GeomInfo& gi = GI[(int) el.geom];
    for (int i = 0; i < gi.ne; i++)
    {
       const int* ev = gi.edges[i];
@@ -2538,17 +2389,17 @@ void ParNCMesh::DecodeMeshIds(std::istream &is, Array<MeshId> ids[])
          id.local = read<char>(is);
 
          // find vertex/edge/face index
-         GeomInfo &gi = GI[el.Geom()];
+         GeomInfo &gi = GI[(int) el.geom];
          switch (type)
          {
             case 0:
             {
-               id.index = nodes[el.node[(int) id.local]].vert_index;
+               id.index = nodes[el.node[id.local]].vert_index;
                break;
             }
             case 1:
             {
-               const int* ev = gi.edges[(int) id.local];
+               const int* ev = gi.edges[id.local];
                Node* node = nodes.Find(el.node[ev[0]], el.node[ev[1]]);
                MFEM_ASSERT(node && node->HasEdge(), "edge not found.");
                id.index = node->edge_index;
@@ -2556,7 +2407,7 @@ void ParNCMesh::DecodeMeshIds(std::istream &is, Array<MeshId> ids[])
             }
             default:
             {
-               const int* fv = gi.faces[(int) id.local];
+               const int* fv = gi.faces[id.local];
                Face* face = faces.Find(el.node[fv[0]], el.node[fv[1]],
                                        el.node[fv[2]], el.node[fv[3]]);
                MFEM_ASSERT(face, "face not found.");

mesh/pncmesh.hpp

@@ -87,12 +87,8 @@ public:
    virtual void Derefine(const Array<int> &derefs);
 
    /** Migrate leaf elements of the global refinement hierarchy (including ghost
-       elements) so that each processor owns the same number of leaves (+-1).
-       The default partitioning strategy is based on equal splitting of the
-       space-filling sequence of leaf elements (custom_partition == NULL).
-       Alternatively, a used-defined element-rank assignemnt array can be
-       passed. */
-   void Rebalance(const Array<int> *custom_partition = NULL);
+       elements) so that each processor owns the same number of leaves (+-1). */
+   void Rebalance();
 
 
    // interface for ParFiniteElementSpace
@@ -104,6 +100,9 @@ public:
    int GetNGhostFaces() const { return NGhostFaces; }
    int GetNGhostElements() const { return NGhostElements; }
 
+   Geometry::Type GetGhostFaceGeometry(int ghost_face_id) const
+   { return Geometry::SQUARE; }
+
    // Return a list of vertices/edges/faces shared by this processor and at
    // least one other processor. These are subsets of NCMesh::<entity>_list. */
    const NCList& GetSharedVertices() { GetVertexList(); return shared_vertices; }
@@ -169,12 +168,6 @@ public:
    /// Return true if the specified vertex/edge/face is a ghost.
    bool IsGhost(int entity, int index) const
    {
-      if (index < 0) // special case prism edge-face constraint
-      {
-         MFEM_ASSERT(entity == 2, "");
-         entity = 1;
-         index = -1 - index;
-      }
       switch (entity)
       {
          case 0: return index >= NVertices;
@@ -263,7 +256,7 @@ protected: // implementation
    GroupList groups;  // comm group list; NOTE: groups[0] = { MyRank }
    GroupMap group_id; // search index over groups
 
-   // owner rank for each vertex, edge and face (encoded as singleton group)
+   // owner rank for each vertex, edge and face (encoded as singleton groups)
    Array<GroupId> entity_owner[3];
    // P matrix comm pattern groups for each vertex/edge/face (0/1/2)
    Array<GroupId> entity_pmat_group[3];
@@ -343,8 +336,7 @@ protected: // implementation
    void UpdateLayers();
 
    void MakeSharedTable(int ngroups, int ent, Array<int> &shared_local,
-                        Table &group_shared, Array<char> *entity_geom = NULL,
-                        char geom = 0);
+                        Table &group_shared);
 
    /** Uniquely encodes a set of leaf elements in the refinement hierarchy of
        an NCMesh. Can be dumped to a stream, sent to another processor, loaded,
@@ -514,10 +506,7 @@ protected: // implementation
 
    /** Assign new Element::rank to leaf elements and send them to their new
        owners, keeping the ghost layer up to date. Used by Rebalance() and
-       Derefine(). 'target_elements' is the number of elements this rank
-       is supposed to own after the exchange. If this number is not known
-       apriori, the parameter can be set to -1, but more expensive communication
-       (synchronous sends and a barrier) will be used in that case. */
+       Derefine(). */
    void RedistributeElements(Array<int> &new_ranks, int target_elements,
                              bool record_comm);
 
@@ -537,6 +526,8 @@ protected: // implementation
 
    long GroupsMemoryUsage() const;
 
+   static bool compare_ranks_indices(const Element* a, const Element* b);
+
    friend class NeighborRowMessage;
 };
 

mesh/point.hpp

@@ -46,14 +46,9 @@ public:
 
    virtual const int *GetEdgeVertices(int ei) const { return NULL; }
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const
    { nFaceVertices = 0; return 0; }
 
-   virtual int GetNFaces() const { return 0; }
-
-   virtual int GetNFaceVertices(int) const { return 0; }
-
    virtual const int *GetFaceVertices(int fi) const { return NULL; }
 
    virtual Element *Duplicate(Mesh *m) const

mesh/quadrilateral.hpp

@@ -53,14 +53,9 @@ public:
    virtual const int *GetEdgeVertices(int ei) const
    { return geom_t::Edges[ei]; }
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const
    { nFaceVertices = 0; return 0; }
 
-   virtual int GetNFaces() const { return 0; }
-
-   virtual int GetNFaceVertices(int) const { return 0; }
-
    virtual const int *GetFaceVertices(int fi) const { return NULL; }
 
    virtual Element *Duplicate(Mesh *m) const

mesh/segment.hpp

@@ -52,14 +52,9 @@ public:
 
    virtual const int *GetEdgeVertices(int ei) const { return NULL; }
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const
    { nFaceVertices = 0; return 0; }
 
-   virtual int GetNFaces() const { return 0; }
-
-   virtual int GetNFaceVertices(int) const { return 0; }
-
    virtual const int *GetFaceVertices(int fi) const { return NULL; }
 
    virtual Element *Duplicate(Mesh *m) const

mesh/tetrahedron.cpp

@@ -40,16 +40,13 @@ Tetrahedron::Tetrahedron(int ind1, int ind2, int ind3, int ind4, int attr)
    transform = 0;
 }
 
-void Tetrahedron::Init(int ind1, int ind2, int ind3, int ind4, int attr,
-                       int ref_flag)
+void Tetrahedron::Init(int ind1, int ind2, int ind3, int ind4, int attr)
 {
    attribute  = attr;
    indices[0] = ind1;
    indices[1] = ind2;
    indices[2] = ind3;
    indices[3] = ind4;
-   refinement_flag = ref_flag;
-   transform = 0;
 }
 
 void Tetrahedron::ParseRefinementFlag(int refinement_edges[2], int &type,

mesh/tetrahedron.hpp

@@ -52,8 +52,7 @@ public:
    Tetrahedron(int ind1, int ind2, int ind3, int ind4, int attr = 1);
 
    /// Initialize the vertex indices and the attribute of a Tetrahedron.
-   void Init(int ind1, int ind2, int ind3, int ind4, int attr = 1,
-             int ref_flag = 0);
+   void Init(int ind1, int ind2, int ind3, int ind4, int attr = 1);
 
    /// Return element's type.
    virtual Type GetType() const { return Element::TETRAHEDRON; }
@@ -100,14 +99,9 @@ public:
    virtual const int *GetEdgeVertices(int ei) const
    { return geom_t::Edges[ei]; }
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const
    { nFaceVertices = 3; return 4; }
 
-   virtual int GetNFaces() const { return 4; }
-
-   virtual int GetNFaceVertices(int) const { return 3; }
-
    virtual const int *GetFaceVertices(int fi) const
    { MFEM_ABORT("not implemented"); return NULL; }
 

mesh/triangle.hpp

@@ -80,14 +80,9 @@ public:
    virtual const int *GetEdgeVertices(int ei) const
    { return geom_t::Edges[ei]; }
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const
    { nFaceVertices = 0; return 0; }
 
-   virtual int GetNFaces() const { return 0; }
-
-   virtual int GetNFaceVertices(int) const { return 0; }
-
    virtual const int *GetFaceVertices(int fi) const
    { MFEM_ABORT("not implemented"); return NULL; }
 

mesh/wedge.hpp

@@ -54,13 +54,10 @@ public:
    virtual const int *GetEdgeVertices(int ei) const
    { return geom_t::Edges[ei]; }
 
-   /// @deprecated Use GetNFaces(void) and GetNFaceVertices(int) instead.
    virtual int GetNFaces(int &nFaceVertices) const;
 
-   virtual int GetNFaces() const { return 5; }
-
-   virtual int GetNFaceVertices(int fi) const
-   { return (fi < 2) ? 3 : 4; }
+   virtual int GetNFaceVerticess(int fi) const
+   { return ( ( fi < 2 ) ? 3 : 4); }
 
    virtual const int *GetFaceVertices(int fi) const
    { return geom_t::FaceVert[fi]; }
@@ -72,7 +69,7 @@ public:
 };
 
 // Defined in fe.cpp to ensure construction after 'mfem::poly1d'.
-extern H1_WedgeElement WedgeFE;
+extern class H1_WedgeElement WedgeFE;
 
 }
 

miniapps/drl4amr/.gitignore

@@ -0,0 +1,3 @@
+drl4amr
+*.so 
+image.jpg

miniapps/drl4amr/drl4amr.cpp

@@ -0,0 +1,392 @@
+//                                MFEM DRL4AMR
+
+#define protected public // FIXME!
+
+#include "mfem.hpp"
+#include "drl4amr.hpp"
+
+#include "linalg/dtensor.hpp"
+
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+#define dbg(...) \
+   { printf("\n\033[33m"); printf(__VA_ARGS__); printf("\033[m"); fflush(0); }
+
+const char *device_config = "cpu";
+
+static int discs;
+static double theta;
+static Array<double> offsets;
+constexpr int nb_discs_max = 6;
+constexpr double sharpness = 100.0;
+
+const bool visualization = true;
+const char *vishost = "localhost";
+const int visport = 19916;
+const int visw = 480;
+const int vish = 480;
+
+// *****************************************************************************
+double x0(const Vector &x)
+{
+   double result = 0.0;
+   const double t = x[0] + tan(theta)*x[1];
+   for (double o : offsets)
+   {
+      result += 1.0 + tanh(sharpness*(o - t));
+   }
+   return result / (discs << 1); // should be in [0,1]
+}
+
+// *****************************************************************************
+Drl4Amr::Drl4Amr(int order):
+   mesh(nx, ny, elem_type, true, sx, sy, false),
+   order(order),
+   dim(mesh.Dimension()),
+   device(device_config),
+   fec(order, dim/*, BasisType::Positive*/),
+   fespace(&mesh, &fec),
+   one(1.0),
+   zero(0.0),
+   integ(new DiffusionIntegrator(one)),
+   xcoeff(x0),
+   x(&fespace),
+   iteration(0),
+   flux_fespace(&mesh, &fec),
+   estimator(*integ, x, flux_fespace),
+   refiner(estimator)
+{
+   //dbg("Drl4Amr order:%d",o);
+   device.Print();
+
+   mesh.EnsureNCMesh();
+   mesh.PrintCharacteristics();
+
+   fespace.Update();
+
+   // Connect to GLVis.
+   if (visualization)
+   {
+      vis[0].open(vishost, visport);
+//      vis[1].open(vishost, visport);
+//      vis[2].open(vishost, visport);
+   }
+
+   // Initialize theta, offsets and x from x0_coeff
+   srand48(4);//time(NULL));
+   theta = M_PI*drand48()/2.0;
+   discs = static_cast<int>(1 + nb_discs_max*drand48());
+   offsets.SetSize(discs);
+   for (int i=0; i < discs; i++)
+   {
+      offsets[i] = drand48();
+   }
+   offsets.Sort();
+   printf("\ntheta = %f, discontinuities:%d", theta, discs);
+   for (double offset: offsets)
+   {
+      printf("\n%f ", offset);
+   }
+   x.ProjectCoefficient(xcoeff); // TODO: call Compute?
+
+/*   if (visualization && vis[0].good())
+   {
+      vis[0].precision(8);
+      vis[0] << "solution" << endl << mesh << x << flush;
+      vis[0] << "window_title '" << "Solution" << "'" << endl
+             << "window_geometry " << 0 << " " << 0 << " " << visw << " " << vish << endl
+             << "keys mgA" << endl;
+
+      vis[1].precision(8);
+      vis[1] << "mesh" << endl << mesh << flush;
+      vis[1] << "window_title '" << "Mesh" << "'" << endl
+             << "window_geometry "
+             << (vish + 10) << " " << 0
+             << " " << visw << " " << vish  << endl
+             << "keys mgA" << endl;
+
+      vis[2].precision(8);
+      vis[2] << "solution" << endl << mesh << x << flush;
+      vis[2] << "window_title '" << "Image" << "'" << endl
+             << "window_geometry "
+             <<  (2 * vish + 10) << " " << 0
+             << " " << visw << " " << vish << endl
+             << "keys RjgA" << endl; // mn
+   }*/
+
+   // Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   // that uses the ComputeElementFlux method of the DiffusionIntegrator to
+   // recover a smoothed flux (gradient) that is subtracted from the element
+   // flux to get an error indicator. We need to supply the space for the
+   // smoothed flux: an (H1)^sdim (i.e., vector-valued) space is used here.
+   //estimator.SetAnisotropic();
+
+   // A refiner selects and refines elements based on a refinement strategy.
+   // The strategy here is to refine elements with errors larger than a
+   // fraction of the maximum element error. Other strategies are possible.
+   // The refiner will call the given error estimator.
+   refiner.SetTotalErrorFraction(0.7);
+}
+
+
+// *****************************************************************************
+int Drl4Amr::Compute()
+{
+   iteration ++;
+   const int cdofs = fespace.GetTrueVSize();
+   cout << "\nAMR iteration " << iteration << endl;
+   cout << "Number of unknowns: " << cdofs << endl;
+
+   // TODO: it would be more proper to actually solve here
+   x.ProjectCoefficient(xcoeff);
+
+   // constrain slave nodes
+   if (fespace.GetProlongationMatrix())
+   {
+      Vector y(fespace.GetTrueVSize());
+      fespace.GetRestrictionMatrix()->Mult(x, y);
+      fespace.GetProlongationMatrix()->Mult(y, x);
+   }
+
+   // Send solution by socket to the GLVis server.
+   if (visualization && vis[0].good())
+   {
+      vis[0] << "solution\n" << mesh << x << flush;
+      fflush(0);
+   }
+   if (cdofs > max_dofs)
+   {
+      cout << "Reached the maximum number of dofs. Stop." << endl;
+      exit(0);
+   }
+   return 0;
+}
+
+
+// *****************************************************************************
+int Drl4Amr::Refine(int el_to_refine)
+{
+   if (el_to_refine >= 0)
+   {
+      //mesh.PrintCharacteristics();
+      mesh.EnsureNCMesh();
+      const int depth = mesh.ncmesh->GetElementDepth(el_to_refine);
+      if (depth == max_depth)
+      {
+         return 0;
+      }
+      MFEM_VERIFY(depth <= max_depth, "max_amr_depth error");
+      //dbg("Refine el:%d, depth:%d", el_to_refine, depth);
+      Array<Refinement> refinements(1);
+      refinements[0] = Refinement(el_to_refine);
+      mesh.GeneralRefinement(refinements, 1, 1);
+      // Send solution by socket to the GLVis server.
+      if (visualization && vis[1].good())
+      {
+         vis[1] << "mesh\n" << mesh << flush;
+         fflush(0);
+      }
+   }
+   else
+   {
+      //dbg("Refine with refiner");
+      // Call the refiner to modify the mesh. The refiner calls the error
+      // estimator to obtain element errors, then it selects elements to be
+      // refined and finally it modifies the mesh. The Stop() method can be
+      // used to determine if a stopping criterion was met.
+      refiner.Apply(mesh);
+      if (refiner.Stop())
+      {
+         cout << "Stopping criterion satisfied. Stop." << endl;
+         return 1;
+      }
+   }
+
+   // Update the space to reflect the new state of the mesh.
+   fespace.Update();
+   x.Update();
+   return 0;
+}
+
+
+void Drl4Amr::RandomRefine(double p)
+{
+   mesh.RandomRefinement(p);
+   fespace.Update();
+   x.Update();
+}
+
+
+// *****************************************************************************
+double Drl4Amr::GetNorm()
+{
+   //dbg("GetNorm");
+   // Setup all integration rules for any element type
+   const int order_quad = max(2, 2*order+1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i=0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const double err_x  = x.ComputeL2Error(xcoeff, irs);
+   const double norm_x = ComputeLpNorm(2., xcoeff, mesh, irs);
+   return err_x / norm_x;
+}
+
+
+// *****************************************************************************
+double *Drl4Amr::GetFullImage()
+{
+   Array<int> vert;
+   Vector sln;
+
+   int width = GetFullWidth();
+   int height = GetFullHeight();
+
+   image.SetSize(width * height);
+
+   // rasterize each element into the image
+   for (int k = 0; k < mesh.GetNE(); k++)
+   {
+      int subdiv = (1 << (max_depth - mesh.ncmesh->GetElementDepth(k))) * order;
+
+      const IntegrationRule &ir =
+         GlobGeometryRefiner.Refine(Geometry::SQUARE, subdiv)->RefPts;
+
+      x.GetValues(k, ir, sln);
+
+      mesh.GetElementVertices(k, vert);
+      const double *v = mesh.GetVertex(vert[0]);
+
+      int ox = int(v[0] * nx*(1 << max_depth) * order);
+      int oy = int(v[1] * ny*(1 << max_depth) * order);
+
+      for (int i = 0; i <= subdiv; i++)
+      for (int j = 0; j <= subdiv; j++)
+      {
+         int n = i*(subdiv+1) + j;
+         int m = (oy + i)*width + (ox + j);
+
+         image(m) = sln(n);
+      }
+   }
+
+   if (visualization) { ShowFullImage(); }
+
+   return image.GetData();
+}
+
+// *****************************************************************************
+double* Drl4Amr::GetLocalImage(int element)
+{
+   NCMesh *ncmesh = mesh.ncmesh;
+   MFEM_ASSERT(ncmesh, "");
+
+   int width = GetLocalWidth();
+   int height = GetLocalHeight();
+   int subdiv = oversample*order;
+
+   local_image.SetSize(width*height*4);
+   local_image = 0.0;
+
+   auto im = Reshape(local_image.Write(false), width, height, 4);
+
+   // find neighbors of the current element
+   // TODO: fix the protected access hack
+   Array<int> neighbors;
+   ncmesh->FindNeighbors(ncmesh->leaf_elements[element], neighbors);
+   for (int i = 0; i < neighbors.Size(); i++)
+   {
+      neighbors[i] = ncmesh->elements[neighbors[i]].index;
+   }
+
+   // add the element itself at the end of the list
+   neighbors.Append(element);
+
+   Array<IsoparametricTransformation*> elemT(neighbors.Size());
+   Array<InverseElementTransformation*> invT(neighbors.Size());
+
+   // prepare forward and inverse transforms for all the elements
+   for (int i = 0; i < neighbors.Size(); i++)
+   {
+      elemT[i] = new IsoparametricTransformation;
+      mesh.GetElementTransformation(neighbors[i], elemT[i]);
+      invT[i] = new InverseElementTransformation(elemT[i]);
+   }
+
+   Vector phys, grad;
+
+   // sample solution at a regular grid of points centered on 'element'
+   for (int y = 0; y < height; y++)
+   for (int x = 0; x < width; x++)
+   {
+      IntegrationPoint ip;
+      ip.Set2((x - context + 0.5) / subdiv,
+              (y - context + 0.5) / subdiv);
+
+      // use the local element to transform forward
+      elemT.Last()->Transform(ip, phys);
+
+      // see what neighbor, if any, the physical point lies in
+      for (int k = 0; k < neighbors.Size(); k++)
+      {
+         if (invT[k]->Transform(phys, ip)
+             == InverseElementTransformation::Inside)
+         {
+            im(x, y, 0) = this->x.GetValue(neighbors[k], ip);
+
+            elemT[k]->SetIntPoint(&ip);
+            this->x.GetGradient(*elemT[k], grad);
+
+            im(x, y, 1) = grad(0);
+            im(x, y, 2) = grad(1);
+            im(x, y, 3) = ncmesh->GetElementDepth(neighbors[k])
+                          - ncmesh->GetElementDepth(element);
+            break;
+         }
+      }
+   }
+
+   for (int i = 0; i < neighbors.Size(); i++)
+   {
+      delete elemT[i];
+      delete invT[i];
+   }
+
+   return local_image.GetData();
+}
+
+
+// *****************************************************************************
+void Drl4Amr::ShowFullImage()
+{
+   if (!vis[2].good()) { return; }
+
+   Mesh imesh(GetFullWidth(), GetFullHeight(), elem_type, false, sx, sy, false);
+
+   L2_FECollection fec(0, imesh.Dimension());
+   FiniteElementSpace fes(&imesh, &fec);
+   GridFunction gridfn(&fes, image.GetData());
+
+   vis[2] << "solution" << endl << imesh << gridfn << flush;
+}
+
+// *****************************************************************************
+extern "C" {
+
+Drl4Amr* Ctrl(int order) { return new Drl4Amr(order); }
+int Compute(Drl4Amr *ctrl) { return ctrl->Compute(); }
+int Refine(Drl4Amr *ctrl, int el) { return ctrl->Refine(el); }
+int GetNDofs(Drl4Amr *ctrl) { return ctrl->GetNDofs(); }
+int GetNE(Drl4Amr *ctrl) { return ctrl->GetNE(); }
+double GetNorm(Drl4Amr *ctrl) { return ctrl->GetNorm(); }
+
+double *GetFullImage(Drl4Amr *ctrl) { return ctrl->GetFullImage(); }
+int GetFullWidth(Drl4Amr *ctrl) { return ctrl->GetFullWidth(); }
+int GetFullHeight(Drl4Amr *ctrl) { return ctrl->GetFullHeight(); }
+
+} // extern "C"

miniapps/drl4amr/drl4amr.hpp

@@ -0,0 +1,70 @@
+#ifndef DRL4AMR_HPP
+#define DRL4AMR_HPP
+
+#include "mfem.hpp"
+
+using namespace std;
+using namespace mfem;
+
+class Drl4Amr
+{
+private:
+   const int nx = 8;
+   const int ny = 8;
+   const int max_dofs = 50000;
+   const int max_depth = 2;
+   const Element::Type elem_type = Element::QUADRILATERAL;
+   const double sx = 1.0;
+   const double sy = 1.0;
+
+   const int oversample = 8;
+   const int context = 4;
+
+   Mesh mesh;
+
+   int order;
+   int dim;
+
+   Device device;
+   socketstream vis[3];
+
+   Vector image;
+   Vector local_image;
+
+   H1_FECollection fec;
+   FiniteElementSpace fespace;
+
+   ConstantCoefficient one;
+   ConstantCoefficient zero;
+   BilinearFormIntegrator *integ;
+   FunctionCoefficient xcoeff;
+
+   GridFunction x;
+   int iteration;
+
+   FiniteElementSpace flux_fespace;
+   ZienkiewiczZhuEstimator estimator;
+   ThresholdRefiner refiner;
+
+public:
+   Drl4Amr(int order);
+
+   int Compute();
+   int Refine(int el =-1);
+   int GetNDofs() { return fespace.GetNDofs();}
+   int GetNE() { return fespace.GetNE(); }
+   double GetNorm();
+
+   double *GetFullImage();
+   int GetFullWidth() const { return 1 + order * (nx << max_depth); }
+   int GetFullHeight() const { return GetFullWidth(); }
+   void ShowFullImage();
+
+   double* GetLocalImage(int element);
+   int GetLocalWidth() const { return oversample*order + 2*context; }
+   int GetLocalHeight() const { return GetLocalWidth(); }
+
+   void RandomRefine(double p = 0.5);
+};
+
+#endif // DRL4AMR_HPP

miniapps/drl4amr/drl4amr.py

@@ -0,0 +1,76 @@
+import numpy as np
+from ctypes import *
+from random import *
+from PIL import Image
+
+c_double_p = POINTER(c_double)
+
+MFEM = cdll.LoadLibrary('./libdrl4amr.so')
+
+def sign(f, args_t, res_t):
+    f.restype = res_t
+    f.argtypes = args_t
+
+class Ctrl(object):
+    def __init__(self, order): self.obj = MFEM.Ctrl(order)
+    sign(MFEM.Ctrl, [c_int], c_void_p)
+
+    def Compute(self): return MFEM.Compute(self.obj)
+    sign(MFEM.Compute, [c_void_p], c_int)
+
+    def Refine(self, el_to_refine): return MFEM.Refine(self.obj, el_to_refine)
+    sign(MFEM.Refine, [c_void_p, c_int], c_int)
+
+    def GetNE(self): return MFEM.GetNE(self.obj)
+    sign(MFEM.GetNE, [c_void_p], c_int)
+
+    def GetNorm(self): return MFEM.GetNorm(self.obj)
+    sign(MFEM.GetNorm, [c_void_p], c_double)
+
+    def GetNDofs(self): return MFEM.GetNDofs(self.obj)
+    sign(MFEM.GetNDofs, [c_void_p], c_int)
+
+    def GetImage(self): return MFEM.GetImage(self.obj)
+    sign(MFEM.GetImage, [c_void_p], c_double_p)
+
+    def GetImageSize(self): return MFEM.GetImageSize(self.obj)
+    sign(MFEM.GetImageSize, [c_void_p], c_int)
+
+    def GetImageX(self): return MFEM.GetImageX(self.obj)
+    sign(MFEM.GetImageX, [c_void_p], c_int)
+
+    def GetImageY(self): return MFEM.GetImageY(self.obj)
+    sign(MFEM.GetImageY, [c_void_p], c_int)
+
+
+order = 2
+
+sim = Ctrl(order)
+
+NE = sim.GetNE()
+NX = sim.GetImageX()
+NY = sim.GetImageY()
+#print(NX,NY)
+
+while sim.GetNorm() > 0.01:
+    sim.Compute()
+
+    #sim.Refine(-1); # Will refine using the internal refiner
+    sim.Refine(int(NE*random()))
+
+    image_d = sim.GetImage()
+    #image_s = sim.GetImageSize()
+    #print("image_s: " + str(image_s))
+
+    # Get the data back, two ways:
+    #data = np.fromiter(image_d, dtype=np.double, count=NX*NY) # copy
+    # or:
+    data = np.frombuffer((c_double*NX*NY).from_address(addressof(image_d.contents))) # address
+
+    # Scale and convert the image
+    image_f = (data * 255 / np.max(data)).astype('uint8')
+    image = Image.fromarray(image_f.reshape(NX,NY),'L')
+    image.save('image.jpg')
+    print("Norm: "+str(sim.GetNorm()))
+
+print("Done, final norm: "+str(sim.GetNorm()))

miniapps/drl4amr/main.cpp

@@ -0,0 +1,66 @@
+#include "mfem.hpp"
+
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+#include "drl4amr.hpp"
+
+
+void ShowImg(socketstream &sock, double *img, int width, int height,
+             const char *title = NULL)
+{
+   Mesh mesh(width, height, Element::QUADRILATERAL, true, 1.0, 1.0, false);
+   L2_FECollection fec(0, 2);
+   FiniteElementSpace fes(&mesh, &fec);
+   GridFunction gf(&fes, img);
+   sock << "solution\n" << mesh << gf;
+   sock << "keys Rjlm\n";
+   if (title)
+   {
+      sock << "window_title '" << title << "'\n";
+   }
+   sock << flush;
+}
+
+
+int main(int argc, char *argv[])
+{
+   const int order = 2;
+   Drl4Amr sim(order);
+
+   socketstream vis1("localhost", 19916);
+   socketstream vis2("localhost", 19916);
+   socketstream vis3("localhost", 19916);
+   socketstream vis4("localhost", 19916);
+
+   //sim.Refine(9);
+   sim.RandomRefine();
+   sim.RandomRefine();
+   sim.RandomRefine();
+
+   sim.Compute();
+
+   while (sim.GetNorm() > 0.01)
+   {
+#if 0
+      const int el = static_cast<int>(drand48()*sim.GetNE());
+      sim.Compute();
+      sim.Refine(el);
+      sim.GetFullImage();
+      sim.GetFullWidth();
+#else
+      double *img = sim.GetLocalImage(142);
+      int w = sim.GetLocalWidth();
+      int h = sim.GetLocalWidth();
+      ShowImg(vis1, img, w, h, "Solution");
+      ShowImg(vis2, img + w*h, w, h, "dx");
+      ShowImg(vis3, img + 2*w*h, w, h, "dy");
+      ShowImg(vis4, img + 3*w*h, w, h, "Depth");
+      break;
+#endif
+   }
+   return 0;
+}

miniapps/drl4amr/makefile

@@ -0,0 +1,69 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/drl4amr/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+# Some choices below are based on the OS type:
+NOTMAC := $(subst Darwin,,$(shell uname -s))
+ifneq ($(NOTMAC),)
+   SOPREFIX = so
+else
+   SOPREFIX = install_
+endif
+
+SEQ_MINIOBJS = drl4amr.o
+PAR_MINIOBJS =
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIOBJS = $(SEQ_MINIOBJS)
+else
+   MINIOBJS = $(PAR_MINIOBJS) $(SEQ_MINIOBJS)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all cln clean clean-build clean-exec
+
+# Remove built-in rule
+%.o: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) -c -fPIC $(MFEM_FLAGS) $< -o $@
+
+all: libdrl4amr.so drl4amr
+
+go:;./drl4amr
+py python:;python drl4amr.py
+tst test:;$(MAKE) go && $(MAKE) python
+
+drl4amr: main.cpp libdrl4amr.so
+	$(MFEM_CXX) $(MFEM_FLAGS) main.cpp -o drl4amr -L. -Wl,-rpath,. -ldrl4amr
+
+libdrl4amr.so: $(MINIOBJS)
+	$(MFEM_CXX) -shared -Wl,-$(SOPREFIX)name,libdrl4amr.so -o libdrl4amr.so $(MINIOBJS) $(MFEM_LIBS)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+cln clean:
+	rm -f *.o *~
+	rm -rf *.dSYM *.TVD.*breakpoints $(SEQ_MINIOBJS) drl4amr libdrl4amr.so

miniapps/electromagnetics/volta_solver.cpp

@@ -41,7 +41,6 @@ VoltaSolver::VoltaSolver(ParMesh & pmesh, int order,
      H1FESpace_(NULL),
      HCurlFESpace_(NULL),
      HDivFESpace_(NULL),
-     L2FESpace_(NULL),
      divEpsGrad_(NULL),
      h1Mass_(NULL),
      h1SurfMass_(NULL),
@@ -50,24 +49,20 @@ VoltaSolver::VoltaSolver(ParMesh & pmesh, int order,
      hCurlHDiv_(NULL),
      weakDiv_(NULL),
      rhod_(NULL),
-     l2_vol_int_(NULL),
-     rt_surf_int_(NULL),
      grad_(NULL),
      phi_(NULL),
-     rho_src_(NULL),
      rho_(NULL),
-     sigma_src_(NULL),
+     sigma_(NULL),
      e_(NULL),
      d_(NULL),
-     p_src_(NULL),
-     oneCoef_(1.0),
+     p_(NULL),
      epsCoef_(&epsCoef),
      phiBCCoef_(NULL),
      rhoCoef_(NULL),
      pCoef_(NULL),
-     phi_bc_func_(phi_bc),
-     rho_src_func_(rho_src),
-     p_src_func_(p_src),
+     phi_bc_(phi_bc),
+     rho_src_(rho_src),
+     p_src_(p_src),
      point_charge_params_(point_charges),
      point_charges_(0)
 {
@@ -81,38 +76,34 @@ VoltaSolver::VoltaSolver(ParMesh & pmesh, int order,
    H1FESpace_    = new H1_ParFESpace(pmesh_,order,pmesh_->Dimension());
    HCurlFESpace_ = new ND_ParFESpace(pmesh_,order,pmesh_->Dimension());
    HDivFESpace_  = new RT_ParFESpace(pmesh_,order,pmesh_->Dimension());
-   L2FESpace_    = new L2_ParFESpace(pmesh_,order-1,pmesh_->Dimension());
 
    // Select surface attributes for Dirichlet BCs
    ess_bdr_.SetSize(pmesh.bdr_attributes.Max());
    ess_bdr_ = 0;   // Deselect all outer surfaces
    for (int i=0; i<dbcs_->Size(); i++)
    {
-      if ((*dbcs_)[i] <= ess_bdr_.Size())
-      {
-         ess_bdr_[(*dbcs_)[i]-1] = 1;
-      }
+      ess_bdr_[(*dbcs_)[i]-1] = 1;
    }
 
    // Setup various coefficients
 
    // Potential on outer surface
-   if ( phi_bc_func_ != NULL )
+   if ( phi_bc_ != NULL )
    {
-      phiBCCoef_ = new FunctionCoefficient(*phi_bc_func_);
+      phiBCCoef_ = new FunctionCoefficient(*phi_bc_);
    }
 
    // Volume Charge Density
-   if ( rho_src_func_ != NULL )
+   if ( rho_src_ != NULL )
    {
-      rhoCoef_ = new FunctionCoefficient(rho_src_func_);
+      rhoCoef_ = new FunctionCoefficient(rho_src_);
    }
 
    // Polarization
-   if ( p_src_func_ != NULL )
+   if ( p_src_ != NULL )
    {
       pCoef_ = new VectorFunctionCoefficient(pmesh_->SpaceDimension(),
-                                             p_src_func_);
+                                             p_src_);
    }
 
    // Bilinear Forms
@@ -125,23 +116,15 @@ VoltaSolver::VoltaSolver(ParMesh & pmesh, int order,
    hCurlHDivEps_ = new ParMixedBilinearForm(HCurlFESpace_,HDivFESpace_);
    hCurlHDivEps_->AddDomainIntegrator(new VectorFEMassIntegrator(*epsCoef_));
 
-   rhod_   = new ParLinearForm(H1FESpace_);
+   rhod_ = new ParLinearForm(H1FESpace_);
 
-   l2_vol_int_ = new ParLinearForm(L2FESpace_);
-   l2_vol_int_->AddDomainIntegrator(new DomainLFIntegrator(oneCoef_));
-
-   rt_surf_int_ = new ParLinearForm(HDivFESpace_);
-   rt_surf_int_->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator);
-
-   // Discrete derivative operator
+   // Discrete Grad operator
    grad_ = new ParDiscreteGradOperator(H1FESpace_, HCurlFESpace_);
-   div_  = new ParDiscreteDivOperator(HDivFESpace_, L2FESpace_);
 
    // Build grid functions
    phi_  = new ParGridFunction(H1FESpace_);
    d_    = new ParGridFunction(HDivFESpace_);
    e_    = new ParGridFunction(HCurlFESpace_);
-   rho_  = new ParGridFunction(L2FESpace_);
 
    if ( point_charge_params_.Size() > 0 )
    {
@@ -166,28 +149,29 @@ VoltaSolver::VoltaSolver(ParMesh & pmesh, int order,
       }
    }
 
-   if ( rho_src_func_ )
+   if ( rho_src_ )
    {
-      rho_src_ = new ParGridFunction(H1FESpace_);
+      rho_ = new ParGridFunction(H1FESpace_);
 
       h1Mass_ = new ParBilinearForm(H1FESpace_);
       h1Mass_->AddDomainIntegrator(new MassIntegrator);
    }
 
-   if ( p_src_func_ )
+   if ( p_src_ )
    {
-      p_src_ = new ParGridFunction(HCurlFESpace_);
+      p_ = new ParGridFunction(HCurlFESpace_);
 
       hCurlHDiv_ = new ParMixedBilinearForm(HCurlFESpace_, HDivFESpace_);
       hCurlHDiv_->AddDomainIntegrator(new VectorFEMassIntegrator);
 
       weakDiv_ = new ParMixedBilinearForm(HCurlFESpace_, H1FESpace_);
       weakDiv_->AddDomainIntegrator(new VectorFEWeakDivergenceIntegrator);
+
    }
 
    if ( nbcs_->Size() > 0 )
    {
-      sigma_src_ = new ParGridFunction(H1FESpace_);
+      sigma_ = new ParGridFunction(H1FESpace_);
 
       h1SurfMass_ = new ParBilinearForm(H1FESpace_);
       h1SurfMass_->AddBoundaryIntegrator(new MassIntegrator);
@@ -201,18 +185,14 @@ VoltaSolver::~VoltaSolver()
    delete pCoef_;
 
    delete phi_;
-   delete rho_src_;
    delete rho_;
    delete rhod_;
-   delete l2_vol_int_;
-   delete rt_surf_int_;
-   delete sigma_src_;
+   delete sigma_;
    delete d_;
    delete e_;
-   delete p_src_;
+   delete p_;
 
    delete grad_;
-   delete div_;
 
    delete divEpsGrad_;
    delete h1Mass_;
@@ -225,7 +205,6 @@ VoltaSolver::~VoltaSolver()
    delete H1FESpace_;
    delete HCurlFESpace_;
    delete HDivFESpace_;
-   delete L2FESpace_;
 
    for (unsigned int i=0; i<point_charges_.size(); i++)
    {
@@ -251,13 +230,11 @@ VoltaSolver::PrintSizes()
    HYPRE_Int size_h1 = H1FESpace_->GlobalTrueVSize();
    HYPRE_Int size_nd = HCurlFESpace_->GlobalTrueVSize();
    HYPRE_Int size_rt = HDivFESpace_->GlobalTrueVSize();
-   HYPRE_Int size_l2 = L2FESpace_->GlobalTrueVSize();
    if (myid_ == 0)
    {
       cout << "Number of H1      unknowns: " << size_h1 << endl;
       cout << "Number of H(Curl) unknowns: " << size_nd << endl;
       cout << "Number of H(Div)  unknowns: " << size_rt << endl;
-      cout << "Number of L2      unknowns: " << size_l2 << endl;
    }
 }
 
@@ -277,15 +254,9 @@ void VoltaSolver::Assemble()
    *rhod_ = 0.0;
    rhod_->Assemble();
 
-   l2_vol_int_->Assemble();
-   rt_surf_int_->Assemble();
-
    grad_->Assemble();
    grad_->Finalize();
 
-   div_->Assemble();
-   div_->Finalize();
-
    if ( h1Mass_ )
    {
       h1Mass_->Assemble();
@@ -321,19 +292,15 @@ VoltaSolver::Update()
    H1FESpace_->Update(false);
    HCurlFESpace_->Update(false);
    HDivFESpace_->Update(false);
-   L2FESpace_->Update(false);
 
    // Inform the grid functions that the space has changed.
    phi_->Update();
    rhod_->Update();
-   l2_vol_int_->Update();
-   rt_surf_int_->Update();
    d_->Update();
    e_->Update();
-   rho_->Update();
-   if ( rho_src_   ) { rho_src_->Update(); }
-   if ( sigma_src_ ) { sigma_src_->Update(); }
-   if ( p_src_     ) { p_src_->Update(); }
+   if ( rho_   ) { rho_->Update(); }
+   if ( sigma_ ) { sigma_->Update(); }
+   if ( p_     ) { p_->Update(); }
 
    // Inform the bilinear forms that the space has changed.
    divEpsGrad_->Update();
@@ -347,7 +314,6 @@ VoltaSolver::Update()
 
    // Inform the other objects that the space has changed.
    grad_->Update();
-   div_->Update();
 }
 
 void
@@ -373,46 +339,40 @@ VoltaSolver::Solve()
          {
             ConstantCoefficient voltage((*dbcv_)[i]);
             dbc_bdr_attr = 0;
-            if ((*dbcs_)[i] <= dbc_bdr_attr.Size())
-            {
-               dbc_bdr_attr[(*dbcs_)[i]-1] = 1;
-            }
+            dbc_bdr_attr[(*dbcs_)[i]-1] = 1;
             phi_->ProjectBdrCoefficient(voltage, dbc_bdr_attr);
          }
       }
    }
 
    // Initialize the volumetric charge density
-   if ( rho_src_ )
+   if ( rho_ )
    {
-      rho_src_->ProjectCoefficient(*rhoCoef_);
-      h1Mass_->AddMult(*rho_src_, *rhod_);
+      rho_->ProjectCoefficient(*rhoCoef_);
+      h1Mass_->AddMult(*rho_, *rhod_);
    }
 
    // Initialize the Polarization
-   if ( p_src_ )
+   if ( p_ )
    {
-      p_src_->ProjectCoefficient(*pCoef_);
-      weakDiv_->AddMult(*p_src_, *rhod_);
+      p_->ProjectCoefficient(*pCoef_);
+      weakDiv_->AddMult(*p_, *rhod_);
    }
 
    // Initialize the surface charge density
-   if ( sigma_src_ )
+   if ( sigma_ )
    {
-      *sigma_src_ = 0.0;
+      *sigma_ = 0.0;
 
       Array<int> nbc_bdr_attr(pmesh_->bdr_attributes.Max());
       for (int i=0; i<nbcs_->Size(); i++)
       {
          ConstantCoefficient sigma_coef((*nbcv_)[i]);
          nbc_bdr_attr = 0;
-         if ((*nbcs_)[i] <= nbc_bdr_attr.Size())
-         {
-            nbc_bdr_attr[(*nbcs_)[i]-1] = 1;
-         }
-         sigma_src_->ProjectBdrCoefficient(sigma_coef, nbc_bdr_attr);
+         nbc_bdr_attr[(*nbcs_)[i]-1] = 1;
+         sigma_->ProjectBdrCoefficient(sigma_coef, nbc_bdr_attr);
       }
-      h1SurfMass_->AddMult(*sigma_src_, *rhod_);
+      h1SurfMass_->AddMult(*sigma_, *rhod_);
    }
 
    // Determine the essential BC degrees of freedom
@@ -464,9 +424,9 @@ VoltaSolver::Solve()
 
    ParGridFunction ed(HDivFESpace_);
    hCurlHDivEps_->Mult(*e_, ed);
-   if ( p_src_ )
+   if ( p_ )
    {
-      hCurlHDiv_->AddMult(*p_src_, ed, -1.0);
+      hCurlHDiv_->AddMult(*p_, ed, -1.0);
    }
 
    HypreParMatrix MassHDiv;
@@ -485,27 +445,8 @@ VoltaSolver::Solve()
 
    hDivMass_->RecoverFEMSolution(D, ed, *d_);
 
-   // Compute charge density from rho = Div(D)
-   div_->Mult(*d_, *rho_);
-
    if (myid_ == 0) { cout << "done." << flush; }
 
-   {
-      // Compute total charge as volume integral of rho
-      double charge_rho = (*l2_vol_int_)(*rho_);
-
-      // Compute total charge as surface integral of D
-      double charge_D = (*rt_surf_int_)(*d_);
-
-      if (myid_ == 0)
-      {
-         cout << endl << "Total charge: \n"
-              << "   Volume integral of charge density:   " << charge_rho
-              << "\n   Surface integral of dielectric flux: " << charge_D
-              << endl << flush;
-      }
-   }
-
    if (myid_ == 0) { cout << "Solver done. " << endl; }
 }
 
@@ -539,10 +480,9 @@ VoltaSolver::RegisterVisItFields(VisItDataCollection & visit_dc)
    visit_dc.RegisterField("Phi", phi_);
    visit_dc.RegisterField("D",     d_);
    visit_dc.RegisterField("E",     e_);
-   visit_dc.RegisterField("Rho", rho_);
-   if ( rho_src_   ) { visit_dc.RegisterField("Rho Source",     rho_src_); }
-   if ( p_src_     ) { visit_dc.RegisterField("P Source",         p_src_); }
-   if ( sigma_src_ ) { visit_dc.RegisterField("Sigma Source", sigma_src_); }
+   if ( rho_   ) { visit_dc.RegisterField("Rho",     rho_); }
+   if ( p_     ) { visit_dc.RegisterField("P",         p_); }
+   if ( sigma_ ) { visit_dc.RegisterField("Sigma", sigma_); }
 }
 
 void
@@ -575,23 +515,20 @@ VoltaSolver::InitializeGLVis()
    socks_["E"] = new socketstream;
    socks_["E"]->precision(8);
 
-   socks_["Rho"] = new socketstream;
-   socks_["Rho"]->precision(8);
-
-   if ( rho_src_ )
+   if ( rho_)
    {
-      socks_["RhoSrc"] = new socketstream;
-      socks_["RhoSrc"]->precision(8);
+      socks_["Rho"] = new socketstream;
+      socks_["Rho"]->precision(8);
    }
-   if ( p_src_ )
+   if ( p_)
    {
-      socks_["PSrc"] = new socketstream;
-      socks_["PSrc"]->precision(8);
+      socks_["P"] = new socketstream;
+      socks_["P"]->precision(8);
    }
-   if ( sigma_src_ )
+   if ( sigma_)
    {
-      socks_["SigmaSrc"] = new socketstream;
-      socks_["SigmaSrc"]->precision(8);
+      socks_["Sigma"] = new socketstream;
+      socks_["Sigma"]->precision(8);
    }
 }
 
@@ -611,36 +548,31 @@ VoltaSolver::DisplayToGLVis()
                   *phi_, "Electric Potential (Phi)", Wx, Wy, Ww, Wh);
    Wx += offx;
 
-   VisualizeField(*socks_["E"], vishost, visport,
-                  *e_, "Electric Field (E)", Wx, Wy, Ww, Wh);
-   Wx += offx;
-
    VisualizeField(*socks_["D"], vishost, visport,
                   *d_, "Electric Displacement (D)", Wx, Wy, Ww, Wh);
    Wx += offx;
 
-   VisualizeField(*socks_["Rho"], vishost, visport,
-                  *rho_, "Charge Density", Wx, Wy, Ww, Wh);
+   VisualizeField(*socks_["E"], vishost, visport,
+                  *e_, "Electric Field (E)", Wx, Wy, Ww, Wh);
+
    Wx = 0; Wy += offy; // next line
 
-   if ( rho_src_ )
+   if ( rho_ )
    {
-      VisualizeField(*socks_["RhoSrc"], vishost, visport,
-                     *rho_src_, "Charge Density Source (Rho)", Wx, Wy, Ww, Wh);
+      VisualizeField(*socks_["Rho"], vishost, visport,
+                     *rho_, "Charge Density (Rho)", Wx, Wy, Ww, Wh);
       Wx += offx;
    }
-   if ( p_src_ )
+   if ( p_ )
    {
-      VisualizeField(*socks_["PSrc"], vishost, visport,
-                     *p_src_, "Electric Polarization Source (P)",
-                     Wx, Wy, Ww, Wh);
+      VisualizeField(*socks_["P"], vishost, visport,
+                     *p_, "Electric Polarization (P)", Wx, Wy, Ww, Wh);
       Wx += offx;
    }
-   if ( sigma_src_ )
+   if ( sigma_ )
    {
-      VisualizeField(*socks_["SigmaSrc"], vishost, visport,
-                     *sigma_src_, "Surface Charge Density Source (Sigma)",
-                     Wx, Wy, Ww, Wh);
+      VisualizeField(*socks_["Sigma"], vishost, visport,
+                     *sigma_, "Surface Charge Density (Sigma)", Wx, Wy, Ww, Wh);
       // Wx += offx; // not used
    }
    if (myid_ == 0) { cout << " done." << endl; }

miniapps/electromagnetics/volta_solver.hpp

@@ -26,9 +26,7 @@ namespace mfem
 using miniapps::H1_ParFESpace;
 using miniapps::ND_ParFESpace;
 using miniapps::RT_ParFESpace;
-using miniapps::L2_ParFESpace;
 using miniapps::ParDiscreteGradOperator;
-using miniapps::ParDiscreteDivOperator;
 
 namespace electromagnetics
 {
@@ -86,7 +84,6 @@ private:
    H1_ParFESpace * H1FESpace_;    // Continuous space for phi
    ND_ParFESpace * HCurlFESpace_; // Tangentially continuous space for E
    RT_ParFESpace * HDivFESpace_;  // Normally continuous space for D
-   L2_ParFESpace * L2FESpace_;    // Discontinuous space for rho
 
    ParBilinearForm * divEpsGrad_; // Laplacian operator
    ParBilinearForm * h1Mass_;     // For Volumetric Charge Density Source
@@ -97,32 +94,26 @@ private:
    ParMixedBilinearForm * hCurlHDiv_;    // For computing D from E and P
    ParMixedBilinearForm * weakDiv_;      // For computing the source term from P
 
-   ParLinearForm * rhod_; // Dual of Volumetric Charge Density Source
-
-   ParLinearForm * l2_vol_int_;  // Integral of L2 field
-   ParLinearForm * rt_surf_int_; // Integral of H(Div) field over boundary
+   ParLinearForm * rhod_; // Dual of Volumetric Charge Density
 
    ParDiscreteGradOperator * grad_; // For Computing E from phi
-   ParDiscreteDivOperator  * div_;  // For Computing rho from D
 
-   ParGridFunction * phi_;       // Electric Scalar Potential
-   ParGridFunction * rho_src_;   // Volumetric Charge Density Source
-   ParGridFunction * rho_;       // Volumetric Charge Density (Div(D))
-   ParGridFunction * sigma_src_; // Surface Charge Density Source
-   ParGridFunction * e_;         // Electric Field
-   ParGridFunction * d_;         // Electric Flux Density (aka Dielectric Flux)
-   ParGridFunction * p_src_;     // Polarization Field Source
+   ParGridFunction * phi_; // Electric Scalar Potential
+   ParGridFunction * rho_; // Volumetric Charge Density
+   ParGridFunction * sigma_; // Surface Charge Density
+   ParGridFunction * e_; // Electric Field
+   ParGridFunction * d_; // Electric Flux Density (aka Dielectric Flux)
+   ParGridFunction * p_; // Polarization Field
 
-   ConstantCoefficient oneCoef_;   // Coefficient equal to 1
    Coefficient       * epsCoef_;   // Dielectric Permittivity Coefficient
    Coefficient       * phiBCCoef_; // Scalar Potential Boundary Condition
    Coefficient       * rhoCoef_;   // Charge Density Coefficient
    VectorCoefficient * pCoef_;     // Polarization Vector Field Coefficient
 
    // Source functions
-   double (*phi_bc_func_ )(const Vector&);          // Scalar Potential BC
-   double (*rho_src_func_)(const Vector&);          // Volumetric Charge Density
-   void   (*p_src_func_  )(const Vector&, Vector&); // Polarization Field
+   double (*phi_bc_ )(const Vector&);          // Scalar Potential BC
+   double (*rho_src_)(const Vector&);          // Volumetric Charge Density
+   void   (*p_src_  )(const Vector&, Vector&); // Polarization Field
 
    const Vector & point_charge_params_;
 

miniapps/meshing/amr-quad-q2.mesh

@@ -1,418 +0,0 @@
-MFEM mesh v1.1
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-28
-1 3 0 9 21 12
-1 3 12 21 11 7
-1 3 21 22 37 25
-1 3 25 37 24 11
-1 3 37 23 8 24
-1 3 22 10 23 37
-1 3 9 4 10 21
-1 3 4 13 26 10
-1 3 13 1 14 26
-1 3 26 14 5 15
-1 3 38 28 15 29
-1 3 27 26 28 38
-1 3 10 27 38 23
-1 3 23 38 29 8
-1 3 8 29 39 33
-1 3 33 39 32 18
-1 3 39 31 30 32
-1 3 29 15 31 39
-1 3 15 5 16 30
-1 3 30 16 3 17
-1 3 18 30 17 6
-1 3 34 18 6 19
-1 3 40 33 18 36
-1 3 24 8 33 40
-1 3 11 24 40 35
-1 3 35 40 36 34
-1 3 7 11 34 20
-1 3 20 34 19 2
-
-boundary
-16
-2 1 0 9
-1 1 12 0
-1 1 7 12
-2 1 9 4
-2 1 4 13
-2 1 13 1
-1 1 1 14
-1 1 14 5
-1 1 5 16
-1 1 16 3
-2 1 3 17
-2 1 17 6
-2 1 6 19
-1 1 20 7
-2 1 19 2
-1 1 2 20
-
-vertex_parents
-37
-4 0 1
-5 1 3
-6 2 3
-7 0 2
-8 4 6
-9 0 4
-10 4 8
-11 7 8
-12 0 7
-13 1 4
-14 1 5
-15 5 8
-16 3 5
-17 3 6
-18 6 8
-19 2 6
-20 2 7
-21 9 11
-22 10 21
-23 8 10
-24 8 11
-25 11 21
-26 13 15
-27 10 26
-28 15 26
-29 8 15
-30 15 17
-31 15 30
-32 18 30
-33 8 18
-34 11 19
-35 11 34
-36 18 34
-37 22 24
-38 27 29
-39 29 32
-40 24 36
-
-coarse_elements
-9
-3 2 5 4 3
-3 0 6 28 1
-3 12 11 10 13
-3 7 8 9 30
-3 14 17 16 15
-3 32 18 19 20
-3 24 23 22 25
-3 26 34 21 27
-3 29 31 33 35
-
-vertices
-41
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P2
-VDim: 2
-Ordering: 0
-
-0
-1
-0
-1
-0.5
-1
-0.5
-0
-0.490917
-0.25
-0.481659
-0.229025
-0
-0.75
-1
-0.75002
-1
-0.75
-0.497731
-0.25
-0
-0.230224
-0.361824
-0.498798
-0.358947
-0.256374
-0.75627
-0.609156
-0.760931
-0.614878
-0.754468
-0.729327
-0.597212
-0.479497
-0.255854
-0.252575
-0.374701
-0.365809
-0.616315
-0.620005
-0.354453
-0.125
-0.241186
-0.113998
-0
-0.256374
-0.0996625
-0
-0.297495
-0.381624
-0.298118
-0.249986
-0.354661
-0.29368
-0.249387
-0.422671
-0.50313
-0.416618
-0.423212
-0.50141
-0.375
-0.498432
-0.361824
-0.625
-0.750551
-0.609156
-0.875
-1
-0.883899
-1
-0.859365
-0.760931
-0.66728
-0.757422
-0.680277
-0.604085
-0.680261
-0.760547
-0.611485
-0.542955
-0.563608
-0.556166
-0.630915
-0.564831
-0.496409
-0.603763
-0.54025
-0.492351
-0.666944
-0.736168
-0.668618
-0.733944
-1
-0.878096
-0.729327
-1
-0.875
-0.731558
-0.597212
-0.625
-0.509544
-0.374701
-0.375
-0.262228
-0.439501
-0.435693
-0.381762
-0.375151
-0.303302
-0.243334
-0.314755
-0.256748
-0.252575
-0.100238
-0
-0.125
-0
-0.0884843
-0.133844
-0.301451
-0.319617
-0.421308
-0.439579
-0.366542
-0.638837
-0.84411
-0.887525
-0.673669
-0.684397
-0.547681
-0.553932
-0.556981
-0.569094
-0.682474
-0.680908
-0.877814
-0.843209
-0.610964
-0.363741
-0.433969
-0.438104
-0.317325
-0.315253
-0.114408
-0.0955121
-0
-0
-1
-1
-0
-0.5
-1
-0.5
-0.500169
-0
-0.230361
-0.506286
-0.25
-0
-0.25
-0.501817
-0.75
-1
-0.751223
-1
-0.75
-0.245442
-0.249774
-0.359053
-0.482625
-0.339768
-0.25622
-0.246257
-0.342759
-0.493736
-0.737081
-0.598044
-0.732223
-0.628407
-0.755429
-0.618147
-0.736346
-0.380657
-0.367339
-0.619478
-0.619469
-0
-0.112152
-0.214709
-0.125
-0.339768
-0.511468
-0.375
-0.250576
-0.312269
-0.354505
-0.283581
-0.425524
-0.495609
-0.414003
-0.367212
-0.438493
-0.482955
-0.243036
-0.314443
-0
-0.130361
-0.249774
-0
-0.1228
-0.246257
-0
-0.125
-0.243197
-0.375
-0.460998
-0.342759
-0.372076
-0.413223
-0.487797
-0.431307
-0.25198
-0.290425
-0.296938
-0.239051
-0.353925
-0.485506
-0.542001
-0.613511
-0.565178
-0.672486
-0.738741
-0.683983
-0.614503
-0.662211
-0.73167
-0.544579
-0.625
-0.717877
-0.598044
-0.875
-1
-0.883366
-0.732223
-1
-0.888992
-0.736346
-1
-0.868768
-0.606623
-0.739539
-0.671004
-0.56527
-0.611998
-0.559039
-0.741642
-0.68361
-0.618147
-0.760055
-0.625
-1
-0.875
-0.110313
-0.337719
-0.306391
-0.440854
-0.431661
-0.308523
-0.0865872
-0.124325
-0.133215
-0.332392
-0.433604
-0.313603
-0.30387
-0.419458
-0.563411
-0.693606
-0.689503
-0.553108
-0.586083
-0.880316
-0.866348
-0.884747
-0.669969
-0.569241
-0.56415
-0.679016
-0.617255
-0.87398

miniapps/meshing/mesh-optimizer.cpp

@@ -32,11 +32,6 @@
 // Compile with: make mesh-optimizer
 //
 // Sample runs:
-//   Adapted analytic Hessian:
-//     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 2 -tid 4 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
-//   Adapted discrete size:
-//     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 5 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
-//
 //   Blade shape:
 //     mesh-optimizer -m blade.mesh -o 4 -rs 0 -mid 2 -tid 1 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
 //   Blade limited shape:
@@ -53,9 +48,6 @@
 //     mesh-optimizer -o 3 -rs 0 -mid 1 -tid 1 -ni 1000 -ls 2 -li 100 -bnd -qt 1 -qo 8 -cmb
 //   3D pinched sphere shape (the mesh is in the mfem/data GitHub repository):
 //   * mesh-optimizer -m ../../../mfem_data/ball-pert.mesh -o 4 -rs 0 -mid 303 -tid 1 -ni 20 -ls 2 -li 500 -fix-bnd
-//   2D non-conforming shape and equal size:
-//     mesh-optimizer -m ./amr-quad-q2.mesh -o 2 -rs 1 -mid 9 -tid 2 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
-
 
 #include "mfem.hpp"
 #include <fstream>
@@ -66,159 +58,189 @@ using namespace std;
 
 double weight_fun(const Vector &x);
 
-double ind_values(const Vector &x)
+// Metric values are visualized by creating an L2 finite element functions and
+// computing the metric values at the nodes.
+void vis_metric(int order, TMOP_QualityMetric &qm, const TargetConstructor &tc,
+                Mesh &mesh, char *title, int position)
 {
-   const int opt = 6;
-   const double small = 0.001, big = 0.01;
-
-   // Sine wave.
-   if (opt==1)
-   {
-      const double X = x(0), Y = x(1);
-      const double ind = std::tanh((10*(Y-0.5) + std::sin(4.0*M_PI*X)) + 1) -
-                         std::tanh((10*(Y-0.5) + std::sin(4.0*M_PI*X)) - 1);
-
-      return ind * small + (1.0 - ind) * big;
-   }
-
-   if (opt==2)
-   {
-      // Circle in the middle.
-      double val = 0.;
-      const double xc = x(0) - 0.5, yc = x(1) - 0.5;
-      const double r = sqrt(xc*xc + yc*yc);
-      double r1 = 0.15; double r2 = 0.35; double sf=30.0;
-      val = 0.5*(std::tanh(sf*(r-r1)) - std::tanh(sf*(r-r2)));
-      if (val > 1.) {val = 1;}
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   if (opt == 3)
-   {
-      // cross
-      const double X = x(0), Y = x(1);
-      const double r1 = 0.45, r2 = 0.55;
-      const double sf = 40.0;
-
-      double val = 0.5 * ( std::tanh(sf*(X-r1)) - std::tanh(sf*(X-r2)) +
-                           std::tanh(sf*(Y-r1)) - std::tanh(sf*(Y-r2)) );
-      if (val > 1.) { val = 1.0; }
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   if (opt==4)
-   {
-      // Multiple circles
-      double r1,r2,val,rval;
-      double sf = 10;
-      val = 0.;
-      // circle 1
-      r1= 0.25; r2 = 0.25; rval = 0.1;
-      double xc = x(0) - r1, yc = x(1) - r2;
-      double r = sqrt(xc*xc+yc*yc);
-      val =  0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*
-                                                               (r-rval)));// std::exp(val1);
-      // circle 2
-      r1= 0.75; r2 = 0.75;
-      xc = x(0) - r1, yc = x(1) - r2;
-      r = sqrt(xc*xc+yc*yc);
-      val +=  (0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*
-                                                                 (r-rval))));// std::exp(val1);
-      // circle 3
-      r1= 0.75; r2 = 0.25;
-      xc = x(0) - r1, yc = x(1) - r2;
-      r = sqrt(xc*xc+yc*yc);
-      val +=  0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*
-                                                                (r-rval)));// std::exp(val1);
-      // circle 4
-      r1= 0.25; r2 = 0.75;
-      xc = x(0) - r1, yc = x(1) - r2;
-      r = sqrt(xc*xc+yc*yc);
-      val +=  0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*(r-rval)));
-      if (val > 1.0) {val = 1.;}
-      if (val < 0.0) {val = 0.;}
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   if (opt==5)
-   {
-      // cross
-      double val = 0.;
-      double X = x(0)-0.5, Y = x(1)-0.5;
-      double rval = std::sqrt(X*X + Y*Y);
-      double thval = 60.*M_PI/180.;
-      double Xmod,Ymod;
-      Xmod = X*std::cos(thval) + Y*std::sin(thval);
-      Ymod= -X*std::sin(thval) + Y*std::cos(thval);
-      X = Xmod+0.5; Y = Ymod+0.5;
-      double r1 = 0.45; double r2 = 0.55; double sf=30.0;
-      val = ( 0.5*(1+std::tanh(sf*(X-r1))) - 0.5*(1+std::tanh(sf*(X-r2)))
-              + 0.5*(1+std::tanh(sf*(Y-r1))) - 0.5*(1+std::tanh(sf*(Y-r2))) );
-      if (rval > 0.4) {val = 0.;}
-      if (val > 1.0) {val = 1.;}
-      if (val < 0.0) {val = 0.;}
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   if (opt==6)
-   {
-      double val = 0.;
-      const double xc = x(0) - 0.0, yc = x(1) - 0.5;
-      const double r = sqrt(xc*xc + yc*yc);
-      double r1 = 0.45; double r2 = 0.55; double sf=30.0;
-      val = 0.5*(1+std::tanh(sf*(r-r1))) - 0.5*(1+std::tanh(sf*(r-r2)));
-      if (val > 1.) {val = 1;}
-      if (val < 0.) {val = 0;}
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   return 0.0;
+   L2_FECollection fec(order, mesh.Dimension(), BasisType::GaussLobatto);
+   FiniteElementSpace fes(&mesh, &fec, 1);
+   GridFunction metric(&fes);
+   InterpolateTMOP_QualityMetric(qm, tc, mesh, metric);
+   osockstream sock(19916, "localhost");
+   sock << "solution\n";
+   mesh.Print(sock);
+   metric.Save(sock);
+   sock.send();
+   sock << "window_title '"<< title << "'\n"
+        << "window_geometry "
+        << position << " " << 0 << " " << 600 << " " << 600 << "\n"
+        << "keys jRmclA" << endl;
 }
 
-class HessianCoefficient : public MatrixCoefficient
+class RelaxedNewtonSolver : public NewtonSolver
 {
 private:
-   int type;
+   // Quadrature points that are checked for negative Jacobians etc.
+   const IntegrationRule &ir;
+   FiniteElementSpace *fes;
+   mutable GridFunction x_gf;
 
 public:
-   HessianCoefficient(int dim, int type_)
-      : MatrixCoefficient(dim), type(type_) { }
+   RelaxedNewtonSolver(const IntegrationRule &irule, FiniteElementSpace *f)
+      : ir(irule), fes(f) { }
 
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
-                     const IntegrationPoint &ip)
+   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
+};
+
+double RelaxedNewtonSolver::ComputeScalingFactor(const Vector &x,
+                                                 const Vector &b) const
+{
+   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
+   MFEM_VERIFY(nlf != NULL, "invalid Operator subclass");
+   const bool have_b = (b.Size() == Height());
+
+   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
+             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
+   Array<int> xdofs(dof * dim);
+   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
+   Vector posV(pos.Data(), dof * dim);
+
+   Vector x_out(x.Size());
+   bool x_out_ok = false;
+   const double energy_in = nlf->GetEnergy(x);
+   double scale = 1.0, energy_out;
+   double norm0 = Norm(r);
+   x_gf.MakeTRef(fes, x_out, 0);
+
+   // Decreases the scaling of the update until the new mesh is valid.
+   for (int i = 0; i < 12; i++)
    {
-      Vector pos(3);
-      T.Transform(ip, pos);
+      add(x, -scale, c, x_out);
+      x_gf.SetFromTrueVector();
 
-      if (type == 0)
+      energy_out = nlf->GetGridFunctionEnergy(x_gf);
+      if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
       {
-         K(0, 0) = 1.0 + 3.0 * std::sin(M_PI*pos(0));
-         K(0, 1) = 0.0;
-         K(1, 0) = 0.0;
-         K(1, 1) = 1.0;
+         if (print_level >= 0)
+         { cout << "Scale = " << scale << " Increasing energy." << endl; }
+         scale *= 0.5; continue;
       }
-      else
+
+      int jac_ok = 1;
+      for (int i = 0; i < NE; i++)
       {
-         const double xc = pos(0) - 0.5, yc = pos(1) - 0.5;
-         const double r = sqrt(xc*xc + yc*yc);
-         double r1 = 0.15; double r2 = 0.35; double sf=30.0;
-         const double eps = 0.5;
+         fes->GetElementVDofs(i, xdofs);
+         x_gf.GetSubVector(xdofs, posV);
+         for (int j = 0; j < nsp; j++)
+         {
+            fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+            MultAtB(pos, dshape, Jpr);
+            if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+         }
+      }
+   break2:
+      if (jac_ok == 0)
+      {
+         if (print_level >= 0)
+         { cout << "Scale = " << scale << " Neg det(J) found." << endl; }
+         scale *= 0.5; continue;
+      }
 
-         const double tan1 = std::tanh(sf*(r-r1)),
-                      tan2 = std::tanh(sf*(r-r2));
+      oper->Mult(x_out, r);
+      if (have_b) { r -= b; }
+      double norm = Norm(r);
 
-         K(0, 0) = eps + 1.0 * (tan1 - tan2);
-         K(0, 1) = 0.0;
-         K(1, 0) = 0.0;
-         K(1, 1) = 1.0;
+      if (norm > 1.2*norm0)
+      {
+         if (print_level >= 0)
+         { cout << "Scale = " << scale << " Norm increased." << endl; }
+         scale *= 0.5; continue;
+      }
+      else { x_out_ok = true; break; }
+   }
+
+   if (print_level >= 0)
+   {
+      cout << "Energy decrease: "
+           << (energy_in - energy_out) / energy_in * 100.0
+           << "% with " << scale << " scaling." << endl;
+   }
+
+   if (x_out_ok == false) { scale = 0.0; }
+
+   return scale;
+}
+
+// Allows negative Jacobians. Used in untangling metrics.
+class DescentNewtonSolver : public NewtonSolver
+{
+private:
+   // Quadrature points that are checked for negative Jacobians etc.
+   const IntegrationRule &ir;
+   FiniteElementSpace *fes;
+   mutable GridFunction x_gf;
+
+public:
+   DescentNewtonSolver(const IntegrationRule &irule, FiniteElementSpace *f)
+      : ir(irule), fes(f) { }
+
+   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
+};
+
+double DescentNewtonSolver::ComputeScalingFactor(const Vector &x,
+                                                 const Vector &b) const
+{
+   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
+   MFEM_VERIFY(nlf != NULL, "invalid Operator subclass");
+
+   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
+             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
+   Array<int> xdofs(dof * dim);
+   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
+   Vector posV(pos.Data(), dof * dim);
+
+   x_gf.MakeTRef(fes, x.GetData());
+   x_gf.SetFromTrueVector();
+
+   double min_detJ = infinity();
+   for (int i = 0; i < NE; i++)
+   {
+      fes->GetElementVDofs(i, xdofs);
+      x_gf.GetSubVector(xdofs, posV);
+      for (int j = 0; j < nsp; j++)
+      {
+         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+         MultAtB(pos, dshape, Jpr);
+         min_detJ = min(min_detJ, Jpr.Det());
       }
    }
-};
+   cout << "Minimum det(J) = " << min_detJ << endl;
+
+   Vector x_out(x.Size());
+   bool x_out_ok = false;
+   const double energy_in = nlf->GetGridFunctionEnergy(x_gf);
+   double scale = 1.0, energy_out;
+
+   for (int i = 0; i < 7; i++)
+   {
+      add(x, -scale, c, x_out);
+
+      energy_out = nlf->GetEnergy(x_out);
+      if (energy_out > energy_in || std::isnan(energy_out) != 0)
+      {
+         scale *= 0.5;
+      }
+      else { x_out_ok = true; break; }
+   }
+
+   cout << "Energy decrease: " << (energy_in - energy_out) / energy_in * 100.0
+        << "% with " << scale << " scaling." << endl;
+
+   if (x_out_ok == false) { return 0.0; }
+
+   return scale;
+}
 
 // Additional IntegrationRules that can be used with the --quad-type option.
 IntegrationRules IntRulesLo(0, Quadrature1D::GaussLobatto);
@@ -282,9 +304,7 @@ int main (int argc, char *argv[])
                   "Target (ideal element) type:\n\t"
                   "1: Ideal shape, unit size\n\t"
                   "2: Ideal shape, equal size\n\t"
-                  "3: Ideal shape, initial size\n\t"
-                  "4: Given full analytic Jacobian (in physical space)\n\t"
-                  "5: Ideal shape, given size (in physical space)");
+                  "3: Ideal shape, initial size");
    args.AddOption(&lim_const, "-lc", "--limit-const", "Limiting constant.");
    args.AddOption(&quad_type, "-qt", "--quad-type",
                   "Quadrature rule type:\n\t"
@@ -355,8 +375,7 @@ int main (int argc, char *argv[])
    // 6. Get the mesh nodes (vertices and other degrees of freedom in the finite
    //    element space) as a finite element grid function in fespace. Note that
    //    changing x automatically changes the shapes of the mesh elements.
-   GridFunction x(fespace);
-   mesh->SetNodalGridFunction(&x);
+   GridFunction *x = mesh->GetNodes();
 
    // 7. Define a vector representing the minimal local mesh size in the mesh
    //    nodes. We index the nodes using the scalar version of the degrees of
@@ -406,9 +425,10 @@ int main (int argc, char *argv[])
       // Set the boundary values to zero.
       for (int j = 0; j < vdofs.Size(); j++) { rdm(vdofs[j]) = 0.0; }
    }
-   x -= rdm;
-   x.SetTrueVector();
-   x.SetFromTrueVector();
+   *x -= rdm;
+   // Set the perturbation of all nodes from the true nodes.
+   x->SetTrueVector();
+   x->SetFromTrueVector();
 
    // 9. Save the starting (prior to the optimization) mesh to a file. This
    //    output can be viewed later using GLVis: "glvis -m perturbed.mesh".
@@ -419,7 +439,7 @@ int main (int argc, char *argv[])
 
    // 10. Store the starting (prior to the optimization) positions.
    GridFunction x0(fespace);
-   x0 = x;
+   x0 = *x;
 
    // 11. Form the integrator that uses the chosen metric and target.
    double tauval = -0.1;
@@ -448,43 +468,15 @@ int main (int argc, char *argv[])
       default: cout << "Unknown metric_id: " << metric_id << endl; return 3;
    }
    TargetConstructor::TargetType target_t;
-   TargetConstructor *target_c = NULL;
-   HessianCoefficient *adapt_coeff = NULL;
-   H1_FECollection ind_fec(3, dim);
-   FiniteElementSpace ind_fes(mesh, &ind_fec);
-   GridFunction size;
    switch (target_id)
    {
       case 1: target_t = TargetConstructor::IDEAL_SHAPE_UNIT_SIZE; break;
       case 2: target_t = TargetConstructor::IDEAL_SHAPE_EQUAL_SIZE; break;
       case 3: target_t = TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE; break;
-      case 4:
-      {
-         target_t = TargetConstructor::GIVEN_FULL;
-         AnalyticAdaptTC *tc = new AnalyticAdaptTC(target_t);
-         adapt_coeff = new HessianCoefficient(dim, 1);
-         tc->SetAnalyticTargetSpec(NULL, NULL, adapt_coeff);
-         target_c = tc;
-         break;
-      }
-      case 5:
-      {
-         target_t = TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE;
-         DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
-         size.SetSpace(&ind_fes);
-         FunctionCoefficient ind_coeff(ind_values);
-         size.ProjectCoefficient(ind_coeff);
-         tc->SetSerialDiscreteTargetSpec(size);
-         target_c = tc;
-         break;
-      }
-      default: cout << "Unknown target_id: " << target_id << endl; return 3;
-   }
-
-   if (target_c == NULL)
-   {
-      target_c = new TargetConstructor(target_t);
+      default: cout << "Unknown target_id: " << target_id << endl;
+         delete metric; return 3;
    }
+   TargetConstructor *target_c = new TargetConstructor(target_t);
    target_c->SetNodes(x0);
    TMOP_Integrator *he_nlf_integ = new TMOP_Integrator(metric, target_c);
 
@@ -550,13 +542,13 @@ int main (int argc, char *argv[])
    }
    else { a.AddDomainIntegrator(he_nlf_integ); }
 
-   const double init_energy = a.GetGridFunctionEnergy(x);
+   const double init_energy = a.GetGridFunctionEnergy(*x);
 
    // 15. Visualize the starting mesh and metric values.
    if (visualization)
    {
       char title[] = "Initial metric values";
-      vis_tmop_metric_s(mesh_poly_deg, *metric, *target_c, *mesh, title, 0);
+      vis_metric(mesh_poly_deg, *metric, *target_c, *mesh, title, 0);
    }
 
    // 16. Fix all boundary nodes, or fix only a given component depending on the
@@ -660,13 +652,8 @@ int main (int argc, char *argv[])
    if (tauval > 0.0)
    {
       tauval = 0.0;
-      TMOPNewtonSolver *tns = new TMOPNewtonSolver(*ir);
-      if (target_id == 5)
-      {
-         tns->SetDiscreteAdaptTC(dynamic_cast<DiscreteAdaptTC *>(target_c));
-      }
-      newton = tns;
-      cout << "TMOPNewtonSolver is used (as all det(J) > 0).\n";
+      newton = new RelaxedNewtonSolver(*ir, fespace);
+      cout << "The RelaxedNewtonSolver is used (as all det(J)>0)." << endl;
    }
    else
    {
@@ -677,8 +664,8 @@ int main (int argc, char *argv[])
          return 3;
       }
       tauval -= 0.01 * h0.Min(); // Slightly below minJ0 to avoid div by 0.
-      newton = new TMOPDescentNewtonSolver(*ir);
-      cout << "The TMOPDescentNewtonSolver is used (as some det(J) < 0).\n";
+      newton = new DescentNewtonSolver(*ir, fespace);
+      cout << "The DescentNewtonSolver is used (as some det(J)<0)." << endl;
    }
    newton->SetPreconditioner(*S);
    newton->SetMaxIter(newton_iter);
@@ -686,9 +673,8 @@ int main (int argc, char *argv[])
    newton->SetAbsTol(0.0);
    newton->SetPrintLevel(verbosity_level >= 1 ? 1 : -1);
    newton->SetOperator(a);
-   newton->Mult(b, x.GetTrueVector());
-   x.SetFromTrueVector();
-
+   newton->Mult(b, x->GetTrueVector());
+   x->SetFromTrueVector();
    if (newton->GetConverged() == false)
    {
       cout << "NewtonIteration: rtol = " << newton_rtol << " not achieved."
@@ -705,12 +691,12 @@ int main (int argc, char *argv[])
    }
 
    // 21. Compute the amount of energy decrease.
-   const double fin_energy = a.GetGridFunctionEnergy(x);
+   const double fin_energy = a.GetGridFunctionEnergy(*x);
    double metric_part = fin_energy;
    if (lim_const != 0.0)
    {
       lim_coeff.constant = 0.0;
-      metric_part = a.GetGridFunctionEnergy(x);
+      metric_part = a.GetGridFunctionEnergy(*x);
       lim_coeff.constant = lim_const;
    }
    cout << "Initial strain energy: " << init_energy
@@ -726,13 +712,13 @@ int main (int argc, char *argv[])
    if (visualization)
    {
       char title[] = "Final metric values";
-      vis_tmop_metric_s(mesh_poly_deg, *metric, *target_c, *mesh, title, 600);
+      vis_metric(mesh_poly_deg, *metric, *target_c, *mesh, title, 600);
    }
 
    // 23. Visualize the mesh displacement.
    if (visualization)
    {
-      x0 -= x;
+      x0 -= *x;
       osockstream sock(19916, "localhost");
       sock << "solution\n";
       mesh->Print(sock);

miniapps/meshing/pmesh-optimizer.cpp

@@ -32,11 +32,6 @@
 // Compile with: make pmesh-optimizer
 //
 // Sample runs:
-//   Adapted analytic Hessian:
-//     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 2 -tid 4 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
-//   Adapted discrete size:
-//     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 5 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
-//
 //   Blade shape:
 //     mpirun -np 4 pmesh-optimizer -m blade.mesh -o 4 -rs 0 -mid 2 -tid 1 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
 //   Blade limited shape:
@@ -53,176 +48,223 @@
 //     mpirun -np 4 pmesh-optimizer -o 3 -rs 0 -mid 1 -tid 1 -ni 1000 -ls 2 -li 100 -bnd -qt 1 -qo 8 -cmb
 //   3D pinched sphere shape (the mesh is in the mfem/data GitHub repository):
 //   * mpirun -np 4 pmesh-optimizer -m ../../../mfem_data/ball-pert.mesh -o 4 -rs 0 -mid 303 -tid 1 -ni 20 -ls 2 -li 500 -fix-bnd
-//   2D non-conforming shape and equal size:
-//     mpirun -np 4 pmesh-optimizer -m ./amr-quad-q2.mesh -o 2 -rs 1 -mid 9 -tid 2 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
 
 #include "mfem.hpp"
-#include <iostream>
 #include <fstream>
+#include <iostream>
 
 using namespace mfem;
 using namespace std;
 
 double weight_fun(const Vector &x);
 
-double ind_values(const Vector &x)
+// Metric values are visualized by creating an L2 finite element functions and
+// computing the metric values at the nodes.
+void vis_metric(int order, TMOP_QualityMetric &qm, const TargetConstructor &tc,
+                ParMesh &pmesh, char *title, int position)
 {
-   const int opt = 6;
-   const double small = 0.001, big = 0.01;
-
-   // Sine wave.
-   if (opt==1)
+   L2_FECollection fec(order, pmesh.Dimension(), BasisType::GaussLobatto);
+   ParFiniteElementSpace fes(&pmesh, &fec, 1);
+   ParGridFunction metric(&fes);
+   InterpolateTMOP_QualityMetric(qm, tc, pmesh, metric);
+   socketstream sock;
+   if (pmesh.GetMyRank() == 0)
    {
-      const double X = x(0), Y = x(1);
-      const double ind = std::tanh((10*(Y-0.5) + std::sin(4.0*M_PI*X)) + 1) -
-                         std::tanh((10*(Y-0.5) + std::sin(4.0*M_PI*X)) - 1);
-
-      return ind * small + (1.0 - ind) * big;
+      sock.open("localhost", 19916);
+      sock << "solution\n";
    }
-
-   if (opt==2)
+   pmesh.PrintAsOne(sock);
+   metric.SaveAsOne(sock);
+   if (pmesh.GetMyRank() == 0)
    {
-      // Circle in the middle.
-      double val = 0.;
-      const double xc = x(0) - 0.5, yc = x(1) - 0.5;
-      const double r = sqrt(xc*xc + yc*yc);
-      double r1 = 0.15; double r2 = 0.35; double sf=30.0;
-      val = 0.5*(std::tanh(sf*(r-r1)) - std::tanh(sf*(r-r2)));
-      if (val > 1.) {val = 1;}
-
-      return val * small + (1.0 - val) * big;
+      sock << "window_title '"<< title << "'\n"
+           << "window_geometry "
+           << position << " " << 0 << " " << 600 << " " << 600 << "\n"
+           << "keys jRmclA" << endl;
    }
-
-   if (opt == 3)
-   {
-      // cross
-      const double X = x(0), Y = x(1);
-      const double r1 = 0.45, r2 = 0.55;
-      const double sf = 40.0;
-
-      double val = 0.5 * ( std::tanh(sf*(X-r1)) - std::tanh(sf*(X-r2)) +
-                           std::tanh(sf*(Y-r1)) - std::tanh(sf*(Y-r2)) );
-      if (val > 1.) { val = 1.0; }
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   if (opt==4)
-   {
-      // Multiple circles
-      double r1,r2,val,rval;
-      double sf = 10;
-      val = 0.;
-      // circle 1
-      r1= 0.25; r2 = 0.25; rval = 0.1;
-      double xc = x(0) - r1, yc = x(1) - r2;
-      double r = sqrt(xc*xc+yc*yc);
-      val =  0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*
-                                                               (r-rval)));// std::exp(val1);
-      // circle 2
-      r1= 0.75; r2 = 0.75;
-      xc = x(0) - r1, yc = x(1) - r2;
-      r = sqrt(xc*xc+yc*yc);
-      val +=  (0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*
-                                                                 (r-rval))));// std::exp(val1);
-      // circle 3
-      r1= 0.75; r2 = 0.25;
-      xc = x(0) - r1, yc = x(1) - r2;
-      r = sqrt(xc*xc+yc*yc);
-      val +=  0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*
-                                                                (r-rval)));// std::exp(val1);
-      // circle 4
-      r1= 0.25; r2 = 0.75;
-      xc = x(0) - r1, yc = x(1) - r2;
-      r = sqrt(xc*xc+yc*yc);
-      val +=  0.5*(1+std::tanh(sf*(r+rval))) - 0.5*(1+std::tanh(sf*(r-rval)));
-      if (val > 1.0) {val = 1.;}
-      if (val < 0.0) {val = 0.;}
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   if (opt==5)
-   {
-      // cross
-      double val = 0.;
-      double X = x(0)-0.5, Y = x(1)-0.5;
-      double rval = std::sqrt(X*X + Y*Y);
-      double thval = 60.*M_PI/180.;
-      double Xmod,Ymod;
-      Xmod = X*std::cos(thval) + Y*std::sin(thval);
-      Ymod= -X*std::sin(thval) + Y*std::cos(thval);
-      X = Xmod+0.5; Y = Ymod+0.5;
-      double r1 = 0.45; double r2 = 0.55; double sf=30.0;
-      val = ( 0.5*(1+std::tanh(sf*(X-r1))) - 0.5*(1+std::tanh(sf*(X-r2)))
-              + 0.5*(1+std::tanh(sf*(Y-r1))) - 0.5*(1+std::tanh(sf*(Y-r2))) );
-      if (rval > 0.4) {val = 0.;}
-      if (val > 1.0) {val = 1.;}
-      if (val < 0.0) {val = 0.;}
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   if (opt==6)
-   {
-      double val = 0.;
-      const double xc = x(0) - 0.0, yc = x(1) - 0.5;
-      const double r = sqrt(xc*xc + yc*yc);
-      double r1 = 0.45; double r2 = 0.55; double sf=30.0;
-      val = 0.5*(1+std::tanh(sf*(r-r1))) - 0.5*(1+std::tanh(sf*(r-r2)));
-      if (val > 1.) {val = 1;}
-      if (val < 0.) {val = 0;}
-
-      return val * small + (1.0 - val) * big;
-   }
-
-   return 0.0;
 }
 
-class HessianCoefficient : public MatrixCoefficient
+class RelaxedNewtonSolver : public NewtonSolver
 {
 private:
-   int type;
+   // Quadrature points that are checked for negative Jacobians etc.
+   const IntegrationRule &ir;
+   ParFiniteElementSpace *pfes;
+   mutable ParGridFunction x_gf;
 
 public:
-   HessianCoefficient(int dim, int type_)
-      : MatrixCoefficient(dim), type(type_) { }
+   RelaxedNewtonSolver(const IntegrationRule &irule, ParFiniteElementSpace *pf)
+      : NewtonSolver(pf->GetComm()), ir(irule), pfes(pf) { }
 
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
-                     const IntegrationPoint &ip)
+   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
+};
+
+double RelaxedNewtonSolver::ComputeScalingFactor(const Vector &x,
+                                                 const Vector &b) const
+{
+   const ParNonlinearForm *nlf = dynamic_cast<const ParNonlinearForm *>(oper);
+   MFEM_VERIFY(nlf != NULL, "invalid Operator subclass");
+   const bool have_b = (b.Size() == Height());
+
+   const int NE = pfes->GetParMesh()->GetNE(), dim = pfes->GetFE(0)->GetDim(),
+             dof = pfes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
+   Array<int> xdofs(dof * dim);
+   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
+   Vector posV(pos.Data(), dof * dim);
+
+   Vector x_out(x.Size());
+   bool x_out_ok = false;
+   const double energy_in = nlf->GetEnergy(x);
+   double scale = 1.0, energy_out;
+   double norm0 = Norm(r);
+   x_gf.MakeTRef(pfes, x_out, 0);
+
+   // Decreases the scaling of the update until the new mesh is valid.
+   for (int i = 0; i < 12; i++)
    {
-      Vector pos(3);
-      T.Transform(ip, pos);
+      add(x, -scale, c, x_out);
+      x_gf.SetFromTrueVector();
 
-      if (type == 0)
+      energy_out = nlf->GetParGridFunctionEnergy(x_gf);
+      if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
       {
-         K(0, 0) = 1.0 + 3.0 * std::sin(M_PI*pos(0));
-         K(0, 1) = 0.0;
-         K(1, 0) = 0.0;
-         K(1, 1) = 1.0;
+         if (print_level >= 0)
+         { cout << "Scale = " << scale << " Increasing energy." << endl; }
+         scale *= 0.5; continue;
       }
-      else
+
+      int jac_ok = 1;
+      for (int i = 0; i < NE; i++)
       {
-         const double xc = pos(0) - 0.5, yc = pos(1) - 0.5;
-         const double r = sqrt(xc*xc + yc*yc);
-         double r1 = 0.15; double r2 = 0.35; double sf=30.0;
-         const double eps = 0.5;
+         pfes->GetElementVDofs(i, xdofs);
+         x_gf.GetSubVector(xdofs, posV);
+         for (int j = 0; j < nsp; j++)
+         {
+            pfes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+            MultAtB(pos, dshape, Jpr);
+            if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+         }
+      }
+   break2:
+      int jac_ok_all;
+      MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
+                    pfes->GetComm());
 
-         const double tan1 = std::tanh(sf*(r-r1)),
-                      tan2 = std::tanh(sf*(r-r2));
+      if (jac_ok_all == 0)
+      {
+         if (print_level >= 0)
+         { cout << "Scale = " << scale << " Neg det(J) found." << endl; }
+         scale *= 0.5; continue;
+      }
 
-         K(0, 0) = eps + 1.0 * (tan1 - tan2);
-         K(0, 1) = 0.0;
-         K(1, 0) = 0.0;
-         K(1, 1) = 1.0;
+      oper->Mult(x_out, r);
+      if (have_b) { r -= b; }
+      double norm = Norm(r);
+
+      if (norm > 1.2*norm0)
+      {
+         if (print_level >= 0)
+         { cout << "Scale = " << scale << " Norm increased." << endl; }
+         scale *= 0.5; continue;
+      }
+      else { x_out_ok = true; break; }
+   }
+
+   if (print_level >= 0)
+   {
+      cout << "Energy decrease: "
+           << (energy_in - energy_out) / energy_in * 100.0
+           << "% with " << scale << " scaling." << endl;
+   }
+
+   if (x_out_ok == false) { scale = 0.0; }
+
+   return scale;
+}
+
+// Allows negative Jacobians. Used in untangling metrics.
+class DescentNewtonSolver : public NewtonSolver
+{
+private:
+   // Quadrature points that are checked for negative Jacobians etc.
+   const IntegrationRule &ir;
+   ParFiniteElementSpace *pfes;
+   mutable ParGridFunction x_gf;
+
+public:
+   DescentNewtonSolver(const IntegrationRule &irule, ParFiniteElementSpace *pf)
+      : NewtonSolver(pf->GetComm()), ir(irule), pfes(pf) { }
+
+   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
+};
+
+double DescentNewtonSolver::ComputeScalingFactor(const Vector &x,
+                                                 const Vector &b) const
+{
+   const ParNonlinearForm *nlf = dynamic_cast<const ParNonlinearForm *>(oper);
+   MFEM_VERIFY(nlf != NULL, "invalid Operator subclass");
+
+   const int NE = pfes->GetParMesh()->GetNE(), dim = pfes->GetFE(0)->GetDim(),
+             dof = pfes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
+   Array<int> xdofs(dof * dim);
+   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
+   Vector posV(pos.Data(), dof * dim);
+
+   x_gf.MakeTRef(pfes, x.GetData());
+   x_gf.SetFromTrueVector();
+
+   double min_detJ = infinity();
+   for (int i = 0; i < NE; i++)
+   {
+      pfes->GetElementVDofs(i, xdofs);
+      x_gf.GetSubVector(xdofs, posV);
+      for (int j = 0; j < nsp; j++)
+      {
+         pfes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+         MultAtB(pos, dshape, Jpr);
+         min_detJ = min(min_detJ, Jpr.Det());
       }
    }
-};
+   double min_detJ_all;
+   MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
+                 pfes->GetComm());
+   if (print_level >= 0)
+   { cout << "Minimum det(J) = " << min_detJ_all << endl; }
+
+   Vector x_out(x.Size());
+   bool x_out_ok = false;
+   const double energy_in = nlf->GetParGridFunctionEnergy(x_gf);
+   double scale = 1.0, energy_out;
+
+   for (int i = 0; i < 7; i++)
+   {
+      add(x, -scale, c, x_out);
+
+      energy_out = nlf->GetEnergy(x_out);
+      if (energy_out > energy_in || std::isnan(energy_out) != 0)
+      {
+         scale *= 0.5;
+      }
+      else { x_out_ok = true; break; }
+   }
+
+   if (print_level >= 0)
+   {
+      cout << "Energy decrease: "
+           << (energy_in - energy_out) / energy_in * 100.0
+           << "% with " << scale << " scaling." << endl;
+   }
+
+   if (x_out_ok == false) { return 0.0; }
+
+   return scale;
+}
 
 // Additional IntegrationRules that can be used with the --quad-type option.
 IntegrationRules IntRulesLo(0, Quadrature1D::GaussLobatto);
 IntegrationRules IntRulesCU(0, Quadrature1D::ClosedUniform);
 
+
 int main (int argc, char *argv[])
 {
    // 0. Initialize MPI.
@@ -289,9 +331,7 @@ int main (int argc, char *argv[])
                   "Target (ideal element) type:\n\t"
                   "1: Ideal shape, unit size\n\t"
                   "2: Ideal shape, equal size\n\t"
-                  "3: Ideal shape, initial size\n\t"
-                  "4: Given full analytic Jacobian (in physical space)\n\t"
-                  "5: Ideal shape, given size (in physical space)");
+                  "3: Ideal shape, initial size");
    args.AddOption(&lim_const, "-lc", "--limit-const", "Limiting constant.");
    args.AddOption(&quad_type, "-qt", "--quad-type",
                   "Quadrature rule type:\n\t"
@@ -341,7 +381,6 @@ int main (int argc, char *argv[])
       cout << endl;
    }
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-
    delete mesh;
    for (int lev = 0; lev < rp_levels; lev++) { pmesh->UniformRefinement(); }
 
@@ -471,47 +510,20 @@ int main (int argc, char *argv[])
          return 3;
    }
    TargetConstructor::TargetType target_t;
-   TargetConstructor *target_c = NULL;
-   HessianCoefficient *adapt_coeff = NULL;
-   H1_FECollection ind_fec(3, dim);
-   ParFiniteElementSpace ind_fes(pmesh, &ind_fec);
-   ParGridFunction size;
    switch (target_id)
    {
       case 1: target_t = TargetConstructor::IDEAL_SHAPE_UNIT_SIZE; break;
       case 2: target_t = TargetConstructor::IDEAL_SHAPE_EQUAL_SIZE; break;
       case 3: target_t = TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE; break;
-      case 4:
-      {
-         target_t = TargetConstructor::GIVEN_FULL;
-         AnalyticAdaptTC *tc = new AnalyticAdaptTC(target_t);
-         adapt_coeff = new HessianCoefficient(dim, 1);
-         tc->SetAnalyticTargetSpec(NULL, NULL, adapt_coeff);
-         target_c = tc;
-         break;
-      }
-      case 5:
-      {
-         target_t = TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE;
-         DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
-         size.SetSpace(&ind_fes);
-         FunctionCoefficient ind_coeff(ind_values);
-         size.ProjectCoefficient(ind_coeff);
-         tc->SetParDiscreteTargetSpec(size);
-         target_c = tc;
-         break;
-      }
       default:
          if (myid == 0) { cout << "Unknown target_id: " << target_id << endl; }
          return 3;
    }
-
-   if (target_c == NULL)
-   {
-      target_c = new TargetConstructor(target_t, MPI_COMM_WORLD);
-   }
+   TargetConstructor *target_c;
+   target_c = new TargetConstructor(target_t, MPI_COMM_WORLD);
    target_c->SetNodes(x0);
-   TMOP_Integrator *he_nlf_integ= new TMOP_Integrator(metric, target_c);
+   TMOP_Integrator *he_nlf_integ;
+   he_nlf_integ = new TMOP_Integrator(metric, target_c);
 
    // 13. Setup the quadrature rule for the non-linear form integrator.
    const IntegrationRule *ir = NULL;
@@ -585,7 +597,7 @@ int main (int argc, char *argv[])
    if (visualization)
    {
       char title[] = "Initial metric values";
-      vis_tmop_metric_p(mesh_poly_deg, *metric, *target_c, *pmesh, title, 0);
+      vis_metric(mesh_poly_deg, *metric, *target_c, *pmesh, title, 0);
    }
 
    // 17. Fix all boundary nodes, or fix only a given component depending on the
@@ -693,14 +705,9 @@ int main (int argc, char *argv[])
    if (tauval > 0.0)
    {
       tauval = 0.0;
-      TMOPNewtonSolver *tns = new TMOPNewtonSolver(pfespace->GetComm(), *ir);
-      if (target_id == 5)
-      {
-         tns->SetDiscreteAdaptTC(dynamic_cast<DiscreteAdaptTC *>(target_c));
-      }
-      newton = tns;
+      newton = new RelaxedNewtonSolver(*ir, pfespace);
       if (myid == 0)
-      { cout << "TMOPNewtonSolver is used (as all det(J) > 0)." << endl; }
+      { cout << "RelaxedNewtonSolver is used (as all det(J) > 0)." << endl; }
    }
    else
    {
@@ -708,15 +715,15 @@ int main (int argc, char *argv[])
            (dim == 3 && metric_id != 352) )
       {
          if (myid == 0)
-         { cout << "The mesh is inverted. Use an untangling metric.\n"; }
+         { cout << "The mesh is inverted. Use an untangling metric." << endl; }
          return 3;
       }
       double h0min = h0.Min(), h0min_all;
       MPI_Allreduce(&h0min, &h0min_all, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
       tauval -= 0.01 * h0min_all; // Slightly below minJ0 to avoid div by 0.
-      newton = new TMOPDescentNewtonSolver(pfespace->GetComm(), *ir);
+      newton = new DescentNewtonSolver(*ir, pfespace);
       if (myid == 0)
-      { cout << "TMOPDescentNewtonSolver is used (as some det(J) < 0).\n"; }
+      { cout << "DescentNewtonSolver is used (as some det(J) < 0)." << endl; }
    }
    newton->SetPreconditioner(*S);
    newton->SetMaxIter(newton_iter);
@@ -768,7 +775,7 @@ int main (int argc, char *argv[])
    if (visualization)
    {
       char title[] = "Final metric values";
-      vis_tmop_metric_p(mesh_poly_deg, *metric, *target_c, *pmesh, title, 600);
+      vis_metric(mesh_poly_deg, *metric, *target_c, *pmesh, title, 600);
    }
 
    // 23. Visualize the mesh displacement.
@@ -798,7 +805,6 @@ int main (int argc, char *argv[])
    delete metric2;
    delete coeff1;
    delete target_c;
-   delete adapt_coeff;
    delete metric;
    delete pfespace;
    delete fec;

miniapps/meshing/square01.mesh

@@ -1,114 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-16
-1 3 0 1 6 5
-1 3 1 2 7 6
-1 3 2 3 8 7
-1 3 3 4 9 8
-1 3 5 6 11 10
-1 3 6 7 12 11
-1 3 7 8 13 12
-1 3 8 9 14 13
-1 3 10 11 16 15
-1 3 11 12 17 16
-1 3 12 13 18 17
-1 3 13 14 19 18
-1 3 15 16 21 20
-1 3 16 17 22 21
-1 3 17 18 23 22
-1 3 18 19 24 23
-
-boundary
-16
-2 1 0 1
-2 1 1 2
-2 1 2 3
-2 1 3 4
-2 1 21 20
-2 1 22 21
-2 1 23 22
-2 1 24 23
-1 1 5 0
-1 1 10 5
-1 1 15 10
-1 1 20 15
-1 1 4 9
-1 1 9 14
-1 1 14 19
-1 1 19 24
-
-vertices
-25
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P1
-VDim: 2
-Ordering: 0
-
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0
-0
-0
-0
-0.25
-0.25
-0.25
-0.25
-0.25
-0.5
-0.5
-0.5
-0.5
-0.5
-0.75
-0.75
-0.75
-0.75
-0.75
-1
-1
-1
-1
-1

tests/unit/fem/test_pa_coeff.cpp

@@ -1,147 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "catch.hpp"
-#include "mfem.hpp"
-
-using namespace mfem;
-
-namespace pa_coeff
-{
-
-int dimension;
-
-double coeffFunction(const Vector& x)
-{
-   if (dimension == 2)
-   {
-      return sin(8.0 * M_PI * x[0]) * cos(6.0 * M_PI * x[1]) + 2.0;
-   }
-   else
-   {
-      return sin(8.0 * M_PI * x[0]) * cos(6.0 * M_PI * x[1]) *
-             sin(4.0 * M_PI * x[2]) +
-             2.0;
-   }
-}
-
-TEST_CASE("pa_coeff")
-{
-   for (dimension = 2; dimension < 4; ++dimension)
-   {
-      for (int coeffType = 0; coeffType < 3; ++coeffType)
-      {
-         for (int integrator = 0; integrator < 2; ++integrator)
-         {
-            const int ne = 2;
-            std::cout << "Testing " << dimension << "D partial assembly with "
-                      << "coeffType " << coeffType << " and "
-                      << "integrator " << integrator << std::endl;
-            for (int order = 1; order < 5; ++order)
-            {
-               Mesh* mesh;
-               if (dimension == 2)
-               {
-                  mesh = new Mesh(ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
-               }
-               else
-               {
-                  mesh = new Mesh(ne, ne, ne, Element::HEXAHEDRON, 1, 1.0, 1.0,
-                                  1.0);
-               }
-               FiniteElementCollection* h1_fec =
-                  new H1_FECollection(order, dimension);
-               FiniteElementSpace h1_fespace(mesh, h1_fec);
-               Array<int> ess_tdof_list;
-
-               BilinearForm paform(&h1_fespace);
-               GridFunction* coeffGridFunction = nullptr;
-               Coefficient* coeff = nullptr;
-               if (coeffType == 0)
-               {
-                  coeff = new ConstantCoefficient(1.0);
-               }
-               else if (coeffType == 1)
-               {
-                  coeff = new FunctionCoefficient(&coeffFunction);
-               }
-               else if (coeffType == 2)
-               {
-                  FunctionCoefficient tmpCoeff(&coeffFunction);
-                  coeffGridFunction = new GridFunction(&h1_fespace);
-                  coeffGridFunction->ProjectCoefficient(tmpCoeff);
-                  coeff = new GridFunctionCoefficient(coeffGridFunction);
-               }
-               paform.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-               if (integrator < 2)
-               {
-                  paform.AddDomainIntegrator(new DiffusionIntegrator(*coeff));
-               }
-               if (integrator > 0)
-               {
-                  paform.AddDomainIntegrator(new MassIntegrator(*coeff));
-               }
-               paform.Assemble();
-               OperatorHandle paopr;
-               paform.FormSystemMatrix(ess_tdof_list, paopr);
-
-               BilinearForm assemblyform(&h1_fespace);
-               if (integrator < 2)
-               {
-                  assemblyform.AddDomainIntegrator(
-                     new DiffusionIntegrator(*coeff));
-               }
-               if (integrator > 0)
-               {
-                  assemblyform.AddDomainIntegrator(new MassIntegrator(*coeff));
-               }
-               assemblyform.SetDiagonalPolicy(Matrix::DIAG_ONE);
-               assemblyform.Assemble();
-               assemblyform.Finalize();
-               const SparseMatrix& A_explicit = assemblyform.SpMat();
-
-               Vector xin(h1_fespace.GetTrueVSize());
-               xin.Randomize();
-               Vector y_mat(xin);
-               y_mat = 0.0;
-               Vector y_assembly(xin);
-               y_assembly = 0.0;
-               Vector y_pa(xin);
-               y_pa = 0.0;
-
-               paopr->Mult(xin, y_pa);
-               assemblyform.Mult(xin, y_assembly);
-               A_explicit.Mult(xin, y_mat);
-
-               y_pa -= y_mat;
-               double pa_error = y_pa.Norml2();
-               std::cout << "  order: " << order
-                         << ", pa error norm: " << pa_error << std::endl;
-               REQUIRE(pa_error < 1.e-12);
-
-               y_assembly -= y_mat;
-               double assembly_error = y_assembly.Norml2();
-               std::cout << "  order: " << order
-                         << ", assembly error norm: " << assembly_error
-                         << std::endl;
-               REQUIRE(assembly_error < 1.e-12);
-
-               delete coeff;
-               delete coeffGridFunction;
-               delete mesh;
-               delete h1_fec;
-            }
-         }
-      }
-   }
-}
-
-} // namespace pa_coeff