minor bug

remove comments
fixing bugs
2023-05-31 11:00:58 -07:00 · 2023-04-07 18:43:48 -07:00 · 2023-04-07 18:43:09 -07:00 · 2023-04-04 16:40:28 -07:00 · 2023-04-04 12:14:57 -07:00 · 2023-04-04 00:09:53 -07:00
255 changed files with 13626 additions and 4249 deletions
@@ -62,7 +62,7 @@ jobs:

      - name: GHCR Login
        if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
@@ -94,7 +94,7 @@ jobs:
    # This external action allows to interrupt a workflow already running on
    # the same branch to save resource
    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
+      uses: styfle/cancel-workflow-action@0.11.0
      with:
        access_token: ${{ github.token }}

@@ -102,7 +102,7 @@ jobs:
    # /home/runner/work/mfem/mfem/mfem
    # Note: Done now to access "install-hypre" and "install-metis" actions.
    - name: checkout mfem
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
      with:
        path: ${{ env.MFEM_TOP_DIR }}
        # Fetch the complete history for codecov to access commits ID
@@ -115,25 +115,25 @@ jobs:
      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install lcov

-    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
+    # Keep the following section in case we need it again in the future,
+    # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
+    # - name: Set up Homebrew
+    #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
+    #   uses: Homebrew/actions/setup-homebrew@master

    - name: get MPI (MacOS)
      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
-        export MAKE_CXX_FLAG="MPICXX=mpic++"

-    - name: get MPI (MacOS)
+    - name: get lcov (MacOS)
      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
@@ -141,14 +141,14 @@ jobs:

    - name: get MPI (Windows)
      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-      uses: mpi4py/setup-mpi@v1.0.3
+      uses: mpi4py/setup-mpi@v1.1.4

    # Get Hypre through cache, or build it.
    # Install will only run on cache miss.
    - name: cache hypre
      id: hypre-cache
      if: matrix.mpi == 'par'
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
@@ -176,7 +176,7 @@ jobs:
    - name: cache metis
      id: metis-cache
      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ${{ env.METIS_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
@@ -190,12 +190,13 @@ jobs:

    - name: cache vcpkg (Windows)
      id: vcpkg-cache
+      if: matrix.os == 'windows-latest'
      uses: actions/cache@v3
      with:
        path: vcpkg_cache
        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1

-    - name: prepare binary cache location
+    - name: prepare vcpkg binary cache location (Windows)
      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
      run: |
        mkdir -p vcpkg_cache
@@ -230,11 +231,7 @@ jobs:
      run: |
        cd ${{ env.MFEM_TOP_DIR }}  && make check

-    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'opt'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make unittest
-
+    # Note: 'tests' include the unit tests
    - name: tests
      if: matrix.build-system == 'make' && matrix.target == 'opt'
      run: |
@@ -247,7 +244,7 @@ jobs:
        cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
      shell: bash

-    - name: cmake unit tests (Ubuntu 20.04)
+    - name: cmake unit tests (Ubuntu)
      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
      run: |
        CTEST_CONFIG="Release"
@@ -265,7 +262,7 @@ jobs:
    # Code coverage (process and upload reports)
    - name: codecov
      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.0
+      uses: mfem/github-actions/upload-coverage@v2.2
      with:
        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
        project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -35,23 +35,22 @@ jobs:

    steps:
    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
+      uses: styfle/cancel-workflow-action@0.11.0
      with:
        access_token: ${{ github.token }}

    - name: checkout MFEM
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
      with:
        path: mfem

    - name: Get MPI (Linux)
      run: |
        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: Cache Hypre Install
      id: hypre-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
@@ -66,7 +65,7 @@ jobs:

    - name: Cache Metis Install
      id: metis-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ${{ env.METIS_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
@@ -34,12 +34,12 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
+      uses: styfle/cancel-workflow-action@0.11.0
      with:
        access_token: ${{ github.token }}

    - name: checkout mfem
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3

    - name: copyright check
      id: copyright
@@ -84,7 +84,7 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3

    - name: get astyle
      run: |
@@ -101,11 +101,14 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3

    - name: get doxygen and graphviz
      run: |
        sudo apt-get install doxygen graphviz
+
+    - name: update doxygen config file
+      run: |
        cd doc
        doxygen -u CodeDocumentation.conf.in

@@ -123,7 +126,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
    - name: checkout mfem
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
      with:
        fetch-depth: 0

@@ -131,6 +131,12 @@ examples/hiop/ex9-mesh.*
 examples/hiop/ex9-init.*
 examples/hiop/ex9-final.*

+examples/ipopt/exContactBlockTL
+examples/ipopt/exContactBlockTL.mesh
+examples/ipopt/exContactBlockTL-mesh.*
+examples/ipopt/exContactBlockTL-init.*
+examples/ipopt/exContactBlockTL-final.*
+
 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
 examples/petsc/mesh.*
@@ -203,6 +209,7 @@ miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
 miniapps/meshing/trimmer
+miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
@@ -214,9 +221,12 @@ miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
 miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
+miniapps/meshing/mesh-explorer-visit*
+miniapps/meshing/mesh-explorer-paraview/
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
+miniapps/meshing/reflected.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh
@@ -275,6 +285,8 @@ miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
+miniapps/tools/check-tmop-metric
+miniapps/tools/tmop-metric-magnitude

 miniapps/toys/automata
 miniapps/toys/life
@@ -93,7 +93,7 @@ report_baseline:
          git pull && \
          git add ${rundir} && \
          git commit -m "${msg}" && \
-          git push origin master
+          ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
        else
          for file in ${rundir}/*; do
            echo "------------------------------"
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Try to push to the remote 5 times. If the push fails, and the local and remote
+# have diverged, then pull from the remote to merge changes, and try pushing
+# again. If some other failure happens
+for i in {1..5}; do
+   git push origin master && exit 0
+   # Wait for 20 seconds in case someone else is pushing to the remote
+   # concurrently
+   sleep 20
+   # Fetch any updates from the remote
+   git remote update
+   # Get the latest commit on the local branch
+   LOCAL=$(git rev-parse @)
+   # Get the latest commit on the remote
+   REMOTE=$(git rev-parse @{u})
+   # Get the common ancestor
+   BASE=$(git merge-base @ @{u})
+   # Have the local and remote diverged?
+   if [[ $LOCAL != $REMOTE && $LOCAL != $BASE && $REMOTE != $BASE ]]; then
+      git pull
+      if [[ $? == 0 ]]; then
+         continue
+      else
+         exit 1 # Something else went wrong trying to pull
+      fi
+   fi
+done
+
+exit 1 # Did not succeed in 5 attempts
@@ -32,7 +32,7 @@ if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
  git pull && \
  git add ${rundir} && \
  git commit -m "${msg}" && \
-  git push origin master
+  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
 else
  for file in ${rundir}/*; do
    echo "------------------------------"
@@ -29,7 +29,7 @@ if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
  git pull && \
  git add ${rundir} && \
  git commit -m "${msg}" && \
-  git push origin master
+  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
 else
  for file in ${rundir}/*; do
    echo "------------------------------"
@@ -8,43 +8,107 @@
                               https://mfem.org


-Version 4.4.1 (development)
+Version 4.5.1 (development)
 ===========================
+- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
+  as the coarsening operator for non-conforming AMR meshes.

 Meshing improvements
 --------------------
- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
+- Added support for pyramids in non-conforming meshes. Currently only isotropic
+  refinement is supported in this case.

+- Updated logic in FindPointsGSLIB to ignore points found near (but outside) the
+  domain boundary.
+
+- Added support for pyramids in Gmsh meshes.
+
+- Fixed a bug in TMOP metric 301.
+
+- Added an option to auto-balance compound TMOP metrics.
+
+Discretization improvements
+---------------------------
+- TBD
+
+Linear and nonlinear solvers
+----------------------------
+- Added a fast normalization-based distance solver, see the Distance miniapp
+  in the miniapps/shifted/ directory.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new meshing miniapp, reflector, which reflects a high-order or NURBS
+  hexahedral mesh about a plane.
+
+- The mesh-explorer miniapp can now save mesh files in the VisIt or ParaView
+  formats using the corresponding DataCollection objects. See option 'D' in the
+  main menu.
+
+Integrations, testing and documentation
+---------------------------------------
+- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
+  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
+
+- The following integrations have updated minimum version requirements:
+  * RAJA >= 2022.10.3
+
+Miscellaneous
+-------------
+- VisItDataCollection now correctly handles collection names containing
+  underscores.
+
+- VisItDataCollection::SetPadDigits() no longer alters the number of digits
+  used to represent the MPI rank because VisIt seems to require 6 digits.
+  This parameter can still be explicitly overridden with
+  VisItDataCollection::SetPadDigitsRank().
+
+API changes
+-----------
+- The implicit cast methods of class Vector to 'double *' and 'const double *'
+  have been deprecated and generate deprecation warnings if used. They will be
+  removed in a future release.
+
+- The methods Mesh::GetFaceBaseGeometry and Mesh::GetFaceGeometryType have been
+  deprecated, and Mesh::GetFaceGeometry (which provides identical functionality)
+  should be used instead.
+
+Version 4.5, released on October 22, 2022
+=========================================
+
+Meshing improvements
+--------------------
 - Added new SubMesh and ParSubMesh classes that can be used to extract a subset
  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
  and work with all existing MFEM interfaces like finite element spaces etc.

+- Added a method, ParMesh::GetSerialMesh(), that reconstructs a partitioned
+  parallel mesh on a given single rank. Also, added ParMesh::PrintAsSerial(),
+  which saves the reconstructed serial mesh to a C++ stream on rank 0.
+
+- Added more 3D TMOP metrics, as well as specialized metrics for mesh
+  untangling and worst-case quality improvement.
+
+- Added a new method, Mesh::NodesUpdated, which should be called after the mesh
+  node coordinates have changed, e.g. after the mesh has moved. This is
+  necessary, for example, with device assembly of linear and bilinear forms.
+
+- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
+
 Discretization improvements
 ---------------------------
- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration with arbitrary user-supplied coefficients.
-
- Added support for partial assembly and fully matrix-free operators on mixed
-  meshes (different element types and p-adaptivity) through libCEED, including
-  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
-  currently limited by MFEM capabilities, i.e. 2D serial meshes. All mixed
-  element topologies are supported in serial and parallel: segment, triangle,
-  square, tetrahedron, cube, prism, and pyramid.
-
 - Added full assembly and device support for several LinearForm integrators:
  * DomainLF: (f, v)
  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
  * DomainLFGrad: (f, grad(v))
  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
+  The device assembly of linear forms has to be explicitly enabled by calling
+  LinearForm::UseFastAssembly(true), otherwise the legacy linear form assembly
+  is used by default.

- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
-  spatial Gaussian white noise.
-
- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
-  See fem/estimators.hpp.
-
- Fixes and improvements in LinearFormExtension.
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration with arbitrary user-supplied coefficients.

 - Added a new class FaceQuadratureSpace that allows for the construction of
  QuadratureFunctions on the interior or boundary faces of a mesh.
@@ -52,13 +116,24 @@ Discretization improvements
 - Added a class CoefficientVector for efficient access of variable coefficient
  values at quadrature points (in particular for GPU/device kernels).

+- Added support for GridFunction::GetGradients() and
+  GriFunction::GetVectorGradient() on face-neighbor elements.
+
+- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
+  spatial Gaussian white noise.
+
+- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
+  See fem/estimators.hpp.
+
+- Various fixes and improvements in LinearFormExtension.
+
 Linear and nonlinear solvers
 ----------------------------
- Added a new class DGMassInverse that performs a local elementwise CG
+- Added a new class DGMassInverse that performs a local element-wise CG
  iteration to solve systems involving the discontinuous Galerkin mass matrix,
  including support for device/GPU acceleration.

- Added more flexibility to the constraint solver classes:
+- Added more flexibility to the constrained solver classes:
  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
    (necessary for penalty contact)
  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
@@ -67,6 +142,9 @@ Linear and nonlinear solvers
 - Added functions to toggle additional options for the SuperLU_Dist and Hypre
   preconditioners (ParaSails, Euclid, ILU).

+- Added boundary elimination with device support for `SparseMatrix` and
+  `HypreParMatrix`.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new elasticity miniapp, Hooke, that showcases a low-level approach of
@@ -75,12 +153,28 @@ New and updated examples and miniapps
  automatic differentiation tools like a native dual number implementation or a
  third party library such as Enzyme. See miniapps/elasticity for more details.

+- Added example for body-fitted volumetric and shape integration using the
+  Algoim library in miniapps/shifted.
+
 - Add a new example code, Example 33/33p, to demonstrate the solution of
  spectral fractional PDEs with MFEM.

 Integrations, testing and documentation
 ---------------------------------------
 - Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
+  More sophisticated developer containers are available in the new repo
+  https://github.com/mfem/containers.
+
+- Added support for the LLVM-based automatic differentiation tool Enzyme, see
+  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
+  header are provided. The functionality and interaction are demonstrated in a
+  new miniapp in miniapps/elasticity.
+
+- Added support for partial assembly and fully matrix-free operators on mixed
+  meshes (different element types and p-adaptivity) through libCEED, including
+  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
+  currently limited to 2D serial meshes. All mixed element topologies are
+  supported in both serial and parallel.

 - Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
  which provides parallel non-conforming, non-matching, variational, volumetric
@@ -88,28 +182,39 @@ Integrations, testing and documentation
  between arbitrarily distributed and unrelated finite element meshes in a
  variationally consistent way.

- Added support for the LLVM-based automatic differentiation tool Enzyme, see
-  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
-  header are provided. The functionality and interaction are demonstrated in a
-  new miniapp in miniapps/elasticity.
+- Fully encapsulated SUNDIALS `N_Vector` object within the `SundialsNVector`
+  class by removing deprecated (e.g. `HypreParVector::ToNVector`) and
+  non-deprecated (e.g. `Vector::ToNVector`) functions in other classes.

- Added example for body-fitted volumetric and shape integration using the
-  Algoim library.
+- New benchmark for the different assembly levels inspired by the CEED
+  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.

 - Added Windows 2022 CI testing with GitHub actions.

 Miscellaneous
 -------------
- Various other simplifications, extensions, and bugfixes in the code.
+- The method SparseMatrix::EnsureMultTranspose() is now automatically called
+  by the methods AddMultTranspose(), MultTranspose(), and AbsMultTranspose().
+  Added a method with the same name to class HypreParMatrix which is also called
+  automatically by the HypreParMatrix::MultTranspose() methods.

- Added boundary elimination with device support for `SparseMatrix` and
-  `HypreParMatrix`.
+- Updated various MemoryUsage methods to return 'std::size_t' instead of 'long'
+  since the latter is 32-bit in Win64 builds.

 - When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
  outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
  `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).

- Added TMOP metrics for mesh untangling and worst-case quality improvement.
+- In various places in the library, replace the use of 'long' with 'long long'
+  to better support Win64 builds where 'long' is 32-bit and 'long long' is
+  64-bit. On Linux and MacOS, both types are typically 64-bit.
+
+- The behavior of GridFunction::GetTrueVector() has been changed to not return
+  an empty true vector.
+
+- Added support for ordering search points byVDIM in FindPointsGSLIB.
+
+- Various other simplifications, extensions, and bugfixes in the code.


 Version 4.4, released on March 21, 2022
@@ -143,11 +248,6 @@ Meshing improvements
 - Added a simpler interface to access mesh face information, see FaceInformation
  and GetFaceInformation in the Mesh class.

- Added the method ParMesh::GetSerialMesh() that reconstructs a partitioned
-  parallel mesh on a given single rank. Also, added the method
-  ParMesh::PrintAsSerial() that saves the reconstructed serial mesh to a C++
-  stream on rank 0.
-
 - Gmsh meshes where all elements have zero physical tag (the default Gmsh output
  format if no physical groups are defined) are now successfully loaded, and
  elements are reassigned attribute number 1.
@@ -237,9 +337,6 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
  formatting. See the "make style" target.

- New benchmark for the different assembly levels inspired by the CEED
-  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
-
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
@@ -250,13 +347,6 @@ Miscellaneous

 - Fixed several MinGW build issues on Windows.

- In various places in the library, replace the use of 'long' with 'long long'
-  to better support Win64 builds where 'long' is 32-bit and 'long long' is
-  64-bit. On Linux and MacOS, both types are typically 64-bit.
-
- Update various "MemoryUsage" methods to return 'std::size_t' instead of 'long'
-  since the latter is 32-bit in Win64 builds.
-
 - Added 'double' atomicAdd implementation for previous versions of CUDA.

 - HypreParVector and Vector now support C++ move semantics, and the copy
@@ -10,7 +10,9 @@
 # CONTRIBUTING.md for details.

 # The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-cmake_minimum_required(VERSION 3.1)
+# Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
+# Version 3.8 or newer is required for direct CUDA support.
+cmake_minimum_required(VERSION 3.8)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
  "Path to optional user configuration file.")

@@ -51,7 +53,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.4.1)
+set(${PROJECT_NAME}_VERSION 4.5.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -66,8 +68,7 @@ set(XSDK_ENABLE_C OFF)
 set(XSDK_ENABLE_Fortran OFF)

 # Check if we need to enable C or Fortran.
-if (CMAKE_VERSION VERSION_LESS 3.2 OR
-    MFEM_USE_CONDUIT OR
+if (MFEM_USE_CONDUIT OR
    MFEM_USE_SIDRE OR
    MFEM_USE_PETSC)
   # This seems to be needed by:
@@ -81,7 +82,13 @@ if (MFEM_USE_STRUMPACK)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
-if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
+# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
+if ((MFEM_USE_SUNDIALS OR
+     MFEM_USE_STRUMPACK OR
+     MFEM_USE_GINKGO OR
+     MFEM_USE_RAJA OR
+     MFEM_USE_UMPIRE) AND
+    ("${CMAKE_CXX_STANDARD}" LESS "14"))
  set(CMAKE_CXX_STANDARD 14)
 endif()

@@ -99,17 +106,12 @@ if (MFEM_USE_CUDA)
   if (MFEM_USE_HIP)
      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
   endif()
-   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
-   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
   if (NOT CMAKE_CUDA_HOST_COMPILER)
      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
   endif()
   enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO)
-     set(CMAKE_CUDA_STANDARD 14)
-   endif()
+   set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
   set(CMAKE_CUDA_EXTENSIONS OFF)
   set(CUDA_FLAGS "--expt-extended-lambda")
@@ -132,12 +134,11 @@ if (MFEM_USE_CUDA)
      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
   endif()
-   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
-       "CUDA flags set for MFEM" FORCE)
+   set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS})
   set(CUSPARSE_FOUND TRUE)
   set(CUSPARSE_LIBRARIES "cusparse")
   set(CUBLAS_FOUND TRUE)
-   set(CUSBLAS_LIBRARIES "cublas")
+   set(CUBLAS_LIBRARIES "cublas")
 endif()

 if (XSDK_ENABLE_C)
@@ -195,15 +196,35 @@ if (MFEM_USE_HIP)
  find_package(HIPSPARSE REQUIRED)
 endif()

+# OpenMP
+if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
+  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
+    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
+  endif()
+  find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
+  if(APPLE)
+    # On macOS, the compiler needs additional help to find the <omp.h> header.
+    # See issue #2642 for more information.
+    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
+  endif(APPLE)
+  if (OPENMP_FOUND)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+    if (MFEM_USE_CUDA)
+      set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+    endif()
+  endif()
+endif()
+
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
  set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
  if (MFEM_MPIEXEC)
-    set(MPIEXEC ${MFEM_MPIEXEC})
+    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
  endif()
  if (MFEM_MPIEXEC_NP)
-    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
+    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
  endif()
  # Parallel MFEM depends on hypre
  find_package(HYPRE REQUIRED)
@@ -260,20 +281,6 @@ if (MFEM_USE_LAPACK)
  find_package(LAPACK REQUIRED)
 endif()

-# OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
-  endif()
-  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
-endif()
-
 # SuiteSparse (before SUNDIALS which may depend on KLU)
 if (MFEM_USE_SUITESPARSE)
  find_package(SuiteSparse REQUIRED
@@ -292,11 +299,6 @@ if (MFEM_USE_SUNDIALS)
  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

-# Mesquite
-if (MFEM_USE_MESQUITE)
-  find_package(Mesquite REQUIRED)
-endif()
-
 # SuperLU_DIST can only be enabled in parallel
 if (MFEM_USE_SUPERLU)
  if (MFEM_USE_MPI)
@@ -402,6 +404,15 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

+# IpOpt optimizer
+if (MFEM_USE_IPOPT)
+  find_package(IPOPT REQUIRED)
+  message(
+      STATUS
+      "IPOPT_INCLUDE_DIRS=${IPOPT_INCLUDE_DIRS}, IPOPT_LIBRARIES=${IPOPT_LIBRARIES}, IPOPT_DIR=${IPOPT_DIR}")
+  # find_package updates IPOPT_FOUND, IPOPT_INCLUDE_DIRS, IPOPT_LIBRARIES
+endif()
+
 # CoDiPack package
 if (MFEM_USE_CODIPACK)
  find_package(CODIPACK REQUIRED)
@@ -484,6 +495,10 @@ endif()
 #   an ALIAS target is missing?
 # Call Stack (most recent call first):
 #   CMakeLists.txt:474 (mfem_add_library)
+#
+# NOTE: We need to figure out which TPL library adds the dependency on
+#       "Threads::Threads" and call the next line only when that TPL library is
+#       enabled.  -V. Dobrev
 find_package(Threads REQUIRED)

 # List all possible libraries in order of dependencies.
@@ -491,9 +506,9 @@ find_package(Threads REQUIRED)
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist METIS SuiteSparse SUNDIALS
-    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
-    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
+set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
+    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
+    NETCDF MPFR PUMI HIOP IPOPT POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

@@ -507,18 +522,12 @@ foreach(TPL IN LISTS MFEM_TPLS)
    list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
  endif()
 endforeach(TPL)
+list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
+list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")

-if (OPENMP_FOUND)
-  message(STATUS "MFEM: using package OpenMP")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-  if (MFEM_USE_CUDA)
-    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-  endif()
-endif()
-
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -697,6 +706,8 @@ add_subdirectory(doc)
 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
 set(INSTALL_INCLUDE_DIR include
  CACHE PATH "Relative path for installing header files.")
+set(INSTALL_BIN_DIR bin
+  CACHE PATH "Relative path for installing the binaries.")
 set(INSTALL_LIB_DIR lib
  CACHE PATH "Relative path for installing the library.")
 # other options: "share/mfem/cmake", "lib/mfem/cmake"
@@ -715,7 +726,9 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Development)
 # Install the library
 install(TARGETS ${PROJECT_NAME}
  EXPORT ${PROJECT_NAME_UC}Targets
-  DESTINATION ${INSTALL_LIB_DIR})
+  RUNTIME DESTINATION ${INSTALL_BIN_DIR}
+  LIBRARY DESTINATION ${INSTALL_LIB_DIR}
+  ARCHIVE DESTINATION ${INSTALL_LIB_DIR})

 # Install the master headers
 foreach(Header mfem.hpp mfem-performance.hpp)
@@ -112,6 +112,7 @@ The MFEM source code has the following structure:
  │   ├── caliper
  │   ├── ginkgo
  │   ├── hiop
+  │   ├── ipopt
  │   ├── jupyter
  │   ├── moonolith
  │   ├── petsc
@@ -7,6 +7,10 @@

                               https://mfem.org

+This file provides a detailed description of how to build and install the MFEM
+library. For a simple build, see the step-by-step instructions on the website
+at https://mfem.org/building.
+
 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
 of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
@@ -19,7 +23,7 @@ requires an MPI C++ compiler, as well as the following external libraries:
  https://github.com/mfem/tpls

  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
-  webpage, https://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
+  webpage, http://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
  and we don't support yet the new repo https://github.com/KarypisLab/METIS.

 The hypre dependency can be downloaded as a tarball from GitHub or from the
@@ -333,10 +337,6 @@ MFEM_USE_SUNDIALS = YES/NO
   library. When enabled, this option uses the SUNDIALS_* library options,
   see below.

-MFEM_USE_MESQUITE = YES/NO
-   Enable MFEM functionality based on the Mesquite library. When enabled, this
-   option uses the MESQUITE_* library options, see below.
-
 MFEM_USE_SUITESPARSE = YES/NO
   Enable MFEM functionality based on the SuiteSparse library. Currently, this
   option adds the classes UMFPackSolver and KLUSolver (both sparse serial
@@ -471,6 +471,9 @@ MFEM_USE_HIOP = YES/NO
   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
   HPC solver for nonlinear optimization problems.

+MFEM_USE_IPOPT = YES/NO
+   Enable the usage of Ipopt in MFEM.
+
 MFEM_USE_CODIPACK = YES/NO
  Enable automatic differentiation using the CoDiPack library.
  www.scicomp.uni-kl.de/codi/
@@ -632,11 +635,6 @@ The specific libraries and their options are:
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.

- Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
-  URL: http://trilinos.org/oldsite/packages/mesquite
-  Options: MESQUITE_OPT, MESQUITE_LIB.
-  The Mesquite support is deprecated and will be removed in the future.
-
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
  URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
  Options: SUITESPARSE_OPT, SUITESPARSE_LIB.
@@ -743,6 +741,11 @@ The specific libraries and their options are:
  Options: HIOP_OPT, HIOP_LIB.
  Versions: HIOP >= 0.4.6.

+- Ipopt (optional), used when MFEM_USE_IPOPT = YES.
+  URL: https://github.com/coin-or/Ipopt
+  Options: IPOPT_OPT, IPOPT_LIB.
+  Versions: IPOPT >= 3.14
+
 - CoDiPack (optional), used with MFEM_USE_CODIPACK = YES
  URL: https://www.scicomp.uni-kl.de/codi/
  Options: CODIPACK_OPT
@@ -758,12 +761,12 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.7.

- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
+- ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
  headers so it just needs to be downloaded at the same level as MFEM. Download
  the specific version we use as:
  "git clone https://github.com/algoim/algoim.git;
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
+  ALGOIM depends on BLITZ and the library must be built prior to the MFEM build.
  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
  "ln -s blitz-1.0.2 blitz".
  Build Blitz using CMake as:
@@ -798,10 +801,10 @@ The specific libraries and their options are:
  Versions: libCEED >= 0.10.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.3, only RAJA v0.14.0+ is supported.
+  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.14.0.
+  Versions: RAJA >= 2022.10.3.

 - Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
  URL: https://bitbucket.org/zulianp/par_moonolith
@@ -965,7 +968,6 @@ MFEM_USE_LEGACY_OPENMP
 MFEM_USE_OPENMP
 MFEM_USE_MEMALLOC
 MFEM_TIMER_TYPE - Set automatically, can be overwritten.
-MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_MUMPS
@@ -978,6 +980,7 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
+MFEM_USE_IPOPT
 MFEM_USE_CODIPACK
 MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
@@ -1030,7 +1033,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - HYPRE
 - METIS - The option MFEM_USE_METIS_5 is auto-detected.
 - ParMETIS
- - MESQUITE
 - SuiteSparse
 - SuperLUDist, STRUMPACK
 - Ginkgo
@@ -1042,6 +1044,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - POSIXCLOCKS
 - PUMI
 - HIOP
+ - IPOPT
 - CoDiPack
 - OCCA
 - RAJA
@@ -212,10 +212,6 @@ IF (DEFINED TPL_ENABLE_SUNDIALS)
  SET(MFEM_USE_SUNDIALS ${TPL_ENABLE_SUNDIALS} CACHE BOOL "Enable SUNDIALS usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_MESQUITE)
-  SET(MFEM_USE_MESQUITE ${TPL_ENABLE_MESQUITE} CACHE BOOL "Enable MESQUITE usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_SUITESPARSE)
  SET(MFEM_USE_SUITESPARSE ${TPL_ENABLE_SUITESPARSE} CACHE BOOL "Enable SuiteSparse usage" FORCE)
 ENDIF()
@@ -29,7 +29,6 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
-set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
@@ -37,6 +36,7 @@ set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
+set(MFEM_USE_IPOPT @MFEM_USE_IPOPT@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
@@ -77,9 +77,6 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC

-// Enable MFEM functionality based on the Mesquite library.
-#cmakedefine MFEM_USE_MESQUITE
-
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE

@@ -134,6 +131,9 @@
 // Enable MFEM functionality based on the HiOp library
 #cmakedefine MFEM_USE_HIOP

+// Enable MFEM functionality based on the Ipopt library
+#cmakedefine MFEM_USE_IPOPT
+
 // Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
@@ -16,7 +16,7 @@

 include(MfemCmakeUtilities)
 mfem_find_package(Algoim ALGOIM ALGOIM_DIR
-       	"include" "algoim_quad.hpp"
+        "include;src" "algoim_quad.hpp"
       	"" ""
-	"Paths to headers required by Algoim."
-	"Libraries required by Algoim.")
+        "Paths to headers required by Algoim."
+        "Libraries required by Algoim.")
@@ -16,7 +16,22 @@

 include(MfemCmakeUtilities)
 mfem_find_package(Caliper CALIPER CALIPER_DIR
-       	"include" "caliper/cali.h"
-       	"lib" "caliper"
-  	"Paths to headers required by Caliper."
-       	"Libraries required by Caliper.")
+      "include" "caliper/cali.h"
+      "lib" "caliper"
+      "Paths to headers required by Caliper."
+      "Libraries required by Caliper.")
+
+# Append adiak path/lib if the user provided ADIAK_DIR
+if(ADIAK_DIR AND EXISTS ${ADIAK_DIR})
+    find_package(adiak NO_DEFAULT_PATH REQUIRED PATHS ${ADIAK_DIR}/lib/cmake/adiak ${ADIAK_DIR})
+    list(APPEND CALIPER_INCLUDE_DIRS ${adiak_INCLUDE_DIRS})
+    list(APPEND CALIPER_LIBRARIES ${adiak_LIBRARIES})
+endif()
+
+# Append gotcha path/lib if the user provided GOTCHA_DIR
+if(GOTCHA_DIR AND EXISTS ${GOTCHA_DIR})
+    find_package(gotcha NO_DEFAULT_PATH REQUIRED PATHS ${GOTCHA_DIR}/lib/cmake/gotcha ${GOTCHA_DIR})
+    list(APPEND CALIPER_INCLUDE_DIRS ${gotcha_INCLUDE_DIRS})
+    list(APPEND CALIPER_LIBRARIES ${gotcha_LIBRARIES})
+endif()
+
@@ -14,9 +14,21 @@
 #   - HDF5_LIBRARIES - The HDF5 libraries
 #   - HDF5_INCLUDE_DIRS - The HDF5 include directories

+# NOTE: Using this FindHDF5.cmake instead of the CMake provided version may lead
+#       to issues with some TPL libraries that depend (or may depend) on HDF5.
+#       For this reason, we should consider removing this file, or at least
+#       making it use the CMake provided version by default and apply the logic
+#       below only when specifically requested by a user.  -V. Dobrev
+
 # First Check for HDF5_DIR
 if(NOT HDF5_DIR)
-    MESSAGE(FATAL_ERROR "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
+  message(FATAL_ERROR
+    "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
+endif()
+
+if (NOT HDF5_FIND_QUIETLY)
+  message(STATUS "Looking for HDF5 ...")
+  message(STATUS "   in HDF5_DIR = ${HDF5_DIR}")
 endif()

 # Find includes
@@ -50,8 +62,9 @@ include(FindPackageHandleStandardArgs)

 # Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
 # listed variables are TRUE
-find_package_handle_standard_args(HDF5  DEFAULT_MSG
+find_package_handle_standard_args(HDF5
+                                  " *** HDF5 not found. Please set HDF5_DIR."
+                                  HDF5_LIBRARIES
                                  HDF5_INCLUDE_DIRS
                                  __HDF5_LIBRARY
-                                  __HDF5_HL_LIBRARY
-                                  HDF5_LIBRARIES )
+                                  __HDF5_HL_LIBRARY)
@@ -9,12 +9,15 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# Defines the following variables:
-#   - MESQUITE_FOUND
-#   - MESQUITE_LIBRARIES
-#   - MESQUITE_INCLUDE_DIRS
+# Sets the following variables:
+#   - IPOPT_FOUND
+#   - IPOPT_INCLUDE_DIRS
+#   - IPOPT_LIBRARIES

 include(MfemCmakeUtilities)
-mfem_find_package(Mesquite MESQUITE MESQUITE_DIR
-  "include" "Mesquite_all_headers.hpp" "lib" "mesquite"
-  "Paths to headers required by Mesquite." "Libraries required by Mesquite.")
+mfem_find_package(IPOPT IPOPT IPOPT_DIR
+  "include" "IpTNLP.hpp"
+  "lib" "ipopt"
+  "Paths to headers required by IPOPT."
+  "Libraries required by IPOPT.")
+
@@ -17,18 +17,24 @@
 include(MfemCmakeUtilities)

 # FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
+# (MFEM's FindHDF5.cmake does not need HDF5_ROOT)
+# set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
+
 # We need to guard against the case where HDF5 was already found but without
 # the HL extensions (in which case mfem_find_package will treat the package
 # as already having been found), so we reset the variable to force FindHDF5.cmake
 # to be called for a second time
 set(HDF5_FOUND OFF)
 enable_language(C) # FindHDF5.cmake uses the C compiler
+
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
  "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
+
 # The HL extension libraries are in a separate variable and must precede
 # the "regular" hdf5 library, as hdf5_hl depends on hdf5
 # The netcdf library will always be the first element of NETCDF_LIBRARIES
 # and we need to insert after that library but before the hdf5 library, so
 # position 1 is used
-list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+# (MFEM's FindHDF5.cmake does not set HDF5_C_LIBRARY_hdf5_hl and the HL library
+# is already added to NETCDF_LIBRARIES)
+# list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
@@ -14,17 +14,27 @@
 #   - RAJA_LIBRARIES
 #   - RAJA_INCLUDE_DIRS

-include(MfemCmakeUtilities)
-mfem_find_package(RAJA RAJA RAJA_DIR "include" "RAJA/RAJA.hpp" "lib" "RAJA"
-  "Paths to headers required by RAJA." "Libraries required by RAJA.")
-
-if (NOT RAJA_CONFIG_CMAKE)
-   set(RAJA_CONFIG_CMAKE "${RAJA_DIR}/share/raja/cmake/raja-config.cmake")
+if (RAJA_FOUND)
+   return()
 endif()
-if (EXISTS "${RAJA_CONFIG_CMAKE}")
-   include("${RAJA_CONFIG_CMAKE}")
-   if (ENABLE_CUDA AND NOT MFEM_USE_CUDA)
-      message(FATAL_ERROR
-         "RAJA is built with CUDA: MFEM_USE_CUDA=YES is required")
+message(STATUS "Looking for RAJA ...")
+if (RAJA_DIR)
+   message(STATUS "   in RAJA_DIR = ${RAJA_DIR}")
+   find_package(RAJA CONFIG NO_DEFAULT_PATH PATHS "${RAJA_DIR}")
+endif()
+if (NOT RAJA_FOUND)
+   message(STATUS "   in standard CMake locations")
+   find_package(RAJA CONFIG)
+endif()
+if (RAJA_FOUND)
+   set(RAJA_LIBRARIES "RAJA" CACHE STRING "RAJA imported target." FORCE)
+   set(RAJA_INCLUDE_DIRS "" CACHE STRING "RAJA include dirs (not used)" FORCE)
+   message(STATUS
+      "Found RAJA target: ${RAJA_LIBRARIES} (version: ${RAJA_VERSION})")
+else()
+   set(msg STATUS)
+   if (RAJA_FIND_REQUIRED)
+      set(msg FATAL_ERROR)
   endif()
+   message(${msg} "RAJA not found. Please set RAJA_DIR to the RAJA prefix.")
 endif()
@@ -14,6 +14,23 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS

-find_package(umpire REQUIRED CONFIG)
+if (NOT umpire_DIR AND UMPIRE_DIR)
+  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
+endif()
+message(STATUS "Looking for UMPIRE ...")
+message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
+message(STATUS "      umpire_DIR = ${umpire_DIR}")
+find_package(umpire CONFIG)
 set(UMPIRE_FOUND ${umpire_FOUND})
 set(UMPIRE_LIBRARIES "umpire")
+if (UMPIRE_FOUND)
+  message(STATUS
+    "Found UMPIRE target: ${UMPIRE_LIBRARIES} (version: ${umpire_VERSION})")
+else()
+  set(msg STATUS)
+  if (UMPIRE_FIND_REQUIRED)
+    set(msg FATAL_ERROR)
+  endif()
+  message(${msg}
+    "UMPIRE not found. Please set UMPIRE_DIR to the install prefix.")
+endif()
@@ -46,6 +46,10 @@ endfunction()
 # Wrapper for add_executable
 macro(mfem_add_executable NAME)
  add_executable(${NAME} ${ARGN})
+  if (MFEM_USE_CUDA)
+    set_target_properties(${NAME} PROPERTIES
+      CUDA_RESOLVE_DEVICE_SYMBOLS ON)
+  endif()
 endmacro()

 # Wrapper for add_library
@@ -158,27 +162,12 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)

  # Append the additional libraries and options
  if (LIBRARIES_LIST)
-    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
-      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
-    else()
-      target_link_libraries(${MFEM_EXE_NAME} ${LIBRARIES_LIST})
-    endif()
+    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
  endif()
  if (EXTRA_OPTIONS_LIST)
    string(REPLACE ";" " " EXTRA_OPTIONS_STRING "${EXTRA_OPTIONS_LIST}")
    message(STATUS "${MFEM_EXE_NAME}: add flags \"${EXTRA_OPTIONS_STRING}\"")
-    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
-      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
-    else()
-      get_target_property(THIS_COMPILE_FLAGS ${MFEM_EXE_NAME} COMPILE_FLAGS)
-      if (THIS_COMPILE_FLAGS)
-        set(THIS_COMPILE_FLAGS "${THIS_COMPILE_FLAGS} ${EXTRA_OPTIONS_STRING}")
-      else()
-        set(THIS_COMPILE_FLAGS "${EXTRA_OPTIONS_STRING}")
-      endif()
-      set_target_properties(${MFEM_EXE_NAME}
-        PROPERTIES COMPILE_FLAGS ${THIS_COMPILE_FLAGS})
-    endif()
+    target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
  endif()
  if (EXTRA_DEFINES_LIST)
    target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
@@ -187,17 +176,15 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
  # Handle the MPI separately
  if (MFEM_USE_MPI)
    # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
-    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
-      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
-    else()
-      target_link_libraries(${MFEM_EXE_NAME} ${MPI_CXX_LIBRARIES})
-    endif()
+    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})

    if (MPI_CXX_INCLUDE_PATH)
      target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
    endif()
    if (MPI_CXX_COMPILE_FLAGS)
-      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_FLAGS})
+      separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
+        "${MPI_CXX_COMPILE_FLAGS}")
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
    endif()

    if (MPI_CXX_LINK_FLAGS)
@@ -878,11 +865,11 @@ function(mfem_export_mk_files)
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_IPOPT MFEM_USE_GSLIB MFEM_USE_CUDA
      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
@@ -85,9 +85,6 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS

-// Enable MFEM functionality based on the Mesquite library.
-// #define MFEM_USE_MESQUITE
-
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE

@@ -144,6 +141,9 @@
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP

+// Enable MFEM functionality based on the IPOPT library.
+// #define MFEM_USE_IPOPT
+
 // Enable MFEM functionality based on the GSLIB library
 // #define MFEM_USE_GSLIB

@@ -29,7 +29,6 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
-MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
@@ -47,6 +46,7 @@ MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
+MFEM_USE_IPOPT          = @MFEM_USE_IPOPT@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
@@ -30,7 +30,6 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
-option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
@@ -49,6 +48,7 @@ option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
+option(MFEM_USE_IPOPT "Enable Ipopt" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
 option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
@@ -124,9 +124,6 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")

-set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
-    "Path to the Mesquite library.")
-
 set(SuiteSparse_DIR "${MFEM_DIR}/../SuiteSparse" CACHE PATH
    "Path to the SuiteSparse library.")
 set(SuiteSparse_REQUIRED_PACKAGES "BLAS" "METIS"
@@ -188,6 +185,7 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")

+set(HDF5_DIR "/usr" CACHE PATH "Path to the HDF5 library.")
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
 set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
    "Additional packages required by NetCDF.")
@@ -223,6 +221,10 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
    "Directory where HiOp is installed")
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
    "Packages that HiOp depends on.")
+set(IPOPT_DIR "${MFEM_DIR}/../ipopt/install" CACHE STRING
+    "Directory where IpOpt is installed")
+set(IPOPT_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
+    "Packages that IpOpt depends on.")

 set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
@@ -230,17 +232,12 @@ set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")

 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-# If RAJA is built with external CAMP:
-# set(RAJA_REQUIRED_PACKAGES "camp"
-#     CACHE STRING "Packages that RAJA depends on.")
-# set(camp_DIR "${MFEM_DIR}/../camp/lib/cmake/camp"
-#     CACHE PATH "Path to CAMP CMake files.")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
 set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
 set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
-set(ALGOIM_REQUIRED_PACKAGES "BLITZ" CACHE STRING
+set(Algoim_REQUIRED_PACKAGES "Blitz" CACHE STRING
    "Packages that ALGOIM depends on.")

 set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
@@ -131,7 +131,6 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
-MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
@@ -149,6 +148,7 @@ MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
+MFEM_USE_IPOPT         = NO
 MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
 MFEM_USE_HIP           = NO
@@ -251,12 +251,16 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
-SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+# SUNDIALS >= 6.4.0 requires C++14:
+ifeq ($(MFEM_USE_SUNDIALS),YES)
+   BASE_FLAGS = -std=c++14
+endif
+SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
+ $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
+ -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
 -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
-
 ifeq ($(MFEM_USE_MPI),YES)
   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
 endif
@@ -266,11 +270,6 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES

-# MESQUITE library configuration
-MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
-MESQUITE_OPT = -I$(MESQUITE_DIR)/include
-MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
-
 # SuiteSparse library configuration
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
@@ -320,6 +319,9 @@ MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\

 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
+ifeq ($(MFEM_USE_STRUMPACK),YES)
+   BASE_FLAGS = -std=c++14
+endif
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 # If STRUMPACK was build with OpenMP support, the following may be need:
 # STRUMPACK_OPT += $(OPENMP_OPT)
@@ -446,6 +448,11 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)

+# IPOPT
+IPOPT_DIR = @MFEM_DIR@/../ipopt/install
+IPOPT_OPT = -I$(IPOPT_DIR)/include
+IPOPT_LIB = -L$(IPOPT_DIR)/lib -lipopt $(LAPACK_LIB)
+
 # CoDiPack
 CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
 CODIPACK_OPT = -I$(CODIPACK_DIR)
@@ -472,7 +479,17 @@ OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 # CALIPER library configuration
 CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
+CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 $(XLINKER)-rpath,$(CALIPER_DIR)/lib -L$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib -lcaliper
+
+ifdef ADIAK_DIR
+   CALIPER_OPT += -I$(ADIAK_DIR)/include
+   CALIPER_LIB += $(XLINKER)-rpath,$(ADIAK_DIR)/lib64 $(XLINKER)-rpath,$(ADIAK_DIR)/lib -L$(ADIAK_DIR)/lib64 -L$(ADIAK_DIR)/lib -ladiak
+endif
+ifdef GOTCHA_DIR
+   CALIPER_OPT += -I$(GOTCHA_DIR)/include
+   CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
+endif
+

 # BLITZ library configuration
 BLITZ_DIR = @MFEM_DIR@/../blitz
@@ -495,20 +512,29 @@ CEED_OPT = -I$(CEED_DIR)/include
 CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed

 # RAJA library configuration
+ifeq ($(MFEM_USE_RAJA),YES)
+   BASE_FLAGS = -std=c++14
+endif
 RAJA_DIR = @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
   RAJA_OPT += -I$(CUB_DIR)
 endif
+
+CAMP_LIB = -lcamp
 ifdef CAMP_DIR
   RAJA_OPT += -I$(CAMP_DIR)/include
+   CAMP_LIB = $(XLINKER)-rpath,$(CAMP_DIR)/lib -L$(CAMP_DIR)/lib -lcamp
 endif
-RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA
+RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA $(CAMP_LIB)

 # UMPIRE library configuration
+ifeq ($(MFEM_USE_UMPIRE),YES)
+   BASE_FLAGS = -std=c++14
+endif
 UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
-UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
+UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire $(CAMP_LIB)

 # MKL CPardiso library configuration
 MKL_CPARDISO_DIR ?=
@@ -58,6 +58,10 @@ groups_serial=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp"'
+'"ipopt"
+   "IpOpt examples:"
+   "examples/ipopt"
+   "ex10.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -215,6 +219,10 @@ groups_all=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp ex9p.cpp"'
+'"ipopt"
+   "IpOpt examples:"
+   "examples/ipopt"
+   "ex10.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -554,15 +562,14 @@ function go()
   local cmd_line="${1##+( )}"
   cmd_line="${cmd_line%%+( )}"
   shopt -u extglob
-   eval local cmd=(${cmd_line})
   local res=""
   echo $sep
   echo "<${group}>" "${cmd_line}"
   echo $sep
   if [ "${timing}" == "yes" ]; then
-      timed_run "${cmd[@]}"
+      timed_run eval "${cmd_line}"
   else
-      "${cmd[@]}"
+      eval "${cmd_line}"
   fi
   if [ "$?" -eq 0 ]; then
      res="${green}  OK  ${none}"
@@ -3,5 +3,5 @@
  "version-string": "5.1.0",
  "port-version": 0,
  "description": "Serial Graph Partitioning and Fill-reducing Matrix Ordering",
-  "homepage": "https://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
+  "homepage": "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
 }
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.4.1
+PROJECT_NUMBER         = v4.5.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -763,36 +763,55 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/config \
                         @MFEM_SOURCE_DIR@/general \
                         @MFEM_SOURCE_DIR@/linalg \
+                         @MFEM_SOURCE_DIR@/linalg/simd \
                         @MFEM_SOURCE_DIR@/mesh \
+                         @MFEM_SOURCE_DIR@/mesh/submesh \
                         @MFEM_SOURCE_DIR@/fem \
-                         @MFEM_SOURCE_DIR@/fem/moonolith \
+                         @MFEM_SOURCE_DIR@/fem/ceed \
+                         @MFEM_SOURCE_DIR@/fem/ceed/integrators \
+                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/convection \
+                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/diffusion \
+                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/mass \
+                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/nlconvection \
+                         @MFEM_SOURCE_DIR@/fem/ceed/interface \
+                         @MFEM_SOURCE_DIR@/fem/ceed/solvers \
                         @MFEM_SOURCE_DIR@/fem/fe \
                         @MFEM_SOURCE_DIR@/fem/lor \
+                         @MFEM_SOURCE_DIR@/fem/moonolith \
+                         @MFEM_SOURCE_DIR@/fem/qinterp \
+                         @MFEM_SOURCE_DIR@/fem/tmop \
                         @MFEM_SOURCE_DIR@/examples \
-                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/amgx \
+                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/ginkgo \
-                         @MFEM_SOURCE_DIR@/examples/moonolith \
                         @MFEM_SOURCE_DIR@/examples/hiop \
+                         @MFEM_SOURCE_DIR@/examples/ipopt \
+                         @MFEM_SOURCE_DIR@/examples/moonolith \
                         @MFEM_SOURCE_DIR@/examples/petsc \
                         @MFEM_SOURCE_DIR@/examples/pumi \
                         @MFEM_SOURCE_DIR@/examples/sundials \
                         @MFEM_SOURCE_DIR@/examples/superlu \
                         @MFEM_SOURCE_DIR@/miniapps/adjoint \
+                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
+                         @MFEM_SOURCE_DIR@/miniapps/hooke \
+                         @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
+                         @MFEM_SOURCE_DIR@/miniapps/hooke/materials \
+                         @MFEM_SOURCE_DIR@/miniapps/hooke/operators \
+                         @MFEM_SOURCE_DIR@/miniapps/hooke/preconditioners \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
                         @MFEM_SOURCE_DIR@/miniapps/mtop \
-                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
+                         @MFEM_SOURCE_DIR@/miniapps/multidomain \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
+                         @MFEM_SOURCE_DIR@/miniapps/parelag \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
                         @MFEM_SOURCE_DIR@/miniapps/shifted \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys \
-                         @MFEM_SOURCE_DIR@/miniapps/parelag
+                         @MFEM_SOURCE_DIR@/miniapps/toys

 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -178,6 +178,11 @@ if (MFEM_USE_HIOP)
  add_subdirectory(hiop)
 endif()

+# Include the examples/ipopt directory if IpOpt is enabled
+if (MFEM_USE_IPOPT)
+  add_subdirectory(ipopt)
+endif()
+
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -33,13 +33,13 @@ add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})
 if (MFEM_ENABLE_TESTING)
  foreach(SRC_FILE ${CALIPER_EXE_SRCS})
    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(REPLACE ".cpp" "" TEST_NAME ${PREFIX}${SRC_FILENAME})

    set(THIS_TEST_OPTIONS "-no-vis")

    if (NOT (${TEST_NAME} MATCHES ".*p$"))
      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+        COMMAND $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS})
    else()
      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
@@ -0,0 +1,810 @@
+//                                Contact example
+//
+// Compile with: make contact
+//
+// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
+// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "nodepair.hpp"
+
+using namespace std;
+using namespace mfem;
+
+bool ifequalarray(const Array<int> a1, const Array<int> a2)
+{
+   if (a1.Size()!=a2.Size())
+   {
+      return false;
+   }
+   for (int i=0; i<a1.Size(); i++)
+   {
+      if (a1[i] != a2[i])
+      {
+         return false;
+      }
+   }
+   return true;
+}
+
+void FindSurfaceToProject(Mesh& mesh, const int elem, int& cbdrface)
+{
+   Array<int> attr;
+   attr.Append(2);
+   Array<int> faces;
+   Array<int> ori;
+   std::vector<Array<int> > facesVertices;
+   std::vector<int > faceid;
+   mesh.GetElementFaces(elem, faces, ori);
+   int face = -1;
+   for (int i=0; i<faces.Size(); i++)
+   {
+      face = faces[i];
+      Array<int> faceVert;
+      if (!mesh.FaceIsInterior(face)) // if on the boundary
+      {
+         mesh.GetFaceVertices(face, faceVert);
+         faceVert.Sort();
+         facesVertices.push_back(faceVert);
+         faceid.push_back(face);
+      }
+   }
+   int bdrface = facesVertices.size();
+
+   Array<int> bdryFaces;
+   // This shoulnd't need to be rebuilt
+   std::vector<Array<int> > bdryVerts;
+   for (int b=0; b<mesh.GetNBE(); ++b)
+   {
+      if (attr.FindSorted(mesh.GetBdrAttribute(b)) >= 0)  // found the contact surface
+      {
+         bdryFaces.Append(b);
+         Array<int> vert;
+         mesh.GetBdrElementVertices(b, vert);
+         vert.Sort();
+         bdryVerts.push_back(vert);
+      }
+   }
+
+   int bdrvert = bdryVerts.size();
+   cbdrface = -1;  // the face number of the contact surface element
+   int count_cbdrface = 0;  // the number of matching surfaces, used for checks
+
+   for (int i=0; i<bdrface; i++)
+   {
+      for (int j=0; j<bdrvert; j++)
+      {
+         if (ifequalarray(facesVertices[i], bdryVerts[j]))
+         {
+            cbdrface = faceid[i];
+            count_cbdrface += 1;
+         }
+      }
+   }
+   MFEM_VERIFY(count_cbdrface == 1,"projection surface not found");
+
+};
+
+Vector GetNormalVector(Mesh & mesh, const int elem, const double *ref,
+                       int & refFace, int & refNormal, bool & interior)
+{
+   ElementTransformation *trans = mesh.GetElementTransformation(elem);
+   const int dim = mesh.Dimension();
+   const int spaceDim = trans->GetSpaceDim();
+
+   MFEM_VERIFY(spaceDim == 3, "");
+
+   Vector n(spaceDim);
+
+   IntegrationPoint ip;
+   ip.Set(ref, dim);
+
+   trans->SetIntPoint(&ip);
+   //CalcOrtho(trans->Jacobian(), n);  // Works only for face transformations
+   const DenseMatrix jac = trans->Jacobian();
+
+   int dimNormal = -1;
+   int normalSide = -1;
+
+   const double tol = 1.0e-8;
+   for (int i=0; i<dim; ++i)
+   {
+      const double d0 = std::abs(ref[i]);
+      const double d1 = std::abs(ref[i] - 1.0);
+
+      const double d = std::min(d0, d1);
+      // TODO: this works only for hexahedral meshes!
+
+      if (d < tol)
+      {
+         MFEM_VERIFY(dimNormal == -1, "");
+         dimNormal = i;
+
+         if (d0 < tol)
+         {
+            normalSide = 0;
+         }
+         else
+         {
+            normalSide = 1;
+         }
+      }
+   }
+   // closest point on the boundary
+   if (dimNormal < 0 || normalSide < 0) // node is inside the element
+   {
+      interior = 1;
+      Vector n(3);
+      n = 0.0;
+      return n;
+   }
+
+   MFEM_VERIFY(dimNormal >= 0 && normalSide >= 0, "");
+   refNormal = dimNormal;
+
+   MFEM_VERIFY(dim == 3, "");
+
+   {
+      // Find the reference face
+      if (dimNormal == 0)
+      {
+         refFace = (normalSide == 1) ? 2 : 4;
+      }
+      else if (dimNormal == 1)
+      {
+         refFace = (normalSide == 1) ? 3 : 1;
+      }
+      else
+      {
+         refFace = (normalSide == 1) ? 5 : 0;
+      }
+   }
+
+   std::vector<Vector> tang(2);
+
+   int tangDir[2] = {-1, -1};
+   {
+      int t = 0;
+      for (int i=0; i<dim; ++i)
+      {
+         if (i != dimNormal)
+         {
+            tangDir[t] = i;
+            t++;
+         }
+      }
+
+      MFEM_VERIFY(t == 2, "");
+   }
+
+   for (int i=0; i<2; ++i)
+   {
+      tang[i].SetSize(3);
+
+      Vector tangRef(3);
+      tangRef = 0.0;
+      tangRef[tangDir[i]] = 1.0;
+
+      jac.Mult(tangRef, tang[i]);
+   }
+
+   Vector c(3);  // Cross product
+
+   c[0] = (tang[0][1] * tang[1][2]) - (tang[0][2] * tang[1][1]);
+   c[1] = (tang[0][2] * tang[1][0]) - (tang[0][0] * tang[1][2]);
+   c[2] = (tang[0][0] * tang[1][1]) - (tang[0][1] * tang[1][0]);
+
+   c /= c.Norml2();
+
+   Vector nref(3);
+   nref = 0.0;
+   nref[dimNormal] = 1.0;
+
+   Vector ndir(3);
+   jac.Mult(nref, ndir);
+
+   ndir /= ndir.Norml2();
+
+   const double dp = ndir * c;
+
+   // TODO: eliminate c?
+   n = c;
+   if (dp < 0.0)
+   {
+      n *= -1.0;
+   }
+   interior = 0;
+   return n;
+}
+
+// WARNING: global variable, just for this little example.
+std::array<std::array<int, 3>, 8> HEX_VERT =
+{
+   {  {0,0,0},
+      {1,0,0},
+      {1,1,0},
+      {0,1,0},
+      {0,0,1},
+      {1,0,1},
+      {1,1,1},
+      {0,1,1}
+   }
+};
+
+int GetHexVertex(int cdim, int c, int fa, int fb, Vector & refCrd)
+{
+   int ref[3];
+   ref[cdim] = c;
+   ref[cdim == 0 ? 1 : 0] = fa;
+   ref[cdim == 2 ? 1 : 2] = fb;
+
+   for (int i=0; i<3; ++i) { refCrd[i] = ref[i]; }
+
+   int refv = -1;
+
+   for (int i=0; i<8; ++i)
+   {
+      bool match = true;
+      for (int j=0; j<3; ++j)
+      {
+         if (ref[j] != HEX_VERT[i][j]) { match = false; }
+      }
+
+      if (match) { refv = i; }
+   }
+
+   MFEM_VERIFY(refv >= 0, "");
+
+   return refv;
+}
+
+// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
+// where X is the list of x-coordinates for all points and so on.
+// conn: connectivity of the target surface elements
+// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
+void FindPointsInMesh(Mesh & mesh, Vector const& xyz, Array<int>& conn,
+                      Vector& xi)
+{
+   const int dim = mesh.Dimension();
+   const int np = xyz.Size() / dim;
+
+   MFEM_VERIFY(np * dim == xyz.Size(), "");
+
+   mesh.EnsureNodes();
+
+   //FindPointsGSLIB finder(MPI_COMM_WORLD);
+   FindPointsGSLIB finder;
+
+   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
+
+   const double bb_t = 0.5;
+   finder.Setup(mesh, bb_t);
+
+   finder.FindPoints(xyz);
+
+   /// Return code for each point searched by FindPoints: inside element (0), on
+   /// element boundary (1), or not found (2).
+   Array<unsigned int> codes = finder.GetCode();
+
+   /// Return element number for each point found by FindPoints.
+   Array<unsigned int> elems = finder.GetElem();
+
+   /// Return reference coordinates for each point found by FindPoints.
+   Vector refcrd = finder.GetReferencePosition();
+
+   /// Return distance between the sought and the found point in physical space,
+   /// for each point found by FindPoints.
+   Vector dist = finder.GetDist();
+
+   MFEM_VERIFY(dist.Size() == np, "");
+   MFEM_VERIFY(refcrd.Size() == np * dim, "");
+   MFEM_VERIFY(elems.Size() == np, "");
+   MFEM_VERIFY(codes.Size() == np, "");
+
+   bool allfound = true;
+   for (auto code : codes)
+      if (code == 2) { allfound = false; }
+
+   MFEM_VERIFY(allfound, "A point was not found");
+
+   cout << "Maximum distance of projected points: " << dist.Max() << endl;
+
+   // extract information
+   for (int i=0; i<np; ++i)
+   {
+      /*cout << "Point " << i << ": (";
+      for (int j=0; j<dim; ++j)
+      {
+         cout << xyz[i + (j*np)];
+         if (j == dim-1) {cout << ")" << endl;}
+         else{cout << ", ";}
+      }*/
+      //cout << "  element: " << elems[i] << endl;
+      //cout << "  element " << elems[i] << " vertices:" << endl;
+      //Array<int> vert;
+      //mesh.GetElementVertices(elems[i], vert);
+      //for (auto v : vert)
+      //{
+      //   cout << "    " << v << endl;
+      //}
+
+      /*cout << "  reference coordinates: (";
+      for (int j=0; j<dim; ++j)
+      {
+         cout << refcrd[(i*dim) + j];
+         if (j == dim-1)
+         {
+            cout << ")" << endl;
+         }
+         else
+         {
+            cout << ", ";
+         }
+      }*/
+
+      int refFace, refNormal, refNormalSide;
+      bool is_interior = -1;
+      Vector normal = GetNormalVector(mesh, elems[i], refcrd.GetData() + (i*dim),
+                                      refFace, refNormal, is_interior);
+      int phyFace;
+      if (is_interior)
+      {
+         phyFace = -1; // the id of the face that has the closest point
+         FindSurfaceToProject(mesh, elems[i], phyFace);
+
+         Array<int> cbdrVert;
+         mesh.GetFaceVertices(phyFace, cbdrVert);
+         Vector xs(dim);
+         xs[0] = xyz[i + 0*np];
+         xs[1] = xyz[i + 1*np];
+         xs[2] = xyz[i + 2*np];
+         Vector xi_tmp(dim-1);
+         // get nodes!
+
+         GridFunction *nodes = mesh.GetNodes();
+         DenseMatrix coords(4,3);
+         for (int i=0; i<4; i++)
+         {
+            for (int j=0; j<3; j++)
+            {
+               coords(i,j) = (*nodes)[cbdrVert[i]*3+j];
+            }
+         }
+         SlaveToMaster(coords, xs, xi_tmp);
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi[i*(dim-1)+j] = xi_tmp[j];
+         }
+         // now get get the projection to the surface
+      }
+      else
+      {
+         Vector faceRefCrd(dim-1);
+         {
+            int fd = 0;
+            for (int j=0; j<dim; ++j)
+            {
+               if (j == refNormal)
+               {
+                  refNormalSide = (refcrd[(i*dim) + j] > 0.5);
+               }
+               else
+               {
+                  faceRefCrd[fd] = refcrd[(i*dim) + j];
+                  fd++;
+               }
+            }
+
+            MFEM_VERIFY(fd == dim-1, "");
+         }
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
+         }
+         //cout << "  face reference coordinates: (";
+         /*for (int j=0; j<dim-1; ++j)
+         {
+            cout << faceRefCrd[j];
+            if (j == dim-2){cout << ")" << endl;}
+            else{cout << ", ";}
+         }*/
+      }
+      //cout << "  normal vector: ";
+      //normal.Print();
+
+      // ask, does this do anything?
+      /*
+      IntegrationPoint ip;
+      ip.Set(refcrd.GetData() + (i*dim), dim);
+      ElementTransformation *trans = mesh.GetElementTransformation(elems[i]);
+      Vector phys(trans->GetSpaceDim());
+      trans->Transform(ip, phys);
+      cout << "  physical coordinates: ";
+      phys.Print();
+      */
+
+      // Get the element face
+      Array<int> faces;
+      Array<int> ori;
+      int face;
+
+      if (is_interior)
+      {
+         face = phyFace;
+      }
+      else
+      {
+         mesh.GetElementFaces(elems[i], faces, ori);
+         face = faces[refFace];
+      }
+
+      Array<int> faceVert;
+      mesh.GetFaceVertices(face, faceVert);
+
+      //cout << "  face " << face << " vertices:" << endl;
+      //for (auto v : faceVert){ cout << "    " << v << endl;}
+
+      for (int p=0; p<4; p++)
+      {
+         conn[4*i+p] = faceVert[p];
+      }
+      /*
+      Vector ref(dim);
+
+      for (int p=0; p<2; ++p)
+         for (int q=0; q<2; ++q)
+         {
+            const int refv = GetHexVertex(refNormal, refNormalSide, p, q, ref);
+            cout << "  face reference vertex (" << p << "," << q
+                 << ") is global vertex " << vert[refv] << endl;
+
+            {
+               // Sanity check
+               ip.Set(ref.GetData(), dim);
+               trans->Transform(ip, phys);
+               for (int j=0; j<dim; ++j)
+               {
+                  phys[j] -= mesh.GetVertex(vert[refv])[j];
+               }
+          phys.Print();
+          cout<<vert[refv]<<endl;
+               cout<<mesh.GetVertex(vert[refv])[0]<<endl;
+               cout<<mesh.GetVertex(vert[refv])[1]<<endl;
+               cout<<mesh.GetVertex(vert[refv])[2]<<endl;
+               MFEM_VERIFY(phys.Norml2() < 1.0e-12, "Sanity check failed");
+            }
+         }*/
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file1 = "block1.mesh";
+   const char *mesh_file2 = "block2.mesh";
+
+   Array<int> attr;
+   Array<int> m_attr;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file1, "-m1", "--mesh1",
+                  "First mesh file to use.");
+   args.AddOption(&mesh_file2, "-m2", "--mesh2",
+                  "Second mesh file to use.");
+   args.AddOption(&attr, "-at", "--attributes-surf",
+                  "Attributes of boundary faces on contact surface for mesh 2.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   Mesh mesh1(mesh_file1, 1, 1);
+   Mesh mesh2(mesh_file2, 1, 1);
+
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mesh1a_sock(vishost, visport);
+      mesh1a_sock.precision(8);
+      mesh1a_sock << "mesh\n" << mesh1 << flush;
+      socketstream mesh2a_sock(vishost, visport);
+      mesh2a_sock.precision(8);
+      mesh2a_sock << "mesh\n" << mesh2 << flush;
+   }
+
+   const int dim = mesh1.Dimension();
+   MFEM_VERIFY(dim == mesh2.Dimension(), "");
+
+   // boundary attribute 2 is the potential contact surface of nodes
+   attr.Append(2);
+   // boundary attribute 2 is the potential contact surface for master surface
+   m_attr.Append(2);
+
+   //  Define a finite element space on the mesh. Here we use vector finite
+   //  elements, i.e. dim copies of a scalar finite element space. The vector
+   //  dimension is specified by the last argument of the FiniteElementSpace
+   //  constructor.
+   FiniteElementCollection *fec1;
+   FiniteElementSpace *fespace1;
+   fec1 = new H1_FECollection(1, dim);
+   fespace1 = new FiniteElementSpace(&mesh1, fec1, dim, Ordering::byVDIM);
+   cout << "Number of finite element unknowns for mesh1: "
+        << fespace1->GetTrueVSize() << endl;
+   mesh1.SetNodalFESpace(fespace1);
+   GridFunction nodes0 = *mesh1.GetNodes(); // undeformed mesh1 nodal grid function
+   GridFunction *nodes1 = mesh1.GetNodes();
+
+   FiniteElementCollection *fec2 = new H1_FECollection(1, dim);
+   FiniteElementSpace *fespace2 = new FiniteElementSpace(&mesh2, fec2, dim,
+                                                         Ordering::byVDIM);
+   cout << "Number of finite element unknowns for mesh2: "
+        << fespace2->GetTrueVSize() << endl;
+
+   // degrees of freedom of both meshes
+   int ndof_1 = fespace1->GetTrueVSize();
+   int ndof_2 = fespace2->GetTrueVSize();
+   int ndofs = ndof_1 + ndof_2;
+   // number of nodes for each mesh
+   int nnd_1 = mesh1.GetNV();
+   int nnd_2 = mesh2.GetNV();
+   int nnd = nnd_1 + nnd_2;
+   // Determine the list of true (i.e. conforming) essential boundary dofs.
+   // In this example, the boundary conditions are defined by marking only
+   // boundary attribute 1 from the mesh as essential and converting it to a
+   // list of true dofs.
+   Array<int> ess_tdof_list1, ess_bdr1(mesh1.bdr_attributes.Max());
+   ess_bdr1 = 0;
+   //ess_bdr1[0] = 1;
+   // Not ready to be passed on yet
+   // fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   Array<int> ess_tdof_list2, ess_bdr2(mesh2.bdr_attributes.Max());
+   ess_bdr2 = 0;
+   //ess_bdr2[0] = 1;
+
+   // Define the displacement vector x as a finite element grid function
+   // corresponding to fespace. GridFunction is a derived class of Vector.
+   GridFunction x1(fespace1);
+   x1 = 0.0;
+   GridFunction x2(fespace2);
+   x2 = 0.0;
+
+   // Generate force
+   LinearForm *b1 = new LinearForm(fespace1);
+   b1->Assemble();
+
+   LinearForm *b2 = new LinearForm(fespace2);
+   b2->Assemble();
+
+   // Set up the bilinear form a(.,.) on the finite element space
+   //  corresponding to the linear elasticity integrator with piece-wise
+   //  constants coefficient lambda and mu.
+   Vector lambda1(mesh1.attributes.Max());
+   lambda1 = 57.6923076923;
+   PWConstCoefficient lambda1_func(lambda1);
+   Vector mu1(mesh1.attributes.Max());
+   mu1 = 38.4615384615;
+   PWConstCoefficient mu1_func(mu1);
+
+   BilinearForm *a1 = new BilinearForm(fespace1);
+   a1->AddDomainIntegrator(new ElasticityIntegrator(lambda1_func,mu1_func));
+
+   Vector lambda2(mesh2.attributes.Max());
+   lambda2 = 57.6923076923;
+   PWConstCoefficient lambda2_func(lambda2);
+   Vector mu2(mesh2.attributes.Max());
+   mu2 = 38.4615384615;
+   PWConstCoefficient mu2_func(mu2);
+
+   BilinearForm *a2 = new BilinearForm(fespace2);
+   a2->AddDomainIntegrator(new ElasticityIntegrator(lambda2_func,mu2_func));
+
+   a1->Assemble();
+   SparseMatrix A1;
+   Vector B1, X1;
+   a1->FormLinearSystem(ess_tdof_list1, x1, *b1, A1, X1, B1);
+
+   a2->Assemble();
+   SparseMatrix A2;
+   Vector B2, X2;
+   a2->FormLinearSystem(ess_tdof_list2, x2, *b2, A2, X2, B2);
+
+   // Combine elasticity operator for two meshes into one.
+   // Block Matrix
+   SparseMatrix K(ndofs,ndofs);
+   for (int i=0; i<A1.Height(); i++)
+   {
+      Array<int> col_tmp;
+      Vector v_tmp;
+      col_tmp = 0;
+      v_tmp = 0.0;
+      A1.GetRow(i, col_tmp, v_tmp);
+      K.SetRow(i, col_tmp, v_tmp);
+   }
+   for (int i=0; i<A2.Height(); i++)
+   {
+      Array<int> col_tmp;
+      Vector v_tmp;
+      col_tmp = 0;
+      v_tmp = 0.0;
+      A2.GetRow(i, col_tmp, v_tmp);
+      for (int j=0; j<col_tmp.Size(); j++)
+      {
+         col_tmp[j] += ndof_1;
+      }
+      K.SetRow(i+ndof_1, col_tmp, v_tmp);  // mesh1 top left corner
+   }
+
+   // Construct node to segment contact constraint.
+
+   attr.Sort();
+   cout << "Boundary attributes for contact surface faces in mesh 2" << endl;
+   for (auto a : attr) { cout << a << endl; }
+
+   Array<int> bdryFaces2;  // TODO: remove this?
+
+   std::set<int> bdryVerts2;
+   for (int b=0; b<mesh2.GetNBE(); ++b)
+   {
+      if (attr.FindSorted(mesh2.GetBdrAttribute(b)) >= 0)
+      {
+         bdryFaces2.Append(b);
+         Array<int> vert;
+         mesh2.GetBdrElementVertices(b, vert);
+         for (auto v : vert)
+         {
+            bdryVerts2.insert(v);
+         }
+      }
+   }
+
+   int npoints = bdryVerts2.size();
+   Array<int> s_conn(npoints); // connectivity of the second/slave mesh
+   Vector xyz(dim * npoints);
+   xyz = 0.0;
+
+   cout << "Boundary vertices for contact surface vertices in mesh 2" << endl;
+
+   // construct the nodal coordinates on mesh2 to be projected, including displacement
+   int count = 0;
+   for (auto v : bdryVerts2)
+   {
+      cout << v << ": " << mesh2.GetVertex(v)[0] << ", "
+           << mesh2.GetVertex(v)[1] << ", "
+           << mesh2.GetVertex(v)[2] << endl;
+
+      for (int i=0; i<dim; ++i)
+      {
+         xyz[count + (i * npoints)] = mesh2.GetVertex(v)[i] + x2[v*dim+i];
+      }
+
+      s_conn[count] = v + nnd_1; // dof1 is the master
+      count++;
+   }
+
+   MFEM_VERIFY(count == npoints, "");
+
+   // gap function
+   Vector g(npoints*dim);
+   g = -1.0;
+   // segment reference coordinates of the closest point
+   Vector m_xi(npoints*(dim-1));
+   m_xi = -1.0;
+   Vector xs(dim*npoints);
+   xs = 0.0;
+   for (int i=0; i<npoints; i++)
+   {
+      for (int j=0; j<dim; j++)
+      {
+         xs[i*dim+j] = xyz[i + (j*npoints)];
+      }
+   }
+
+   Array<int> m_conn(
+      npoints*4); // only works for linear elements that have 4 vertices!
+   DenseMatrix coordsm(npoints*4, dim);
+
+   // adding displacement to mesh1 using a fixed grid function from mesh1
+   x1 = 1e-4; // x1 order: [xyz xyz... xyz]
+   add(nodes0, x1, *nodes1);
+
+   FindPointsInMesh(mesh1, xyz, m_conn, m_xi);
+
+   for (int i=0; i<npoints; i++)
+   {
+      for (int j=0; j<4; j++)
+      {
+         for (int k=0; k<dim; k++)
+         {
+            coordsm(i*4+j,k) = mesh1.GetVertex(m_conn[i*4+j])[k]+x1[dim*m_conn[i*4+j]+k];
+         }
+      }
+   }
+   //coordsm.Print();
+   SparseMatrix M(nnd,ndofs);
+   std::vector<SparseMatrix> dM(nnd, SparseMatrix(ndofs,ndofs));
+
+   Assemble_Contact(nnd, npoints, ndofs, xs, m_xi, coordsm,
+                    s_conn, m_conn, g, M, dM);
+
+   std::set<int> dirbdryv2;
+   for (int b=0; b<mesh2.GetNBE(); ++b)
+   {
+      if (mesh2.GetBdrAttribute(b) == 1)
+      {
+         Array<int> vert;
+         mesh2.GetBdrElementVertices(b, vert);
+         for (auto v : vert)
+         {
+            dirbdryv2.insert(v);
+         }
+      }
+   }
+   std::set<int> dirbdryv1;
+   for (int b=0; b<mesh1.GetNBE(); ++b)
+   {
+      if (mesh1.GetBdrAttribute(b) == 1)
+      {
+         Array<int> vert;
+         mesh1.GetBdrElementVertices(b, vert);
+         for (auto v : vert)
+         {
+            dirbdryv1.insert(v);
+         }
+      }
+   }
+
+   Array<int> Dirichlet_dof;
+   Array<double> Dirichlet_val;
+
+   for (auto v : dirbdryv2)
+   {
+      for (int i=0; i<dim; ++i)
+      {
+         Dirichlet_dof.Append(v*dim + i + ndof_1);
+         Dirichlet_val.Append(0.);
+      }
+   }
+   double delta = 0.1;
+   for (auto v : dirbdryv1)
+   {
+      Dirichlet_dof.Append(v*dim + 0);
+      Dirichlet_val.Append(delta);
+      Dirichlet_dof.Append(v*dim + 1);
+      Dirichlet_val.Append(0.);
+      Dirichlet_dof.Append(v*dim + 2);
+      Dirichlet_val.Append(0.);
+   }
+
+
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mesh1_sock(vishost, visport);
+      mesh1_sock.precision(8);
+      mesh1_sock << "mesh\n" << mesh1 << flush;
+      socketstream mesh2_sock(vishost, visport);
+      mesh2_sock.precision(8);
+      mesh2_sock << "mesh\n" << mesh2 << flush;
+   }
+
+   //M.Print();
+   /*Vector eps(ndofs);
+   Vector sol(ndofs); sol = 0.;
+   for(int i=0;i<ndofs;i++) eps[i] = 1e-5 * i ;
+   for(int i=0;i<9;i++)
+   {
+     cout<<i<<endl;
+     dM[s_conn[i]].Mult(eps,sol);
+     sol.Print();
+   }
+   */
+   return 0;
+}
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
-//             * ex1 -fa -d cuda
+//               ex1 -fa -d cuda
 //               ex1 -pa -d raja-cuda
 //             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
@@ -192,7 +192,14 @@ int main(int argc, char *argv[])
   //    domain integrator.
   BilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
+   if (fa)
+   {
+      a.SetAssemblyLevel(AssemblyLevel::FULL);
+      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
+      // when Device::IsEnabled() returns true). This makes the results
+      // bit-for-bit deterministic at the cost of somewhat longer run time.
+      a.EnableSparseMatrixSorting(Device::IsEnabled());
+   }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex1p -pa -d cuda
-//             * mpirun -np 4 ex1p -fa -d cuda
+//               mpirun -np 4 ex1p -fa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
@@ -219,7 +219,14 @@ int main(int argc, char *argv[])
   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
+   if (fa)
+   {
+      a.SetAssemblyLevel(AssemblyLevel::FULL);
+      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
+      // when Device::IsEnabled() returns true). This makes the results
+      // bit-for-bit deterministic at the cost of somewhat longer run time.
+      a.EnableSparseMatrixSorting(Device::IsEnabled());
+   }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
-      B.EnsureMultTranspose();
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -0,0 +1,60 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(IPOPT_EXAMPLES_SRCS)
+list(APPEND IPOPT_EXAMPLES_SRCS exContactBlockTL.cpp)
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_ipopt" target, see below.
+add_custom_target(test_ipopt
+  ${CMAKE_CTEST_COMMAND} -R ipopt USES_TERMINAL)
+
+# Add one executable per cpp file, adding "ipopt_" as prefix. Sets
+# "test_ipopt" as a target that depends on the given examples.
+set(PFX ipopt_)
+add_mfem_examples(IPOPT_EXAMPLES_SRCS ${PFX} "" test_ipopt)
+
+# Testing.
+# The IPOPT tests can be run separately using the target "test_ipopt"
+# which builds the examples and runs:
+#   ctest -R ipopt
+
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 9:
+  set(EXCONTACTBTL_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
+  set(EXCONTACTBTL_TEST_OPTS ${EXCONTACTBTL_COMMON_OPTS}  -r  2 )
+
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${IPOPT_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})
+
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -0,0 +1,19 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM for solving nonlinear constrained optimization problems, including
+features based on the IpOpt, a lightweight HPC solver for nonlinear optimization
+problems.
+
+To use the Ipopt features, make sure that MFEM is configured with the option
+"MFEM_USE_IPOPT = YES", see the top-level INSTALL file for details.
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.
@@ -0,0 +1,103 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+9
+1   5     0      1     3     2     8    9     11    10
+1   5     2      3     5     4    10    11    13    12
+1   5     4      5     7     6    12    13    15    14
+1   5     8      9    11    10    16    17    19    18
+1   5     10    11    13    12    18    19    21    20
+1   5     12    13    15    14    20    21    23    22
+1   5     16    17    19    18    24    25    27    26
+1   5     18    19    21    20    26    27    29    28
+1   5     20    21    23    22    28    29    31    30
+
+
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+30
+0   3    1     0     2     3
+0   3    3     2     4     5
+0   3    5     4     6     7
+0   3    24    25    27    26
+0   3    26    27    29    28
+0   3    28    29    31    30
+1   3    2     0     8    10
+1   3    4     2    10    12
+1   3    6     4    12    14
+1  3    10    8    16    18
+1  3    12    10    18    20
+1  3    14    12    20    22
+1  3    18    16    24    26
+1  3    20    18    26    28
+1  3    22    20    28    30
+2  3    1     3    11    9
+2  3     3     5    13    11
+2  3     5     7    15    13
+2  3    9    11    19    17
+2  3    11    13    21    19
+2  3    13    15    23    21
+2  3    17    19    27    25
+2  3    19    21    29    27
+2  3    21    23    31    29
+0  3     8     0     1    9
+0  3    16     8    9    17
+0  3    24    16    17    25
+0  3     6    14    15     7
+0  3    14    22    23    15
+0  3    22    30    31    23
+
+
+vertices
+32
+3
+   -1.0000         0         0
+         0         0         0
+   -1.0000    0.3333         0
+         0    0.3333         0
+   -1.0000    0.6667         0
+         0    0.6667         0
+   -1.0000    1.0000         0
+         0    1.0000         0
+   -1.0000         0    0.3333
+         0         0    0.3333
+   -1.0000    0.3333    0.3333
+         0    0.3333    0.3333
+   -1.0000    0.6667    0.3333
+         0    0.6667    0.3333
+   -1.0000    1.0000    0.3333
+         0    1.0000    0.3333
+   -1.0000         0    0.6667
+         0         0    0.6667
+   -1.0000    0.3333    0.6667
+         0    0.3333    0.6667
+   -1.0000    0.6667    0.6667
+         0    0.6667    0.6667
+   -1.0000    1.0000    0.6667
+         0    1.0000    0.6667
+   -1.0000         0    1.0000
+         0         0    1.0000
+   -1.0000    0.3333    1.0000
+         0    0.3333    1.0000
+   -1.0000    0.6667    1.0000
+         0    0.6667    1.0000
+   -1.0000    1.0000    1.0000
+         0    1.0000    1.0000
@@ -0,0 +1,68 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+# 1 nothing
+elements
+4
+1   5     0     1     3     2     6     7    9     8
+1   5     2     3     5     4     8     9    11    10
+1   5     6     7     9     8    12    13    15    14
+1   5     8     9    11    10    14    15    17    16
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+16
+0   3     1     0     2     3
+0   3     3     2     4     5
+0   3    12    13    15    14
+0   3    14    15    17    16
+2   3     2     0     6     8
+2   3     4     2     8    10
+2   3     8     6    12    14
+2   3    10     8    14    16
+1   3     1     3     9     7
+1   3     3     5    11    9
+1   3     7    9    15    13
+1   3     9    11    17    15
+0   3     6     0     1     7
+0   3    12     6     7    13
+0   3     4    10    11     5
+0   3    10    16    17    11
+
+vertices
+18
+3
+         0    0.2464    0.2464
+    0.5071    0.2464    0.2464
+         0    0.5000    0.2464
+    0.5071    0.5000    0.2464
+         0    0.7536    0.2464
+    0.5071    0.7536    0.2464
+         0    0.2464    0.5000
+    0.5071    0.2464    0.5000
+         0    0.5000    0.5000
+    0.5071    0.5000    0.5000
+         0    0.7536    0.5000
+    0.5071    0.7536    0.5000
+         0    0.2464    0.7536
+    0.5071    0.2464    0.7536
+         0    0.5000    0.7536
+    0.5071    0.5000    0.7536
+         0    0.7536    0.7536
+    0.5071    0.7536    0.7536
@@ -0,0 +1,742 @@
+//                                Contact example
+//
+// Compile with: make contact
+//
+// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
+// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "nodepair.hpp"
+
+using namespace std;
+using namespace mfem;
+
+bool ifequalarray(const Array<int> a1, const Array<int> a2)
+{
+   if (a1.Size()!=a2.Size())
+   {
+      return false;
+   }
+   for (int i=0; i<a1.Size(); i++)
+   {
+      if (a1[i] != a2[i])
+      {
+         return false;
+      }
+   }
+   return true;
+}
+
+void FindSurfaceToProject(Mesh& mesh, const int elem, int& cbdrface)
+{
+   Array<int> attr;
+   attr.Append(2);
+   Array<int> faces;
+   Array<int> ori;
+   std::vector<Array<int> > facesVertices;
+   std::vector<int > faceid;
+   mesh.GetElementFaces(elem, faces, ori);
+   int face = -1;
+   for (int i=0; i<faces.Size(); i++)
+   {
+      face = faces[i];
+      Array<int> faceVert;
+      if (!mesh.FaceIsInterior(face)) // if on the boundary
+      {
+         mesh.GetFaceVertices(face, faceVert);
+         faceVert.Sort();
+         facesVertices.push_back(faceVert);
+         faceid.push_back(face);
+      }
+   }
+   int bdrface = facesVertices.size();
+
+   Array<int> bdryFaces;
+   // This shoulnd't need to be rebuilt
+   std::vector<Array<int> > bdryVerts;
+   for (int b=0; b<mesh.GetNBE(); ++b)
+   {
+      if (attr.FindSorted(mesh.GetBdrAttribute(b)) >= 0)  // found the contact surface
+      {
+         bdryFaces.Append(b);
+         Array<int> vert;
+         mesh.GetBdrElementVertices(b, vert);
+         vert.Sort();
+         bdryVerts.push_back(vert);
+      }
+   }
+
+   int bdrvert = bdryVerts.size();
+   cbdrface = -1;  // the face number of the contact surface element
+   int count_cbdrface = 0;  // the number of matching surfaces, used for checks
+
+   for (int i=0; i<bdrface; i++)
+   {
+      for (int j=0; j<bdrvert; j++)
+      {
+         if (ifequalarray(facesVertices[i], bdryVerts[j]))
+         {
+            cbdrface = faceid[i];
+            count_cbdrface += 1;
+         }
+      }
+   }
+   MFEM_VERIFY(count_cbdrface == 1,"projection surface not found");
+
+};
+
+Vector GetNormalVector(Mesh & mesh, const int elem, const double *ref,
+                       int & refFace, int & refNormal, bool & interior)
+{
+   ElementTransformation *trans = mesh.GetElementTransformation(elem);
+   const int dim = mesh.Dimension();
+   const int spaceDim = trans->GetSpaceDim();
+
+   MFEM_VERIFY(spaceDim == 3, "");
+
+   Vector n(spaceDim);
+
+   IntegrationPoint ip;
+   ip.Set(ref, dim);
+
+   trans->SetIntPoint(&ip);
+   //CalcOrtho(trans->Jacobian(), n);  // Works only for face transformations
+   const DenseMatrix jac = trans->Jacobian();
+
+   int dimNormal = -1;
+   int normalSide = -1;
+
+   const double tol = 1.0e-8;
+   for (int i=0; i<dim; ++i)
+   {
+      const double d0 = std::abs(ref[i]);
+      const double d1 = std::abs(ref[i] - 1.0);
+
+      const double d = std::min(d0, d1);
+      // TODO: this works only for hexahedral meshes!
+
+      if (d < tol)
+      {
+         MFEM_VERIFY(dimNormal == -1, "");
+         dimNormal = i;
+
+         if (d0 < tol)
+         {
+            normalSide = 0;
+         }
+         else
+         {
+            normalSide = 1;
+         }
+      }
+   }
+   // closest point on the boundary
+   if (dimNormal < 0 || normalSide < 0) // node is inside the element
+   {
+      interior = 1;
+      Vector n(3);
+      n = 0.0;
+      return n;
+   }
+
+   MFEM_VERIFY(dimNormal >= 0 && normalSide >= 0, "");
+   refNormal = dimNormal;
+
+   MFEM_VERIFY(dim == 3, "");
+
+   {
+      // Find the reference face
+      if (dimNormal == 0)
+      {
+         refFace = (normalSide == 1) ? 2 : 4;
+      }
+      else if (dimNormal == 1)
+      {
+         refFace = (normalSide == 1) ? 3 : 1;
+      }
+      else
+      {
+         refFace = (normalSide == 1) ? 5 : 0;
+      }
+   }
+
+   std::vector<Vector> tang(2);
+
+   int tangDir[2] = {-1, -1};
+   {
+      int t = 0;
+      for (int i=0; i<dim; ++i)
+      {
+         if (i != dimNormal)
+         {
+            tangDir[t] = i;
+            t++;
+         }
+      }
+
+      MFEM_VERIFY(t == 2, "");
+   }
+
+   for (int i=0; i<2; ++i)
+   {
+      tang[i].SetSize(3);
+
+      Vector tangRef(3);
+      tangRef = 0.0;
+      tangRef[tangDir[i]] = 1.0;
+
+      jac.Mult(tangRef, tang[i]);
+   }
+
+   Vector c(3);  // Cross product
+
+   c[0] = (tang[0][1] * tang[1][2]) - (tang[0][2] * tang[1][1]);
+   c[1] = (tang[0][2] * tang[1][0]) - (tang[0][0] * tang[1][2]);
+   c[2] = (tang[0][0] * tang[1][1]) - (tang[0][1] * tang[1][0]);
+
+   c /= c.Norml2();
+
+   Vector nref(3);
+   nref = 0.0;
+   nref[dimNormal] = 1.0;
+
+   Vector ndir(3);
+   jac.Mult(nref, ndir);
+
+   ndir /= ndir.Norml2();
+
+   const double dp = ndir * c;
+
+   // TODO: eliminate c?
+   n = c;
+   if (dp < 0.0)
+   {
+      n *= -1.0;
+   }
+   interior = 0;
+   return n;
+}
+
+// WARNING: global variable, just for this little example.
+std::array<std::array<int, 3>, 8> HEX_VERT =
+{
+   {  {0,0,0},
+      {1,0,0},
+      {1,1,0},
+      {0,1,0},
+      {0,0,1},
+      {1,0,1},
+      {1,1,1},
+      {0,1,1}
+   }
+};
+
+int GetHexVertex(int cdim, int c, int fa, int fb, Vector & refCrd)
+{
+   int ref[3];
+   ref[cdim] = c;
+   ref[cdim == 0 ? 1 : 0] = fa;
+   ref[cdim == 2 ? 1 : 2] = fb;
+
+   for (int i=0; i<3; ++i) { refCrd[i] = ref[i]; }
+
+   int refv = -1;
+
+   for (int i=0; i<8; ++i)
+   {
+      bool match = true;
+      for (int j=0; j<3; ++j)
+      {
+         if (ref[j] != HEX_VERT[i][j]) { match = false; }
+      }
+
+      if (match) { refv = i; }
+   }
+
+   MFEM_VERIFY(refv >= 0, "");
+
+   return refv;
+}
+
+// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
+// where X is the list of x-coordinates for all points and so on.
+// conn: connectivity of the target surface elements
+// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
+void FindPointsInMesh(Mesh & mesh, Vector const& xyz, Array<int>& conn,
+                      Vector& xi)
+{
+   const int dim = mesh.Dimension();
+   const int np = xyz.Size() / dim;
+
+   MFEM_VERIFY(np * dim == xyz.Size(), "");
+
+   mesh.EnsureNodes();
+
+   //FindPointsGSLIB finder(MPI_COMM_WORLD);
+   FindPointsGSLIB finder;
+
+   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
+
+   const double bb_t = 0.5;
+   finder.Setup(mesh, bb_t);
+
+   finder.FindPoints(xyz);
+
+   /// Return code for each point searched by FindPoints: inside element (0), on
+   /// element boundary (1), or not found (2).
+   Array<unsigned int> codes = finder.GetCode();
+
+   /// Return element number for each point found by FindPoints.
+   Array<unsigned int> elems = finder.GetElem();
+
+   /// Return reference coordinates for each point found by FindPoints.
+   Vector refcrd = finder.GetReferencePosition();
+
+   /// Return distance between the sought and the found point in physical space,
+   /// for each point found by FindPoints.
+   Vector dist = finder.GetDist();
+
+   MFEM_VERIFY(dist.Size() == np, "");
+   MFEM_VERIFY(refcrd.Size() == np * dim, "");
+   MFEM_VERIFY(elems.Size() == np, "");
+   MFEM_VERIFY(codes.Size() == np, "");
+
+   bool allfound = true;
+   for (auto code : codes)
+      if (code == 2) { allfound = false; }
+
+   MFEM_VERIFY(allfound, "A point was not found");
+
+   cout << "Maximum distance of projected points: " << dist.Max() << endl;
+
+   // extract information
+   for (int i=0; i<np; ++i)
+   {
+      cout << "Point " << i << ": (";
+      for (int j=0; j<dim; ++j)
+      {
+         cout << xyz[i + (j*np)];
+         if (j == dim-1) {cout << ")" << endl;}
+         else {cout << ", ";}
+      }
+      //cout << "  element: " << elems[i] << endl;
+      //cout << "  element " << elems[i] << " vertices:" << endl;
+      //Array<int> vert;
+      //mesh.GetElementVertices(elems[i], vert);
+      //for (auto v : vert)
+      //{
+      //   cout << "    " << v << endl;
+      //}
+
+      /*cout << "  reference coordinates: (";
+      for (int j=0; j<dim; ++j)
+      {
+         cout << refcrd[(i*dim) + j];
+         if (j == dim-1)
+         {
+            cout << ")" << endl;
+         }
+         else
+         {
+            cout << ", ";
+         }
+      }*/
+
+      int refFace, refNormal, refNormalSide;
+      bool is_interior = -1;
+      Vector normal = GetNormalVector(mesh, elems[i], refcrd.GetData() + (i*dim),
+                                      refFace, refNormal, is_interior);
+      int phyFace;
+      if (is_interior)
+      {
+         phyFace = -1; // the id of the face that has the closest point
+         FindSurfaceToProject(mesh, elems[i], phyFace);
+
+         Array<int> cbdrVert;
+         mesh.GetFaceVertices(phyFace, cbdrVert);
+         Vector xs(dim);
+         xs[0] = xyz[i + 0*np];
+         xs[1] = xyz[i + 1*np];
+         xs[2] = xyz[i + 2*np];
+         Vector xi_tmp(dim-1);
+         // get nodes!
+
+         GridFunction *nodes = mesh.GetNodes();
+         DenseMatrix coords(4,3);
+         for (int i=0; i<4; i++)
+         {
+            for (int j=0; j<3; j++)
+            {
+               coords(i,j) = (*nodes)[cbdrVert[i]*3+j];
+            }
+         }
+         SlaveToMaster(coords, xs, xi_tmp);
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi[i*(dim-1)+j] = xi_tmp[j];
+         }
+         // now get get the projection to the surface
+      }
+      else
+      {
+         Vector faceRefCrd(dim-1);
+         {
+            int fd = 0;
+            for (int j=0; j<dim; ++j)
+            {
+               if (j == refNormal)
+               {
+                  refNormalSide = (refcrd[(i*dim) + j] > 0.5);
+               }
+               else
+               {
+                  faceRefCrd[fd] = refcrd[(i*dim) + j];
+                  fd++;
+               }
+            }
+
+            MFEM_VERIFY(fd == dim-1, "");
+         }
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
+         }
+         //cout << "  face reference coordinates: (";
+         for (int j=0; j<dim-1; ++j)
+         {
+            cout << faceRefCrd[j];
+            if (j == dim-2) {cout << ")" << endl;}
+            else {cout << ", ";}
+         }
+      }
+      //cout << "  normal vector: ";
+      //normal.Print();
+
+      // ask, does this do anything?
+      /*
+      IntegrationPoint ip;
+      ip.Set(refcrd.GetData() + (i*dim), dim);
+      ElementTransformation *trans = mesh.GetElementTransformation(elems[i]);
+      Vector phys(trans->GetSpaceDim());
+      trans->Transform(ip, phys);
+      cout << "  physical coordinates: ";
+      phys.Print();
+      */
+
+      // Get the element face
+      Array<int> faces;
+      Array<int> ori;
+      int face;
+
+      if (is_interior)
+      {
+         face = phyFace;
+      }
+      else
+      {
+         mesh.GetElementFaces(elems[i], faces, ori);
+         face = faces[refFace];
+      }
+
+      Array<int> faceVert;
+      mesh.GetFaceVertices(face, faceVert);
+
+      //cout << "  face " << face << " vertices:" << endl;
+      //for (auto v : faceVert){ cout << "    " << v << endl;}
+
+      for (int p=0; p<4; p++)
+      {
+         conn[4*i+p] = faceVert[p];
+      }
+      /*
+      Vector ref(dim);
+
+      for (int p=0; p<2; ++p)
+         for (int q=0; q<2; ++q)
+         {
+            const int refv = GetHexVertex(refNormal, refNormalSide, p, q, ref);
+            cout << "  face reference vertex (" << p << "," << q
+                 << ") is global vertex " << vert[refv] << endl;
+
+            {
+               // Sanity check
+               ip.Set(ref.GetData(), dim);
+               trans->Transform(ip, phys);
+               for (int j=0; j<dim; ++j)
+               {
+                  phys[j] -= mesh.GetVertex(vert[refv])[j];
+               }
+          phys.Print();
+          cout<<vert[refv]<<endl;
+               cout<<mesh.GetVertex(vert[refv])[0]<<endl;
+               cout<<mesh.GetVertex(vert[refv])[1]<<endl;
+               cout<<mesh.GetVertex(vert[refv])[2]<<endl;
+               MFEM_VERIFY(phys.Norml2() < 1.0e-12, "Sanity check failed");
+            }
+         }*/
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file1 = "block1.mesh";
+   const char *mesh_file2 = "block2.mesh";
+
+   Array<int> attr;
+   Array<int> m_attr;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file1, "-m1", "--mesh1",
+                  "First mesh file to use.");
+   args.AddOption(&mesh_file2, "-m2", "--mesh2",
+                  "Second mesh file to use.");
+   args.AddOption(&attr, "-at", "--attributes-surf",
+                  "Attributes of boundary faces on contact surface for mesh 2.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   Mesh mesh1(mesh_file1, 1, 1);
+   Mesh mesh2(mesh_file2, 1, 1);
+
+   const int dim = mesh1.Dimension();
+   MFEM_VERIFY(dim == mesh2.Dimension(), "");
+
+   // boundary attribute 2 is the potential contact surface of nodes
+   attr.Append(2);
+   // boundary attribute 2 is the potential contact surface for master surface
+   m_attr.Append(2);
+
+   //  Define a finite element space on the mesh. Here we use vector finite
+   //  elements, i.e. dim copies of a scalar finite element space. The vector
+   //  dimension is specified by the last argument of the FiniteElementSpace
+   //  constructor.
+   FiniteElementCollection *fec1;
+   FiniteElementSpace *fespace1;
+   fec1 = new H1_FECollection(1, dim);
+   fespace1 = new FiniteElementSpace(&mesh1, fec1, dim, Ordering::byVDIM);
+   cout << "Number of finite element unknowns for mesh1: "
+        << fespace1->GetTrueVSize() << endl;
+   mesh1.SetNodalFESpace(fespace1);
+   GridFunction nodes0 = *mesh1.GetNodes(); // undeformed mesh1 nodal grid function
+   GridFunction *nodes1 = mesh1.GetNodes();
+
+   FiniteElementCollection *fec2 = new H1_FECollection(1, dim);
+   FiniteElementSpace *fespace2 = new FiniteElementSpace(&mesh2, fec2, dim,
+                                                         Ordering::byVDIM);
+   cout << "Number of finite element unknowns for mesh2: "
+        << fespace2->GetTrueVSize() << endl;
+
+   // degrees of freedom of both meshes
+   int ndof_1 = fespace1->GetTrueVSize();
+   int ndof_2 = fespace2->GetTrueVSize();
+   int ndofs = ndof_1 + ndof_2;
+   // number of nodes for each mesh
+   int nnd_1 = mesh1.GetNV();
+   int nnd_2 = mesh2.GetNV();
+   int nnd = nnd_1 + nnd_2;
+   // Determine the list of true (i.e. conforming) essential boundary dofs.
+   // In this example, the boundary conditions are defined by marking only
+   // boundary attribute 1 from the mesh as essential and converting it to a
+   // list of true dofs.
+   Array<int> ess_tdof_list1, ess_bdr1(mesh1.bdr_attributes.Max());
+   cout<<mesh1.bdr_attributes.Max()<<endl;
+   ess_bdr1 = 0;
+   //ess_bdr1[0] = 1;
+   // Not ready to be passed on yet
+   // fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   Array<int> ess_tdof_list2, ess_bdr2(mesh2.bdr_attributes.Max());
+   ess_bdr2 = 0;
+   //ess_bdr2[0] = 1;
+
+   // Define the displacement vector x as a finite element grid function
+   // corresponding to fespace. GridFunction is a derived class of Vector.
+   GridFunction x1(fespace1);
+   x1 = 0.0;
+   GridFunction x2(fespace2);
+   x2 = 0.0;
+
+   // Generate force
+   LinearForm *b1 = new LinearForm(fespace1);
+   b1->Assemble();
+
+   LinearForm *b2 = new LinearForm(fespace2);
+   b2->Assemble();
+
+   // Set up the bilinear form a(.,.) on the finite element space
+   //  corresponding to the linear elasticity integrator with piece-wise
+   //  constants coefficient lambda and mu.
+   Vector lambda1(mesh1.attributes.Max());
+   lambda1 = 57.6923076923;
+   PWConstCoefficient lambda1_func(lambda1);
+   Vector mu1(mesh1.attributes.Max());
+   mu1 = 38.4615384615;
+   PWConstCoefficient mu1_func(mu1);
+
+   BilinearForm *a1 = new BilinearForm(fespace1);
+   a1->AddDomainIntegrator(new ElasticityIntegrator(lambda1_func,mu1_func));
+
+   Vector lambda2(mesh2.attributes.Max());
+   lambda2 = 57.6923076923;
+   PWConstCoefficient lambda2_func(lambda2);
+   Vector mu2(mesh2.attributes.Max());
+   mu2 = 38.4615384615;
+   PWConstCoefficient mu2_func(mu2);
+
+   BilinearForm *a2 = new BilinearForm(fespace2);
+   a2->AddDomainIntegrator(new ElasticityIntegrator(lambda2_func,mu2_func));
+
+   a1->Assemble();
+   SparseMatrix A1;
+   Vector B1, X1;
+   a1->FormLinearSystem(ess_tdof_list1, x1, *b1, A1, X1, B1);
+
+   a2->Assemble();
+   SparseMatrix A2;
+   Vector B2, X2;
+   a2->FormLinearSystem(ess_tdof_list2, x2, *b2, A2, X2, B2);
+
+   // Combine elasticity operator for two meshes into one.
+   // Block Matrix
+   SparseMatrix K(ndofs,ndofs);
+   for (int i=0; i<A1.Height(); i++)
+   {
+      Array<int> col_tmp;
+      Vector v_tmp;
+      col_tmp = 0;
+      v_tmp = 0.0;
+      A1.GetRow(i, col_tmp, v_tmp);
+      K.SetRow(i, col_tmp, v_tmp);
+   }
+   for (int i=0; i<A2.Height(); i++)
+   {
+      Array<int> col_tmp;
+      Vector v_tmp;
+      col_tmp = 0;
+      v_tmp = 0.0;
+      A2.GetRow(i, col_tmp, v_tmp);
+      for (int j=0; j<col_tmp.Size(); j++)
+      {
+         col_tmp[j] += ndof_1;
+      }
+      K.SetRow(i+ndof_1, col_tmp, v_tmp);  // mesh1 top left corner
+   }
+
+   // Construct node to segment contact constraint.
+
+   attr.Sort();
+   cout << "Boundary attributes for contact surface faces in mesh 2" << endl;
+   for (auto a : attr)
+   {
+      cout << a << endl;
+   }
+
+   Array<int> bdryFaces2;  // TODO: remove this?
+
+   std::set<int> bdryVerts2;
+   for (int b=0; b<mesh2.GetNBE(); ++b)
+   {
+      if (attr.FindSorted(mesh2.GetBdrAttribute(b)) >= 0)
+      {
+         bdryFaces2.Append(b);
+         Array<int> vert;
+         mesh2.GetBdrElementVertices(b, vert);
+         for (auto v : vert)
+         {
+            bdryVerts2.insert(v);
+         }
+      }
+   }
+
+   int npoints = bdryVerts2.size();
+   Array<int> s_conn(npoints); // connectivity of the second/slave mesh
+   Vector xyz(dim * npoints);
+   xyz = 0.0;
+
+   cout << "Boundary vertices for contact surface vertices in mesh 2" << endl;
+
+   // construct the nodal coordinates on mesh2 to be projected, including displacement
+   int count = 0;
+   for (auto v : bdryVerts2)
+   {
+      cout << v << ": " << mesh2.GetVertex(v)[0] << ", "
+           << mesh2.GetVertex(v)[1] << ", "
+           << mesh2.GetVertex(v)[2] << endl;
+
+      for (int i=0; i<dim; ++i)
+      {
+         xyz[count + (i * npoints)] = mesh2.GetVertex(v)[i] + x2[v*dim+i];
+      }
+
+      s_conn[count] = v + nnd_1; // dof1 is the master
+      count++;
+   }
+
+   MFEM_VERIFY(count == npoints, "");
+
+   // gap function
+   Vector g(npoints*dim);
+   g = -1.0;
+   // segment reference coordinates of the closest point
+   Vector m_xi(npoints*(dim-1));
+   m_xi = -1.0;
+   Vector xs(dim*npoints);
+   xs = 0.0;
+   for (int i=0; i<npoints; i++)
+   {
+      for (int j=0; j<dim; j++)
+      {
+         xs[i*dim+j] = xyz[i + (j*npoints)];
+      }
+   }
+
+   Array<int> m_conn(
+      npoints*4); // only works for linear elements that have 4 vertices!
+   DenseMatrix coordsm(npoints*4, dim);
+
+   // adding displacement to mesh1 using a fixed grid function from mesh1
+   x1 = 1e-4; // x1 order: [xyz xyz... xyz]
+   add(nodes0, x1, *nodes1);
+
+   FindPointsInMesh(mesh1, xyz, m_conn, m_xi);
+
+   for (int i=0; i<npoints; i++)
+   {
+      for (int j=0; j<4; j++)
+      {
+         for (int k=0; k<dim; k++)
+         {
+            coordsm(i*4+j,k) = mesh1.GetVertex(m_conn[i*4+j])[k]+x1[dim*m_conn[i*4+j]+k];
+         }
+      }
+   }
+   //coordsm.Print();
+   SparseMatrix M(nnd,ndofs);
+   std::vector<SparseMatrix> dM(nnd, SparseMatrix(ndofs,ndofs));
+
+   Assemble_Contact(nnd, npoints, ndofs, xs, m_xi, coordsm,
+                    s_conn, m_conn, g, M, dM);
+
+   //M.Print();
+   /*Vector eps(ndofs);
+   Vector sol(ndofs); sol = 0.;
+   for(int i=0;i<ndofs;i++) eps[i] = 1e-5 * i ;
+   for(int i=0;i<9;i++)
+   {
+     cout<<i<<endl;
+     dM[s_conn[i]].Mult(eps,sol);
+     sol.Print();
+   }
+   */
+   return 0;
+}
@@ -0,0 +1,230 @@
+//                                Contact example
+//
+// Compile with: make exContactBlockTL
+//
+// Sample runs:  ./exContactBlockTL -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
+// Sample runs:  ./exContactBlockTL -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
+
+#ifndef EXCONTACTBLOCKTL_HPP
+#define EXCONTACTBLOCKTL_HPP
+
+#include "mfem.hpp"
+#include "IpTNLP.hpp"
+
+using namespace std;
+using namespace mfem;
+using namespace Ipopt;
+
+
+class ExContactBlockTL: public TNLP
+{
+public:
+   /** default constructor */
+   ExContactBlockTL(int argc, char *argv[]);
+
+   /** default destructor */
+   virtual ~ExContactBlockTL();
+
+   /**@name Overloaded from TNLP */
+   /** Method to return some info about the nlp */
+   virtual bool get_nlp_info(
+      Index&          n,
+      Index&          m,
+      Index&          nnz_jac_g,
+      Index&          nnz_h_lag,
+      IndexStyleEnum& index_style
+   );
+
+   /** Method to return the bounds for my problem */
+   virtual bool get_bounds_info(
+      Index   n,
+      Number* x_l,
+      Number* x_u,
+      Index   m,
+      Number* g_l,
+      Number* g_u
+   );
+
+   /** Method to return the starting point for the algorithm */
+   virtual bool get_starting_point(
+      Index   n,
+      bool    init_x,
+      Number* x,
+      bool    init_z,
+      Number* z_L,
+      Number* z_U,
+      Index   m,
+      bool    init_lambda,
+      Number* lambda
+   );
+
+   /** Method to return the objective value */
+   virtual bool eval_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number&       obj_value
+   );
+
+   /** Method to return the gradient of the objective */
+   virtual bool eval_grad_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number*       grad_f
+   );
+
+   /** Method to return the constraint residuals */
+   virtual bool eval_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Number*       cons
+   );
+
+   /** Method to return:
+    *   1) The structure of the Jacobian (if "values" is NULL)
+    *   2) The values of the Jacobian (if "values" is not NULL)
+    */
+   virtual bool eval_jac_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Index         nele_jac,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   );
+
+   /** Method to return:
+    *   1) The structure of the Hessian of the Lagrangian (if "values" is NULL)
+    *   2) The values of the Hessian of the Lagrangian (if "values" is not NULL)
+    */
+   virtual bool eval_h(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number        obj_factor,
+      Index         m,
+      const Number* lambda,
+      bool          new_lambda,
+      Index         nele_hess,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   );
+
+   /** This method is called when the algorithm is complete so the TNLP can store/write the solution */
+   virtual void finalize_solution(
+      SolverReturn               status,
+      Index                      n,
+      const Number*              x,
+      const Number*              z_L,
+      const Number*              z_U,
+      Index                      m,
+      const Number*              g,
+      const Number*              lambda,
+      Number                     obj_value,
+      const IpoptData*           ip_data,
+      IpoptCalculatedQuantities* ip_cq
+   );
+
+private:
+   void update_g();
+   void update_jac();
+   void update_hess();
+
+private:
+   /**@name Methods to block default compiler methods.
+    *
+    * The compiler automatically generates the following three methods.
+    *  Since the default compiler implementation is generally not what
+    *  you want (for all but the most simple classes), we usually
+    *  put the declarations of these methods in the private section
+    *  and never implement them. This prevents the compiler from
+    *  implementing an incorrect "default" behavior without us
+    *  knowing. (See Scott Meyers book, "Effective C++")
+    */
+   ExContactBlockTL(
+      const ExContactBlockTL&
+   );
+
+   ExContactBlockTL& operator=(
+      const ExContactBlockTL&
+   );
+
+   Array<int> attr;
+   Array<int> m_attr;
+   Array<int> s_conn; // connectivity of the second/slave mesh
+   std::string mesh_file1;
+   std::string mesh_file2;
+   Mesh* mesh1;
+   Mesh* mesh2;
+   FiniteElementCollection* fec1;
+   FiniteElementCollection* fec2;
+   FiniteElementSpace* fespace1;
+   FiniteElementSpace* fespace2;
+   Array<int> ess_tdof_list1;
+   Array<int> ess_tdof_list2;
+   GridFunction nodes0;
+   GridFunction* nodes1;
+   GridFunction* nodes2;
+   GridFunction* x1;
+   GridFunction* x2;
+   LinearForm* b1;
+   LinearForm* b2;
+   PWConstCoefficient* lambda1_func;
+   PWConstCoefficient* lambda2_func;
+   PWConstCoefficient* mu1_func;
+   PWConstCoefficient* mu2_func;
+   BilinearForm* a1;
+   BilinearForm* a2;
+
+   mfem::Vector lambda1;
+   mfem::Vector lambda2;
+   mfem::Vector mu1;
+   mfem::Vector mu2;
+   mfem::Vector xyz;
+
+   std::set<int> bdryVerts2;
+
+   int dim;
+   // degrees of freedom of both meshes
+   int ndof_1;
+   int ndof_2;
+   int ndofs;
+   // number of nodes for each mesh
+   int nnd_1;
+   int nnd_2;
+   int nnd;
+
+   int npoints;
+
+   SparseMatrix A1;
+   mfem::Vector B1, X1;
+   SparseMatrix A2;
+   mfem::Vector B2, X2;
+
+   SparseMatrix* K;
+   mfem::Vector g;
+   mfem::Vector m_xi;
+   mfem::Vector xs;
+
+   Array<int> m_conn; // only works for linear elements that have 4 vertices!
+   DenseMatrix* coordsm;
+   SparseMatrix* M;
+
+   std::vector<SparseMatrix>* dM;
+
+   Array<int> Dirichlet_dof;
+   Array<double> Dirichlet_val;
+
+public: 
+   Mesh * GetMesh1() {return mesh1;}
+   Mesh * GetMesh2() {return mesh2;}
+
+};
+
+#endif
@@ -0,0 +1,68 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ipopt/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = exContactBlockTL
+EXAMPLES = $(SEQ_EXAMPLES)
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_IPOPT),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with IPOPT)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f exContactBlockTL.mesh exContactBlockTL-mesh.* exContactBlockTL-init.* exContactBlockTL-final.* ExampleContactBlockTL*
@@ -0,0 +1,888 @@
+
+using namespace std;
+using namespace mfem;
+
+void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]);
+   dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);
+   dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]);
+   dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);
+   dNdxi(1,3) = 0.25*(1-xi[0]);
+}
+
+
+void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
+                     DenseMatrix& dN2dxi)
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(2,4); dNdxi = 0.0;
+   dN2dxi.SetSize(3,4);
+   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
+
+   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
+   dN2dxi(1,3) = -0.25;
+}
+
+// returns the vector and matrix form of the shape functions and its derivative
+void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
+                       DenseMatrix& ddNdxi)
+{
+   N.SetSize(3,12); N = 0.0;
+   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
+   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
+   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
+   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
+   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
+   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
+   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
+
+   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
+   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
+   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
+   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
+   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
+   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
+
+   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
+   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
+   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
+   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
+   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
+   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
+   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
+
+   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
+   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
+   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
+   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
+   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
+   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
+}
+
+
+void cross(const Vector a, const Vector b, Vector& c)
+{
+   assert(a.Size()==3);
+   c.SetSize(3);
+   c[0] = a[1]*b[2] - a[2]*b[1];
+   c[1] = -a[0]*b[2] + b[0]*a[2];
+   c[2] = a[0]*b[1] - a[1]*b[0];
+
+}
+// a outer b
+void outer(const Vector a, const Vector b, DenseMatrix& c)
+{
+   int m = a.Size();
+   int n = b.Size();
+   assert(c.Height()==m);
+   assert(c.Width() ==n);
+   for (int i=0; i<m; i++)
+   {
+      for (int j=0; j<n; j++)
+      {
+         c(i,j) = a[i]*b[j];
+      }
+   }
+}
+// dphidxi 2*4
+// coords 4*3
+void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
+                   Vector& normal, double& nnorm)
+{
+
+   DenseMatrix dxdxi(2,3);
+   Mult(dphidxi, coords, dxdxi);
+   Vector dxdxi1(3);
+   Vector dxdxi2(3);
+
+   dxdxi.GetRow(0,dxdxi1);
+   dxdxi.GetRow(1,dxdxi2);
+
+   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
+   // VectorCrossProductCoefficient::Eval has hard-coded cross product
+   nnorm = normal.Norml2( );
+   normal /=  nnorm;
+}
+
+void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
+{
+   bool converged = false;
+   bool pt_on_elem = false;
+   int dim = 3;
+   xi.SetSize(dim-1);
+   xi = 0.0;
+   double r = 1e10;
+   int max_iter = 15;
+   double off_el_xi = 1e-2;
+   double proj_newton_tol = 1e-13;
+   double proj_max_gap = 0.5;
+   Vector gap_v(dim);
+   // warm start from linear solution
+
+   for (int it=0; it<max_iter; it++)
+   {
+      //cout<<it<<endl;
+      Vector m_N(4);
+      m_N = 0.;
+      DenseMatrix m_dN(2,4);
+      m_dN = 0.;
+      DenseMatrix m_dN2(3,4);
+      m_dN2 = 0.;
+      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+      Vector x_c(dim);
+      m_coords.MultTranspose(m_N, x_c);
+
+      gap_v = s_x;
+      gap_v -= x_c;
+
+      DenseMatrix m_dx(2,3);
+      m_dx = 0.;
+      Mult(m_dN, m_coords, m_dx);
+
+      Vector r(dim-1);
+      r = 0.0;
+      m_dx.Mult(gap_v, r);
+
+      if (r.Normlinf() < proj_newton_tol)
+      {
+         converged = true;
+         break;
+      }
+
+      DenseMatrix drdxi(dim-1,dim-1);
+      drdxi = 0.;
+      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
+      drdxi *= -1.0;
+
+      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+      Mult(m_dN2,m_coords, m_dx2);
+
+      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
+      for (int d=0; d<3; d++)
+      {
+         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+         drdxi.Add(gap_v[d], Mtemp);
+      }
+
+      //cond_num = rcond(drdxi);  condition number?
+      //drdxi.TestInversion();
+      DenseMatrixInverse drdxi_inv(drdxi);
+      Vector xi_tmp(dim-1);
+
+      drdxi_inv.Mult(r,xi_tmp);
+      xi -= xi_tmp;
+   }
+   if (!converged)
+   {
+      xi = 0.0;
+   }
+   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
+
+   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
+   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
+   {
+      pt_on_elem = true;
+   }
+
+   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
+   MFEM_VERIFY(converged == true, "projection didn't converge");
+}
+
+
+
+// m_coords is expected to be 4 * 3
+void  ComputeGapJacobian(const Vector x_s, const Vector xi,
+                         const DenseMatrix m_coords,
+                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N, x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   double nnorm = 0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   gap = gap_v * normal; // gap function value, dot product between vectors
+
+   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
+
+   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
+   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
+
+   Vector m_dxrow1(3);
+   m_dx.GetRow(0, m_dxrow1);
+   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
+   dr_dx_res1 *= -1.0;
+
+   Vector m_dxrow2(3);
+   m_dx.GetRow(1, m_dxrow2);
+   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
+   dr_dx_res2 *= -1.0;
+
+   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
+   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
+
+   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
+   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
+   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
+   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
+
+   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
+   dr_dx_res2 += dr_dx_res2_tmp;
+
+
+   DenseMatrix K_dxidx1(2,2); // 2*2
+   K_dxidx1 = 0.;
+   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   // how to get 2nd order? multidimensional matrix?
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= K_dxidx1;
+   K_dxidx += K_dxidx2;
+
+   // resize the vectors and matrices
+   Vector dxidx(24); dxidx = 0.0;
+   Vector drdx_r(24); drdx_r = 0.0;
+
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         drdx_r[4*j+i] = dr_dx_res1(i,j);
+         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
+
+      }
+   }
+   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
+   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
+   DenseMatrix drdx_K(24,24); drdx_K = 0.;
+   for (int i =0; i<12; i++)
+   {
+      drdx_K(i,i) = K_dxidx(0,0);
+      drdx_K(i,12+i) = K_dxidx(0,1);
+      drdx_K(12+i,i) = K_dxidx(1,0);
+      drdx_K(12+i,12+i) = K_dxidx(1,1);
+   }
+
+   DenseMatrixInverse drdxK_inv(drdx_K);
+   drdxK_inv.Mult(drdx_r,dxidx);
+   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
+   dxidx *= -1.0;
+
+
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+
+   Vector dxidxs(6); dxidxs = 0.0;
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
+
+   dgdxm.SetSize(12); dgdxm = 0.;
+   DenseMatrix dgdxm_tmp(4,3);
+   outer(m_N, normal,dgdxm_tmp);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
+      }
+   }
+   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
+
+   dgdxs.SetSize(3);
+   dgdxs += normal;
+   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
+};
+
+void ComputeGapHessian(const Vector x_s, const Vector xi,
+                       const DenseMatrix m_coords,
+                       DenseMatrix& dg2dx)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   int dim = 3;
+   int num_dofs1 = dim;
+   int num_dofs2 = 4*dim;
+   int num_dofs = num_dofs1 + num_dofs2;
+   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N,x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+   Mult(m_dN2,m_coords, m_dx2);
+   double nnorm = 0.0;
+   Vector normal(3); normal = 0.0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   double gap = gap_v * normal; // gap function value, dot product between vectors
+
+   DenseMatrix M(2,2); M = 0.0;
+   MultABt(m_dx, m_dx, M);
+
+   DenseMatrix f(2, num_dofs2); f = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+      M.Add(-gap_v[d], Mtemp);
+
+      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
+      DenseMatrix ftmp(2,4);
+      outer(m_dxcol, m_N, ftmp);
+      ftmp *= -1;
+      ftmp.Add( gap_v[d], m_dN);   // 2*4
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         f(0,d+j*3) = ftmp(0,j);
+         f(1,d+j*3) = ftmp(1,j);
+      }
+   }
+   //fprintf('hess dxidxm\n');
+   DenseMatrixInverse Minv(M);
+   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
+   Minv.Mult(f, dxidxm);
+   //LinearSolve??
+   //dxidxm = M\f;
+
+   DenseMatrix nde2(2,2); nde2 = 0.0;
+   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
+      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
+      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
+
+      nde2 += ndetmp;
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
+         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
+      }
+
+   }
+
+   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
+   Ndn += Nndx2;
+   AddMult(nde2, dxidxm, Ndn);
+
+
+   DenseMatrix M2(2,2); M2 = 0.0;
+   MultABt(m_dx, m_dx, M2);
+   DenseMatrixInverse M2inv(M2);
+   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
+   DenseMatrix m_con(2,2); m_con = 0.0;
+
+   M2inv.Mult(diag2, m_con);
+
+   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
+
+   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
+   MultAtB(Ndn, m_con, dg2dxm_tmp);
+   Mult(dg2dxm_tmp, Ndn, dg2dxm);
+   dg2dxm *= gap;
+
+   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
+   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   dg2dxm_tmp = 0.0;
+   MultAtB(dxidxm, nde2, dg2dxm_tmp);
+
+   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
+
+   dg2dxm_tmp2 = 0.0;
+   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= M2;
+   K_dxidx += K_dxidx2;
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+   Vector dxidxs(6);
+
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
+   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
+   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
+
+   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
+   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
+
+   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
+   dxidxs_row2 = 0.0;
+   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
+   Vector mdx2_row3(3); mdx2_row3 = 0.0;
+   dxidxs_m.GetRow(0,dxidxs_row1);
+   dxidxs_m.GetRow(1,dxidxs_row2);
+   m_dx2.GetRow(0,mdx2_row1);
+   m_dx2.GetRow(1,mdx2_row2);
+   m_dx2.GetRow(2,mdx2_row3);
+
+   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
+   outer(mdx2_row1, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row2, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+
+   outer(mdx2_row2, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row3, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+
+   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
+
+   for (int d=0; d<3; d++)
+   {
+      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
+      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
+      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
+
+      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
+      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
+
+      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
+      m_dx.GetRow(0, m_dxrow);
+      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
+      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
+      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
+
+      dtaodxs_tmp2 += dtaodxs_tmp;
+      dtaodxs.SetCol(d, dtaodxs_tmp2);
+   }
+
+   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
+   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
+   outer(normal, normal, dndxs_tmp);
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
+
+   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
+   MultAtB(m_dx, dxidxs_m, dgvdxs);
+   dgvdxs *= -1;
+   for (int d=0; d<3; d++)
+   {
+      dgvdxs(d,d) += 1.0;
+   }
+   //dxidxs: 2*3
+
+   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
+   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
+   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
+   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
+   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
+   dg2dxs += dg2dxs_tmp2;
+   dg2dxs_tmp2 = 0.0;
+   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
+   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
+
+   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
+   BasisVectorDerivs(xi, Ne, Be, dBe);
+
+   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
+   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
+
+   Vector m_coords_v(12);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         m_coords_v[i*3+j] = m_coords(i,j);
+      }
+   }
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix dBe_tmp(3,12);
+      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
+
+      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
+
+      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
+      dBe_tmp = 0.0;
+      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
+
+   }
+
+   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
+
+   for (int d=0; d<12; d++)
+   {
+      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
+      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
+      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
+      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
+      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
+      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
+
+      cross(tmp1, m_dxrow2, dtaodxm_tmp);
+      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
+      dtaodxm_tmp += dtaodxm_tmp2;
+
+      dtaodxm.SetCol(d, dtaodxm_tmp);
+   }
+
+   DenseMatrix dndxm(3,12); dndxm = 0.0;
+   dndxm += dtaodxm;
+   dndxm *= 1.0/nnorm;
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
+
+   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
+   dgvdxm -= Ne;
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
+      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
+
+   }
+
+   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
+   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
+   MultAtB(dgvdxs, dndxm, dg2dxsxm);
+
+   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
+   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
+
+   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
+   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
+
+   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
+
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
+      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
+      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
+   }
+
+   dg2dxsxm += dgvdxsxmn;
+
+   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
+   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
+   MultAtB(dgvdxm, dndxs, dg2dxmxs);
+   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
+   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
+
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
+
+   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
+   dgvdxmxsn_tmp *= -1.0;
+   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+      Be_tmp.Transpose(); // Be is now 12*3
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
+      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
+      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
+
+   }
+
+   dg2dxmxs += dgvdxmxsn;
+
+   dg2dx.CopyMN(dg2dxs, 0, 0);
+   dg2dx.CopyMN(dg2dxm, 3, 3);
+   dg2dx.CopyMN(dg2dxsxm, 0, 3);
+   dg2dx.CopyMN(dg2dxmxs, 3, 0);
+
+};
+
+
+
+void NodeSegConPairs(const Vector x1, const Vector xi2,
+                     const DenseMatrix coords2,
+                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
+{
+   double gap = 0.0;
+   Vector normal(3); normal = 0.0;
+   Vector dgdxm(12); dgdxm = 0.0;
+   Vector dgdxs(3); dgdxs = 0.0;
+
+   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
+   node_g = gap;
+
+   node_dg.SetSize(12+3);
+   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
+   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
+
+   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   ComputeGapHessian(x1, xi2, coords2, dg2dx);
+
+   node_dg2.SetSize(15,15);
+   node_dg2 = dg2dx;
+
+   /*
+   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
+
+   v1 = 1:3;
+   v2 = 1:12;
+   %v1 = ones(1,3)
+   %v2 = ones(1,12)
+   v2 = reshape(v2,4,3);
+   x1n1 = x1 + 0.01*v1;
+   coords2n1 = coords2 + 0.001*v2;
+      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
+      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
+   x1n2 = x1 - 0.01*v1;
+   coords2n2 = coords2 - 0.001*v2;
+      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
+      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
+   fprintf('fd\n');
+      %gapv1-gapv2
+   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
+
+   %dgdxsn1-dgdxsn2
+   fprintf('code\n');
+   v2n = v2';
+   %dg2dx(1:3,1:3)*0.04*ones(3,1)
+   temp = zeros(12,3);
+   for i = 1:4
+   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
+   temp((i-1)*3+1:i*3,:) = temp1';
+   end
+   temp2 = zeros(3,12);
+   for i = 1:4
+   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
+   temp2(:,(i-1)*3+1:i*3) = temp3';
+   end
+   %dg2dx
+   %dg2dx(4:end,1:3) = temp;
+   %dg2dx(1:3,4:end) = temp2;
+   %dgvdxm * 0.002*v2n(:)
+   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
+   %dg2dx(4:end,1:3)
+   end*/
+
+};
+
+
+// coordsm : (npoints*4, 3) use what class?
+// m_conn: (npoints*4)
+void Assemble_Contact(const int m, const int npoints, const int ndofs,
+                      const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
+                      std::vector<SparseMatrix>& dM)
+{
+   int n = ndofs;
+   int ndim = 3;
+
+   g.SetSize(m);
+   g = 0.0;
+
+   //SparseMatrix M(m, n);  // M needs to be the correct size
+
+   //dM.resize(m); // needs to clear?
+
+   double g_tmp = 0.;
+   Vector dg(4*ndim+ndim);
+   dg = 0.;
+   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
+   dg2 = 0.;
+
+   for (int i=0; i<npoints; i++)
+   {
+      Vector x1(ndim);
+      x1[0] = x_s[i*ndim];
+      x1[1] = x_s[i*ndim+1];
+      x1[2] = x_s[i*ndim+2];
+
+      Vector xi2(ndim-1);
+      xi2[0] = xi[i*(ndim-1)];
+      xi2[1] = xi[i*(ndim-1)+1];
+
+      DenseMatrix coords2(4,3);
+      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
+
+      //how to get coords2?
+      dg = 0.0;
+      dg2 = 0.;
+      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
+      //x1.Print();
+      //xi2.Print();
+      //coords2.Print();
+      g[s_conn[i]] = g_tmp; // should be unique
+      Array<int> m_conn_i(4);
+      m_conn.GetSubArray(4*i, 4, m_conn_i);
+
+      Array<int> node_conn(5);
+      node_conn[0] = s_conn[i];
+      for (int j=0; j<4; j++)
+      {
+         node_conn[j+1] = m_conn_i[j];
+      }
+
+      Array<int>  M_i_tmp(1);
+      M_i_tmp[0] = s_conn[i];
+
+      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
+      Array<int> j_idx(5*ndim); j_idx = 0;
+      for (int j=0; j< 5; j++)
+      {
+         for (int k=0; k<ndim; k++)
+         {
+            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
+         }
+      }
+      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
+      M_v_tmp.SetRow(0, dg);
+
+      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
+
+      Array<int> dM_i(ndim*(4+1));
+      Array<int> dM_j(ndim*(4+1));
+
+      for (int j=0; j< ndim*(4+1); j++)
+      {
+         dM_i[j] = j_idx[j];
+         dM_j[j] = j_idx[j];
+      }
+      //dg2.Print();
+      //dM[s_conn[i]].Print();
+      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
+   }
+};
+
@@ -46,6 +46,9 @@ endif
 ifeq ($(MFEM_USE_HIOP),YES)
   SUBDIRS += hiop
 endif
+ifeq ($(MFEM_USE_IPOPT),YES)
+   SUBDIRS += ipopt
+endif
 ifeq ($(MFEM_USE_PETSC),YES)
   SUBDIRS += petsc
 endif
@@ -0,0 +1,888 @@
+
+using namespace std;
+using namespace mfem;
+
+void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]);
+   dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);
+   dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]);
+   dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);
+   dNdxi(1,3) = 0.25*(1-xi[0]);
+}
+
+
+void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
+                     DenseMatrix& dN2dxi)
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(2,4); dNdxi = 0.0;
+   dN2dxi.SetSize(3,4);
+   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
+
+   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
+   dN2dxi(1,3) = -0.25;
+}
+
+// returns the vector and matrix form of the shape functions and its derivative
+void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
+                       DenseMatrix& ddNdxi)
+{
+   N.SetSize(3,12); N = 0.0;
+   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
+   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
+   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
+   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
+   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
+   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
+   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
+
+   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
+   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
+   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
+   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
+   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
+   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
+
+   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
+   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
+   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
+   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
+   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
+   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
+   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
+
+   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
+   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
+   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
+   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
+   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
+   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
+}
+
+
+void cross(const Vector a, const Vector b, Vector& c)
+{
+   assert(a.Size()==3);
+   c.SetSize(3);
+   c[0] = a[1]*b[2] - a[2]*b[1];
+   c[1] = -a[0]*b[2] + b[0]*a[2];
+   c[2] = a[0]*b[1] - a[1]*b[0];
+
+}
+// a outer b
+void outer(const Vector a, const Vector b, DenseMatrix& c)
+{
+   int m = a.Size();
+   int n = b.Size();
+   assert(c.Height()==m);
+   assert(c.Width() ==n);
+   for (int i=0; i<m; i++)
+   {
+      for (int j=0; j<n; j++)
+      {
+         c(i,j) = a[i]*b[j];
+      }
+   }
+}
+// dphidxi 2*4
+// coords 4*3
+void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
+                   Vector& normal, double& nnorm)
+{
+
+   DenseMatrix dxdxi(2,3);
+   Mult(dphidxi, coords, dxdxi);
+   Vector dxdxi1(3);
+   Vector dxdxi2(3);
+
+   dxdxi.GetRow(0,dxdxi1);
+   dxdxi.GetRow(1,dxdxi2);
+
+   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
+   // VectorCrossProductCoefficient::Eval has hard-coded cross product
+   nnorm = normal.Norml2( );
+   normal /=  nnorm;
+}
+
+void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
+{
+   bool converged = false;
+   bool pt_on_elem = false;
+   int dim = 3;
+   xi.SetSize(dim-1);
+   xi = 0.0;
+   double r = 1e10;
+   int max_iter = 15;
+   double off_el_xi = 1e-2;
+   double proj_newton_tol = 1e-13;
+   double proj_max_gap = 0.5;
+   Vector gap_v(dim);
+   // warm start from linear solution
+
+   for (int it=0; it<max_iter; it++)
+   {
+      //cout<<it<<endl;
+      Vector m_N(4);
+      m_N = 0.;
+      DenseMatrix m_dN(2,4);
+      m_dN = 0.;
+      DenseMatrix m_dN2(3,4);
+      m_dN2 = 0.;
+      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+      Vector x_c(dim);
+      m_coords.MultTranspose(m_N, x_c);
+
+      gap_v = s_x;
+      gap_v -= x_c;
+
+      DenseMatrix m_dx(2,3);
+      m_dx = 0.;
+      Mult(m_dN, m_coords, m_dx);
+
+      Vector r(dim-1);
+      r = 0.0;
+      m_dx.Mult(gap_v, r);
+
+      if (r.Normlinf() < proj_newton_tol)
+      {
+         converged = true;
+         break;
+      }
+
+      DenseMatrix drdxi(dim-1,dim-1);
+      drdxi = 0.;
+      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
+      drdxi *= -1.0;
+
+      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+      Mult(m_dN2,m_coords, m_dx2);
+
+      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
+      for (int d=0; d<3; d++)
+      {
+         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+         drdxi.Add(gap_v[d], Mtemp);
+      }
+
+      //cond_num = rcond(drdxi);  condition number?
+      //drdxi.TestInversion();
+      DenseMatrixInverse drdxi_inv(drdxi);
+      Vector xi_tmp(dim-1);
+
+      drdxi_inv.Mult(r,xi_tmp);
+      xi -= xi_tmp;
+   }
+   if (!converged)
+   {
+      xi = 0.0;
+   }
+   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
+
+   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
+   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
+   {
+      pt_on_elem = true;
+   }
+
+   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
+   MFEM_VERIFY(converged == true, "projection didn't converge");
+}
+
+
+
+// m_coords is expected to be 4 * 3
+void  ComputeGapJacobian(const Vector x_s, const Vector xi,
+                         const DenseMatrix m_coords,
+                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N, x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   double nnorm = 0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   gap = gap_v * normal; // gap function value, dot product between vectors
+
+   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
+
+   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
+   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
+
+   Vector m_dxrow1(3);
+   m_dx.GetRow(0, m_dxrow1);
+   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
+   dr_dx_res1 *= -1.0;
+
+   Vector m_dxrow2(3);
+   m_dx.GetRow(1, m_dxrow2);
+   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
+   dr_dx_res2 *= -1.0;
+
+   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
+   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
+
+   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
+   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
+   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
+   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
+
+   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
+   dr_dx_res2 += dr_dx_res2_tmp;
+
+
+   DenseMatrix K_dxidx1(2,2); // 2*2
+   K_dxidx1 = 0.;
+   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   // how to get 2nd order? multidimensional matrix?
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= K_dxidx1;
+   K_dxidx += K_dxidx2;
+
+   // resize the vectors and matrices
+   Vector dxidx(24); dxidx = 0.0;
+   Vector drdx_r(24); drdx_r = 0.0;
+
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         drdx_r[4*j+i] = dr_dx_res1(i,j);
+         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
+
+      }
+   }
+   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
+   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
+   DenseMatrix drdx_K(24,24); drdx_K = 0.;
+   for (int i =0; i<12; i++)
+   {
+      drdx_K(i,i) = K_dxidx(0,0);
+      drdx_K(i,12+i) = K_dxidx(0,1);
+      drdx_K(12+i,i) = K_dxidx(1,0);
+      drdx_K(12+i,12+i) = K_dxidx(1,1);
+   }
+
+   DenseMatrixInverse drdxK_inv(drdx_K);
+   drdxK_inv.Mult(drdx_r,dxidx);
+   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
+   dxidx *= -1.0;
+
+
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+
+   Vector dxidxs(6); dxidxs = 0.0;
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
+
+   dgdxm.SetSize(12); dgdxm = 0.;
+   DenseMatrix dgdxm_tmp(4,3);
+   outer(m_N, normal,dgdxm_tmp);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
+      }
+   }
+   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
+
+   dgdxs.SetSize(3);
+   dgdxs += normal;
+   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
+};
+
+void ComputeGapHessian(const Vector x_s, const Vector xi,
+                       const DenseMatrix m_coords,
+                       DenseMatrix& dg2dx)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   int dim = 3;
+   int num_dofs1 = dim;
+   int num_dofs2 = 4*dim;
+   int num_dofs = num_dofs1 + num_dofs2;
+   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N,x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+   Mult(m_dN2,m_coords, m_dx2);
+   double nnorm = 0.0;
+   Vector normal(3); normal = 0.0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   double gap = gap_v * normal; // gap function value, dot product between vectors
+
+   DenseMatrix M(2,2); M = 0.0;
+   MultABt(m_dx, m_dx, M);
+
+   DenseMatrix f(2, num_dofs2); f = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+      M.Add(-gap_v[d], Mtemp);
+
+      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
+      DenseMatrix ftmp(2,4);
+      outer(m_dxcol, m_N, ftmp);
+      ftmp *= -1;
+      ftmp.Add( gap_v[d], m_dN);   // 2*4
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         f(0,d+j*3) = ftmp(0,j);
+         f(1,d+j*3) = ftmp(1,j);
+      }
+   }
+   //fprintf('hess dxidxm\n');
+   DenseMatrixInverse Minv(M);
+   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
+   Minv.Mult(f, dxidxm);
+   //LinearSolve??
+   //dxidxm = M\f;
+
+   DenseMatrix nde2(2,2); nde2 = 0.0;
+   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
+      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
+      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
+
+      nde2 += ndetmp;
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
+         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
+      }
+
+   }
+
+   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
+   Ndn += Nndx2;
+   AddMult(nde2, dxidxm, Ndn);
+
+
+   DenseMatrix M2(2,2); M2 = 0.0;
+   MultABt(m_dx, m_dx, M2);
+   DenseMatrixInverse M2inv(M2);
+   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
+   DenseMatrix m_con(2,2); m_con = 0.0;
+
+   M2inv.Mult(diag2, m_con);
+
+   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
+
+   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
+   MultAtB(Ndn, m_con, dg2dxm_tmp);
+   Mult(dg2dxm_tmp, Ndn, dg2dxm);
+   dg2dxm *= gap;
+
+   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
+   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   dg2dxm_tmp = 0.0;
+   MultAtB(dxidxm, nde2, dg2dxm_tmp);
+
+   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
+
+   dg2dxm_tmp2 = 0.0;
+   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= M2;
+   K_dxidx += K_dxidx2;
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+   Vector dxidxs(6);
+
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
+   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
+   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
+
+   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
+   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
+
+   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
+   dxidxs_row2 = 0.0;
+   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
+   Vector mdx2_row3(3); mdx2_row3 = 0.0;
+   dxidxs_m.GetRow(0,dxidxs_row1);
+   dxidxs_m.GetRow(1,dxidxs_row2);
+   m_dx2.GetRow(0,mdx2_row1);
+   m_dx2.GetRow(1,mdx2_row2);
+   m_dx2.GetRow(2,mdx2_row3);
+
+   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
+   outer(mdx2_row1, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row2, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+
+   outer(mdx2_row2, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row3, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+
+   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
+
+   for (int d=0; d<3; d++)
+   {
+      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
+      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
+      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
+
+      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
+      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
+
+      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
+      m_dx.GetRow(0, m_dxrow);
+      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
+      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
+      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
+
+      dtaodxs_tmp2 += dtaodxs_tmp;
+      dtaodxs.SetCol(d, dtaodxs_tmp2);
+   }
+
+   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
+   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
+   outer(normal, normal, dndxs_tmp);
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
+
+   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
+   MultAtB(m_dx, dxidxs_m, dgvdxs);
+   dgvdxs *= -1;
+   for (int d=0; d<3; d++)
+   {
+      dgvdxs(d,d) += 1.0;
+   }
+   //dxidxs: 2*3
+
+   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
+   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
+   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
+   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
+   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
+   dg2dxs += dg2dxs_tmp2;
+   dg2dxs_tmp2 = 0.0;
+   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
+   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
+
+   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
+   BasisVectorDerivs(xi, Ne, Be, dBe);
+
+   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
+   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
+
+   Vector m_coords_v(12);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         m_coords_v[i*3+j] = m_coords(i,j);
+      }
+   }
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix dBe_tmp(3,12);
+      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
+
+      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
+
+      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
+      dBe_tmp = 0.0;
+      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
+
+   }
+
+   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
+
+   for (int d=0; d<12; d++)
+   {
+      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
+      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
+      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
+      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
+      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
+      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
+
+      cross(tmp1, m_dxrow2, dtaodxm_tmp);
+      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
+      dtaodxm_tmp += dtaodxm_tmp2;
+
+      dtaodxm.SetCol(d, dtaodxm_tmp);
+   }
+
+   DenseMatrix dndxm(3,12); dndxm = 0.0;
+   dndxm += dtaodxm;
+   dndxm *= 1.0/nnorm;
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
+
+   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
+   dgvdxm -= Ne;
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
+      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
+
+   }
+
+   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
+   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
+   MultAtB(dgvdxs, dndxm, dg2dxsxm);
+
+   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
+   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
+
+   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
+   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
+
+   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
+
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
+      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
+      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
+   }
+
+   dg2dxsxm += dgvdxsxmn;
+
+   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
+   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
+   MultAtB(dgvdxm, dndxs, dg2dxmxs);
+   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
+   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
+
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
+
+   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
+   dgvdxmxsn_tmp *= -1.0;
+   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+      Be_tmp.Transpose(); // Be is now 12*3
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
+      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
+      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
+
+   }
+
+   dg2dxmxs += dgvdxmxsn;
+
+   dg2dx.CopyMN(dg2dxs, 0, 0);
+   dg2dx.CopyMN(dg2dxm, 3, 3);
+   dg2dx.CopyMN(dg2dxsxm, 0, 3);
+   dg2dx.CopyMN(dg2dxmxs, 3, 0);
+
+};
+
+
+
+void NodeSegConPairs(const Vector x1, const Vector xi2,
+                     const DenseMatrix coords2,
+                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
+{
+   double gap = 0.0;
+   Vector normal(3); normal = 0.0;
+   Vector dgdxm(12); dgdxm = 0.0;
+   Vector dgdxs(3); dgdxs = 0.0;
+
+   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
+   node_g = gap;
+
+   node_dg.SetSize(12+3);
+   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
+   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
+
+   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   ComputeGapHessian(x1, xi2, coords2, dg2dx);
+
+   node_dg2.SetSize(15,15);
+   node_dg2 = dg2dx;
+
+   /*
+   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
+
+   v1 = 1:3;
+   v2 = 1:12;
+   %v1 = ones(1,3)
+   %v2 = ones(1,12)
+   v2 = reshape(v2,4,3);
+   x1n1 = x1 + 0.01*v1;
+   coords2n1 = coords2 + 0.001*v2;
+      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
+      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
+   x1n2 = x1 - 0.01*v1;
+   coords2n2 = coords2 - 0.001*v2;
+      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
+      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
+   fprintf('fd\n');
+      %gapv1-gapv2
+   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
+
+   %dgdxsn1-dgdxsn2
+   fprintf('code\n');
+   v2n = v2';
+   %dg2dx(1:3,1:3)*0.04*ones(3,1)
+   temp = zeros(12,3);
+   for i = 1:4
+   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
+   temp((i-1)*3+1:i*3,:) = temp1';
+   end
+   temp2 = zeros(3,12);
+   for i = 1:4
+   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
+   temp2(:,(i-1)*3+1:i*3) = temp3';
+   end
+   %dg2dx
+   %dg2dx(4:end,1:3) = temp;
+   %dg2dx(1:3,4:end) = temp2;
+   %dgvdxm * 0.002*v2n(:)
+   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
+   %dg2dx(4:end,1:3)
+   end*/
+
+};
+
+
+// coordsm : (npoints*4, 3) use what class?
+// m_conn: (npoints*4)
+void Assemble_Contact(const int m, const int npoints, const int ndofs,
+                      const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
+                      std::vector<SparseMatrix>& dM)
+{
+   int n = ndofs;
+   int ndim = 3;
+
+   g.SetSize(m);
+   g = 0.0;
+
+   //SparseMatrix M(m, n);  // M needs to be the correct size
+
+   //dM.resize(m); // needs to clear?
+
+   double g_tmp = 0.;
+   Vector dg(4*ndim+ndim);
+   dg = 0.;
+   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
+   dg2 = 0.;
+
+   for (int i=0; i<npoints; i++)
+   {
+      Vector x1(ndim);
+      x1[0] = x_s[i*ndim];
+      x1[1] = x_s[i*ndim+1];
+      x1[2] = x_s[i*ndim+2];
+
+      Vector xi2(ndim-1);
+      xi2[0] = xi[i*(ndim-1)];
+      xi2[1] = xi[i*(ndim-1)+1];
+
+      DenseMatrix coords2(4,3);
+      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
+
+      //how to get coords2?
+      dg = 0.0;
+      dg2 = 0.;
+      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
+      //x1.Print();
+      //xi2.Print();
+      //coords2.Print();
+      g[s_conn[i]] = g_tmp; // should be unique
+      Array<int> m_conn_i(4);
+      m_conn.GetSubArray(4*i, 4, m_conn_i);
+
+      Array<int> node_conn(5);
+      node_conn[0] = s_conn[i];
+      for (int j=0; j<4; j++)
+      {
+         node_conn[j+1] = m_conn_i[j];
+      }
+
+      Array<int>  M_i_tmp(1);
+      M_i_tmp[0] = s_conn[i];
+
+      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
+      Array<int> j_idx(5*ndim); j_idx = 0;
+      for (int j=0; j< 5; j++)
+      {
+         for (int k=0; k<ndim; k++)
+         {
+            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
+         }
+      }
+      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
+      M_v_tmp.SetRow(0, dg);
+
+      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
+
+      Array<int> dM_i(ndim*(4+1));
+      Array<int> dM_j(ndim*(4+1));
+
+      for (int j=0; j< ndim*(4+1); j++)
+      {
+         dM_i[j] = j_idx[j];
+         dM_j[j] = j_idx[j];
+      }
+      //dg2.Print();
+      //dM[s_conn[i]].Print();
+      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
+   }
+};
+
@@ -187,7 +187,6 @@ int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();

@@ -248,7 +248,10 @@ int main(int argc, char *argv[])
   //     constraints for non-conforming AMR, static condensation, etc.
   if (myid == 0) { cout << "matrix ... " << flush; }
   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
+   // Here we want to try out block-size aware AMG solver in PETSc.
+   // For that to work properly, we need a fully-compliant block-size
+   // structure and we do not skip zeros when assembling.
+   a->Assemble(use_petsc ? 0 : 1);

   Vector B, X;
   if (!use_petsc)
@@ -294,13 +297,14 @@ int main(int argc, char *argv[])
         cout << "done." << endl;
         cout << "Size of linear system: " << A.M() << endl;
      }
-      PetscPCGSolver *pcg = new PetscPCGSolver(A);
+      // Tell PETSc the matrix has a block structure
+      A.SetBlockSize(dim);

-      // The preconditioner for the PCG solver defined below is specified in the
-      // PETSc config file, rc_ex2p, since a Krylov solver in PETSc can also
-      // customize its preconditioner.
+      // The preconditioner for the PCG solver can be specified in the
+      // PETSc config file
+      PetscPCGSolver *pcg = new PetscPCGSolver(A);
      PetscPreconditioner *prec = NULL;
-      if (use_nonoverlapping)
+      if (use_nonoverlapping) // Specialized BDDC construction
      {
         // Compute dofs belonging to the natural boundary
         Array<int> nat_tdof_list, nat_bdr(pmesh->bdr_attributes.Max());
@@ -78,6 +78,7 @@ EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly -
 EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
 EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
+EX2_ARGS_ASM          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm
 EX3_ARGS              := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
 EX4_ARGS              := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
 EX4_HYB_ARGS          := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
@@ -109,6 +110,7 @@ endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_BDDC))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_ASM))
 ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p
@@ -14,4 +14,5 @@
 -mg_levels_esteig_ksp_type cg
 -mg_levels_esteig_ksp_max_it 10
 -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05
+-pc_gamg_use_sa_esteig 0
 -mg_levels_pc_type sor
@@ -1,8 +1,7 @@
 -ksp_converged_reason

 # GAMG is still not used at its best,
-# since we are not exploiting the
-# block size (Ordering::byVDIM) and the RBMs
+# since we are not exploiting the RBMs

 -ksp_view
 -pc_type gamg
@@ -0,0 +1,10 @@
+# Additive Schwarz with Overlap
+# This is not a good solver for elasticity
+# These options are here only to describe
+# the setup of the solver
+-ksp_converged_reason
+-ksp_view
+-ksp_max_it 10
+-pc_type asm
+-pc_asm_overlap 1
+-sub_pc_type icc
@@ -210,6 +210,9 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,

 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
   // 1. Parse command-line options.
   const char *mesh_file = "../../data/beam-quad.mesh";
   int ref_levels = 2;
@@ -215,10 +215,11 @@ void visualize(ostream &os, ParMesh *mesh, ParGridFunction *deformed_nodes,

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
   Mpi::Init(argc, argv);
   int myid = Mpi::WorldRank();
   Hypre::Init();
+   Sundials::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../../data/beam-quad.mesh";
@@ -109,6 +109,9 @@ double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
   // 1. Parse command-line options.
   const char *mesh_file = "../../data/star.mesh";
   int ref_levels = 2;
@@ -290,7 +293,10 @@ int main(int argc, char *argv[])
         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 11)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
         ode_solver = arkode; break;
      case 12:
         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
@@ -101,11 +101,12 @@ double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
   Mpi::Init(argc, argv);
   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();
+   Sundials::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../../data/star.mesh";
@@ -327,7 +328,10 @@ int main(int argc, char *argv[])
         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 11)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
         ode_solver = arkode; break;
      case 12:
         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
@@ -140,6 +140,9 @@ public:

 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
   // 1. Parse command-line options.
   problem = 0;
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
@@ -408,7 +411,7 @@ int main(int argc, char *argv[])
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(FEHLBERG_13_7_8);
+         arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
         ode_solver = arkode; break;
   }

@@ -152,11 +152,12 @@ public:

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
   Mpi::Init(argc, argv);
   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();
+   Sundials::Init();

   // 2. Parse command-line options.
   problem = 0;
@@ -487,7 +488,10 @@ int main(int argc, char *argv[])
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 9)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
         ode_solver = arkode; break;
   }

@@ -35,7 +35,7 @@ add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
 if (MFEM_ENABLE_TESTING)
  # Command line options for the tests.
  # Example 1: Test SuperLU on the simple Poisson problem
-  set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
+  set(EX1_COMMON_OPTS -m ../../data/star.mesh)
  set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})

  # Add the tests: one test per source file.
@@ -124,6 +124,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
      case AssemblyLevel::LEGACY:
         break;
      case AssemblyLevel::FULL:
+         SetDiagonalPolicy( DIAG_ONE ); // Only diagonal policy supported on device
         ext = new FABilinearFormExtension(this);
         break;
      case AssemblyLevel::ELEMENT:
@@ -136,7 +137,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
         ext = new MFBilinearFormExtension(this);
         break;
      default:
-         mfem_error("Unknown assembly level");
+         MFEM_ABORT("BilinearForm: unknown assembly level");
   }
 }

@@ -26,7 +26,8 @@ namespace mfem
 {

 /** @brief Enumeration defining the assembly level for bilinear and nonlinear
-    form classes derived from Operator. */
+    form classes derived from Operator. For more details, see
+    https://mfem.org/howto/assembly_levels */
 enum class AssemblyLevel
 {
   /// In the case of a BilinearForm LEGACY corresponds to a fully assembled
@@ -79,6 +80,9 @@ protected:
   /** @brief Extension for supporting Full Assembly (FA), Element Assembly (EA),
       Partial Assembly (PA), or Matrix Free assembly (MF). */
   BilinearFormExtension *ext;
+   /** Indicates if the sparse matrix is sorted after assembly when using
+       Full Assembly (FA). */
+   bool sort_sparse_matrix = false;

   /** @brief Indicates the Mesh::sequence corresponding to the current state of
       the BilinearForm. */
@@ -177,9 +181,24 @@ public:
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE

-       This method must be called before assembly. */
+       If used, this method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

+   /** @brief Force the sparse matrix column indices to be sorted when using
+       AssemblyLevel::FULL.
+
+       When assembling on device the assembly algorithm uses atomic operations
+       to insert values in the sparse matrix, which can result in different
+       column index orderings across runs. Calling this method with @a enable_it
+       set to @a true forces a sorting algorithm to be called at the end of the
+       assembly procedure to ensure sorted column indices (and therefore
+       deterministic results).
+   */
+   void EnableSparseMatrixSorting(bool enable_it)
+   {
+      sort_sparse_matrix = enable_it;
+   }
+
   /// Returns the assembly level
   AssemblyLevel GetAssemblyLevel() const { return assembly; }

@@ -529,7 +529,7 @@ void EABilinearFormExtension::Assemble()
   }

   faceDofs = trial_fes ->
-              GetTraceElement(0, trial_fes->GetMesh()->GetFaceBaseGeometry(0)) ->
+              GetTraceElement(0, trial_fes->GetMesh()->GetFaceGeometry(0)) ->
              GetDof();

   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
@@ -955,6 +955,10 @@ void FABilinearFormExtension::Assemble()
      }
      a->mat = mat;
   }
+   if ( a->sort_sparse_matrix )
+   {
+      a->mat->SortColumnIndices();
+   }
 }


@@ -208,10 +208,6 @@ public:
                                            Vector &x, Vector &b,
                                            OperatorHandle &A, Vector &X, Vector &B) = 0;

-   virtual void AddMult(const Vector &x, Vector &y, const double c=1.0) const = 0;
-   virtual void AddMultTranspose(const Vector &x, Vector &y,
-                                 const double c=1.0) const = 0;
-
   virtual void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const = 0;

   virtual void Update() = 0;
@@ -287,7 +283,7 @@ public:
   /// Partial assembly of all internal integrators
   void Assemble();

-   void AddMult(const Vector &x, Vector &y, const double c) const;
+   void AddMult(const Vector &x, Vector &y, const double c=1.0) const;

   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;

@@ -2338,7 +2338,7 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
   if (spaceH1)
   {
      dshape.SetSize(trial_dof,dim);
-      curlshape.SetSize(dim*trial_dof,1);
+      curlshape.SetSize(trial_dof,dim);
      dimc = dim;
   }
   else
@@ -2367,7 +2367,7 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
      if (spaceH1)
      {
         trial_fe.CalcPhysDShape(Trans, dshape);
-         dshape.GradToCurl(curlshape);
+         dshape.GradToVectorCurl2D(curlshape);
      }
      else
      {
@@ -215,10 +215,10 @@ public:
                             function by any coefficients describing the
                             integrator.
       @param[in] ir  If passed (the default value is NULL), the implementation
-                        of the method will ignore the integration rule provided
-                        by the @a fluxelem parameter and, instead, compute the
-                        discrete flux at the points specified by the integration
-                        rule @a ir.
+                      of the method will ignore the integration rule provided
+                      by the @a fluxelem parameter and, instead, compute the
+                      discrete flux at the points specified by the integration
+                      rule @a ir.
    */
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
@@ -3059,8 +3059,8 @@ public:

 /** Integrator for the DG form:

-    - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
-    + kappa < {h^{-1} Q} [u], [v] >,
+        - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
+        + kappa < {h^{-1} Q} [u], [v] >

    where Q is a scalar or matrix diffusion coefficient and u, v are the trial
    and test spaces, respectively. The parameters sigma and kappa determine the
@@ -145,7 +145,7 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   // Assumes tensor-product elements
   Mesh *mesh = fes.GetMesh();
   const FiniteElement &el =
-      *fes.GetTraceElement(0, fes.GetMesh()->GetFaceBaseGeometry(0));
+      *fes.GetTraceElement(0, fes.GetMesh()->GetFaceGeometry(0));
   FaceElementTransformations &T0 =
      *fes.GetMesh()->GetFaceElementTransformations(0);
   const IntegrationRule *ir = IntRule?
@@ -2710,7 +2710,7 @@ void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
   }
 }

-// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply3D(const int D1D,
                      const int Q1D,
@@ -2900,7 +2900,7 @@ void PAHcurlH1Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(curl), integrated
+// Apply to x corresponding to DOFs in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose3D(const int D1D,
                               const int Q1D,
@@ -3099,7 +3099,7 @@ void PAHcurlH1ApplyTranspose3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply2D(const int D1D,
                      const int Q1D,
@@ -3223,7 +3223,7 @@ void PAHcurlH1Apply2D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(curl), integrated
+// Apply to x corresponding to DOFs in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose2D(const int D1D,
                               const int Q1D,
@@ -3543,7 +3543,7 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   }
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3D(const int D1D,
@@ -3906,7 +3906,7 @@ static void PAHcurlL2Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void SmemPAHcurlL2Apply3D(const int D1D,
@@ -4216,7 +4216,7 @@ static void SmemPAHcurlL2Apply3D(const int D1D,
   ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
 // integrated against H(div) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3D(const int D1D,
@@ -4572,7 +4572,7 @@ static void PAHcurlHdivApply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(div) (test), integrated against the
+// Apply to x corresponding to DOFs in H(div) (test), integrated against the
 // curl of H(curl) trial functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3DTranspose(const int D1D,
@@ -5067,7 +5067,7 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
   }
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), integrated against curl
+// Apply to x corresponding to DOFs in H(curl) (trial), integrated against curl
 // of H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3DTranspose(const int D1D,
@@ -1797,7 +1797,7 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   }
 }

-// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply3D(const int D1D,
                            const int Q1D,
@@ -1960,7 +1960,7 @@ static void PAHdivL2Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply2D(const int D1D,
                            const int Q1D,
@@ -56,8 +56,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
-                                    GeometricFactors::JACOBIANS, mt);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
@@ -74,7 +73,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
      const bool const_c = coeff.Size() == 1;
      const bool by_val = map_type == FiniteElement::VALUE;
      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
-      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
+      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,NE);
      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
                     Reshape(coeff.Read(), Q1D,Q1D,NE);
      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
@@ -84,11 +83,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
         {
            MFEM_FOREACH_THREAD(qy,y,Q1D)
            {
-               const double J11 = J(qx,qy,0,0,e);
-               const double J12 = J(qx,qy,1,0,e);
-               const double J21 = J(qx,qy,0,1,e);
-               const double J22 = J(qx,qy,1,1,e);
-               const double detJ = (J11*J22)-(J21*J12);
+               const double detJ = J(qx,qy,e);
               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
            }
@@ -102,7 +97,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
      const bool const_c = coeff.Size() == 1;
      const bool by_val = map_type == FiniteElement::VALUE;
      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
-      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
+      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,Q1D,NE);
      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
@@ -114,18 +109,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
            {
               MFEM_FOREACH_THREAD(qz,z,Q1D)
               {
-                  const double J11 = J(qx,qy,qz,0,0,e);
-                  const double J21 = J(qx,qy,qz,1,0,e);
-                  const double J31 = J(qx,qy,qz,2,0,e);
-                  const double J12 = J(qx,qy,qz,0,1,e);
-                  const double J22 = J(qx,qy,qz,1,1,e);
-                  const double J32 = J(qx,qy,qz,2,1,e);
-                  const double J13 = J(qx,qy,qz,0,2,e);
-                  const double J23 = J(qx,qy,qz,1,2,e);
-                  const double J33 = J(qx,qy,qz,2,2,e);
-                  const double detJ = J11 * (J22 * J33 - J32 * J23) -
-                  /* */               J21 * (J12 * J33 - J32 * J13) +
-                  /* */               J31 * (J12 * J23 - J22 * J13);
+                  const double detJ = J(qx,qy,qz,e);
                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
               }
@@ -75,7 +75,7 @@ void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
      Vector ea_data_ext_tmp(ea_data_ext.Size());
      bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
      auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
      auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
      auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
@@ -102,7 +102,7 @@ void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
   {
      bfi->AssembleEAInteriorFaces(fes, ea_data_int, ea_data_ext, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
      auto A_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
      auto A_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
      MFEM_FORALL(f, nf,
@@ -146,7 +146,7 @@ void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
      Vector ea_data_bdr_tmp(ea_data_bdr.Size());
      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
      auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
      auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
      MFEM_FORALL(f, nf,
@@ -165,7 +165,7 @@ void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
   {
      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
      auto A_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
      MFEM_FORALL(f, nf,
      {
@@ -69,9 +69,11 @@ void Operator::Mult(const mfem::Vector &x, mfem::Vector &y) const
 #endif
 }

-void Operator::AddMult(const mfem::Vector &x, mfem::Vector &y) const
+void Operator::AddMult(const mfem::Vector &x, mfem::Vector &y,
+                       const double a) const
 {
 #ifdef MFEM_USE_CEED
+   MFEM_VERIFY(a == 1.0, "General coefficient case is not yet supported!");
   const CeedScalar *x_ptr;
   CeedScalar *y_ptr;
   CeedMemType mem;
@@ -38,7 +38,8 @@ public:
   Operator(CeedOperator op);
 #endif
   void Mult(const mfem::Vector &x, mfem::Vector &y) const override;
-   void AddMult(const mfem::Vector &x, mfem::Vector &y) const;
+   void AddMult(const mfem::Vector &x, mfem::Vector &y,
+                const double a = 1.0) const override;
   void GetDiagonal(mfem::Vector &diag) const;
   using mfem::Operator::SetupRAP;
   virtual ~Operator()
@@ -47,6 +47,7 @@ void InitRestrictionWithIndices(const FiniteElementSpace &fes,

 /** @brief Initialize a strided CeedElemRestriction

+    @param[in] fes Input finite element space.
    @param[in] nelem is the number of elements.
    @param[in] nqpts is the total number of quadrature points.
    @param[in] qdatasize is the number of data per quadrature point.
@@ -140,7 +140,11 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   CeedOperator *subops;
   if (isComposite)
   {
+#if CEED_VERSION_GE(0, 10, 2)
+      ierr = CeedCompositeOperatorGetSubList(oper, &subops); CeedChk(ierr);
+#else
      ierr = CeedOperatorGetSubList(oper, &subops); CeedChk(ierr);
+#endif
      ierr = CeedOperatorGetQFunction(subops[0], &qf); CeedChk(ierr);
   }
   else
@@ -275,9 +275,14 @@ CeedOperator CoarsenCeedCompositeOperator(
                                      &op_coarse); PCeedChk(ierr);

   int nsub;
-   ierr = CeedOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
   CeedOperator *subops;
+#if CEED_VERSION_GE(0, 10, 2)
+   ierr = CeedCompositeOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
+   ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#else
+   ierr = CeedOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
   ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#endif
   for (int isub=0; isub<nsub; ++isub)
   {
      CeedOperator subop = subops[isub];
@@ -676,7 +681,6 @@ AlgebraicSpaceHierarchy::AlgebraicSpaceHierarchy(FiniteElementSpace &fes)
      const SparseMatrix *R = fespaces[ilevel+1]->GetRestrictionMatrix();
      if (R)
      {
-         R->EnsureMultTranspose();
         R_tr[ilevel] = new TransposeOperator(*R);
      }
      else
@@ -310,8 +310,13 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
   {
      CeedInt numsub;
      CeedOperator *subops;
+#if CEED_VERSION_GE(0, 10, 2)
+      CeedCompositeOperatorGetNumSub(op, &numsub);
+      ierr = CeedCompositeOperatorGetSubList(op, &subops); CeedChk(ierr);
+#else
      CeedOperatorGetNumSub(op, &numsub);
      ierr = CeedOperatorGetSubList(op, &subops); CeedChk(ierr);
+#endif
      for (int i = 0; i < numsub; ++i)
      {
         ierr = CeedSingleOperatorFullAssemble(subops[i], out); CeedChk(ierr);
@@ -66,6 +66,7 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin, CeedBasis* basisout,
    @param[in] coarse_er         CeedElemRestriction for coarse operator
                                 (see CeedATPMGElemRestriction)
    @param[out] coarse_basis_out CeedBasis for coarser operator
+    @param[out] basis_ctof_out   CeedBasis describing interpolation from coarse to fine
    @param[out] out              coarsened CeedOperator
 */
 int CeedATPMGOperator(CeedOperator oper, int order_reduction,
@@ -1319,7 +1319,6 @@ public:
   virtual void Eval(DenseSymmetricMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip) = 0;

-   using MatrixCoefficient::Eval;
   /** @brief Evaluate the matrix coefficient in the element described by @a T
       at the point @a ip, storing the result as a dense matrix @a K. */
   /** This function allows the use of SymmetricMatrixCoefficient in situations
@@ -1348,7 +1347,6 @@ public:
   ///Construct using matrix @a m for the constant.
   SymmetricMatrixConstantCoefficient(const DenseSymmetricMatrix &m)
      : SymmetricMatrixCoefficient(m.Height()), mat(m) { }
-   using MatrixCoefficient::Eval;
   using SymmetricMatrixCoefficient::Eval;
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
@@ -1400,7 +1398,6 @@ public:
   /// Set the time for internally stored coefficients
   void SetTime(double t);

-   using MatrixCoefficient::Eval;
   using SymmetricMatrixCoefficient::Eval;
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &K, ElementTransformation &T,
@@ -2239,7 +2236,7 @@ public:
   /// @sa CoefficientVector for a description of the @a compress argument.
   void Project(MatrixCoefficient &coeff, bool transpose=false);

-   /// @brief Project the tranpose of @a coeff.
+   /// @brief Project the transpose of @a coeff.
   ///
   /// @sa Project(MatrixCoefficient&, QuadratureSpace&, bool, bool)
   void ProjectTranspose(MatrixCoefficient &coeff);
@@ -110,8 +110,8 @@ DataCollection::DataCollection(const std::string& collection_name, Mesh *mesh_)
   precision = precision_default;
   pad_digits_cycle = pad_digits_rank = pad_digits_default;
   format = SERIAL_FORMAT; // use serial mesh format
-   compression = false;
-   error = NO_ERROR;
+   compression = 0;
+   error = No_Error;
 }

 void DataCollection::SetMesh(Mesh *new_mesh)
@@ -494,7 +494,7 @@ void VisItDataCollection::Load(int cycle_)
 {
   DeleteAll();
   time_step = 0.0;
-   error = NO_ERROR;
+   error = No_Error;
   cycle = cycle_;
   std::string root_name = prefix_path + name + "_" +
                           to_padded_string(cycle, pad_digits_cycle) +
@@ -724,7 +724,7 @@ void VisItDataCollection::ParseVisItRootString(const std::string& json)

   // Set the DataCollection::name using the mesh path
   std::string path = mesh.get("path").get<std::string>();
-   size_t right_sep = path.find('_');
+   size_t right_sep = path.rfind('_');
   if (right_sep == std::string::npos)
   {
      error = READ_ERROR;
@@ -767,10 +767,13 @@ ParaViewDataCollection::ParaViewDataCollection(const std::string&
     high_order_output(false),
     restart_mode(false)
 {
+   cycle = 0; // always include a valid cycle index in file names
+
+   compression_level = -1;  // default zlib compression level, equivalent to 6
 #ifdef MFEM_USE_ZLIB
-   compression = -1; // default zlib compression level, equivalent to 6
+   compression = true; // if we have zlib, enable compression
 #else
-   compression = 0;
+   compression = false; // otherwise, disable compression
 #endif
 }

@@ -919,7 +922,7 @@ void ParaViewDataCollection::Save()
   {
      const std::string &field_name = qfield.first;
      std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
-      qfield.second->SaveVTU(os, pv_data_format, compression);
+      qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel());
   }

   // MPI rank 0 also creates a "PVTU" file that points to all of the separately
@@ -1033,13 +1036,13 @@ void ParaViewDataCollection::WritePVTUFooter(std::ostream &os,
 void ParaViewDataCollection::SaveDataVTU(std::ostream &os, int ref)
 {
   os << "<VTKFile type=\"UnstructuredGrid\"";
-   if (compression != 0)
+   if (GetCompressionLevel() != 0)
   {
      os << " compressor=\"vtkZLibDataCompressor\"";
   }
   os << " version=\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
   os << "<UnstructuredGrid>\n";
-   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,compression);
+   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel());

   // dump out the grid functions as point data
   os << "<PointData >\n";
@@ -1103,7 +1106,7 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,

   if (IsBinaryFormat())
   {
-      WriteVTKEncodedCompressed(os,buf.data(),buf.size(),compression);
+      WriteVTKEncodedCompressed(os,buf.data(),buf.size(),GetCompressionLevel());
      os << '\n';
   }
   os << "</DataArray>" << std::endl;
@@ -1128,18 +1131,13 @@ void ParaViewDataCollection::SetCompressionLevel(int compression_level_)
 {
   MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9,
               "Compression level must be between -1 and 9 (inclusive).");
-   compression = compression_level_;
+   compression_level = compression_level_;
+   compression = compression_level_ != 0;
 }

 void ParaViewDataCollection::SetCompression(bool compression_)
 {
-   // If we are enabling compression, and it was disabled previously, use the
-   // default compression level. Otherwise, leave the compression level
-   // unchanged.
-   if (compression_ && compression == 0)
-   {
-      SetCompressionLevel(-1);
-   }
+   compression = compression_;
 }

 void ParaViewDataCollection::UseRestartMode(bool restart_mode_)
@@ -1171,4 +1169,9 @@ const char *ParaViewDataCollection::GetDataTypeString() const
   }
 }

+int ParaViewDataCollection::GetCompressionLevel() const
+{
+   return compression ? compression_level : 0;
+}
+
 }  // end namespace MFEM
@@ -338,11 +338,12 @@ public:
   /// Set the precision (number of digits) used for the text output of doubles
   void SetPrecision(int prec) { precision = prec; }
   /// Set the number of digits used for both the cycle and the MPI rank
-   void SetPadDigits(int digits) { pad_digits_cycle=pad_digits_rank = digits; }
+   virtual void SetPadDigits(int digits)
+   { pad_digits_cycle=pad_digits_rank = digits; }
   /// Set the number of digits used for the cycle
-   void SetPadDigitsCycle(int digits) { pad_digits_cycle = digits; }
+   virtual void SetPadDigitsCycle(int digits) { pad_digits_cycle = digits; }
   /// Set the number of digits used for the MPI rank in filenames
-   void SetPadDigitsRank(int digits) { pad_digits_rank = digits; }
+   virtual void SetPadDigitsRank(int digits) { pad_digits_rank = digits; }
   /// Set the desired output mesh and data format.
   /** See the enumeration #Format for valid options. Derived classes can define
       their own format enumerations and override this method to perform input
@@ -377,12 +378,24 @@ public:
   virtual ~DataCollection();

   /// Errors returned by Error()
-   enum { NO_ERROR = 0, READ_ERROR = 1, WRITE_ERROR = 2 };
+   enum
+   {
+      // Workaround for use with headers that define NO_ERROR as a macro,
+      // e.g. winerror.h (which is included by Windows.h):
+#ifndef NO_ERROR
+      NO_ERROR = 0,
+#endif
+      // Use the following identifier if NO_ERROR is defined as a macro,
+      // e.g. winerror.h (which is included by Windows.h):
+      No_Error    = 0,
+      READ_ERROR  = 1,
+      WRITE_ERROR = 2
+   };

   /// Get the current error state
   int Error() const { return error; }
   /// Reset the error state
-   void ResetError(int err_state = NO_ERROR) { error = err_state; }
+   void ResetError(int err_state = No_Error) { error = err_state; }

 #ifdef MFEM_USE_MPI
   friend class ParMesh;
@@ -441,21 +454,29 @@ public:
 #endif

   /// Set/change the mesh associated with the collection
-   virtual void SetMesh(Mesh *new_mesh);
+   virtual void SetMesh(Mesh *new_mesh) override;

 #ifdef MFEM_USE_MPI
   /// Set/change the mesh associated with the collection.
-   virtual void SetMesh(MPI_Comm comm, Mesh *new_mesh);
+   virtual void SetMesh(MPI_Comm comm, Mesh *new_mesh) override;
 #endif

   /// Add a grid function to the collection and update the root file
-   virtual void RegisterField(const std::string& field_name, GridFunction *gf);
+   virtual void RegisterField(const std::string& field_name,
+                              GridFunction *gf) override;

   /// Add a quadrature function to the collection and update the root file.
   /** Visualization of quadrature function is not supported in VisIt(3.12).
       A patch has been sent to VisIt developers in June 2020. */
   virtual void RegisterQField(const std::string& q_field_name,
-                               QuadratureFunction *qf);
+                               QuadratureFunction *qf) override;
+
+   /// Set the number of digits used for both the cycle and the MPI rank
+   /// @note VisIt seems to require 6 pad digits for the MPI rank. Therefore,
+   /// this function uses this default value. This behavior can be overridden
+   /// by calling SetPadDigitsCycle() and SetPadDigitsRank() instead.
+   virtual void SetPadDigits(int digits) override
+   { pad_digits_cycle=digits; pad_digits_rank=6; }

   /// Set VisIt parameter: default levels of detail for the MultiresControl
   void SetLevelsOfDetail(int levels_of_detail);
@@ -468,13 +489,13 @@ public:
   void DeleteAll();

   /// Save the collection and a VisIt root file
-   virtual void Save();
+   virtual void Save() override;

   /// Save a VisIt root file for the collection
   void SaveRootFile();

   /// Load the collection based on its VisIt data (described in its root file)
-   virtual void Load(int cycle_ = 0);
+   virtual void Load(int cycle_ = 0) override;

   /// We will delete the mesh and fields if we own them
   virtual ~VisItDataCollection() {}
@@ -486,6 +507,7 @@ class ParaViewDataCollection : public DataCollection
 {
 private:
   int levels_of_detail;
+   int compression_level;
   std::fstream pvd_stream;
   VTKFormat pv_data_format;
   bool high_order_output;
@@ -498,6 +520,9 @@ protected:
   void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
   const char *GetDataFormatString() const;
   const char *GetDataTypeString() const;
+   /// @brief If compression is enabled, return the compression level, otherwise
+   /// return 0.
+   int GetCompressionLevel() const;

   std::string GenerateCollectionPath();
   std::string GenerateVTUFileName(const std::string &prefix, int rank);
@@ -516,7 +541,7 @@ public:
                          mfem::Mesh *mesh_ = NULL);

   /// Set refinement levels - every element is uniformly split based on
-   /// levels_of_detail_
+   /// levels_of_detail_. The initial value is 1.
   void SetLevelsOfDetail(int levels_of_detail_);

   /// Save the collection - the directory name is constructed based on the
@@ -527,18 +552,27 @@ public:
   /// VTKFormat::ASCII, VTKFormat::BINARY, and VTKFormat::BINARY32.
   /// The ASCII and BINARY options output double precision data, whereas the
   /// BINARY32 option outputs single precision data.
+   ///
+   /// The initial format is VTKFormat::BINARY.
   void SetDataFormat(VTKFormat fmt);

-   /// Set the zlib compression level. 0 indicates no compression, -1 indicates
-   /// the default compression level. Otherwise, specify a number between 1 and
-   /// 9, 1 being the fastest, and 9 being the best compression. Compression
-   /// only takes effect if the output format is BINARY or BINARY32. MFEM must
-   /// be compiled with MFEM_USE_ZLIB = YES.
+   /// @brief Set the zlib compression level.
+   ///
+   /// 0 indicates no compression, -1 indicates the default compression level.
+   /// Otherwise, specify a number between 1 and 9, 1 being the fastest, and 9
+   /// being the best compression. Compression only takes effect if the output
+   /// format is BINARY or BINARY32. MFEM must be compiled with MFEM_USE_ZLIB =
+   /// YES.
+   ///
+   /// The initial compression level is 0 if MFEM is compiled with MFEM_USE_ZLIB
+   /// turned off, and -1 otherwise.
+   ///
+   /// Any nonzero compression level will enable compression.
   void SetCompressionLevel(int compression_level_);

   /// Enable or disable zlib compression. If the input is true, use the default
   /// zlib compression level (unless the compression level has previously been
-   /// set by calling SetCompressionLevel).
+   /// set by calling SetCompressionLevel()).
   void SetCompression(bool compression_) override;

   /// Returns true if the output format is BINARY or BINARY32, false if ASCII.
@@ -551,6 +585,8 @@ public:
   /// Enable or disable restart mode. If restart is enabled, new writes will
   /// preserve timestep metadata for any solutions prior to the currently
   /// defined time.
+   ///
+   /// Initially, restart mode is disabled.
   void UseRestartMode(bool restart_mode_);

   /// Load the collection - not implemented in the ParaView writer
@@ -339,8 +339,11 @@ ND_TriDofTransformation::TransformDual(double *v) const
 void
 ND_TriDofTransformation::InvTransformDual(double *v) const
 {
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
+   // Return immediately when no face DoFs are present
+   if (nfdofs < 2) { return; }
+
+   MFEM_VERIFY(Fo.Size() >= 1,
+               "Face orientations are unset in ND_TriDofTransformation");

   double data[2];
   Vector v2(data, 2);
@@ -432,8 +435,11 @@ ND_TetDofTransformation::TransformDual(double *v) const
 void
 ND_TetDofTransformation::InvTransformDual(double *v) const
 {
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
+   // Return immediately when no face DoFs are present
+   if (nfdofs < 2) { return; }
+
+   MFEM_VERIFY(Fo.Size() >= 4,
+               "Face orientations are unset in ND_TetDofTransformation");

   double data[2];
   Vector v2(data, 2);
@@ -159,7 +159,7 @@ void KellyErrorEstimator::ComputeEstimates()
   // the FaceInfo class [1]. Also, the FaceElementTransformations
   // documentation [2] may be helpful to grasp what is going on. Note
   // that the FaceElementTransformations also works in the non-
-   // conforming case to transfer the gauss points from the slave to
+   // conforming case to transfer the Gauss points from the slave to
   // the master element.
   // [1]
   // https://github.com/mfem/mfem/blob/02d0bfe9c18ce049c3c93a6a4208080fcfc96991/mesh/mesh.hpp#L94
@@ -417,7 +417,7 @@ void KellyErrorEstimator::ComputeEstimates()
         Vector val(flux_space->GetVDim());
         flux->GetVectorValue(FT->Elem2No, ip, val);

-         // Evaluate gauss point
+         // Evaluate Gauss point
         Vector normal(mesh->SpaceDimension());
         FT->Face->SetIntPoint(&fip);
         if (mesh->Dimension() == mesh->SpaceDimension())
@@ -36,19 +36,19 @@ FiniteElement::FiniteElement(int D, Geometry::Type G,
 #endif
 }

-void FiniteElement::CalcVShape (
+void FiniteElement::CalcVShape(
   const IntegrationPoint &ip, DenseMatrix &shape) const
 {
   MFEM_ABORT("method is not implemented for this class");
 }

-void FiniteElement::CalcVShape (
+void FiniteElement::CalcVShape(
   ElementTransformation &Trans, DenseMatrix &shape) const
 {
   MFEM_ABORT("method is not implemented for this class");
 }

-void FiniteElement::CalcDivShape (
+void FiniteElement::CalcDivShape(
   const IntegrationPoint &ip, Vector &divshape) const
 {
   MFEM_ABORT("method is not implemented for this class");
@@ -97,14 +97,14 @@ void FiniteElement::GetFaceDofs(int face, int **dofs, int *ndofs) const
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::CalcHessian (const IntegrationPoint &ip,
-                                 DenseMatrix &h) const
+void FiniteElement::CalcHessian(const IntegrationPoint &ip,
+                                DenseMatrix &h) const
 {
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::GetLocalInterpolation (ElementTransformation &Trans,
-                                           DenseMatrix &I) const
+void FiniteElement::GetLocalInterpolation(ElementTransformation &Trans,
+                                          DenseMatrix &I) const
 {
   MFEM_ABORT("method is not overloaded");
 }
@@ -122,13 +122,13 @@ void FiniteElement::GetTransferMatrix(const FiniteElement &fe,
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::Project (
+void FiniteElement::Project(
   Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::Project (
+void FiniteElement::Project(
   VectorCoefficient &vc, ElementTransformation &Trans, Vector &dofs) const
 {
   MFEM_ABORT("method is not overloaded");
@@ -137,7 +137,7 @@ void FiniteElement::Project (
 void FiniteElement::ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
                                     Vector &dofs) const
 {
-   mfem_error ("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
+   mfem_error("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
 }

 void FiniteElement::ProjectMatrixCoefficient(
@@ -239,7 +239,6 @@ void FiniteElement::CalcPhysLaplacian(ElementTransformation &Trans,
   }
 }

-
 // Assume a linear mapping
 void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
                                         Vector &Laplacian) const
@@ -250,7 +249,7 @@ void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
   DenseMatrix Gij(dim,dim);
   Vector scale(size);

-   CalcHessian (Trans.GetIntPoint(), hess);
+   CalcHessian(Trans.GetIntPoint(), hess);
   MultAAt(Trans.InverseJacobian(), Gij);

   if (dim == 3)
@@ -283,7 +282,6 @@ void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
         Laplacian[nd] += hess(nd,ii)*scale[ii];
      }
   }
-
 }

 void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
@@ -363,11 +361,128 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
   Mult( hess, lhm, Hessian);
 }

-const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &,
-                                             DofToQuad::Mode) const
+const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
+                                             DofToQuad::Mode mode) const
 {
-   MFEM_ABORT("method is not implemented for this element");
-   return *dof2quad_array[0]; // suppress a warning
+   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");
+
+   for (int i = 0; i < dof2quad_array.Size(); i++)
+   {
+      const DofToQuad &d2q = *dof2quad_array[i];
+      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+   }
+
+#ifdef MFEM_THREAD_SAFE
+   DenseMatrix vshape(dof, dim);
+#endif
+
+   DofToQuad *d2q = new DofToQuad;
+   const int nqpt = ir.GetNPoints();
+   d2q->FE = this;
+   d2q->IntRule = &ir;
+   d2q->mode = mode;
+   d2q->ndof = dof;
+   d2q->nqpt = nqpt;
+   if (range_type == SCALAR)
+   {
+      d2q->B.SetSize(nqpt*dof);
+      d2q->Bt.SetSize(dof*nqpt);
+
+      Vector shape;
+      vshape.GetColumnReference(0, shape);
+      for (int i = 0; i < nqpt; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         CalcShape(ip, shape);
+         for (int j = 0; j < dof; j++)
+         {
+            d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
+         }
+      }
+   }
+   else
+   {
+      d2q->B.SetSize(nqpt*dim*dof);
+      d2q->Bt.SetSize(dof*nqpt*dim);
+
+      for (int i = 0; i < nqpt; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         CalcVShape(ip, vshape);
+         for (int d = 0; d < dim; d++)
+         {
+            for (int j = 0; j < dof; j++)
+            {
+               d2q->B[i+nqpt*(d+dim*j)] = d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
+            }
+         }
+      }
+   }
+   switch (deriv_type)
+   {
+      case GRAD:
+      {
+         d2q->G.SetSize(nqpt*dim*dof);
+         d2q->Gt.SetSize(dof*nqpt*dim);
+
+         for (int i = 0; i < nqpt; i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+            CalcDShape(ip, vshape);
+            for (int d = 0; d < dim; d++)
+            {
+               for (int j = 0; j < dof; j++)
+               {
+                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+               }
+            }
+         }
+         break;
+      }
+      case DIV:
+      {
+         d2q->G.SetSize(nqpt*dof);
+         d2q->Gt.SetSize(dof*nqpt);
+
+         Vector divshape;
+         vshape.GetColumnReference(0, divshape);
+         for (int i = 0; i < nqpt; i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+            CalcDivShape(ip, divshape);
+            for (int j = 0; j < dof; j++)
+            {
+               d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
+            }
+         }
+         break;
+      }
+      case CURL:
+      {
+         d2q->G.SetSize(nqpt*cdim*dof);
+         d2q->Gt.SetSize(dof*nqpt*cdim);
+
+         DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
+         for (int i = 0; i < nqpt; i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+            CalcCurlShape(ip, curlshape);
+            for (int d = 0; d < cdim; d++)
+            {
+               for (int j = 0; j < dof; j++)
+               {
+                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+               }
+            }
+         }
+         break;
+      }
+      case NONE:
+      default:
+         MFEM_ABORT("invalid finite element derivative type");
+   }
+   dof2quad_array.Append(d2q);
+   return *d2q;
 }

 FiniteElement::~FiniteElement()
@@ -379,16 +494,19 @@ FiniteElement::~FiniteElement()
 }


-void ScalarFiniteElement::NodalLocalInterpolation (
+void ScalarFiniteElement::NodalLocalInterpolation(
   ElementTransformation &Trans, DenseMatrix &I,
   const ScalarFiniteElement &fine_fe) const
 {
   double v[Geometry::MaxDim];
-   Vector vv (v, dim);
+   Vector vv(v, dim);
   IntegrationPoint f_ip;

 #ifdef MFEM_THREAD_SAFE
-   Vector c_shape(dof);
+   Vector shape(dof);
+#else
+   Vector shape;
+   vshape.GetColumnReference(0, shape);
 #endif

   MFEM_ASSERT(map_type == fine_fe.GetMapType(), "");
@@ -398,10 +516,10 @@ void ScalarFiniteElement::NodalLocalInterpolation (
   {
      Trans.Transform(fine_fe.Nodes.IntPoint(i), vv);
      f_ip.Set(v, dim);
-      CalcShape(f_ip, c_shape);
+      CalcShape(f_ip, shape);
      for (int j = 0; j < dof; j++)
      {
-         if (fabs(I(i,j) = c_shape(j)) < 1.0e-12)
+         if (fabs(I(i,j) = shape(j)) < 1.0e-12)
         {
            I(i,j) = 0.0;
         }
@@ -422,7 +540,7 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
   // General "interpolation", defined by L2 projection

   double v[Geometry::MaxDim];
-   Vector vv (v, dim);
+   Vector vv(v, dim);
   IntegrationPoint f_ip;

   const int fs = fine_fe.GetDof(), cs = this->GetDof();
@@ -456,14 +574,13 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
   }
 }

-void ScalarFiniteElement::ScalarLocalRestriction(
+void ScalarFiniteElement::ScalarLocalL2Restriction(
   ElementTransformation &Trans, DenseMatrix &R,
   const ScalarFiniteElement &coarse_fe) const
 {
   // General "restriction", defined by L2 projection
   double v[Geometry::MaxDim];
-   Vector vv (v, dim);
-   IntegrationPoint f_ip;
+   Vector vv(v, dim);

   const int cs = coarse_fe.GetDof(), fs = this->GetDof();
   R.SetSize(cs, fs);
@@ -472,16 +589,27 @@ void ScalarFiniteElement::ScalarLocalRestriction(
   const int ir_order = GetOrder() + coarse_fe.GetOrder();
   const IntegrationRule &ir = IntRules.Get(coarse_fe.GetGeomType(), ir_order);

+   // integrate coarse_mass in the coarse space
   for (int i = 0; i < ir.GetNPoints(); i++)
   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
-      this->CalcShape(ip, fine_shape);
-      Trans.Transform(ip, vv);
-      f_ip.Set(v, dim);
-      coarse_fe.CalcShape(f_ip, coarse_shape);
+      const IntegrationPoint &c_ip = ir.IntPoint(i);
+      coarse_fe.CalcShape(c_ip, coarse_shape);
+      AddMult_a_VVt(c_ip.weight, coarse_shape, coarse_mass);
+   }

-      AddMult_a_VVt(ip.weight, coarse_shape, coarse_mass);
-      AddMult_a_VWt(ip.weight, coarse_shape, fine_shape, coarse_fine_mass);
+   // integrate coarse_fine_mass in the fine space
+   Trans.SetIntPoint(&Geometries.GetCenter(geom_type));
+   for (int i = 0; i < ir.GetNPoints(); i++)
+   {
+      const IntegrationPoint &f_ip = ir.IntPoint(i);
+      this->CalcShape(f_ip, fine_shape);
+      Trans.Transform(f_ip, vv);
+
+      IntegrationPoint c_ip;
+      c_ip.Set(v, dim);
+      coarse_fe.CalcShape(c_ip, coarse_shape);
+      AddMult_a_VWt(f_ip.weight*Trans.Weight(), coarse_shape, fine_shape,
+                    coarse_fine_mass);
   }

   DenseMatrixInverse coarse_mass_inv(coarse_mass);
@@ -494,95 +622,6 @@ void ScalarFiniteElement::ScalarLocalRestriction(
      R *= 1.0 / Trans.Weight();
   }
 }
-const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
-                                                   DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = dof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*dof);
-   d2q->Bt.SetSize(dof*nqpt);
-   d2q->G.SetSize(nqpt*dim*dof);
-   d2q->Gt.SetSize(dof*nqpt*dim);
-#ifdef MFEM_THREAD_SAFE
-   Vector c_shape(dof);
-   DenseMatrix vshape(dof, dim);
-#endif
-   for (int i = 0; i < nqpt; i++)
-   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
-      CalcShape(ip, c_shape);
-      for (int j = 0; j < dof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = c_shape(j);
-      }
-      CalcDShape(ip, vshape);
-      for (int d = 0; d < dim; d++)
-      {
-         for (int j = 0; j < dof; j++)
-         {
-            d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j,d);
-         }
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-
-// protected method
-const DofToQuad &ScalarFiniteElement::GetTensorDofToQuad(
-   const TensorBasisElement &tb,
-   const IntegrationRule &ir, DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const Poly_1D::Basis &basis_1d = tb.GetBasis1D();
-   const int ndof = order + 1;
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/dim) + 0.5);
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis_1d.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-

 void NodalFiniteElement::ProjectCurl_2D(
   const FiniteElement &fe, ElementTransformation &Trans,
@@ -618,7 +657,7 @@ void InvertLinearTrans(ElementTransformation &trans,

   double store[3];
   Vector v(store, x.Size());
-   pt.Get(v, x.Size());
+   pt.Get(store, x.Size());
   v -= x;

   trans.InverseJacobian().Mult(v, x);
@@ -631,7 +670,10 @@ void NodalFiniteElement::GetLocalRestriction(ElementTransformation &Trans,
   Vector pt(&ipt.x, dim);

 #ifdef MFEM_THREAD_SAFE
-   Vector c_shape(dof);
+   Vector shape(dof);
+#else
+   Vector shape;
+   vshape.GetColumnReference(0, shape);
 #endif

   Trans.SetIntPoint(&Nodes[0]);
@@ -641,8 +683,8 @@ void NodalFiniteElement::GetLocalRestriction(ElementTransformation &Trans,
      InvertLinearTrans(Trans, Nodes[j], pt);
      if (Geometries.CheckPoint(geom_type, ipt)) // do we need an epsilon here?
      {
-         CalcShape(ipt, c_shape);
-         R.SetRow(j, c_shape);
+         CalcShape(ipt, shape);
+         R.SetRow(j, shape);
      }
      else
      {
@@ -653,7 +695,7 @@ void NodalFiniteElement::GetLocalRestriction(ElementTransformation &Trans,
   R.Threshold(1e-12);
 }

-void NodalFiniteElement::Project (
+void NodalFiniteElement::Project(
   Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {
   for (int i = 0; i < dof; i++)
@@ -662,7 +704,7 @@ void NodalFiniteElement::Project (
      // some coefficients expect that Trans.IntPoint is the same
      // as the second argument of Eval
      Trans.SetIntPoint(&ip);
-      dofs(i) = coeff.Eval (Trans, ip);
+      dofs(i) = coeff.Eval(Trans, ip);
      if (map_type == INTEGRAL)
      {
         dofs(i) *= Trans.Weight();
@@ -670,7 +712,7 @@ void NodalFiniteElement::Project (
   }
 }

-void NodalFiniteElement::Project (
+void NodalFiniteElement::Project(
   VectorCoefficient &vc, ElementTransformation &Trans, Vector &dofs) const
 {
   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
@@ -849,18 +891,18 @@ VectorFiniteElement::VectorFiniteElement(int D, Geometry::Type G,
   }
 }

-void VectorFiniteElement::CalcShape (
+void VectorFiniteElement::CalcShape(
   const IntegrationPoint &ip, Vector &shape ) const
 {
-   mfem_error ("Error: Cannot use scalar CalcShape(...) function with\n"
-               "   VectorFiniteElements!");
+   mfem_error("Error: Cannot use scalar CalcShape(...) function with\n"
+              "   VectorFiniteElements!");
 }

-void VectorFiniteElement::CalcDShape (
+void VectorFiniteElement::CalcDShape(
   const IntegrationPoint &ip, DenseMatrix &dshape ) const
 {
-   mfem_error ("Error: Cannot use scalar CalcDShape(...) function with\n"
-               "   VectorFiniteElements!");
+   mfem_error("Error: Cannot use scalar CalcDShape(...) function with\n"
+              "   VectorFiniteElements!");
 }

 void VectorFiniteElement::SetDerivMembers()
@@ -900,7 +942,7 @@ void VectorFiniteElement::SetDerivMembers()
   }
 }

-void VectorFiniteElement::CalcVShape_RT (
+void VectorFiniteElement::CalcVShape_RT(
   ElementTransformation &Trans, DenseMatrix &shape) const
 {
   MFEM_ASSERT(map_type == H_DIV, "");
@@ -912,7 +954,7 @@ void VectorFiniteElement::CalcVShape_RT (
   shape *= (1.0 / Trans.Weight());
 }

-void VectorFiniteElement::CalcVShape_ND (
+void VectorFiniteElement::CalcVShape_ND(
   ElementTransformation &Trans, DenseMatrix &shape) const
 {
   MFEM_ASSERT(map_type == H_CURL, "");
@@ -954,8 +996,8 @@ void VectorFiniteElement::Project_RT(
   {
      Trans.SetIntPoint(&Nodes.IntPoint(k));
      // dof_k = nk^t adj(J) xk
-      Vector vk(vc.GetData()+k*sdim, sdim);
-      dofs(k) = Trans.AdjugateJacobian().InnerProduct(vk, nk + d2n[k]*dim);
+      dofs(k) = Trans.AdjugateJacobian().InnerProduct(
+                   &vc[k*sdim], nk + d2n[k]*dim);
      if (!square_J) { dofs(k) /= Trans.Weight(); }
   }
 }
@@ -1171,9 +1213,8 @@ void VectorFiniteElement::Project_ND(
   for (int k = 0; k < dof; k++)
   {
      Trans.SetIntPoint(&Nodes.IntPoint(k));
-      Vector vk(vc.GetData()+k*dim, dim);
      // dof_k = xk^t J tk
-      dofs(k) = Trans.Jacobian().InnerProduct(tk + d2t[k]*dim, vk);
+      dofs(k) = Trans.Jacobian().InnerProduct(tk + d2t[k]*dim, &vc[k*dim]);
   }
 }

@@ -1320,7 +1361,7 @@ void VectorFiniteElement::LocalL2Projection_RT(
      double w = ip.weight;
      this->CalcVShape(ip, fine_shape);
      Trans.Transform(ip, v);
-      tr_ip.Set(v, dim);
+      tr_ip.Set(v.GetData(), dim);
      cfe.CalcVShape(tr_ip, coarse_shape);

      AddMult_a_AAt(w, fine_shape, fine_mass);
@@ -1407,7 +1448,7 @@ void VectorFiniteElement::LocalL2Projection_ND(
      const IntegrationPoint &ip = ir.IntPoint(i);
      this->CalcVShape(ip, fine_shape);
      Trans.Transform(ip, v);
-      tr_ip.Set(v, dim);
+      tr_ip.Set(v.GetData(), dim);
      cfe.CalcVShape(tr_ip, coarse_shape);

      AddMult_a_AAt(ip.weight, fine_shape, fine_mass);
@@ -2403,6 +2444,46 @@ TensorBasisElement::TensorBasisElement(const int dims, const int p,
   }
 }

+const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
+   const FiniteElement &fe, const IntegrationRule &ir,
+   DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
+   Array<DofToQuad*> &dof2quad_array)
+{
+   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
+
+   for (int i = 0; i < dof2quad_array.Size(); i++)
+   {
+      const DofToQuad &d2q = *dof2quad_array[i];
+      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+   }
+
+   DofToQuad *d2q = new DofToQuad;
+   const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
+   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+   d2q->FE = &fe;
+   d2q->IntRule = &ir;
+   d2q->mode = mode;
+   d2q->ndof = ndof;
+   d2q->nqpt = nqpt;
+   d2q->B.SetSize(nqpt*ndof);
+   d2q->Bt.SetSize(ndof*nqpt);
+   d2q->G.SetSize(nqpt*ndof);
+   d2q->Gt.SetSize(ndof*nqpt);
+   Vector val(ndof), grad(ndof);
+   for (int i = 0; i < nqpt; i++)
+   {
+      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
+      // of the 1D rule.
+      basis.Eval(ir.IntPoint(i).x, val, grad);
+      for (int j = 0; j < ndof; j++)
+      {
+         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
+         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+      }
+   }
+   dof2quad_array.Append(d2q);
+   return *d2q;
+}

 NodalTensorFiniteElement::NodalTensorFiniteElement(const int dims,
                                                   const int p,
@@ -2437,8 +2518,7 @@ VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                                                     const DofMapType dmtype)
   : VectorFiniteElement(dims, GetTensorProductGeometry(dims), d,
                         p, M, FunctionSpace::Qk),
-     TensorBasisElement(dims, p, VerifyNodal(cbtype), dmtype),
-     cbasis1d(poly1d.GetBasis(p, VerifyClosed(cbtype))),
+     TensorBasisElement(dims, p, VerifyNodal(VerifyClosed(cbtype)), dmtype),
     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(obtype)))
 {
   MFEM_VERIFY(dims > 1, "Constructor for VectorTensorFiniteElement with both "
@@ -2453,93 +2533,13 @@ VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                                                     const DofMapType dmtype)
   : VectorFiniteElement(dims, GetTensorProductGeometry(dims), d,
                         p, M, FunctionSpace::Pk),
-     TensorBasisElement(dims, p, obtype, dmtype),
-     cbasis1d(poly1d.GetBasis(p, VerifyOpen(obtype))),
+     TensorBasisElement(dims, p, VerifyOpen(obtype), dmtype),
     obasis1d(poly1d.GetBasis(p, VerifyOpen(obtype)))
 {
   MFEM_VERIFY(dims == 1, "Constructor for VectorTensorFiniteElement without "
               "closed basis is only valid for 1D elements.");
 }

-const DofToQuad &VectorTensorFiniteElement::GetDofToQuad(
-   const IntegrationRule &ir,
-   DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
-
-   return GetTensorDofToQuad(ir, mode, true);
-}
-
-const DofToQuad &VectorTensorFiniteElement::GetDofToQuadOpen(
-   const IntegrationRule &ir,
-   DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
-
-   return GetTensorDofToQuad(ir, mode, false);
-}
-
-const DofToQuad &VectorTensorFiniteElement::GetTensorDofToQuad(
-   const IntegrationRule &ir,
-   DofToQuad::Mode mode,
-   const bool closed) const
-{
-   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
-
-   for (int i = 0;
-        i < (closed ? dof2quad_array.Size() : dof2quad_array_open.Size());
-        i++)
-   {
-      const DofToQuad &d2q = closed ? *dof2quad_array[i] : *dof2quad_array_open[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int ndof = closed ? order + 1 : order;
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/dim) + 0.5);
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-
-      if (closed)
-      {
-         cbasis1d.Eval(ir.IntPoint(i).x, val, grad);
-      }
-      else
-      {
-         obasis1d.Eval(ir.IntPoint(i).x, val, grad);
-      }
-
-      for (int j = 0; j < ndof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
-      }
-   }
-
-   if (closed)
-   {
-      dof2quad_array.Append(d2q);
-   }
-   else
-   {
-      dof2quad_array_open.Append(d2q);
-   }
-
-   return *d2q;
-}
-
 VectorTensorFiniteElement::~VectorTensorFiniteElement()
 {
   for (int i = 0; i < dof2quad_array_open.Size(); i++)
@@ -127,7 +127,6 @@ public:
   }
 };

-
 /** @brief Structure representing the matrices/tensors needed to evaluate (in
    reference space) the values, gradients, divergences, or curls of a
    FiniteElement at a the quadrature points of a given IntegrationRule. */
@@ -157,8 +156,7 @@ public:
          dimensions using 1D number of quadrature points and degrees of
          freedom. */
      /** When representing a vector-valued FiniteElement, two DofToQuad objects
-          are used to describe the "closed" and "open" 1D basis functions
-          (TODO). */
+          are used to describe the "closed" and "open" 1D basis functions. */
      TENSOR
   };

@@ -176,7 +174,7 @@ public:
   /// Basis functions evaluated at quadrature points.
   /** The storage layout is column-major with dimensions:
       - #nqpt x #ndof, for scalar elements, or
-       - #nqpt x dim x #ndof, for vector elements, (TODO)
+       - #nqpt x dim x #ndof, for vector elements,

       where

@@ -187,15 +185,15 @@ public:
   /// Transpose of #B.
   /** The storage layout is column-major with dimensions:
       - #ndof x #nqpt, for scalar elements, or
-       - #ndof x #nqpt x dim, for vector elements (TODO). */
+       - #ndof x #nqpt x dim, for vector elements. */
   Array<double> Bt;

   /** @brief Gradients/divergences/curls of basis functions evaluated at
       quadrature points. */
   /** The storage layout is column-major with dimensions:
       - #nqpt x dim x #ndof, for scalar elements, or
-       - #nqpt x #ndof, for H(div) vector elements (TODO), or
-       - #nqpt x cdim x #ndof, for H(curl) vector elements (TODO),
+       - #nqpt x #ndof, for H(div) vector elements, or
+       - #nqpt x cdim x #ndof, for H(curl) vector elements,

       where

@@ -208,12 +206,11 @@ public:
   /// Transpose of #G.
   /** The storage layout is column-major with dimensions:
       - #ndof x #nqpt x dim, for scalar elements, or
-       - #ndof x #nqpt, for H(div) vector elements (TODO), or
-       - #ndof x #nqpt x cdim, for H(curl) vector elements (TODO). */
+       - #ndof x #nqpt, for H(div) vector elements, or
+       - #ndof x #nqpt x cdim, for H(curl) vector elements. */
   Array<double> Gt;
 };

-
 /// Describes the function space on each element
 class FunctionSpace
 {
@@ -247,7 +244,7 @@ protected:
   mutable int orders[Geometry::MaxDim]; ///< Anisotropic orders
   IntegrationRule Nodes;
 #ifndef MFEM_THREAD_SAFE
-   mutable DenseMatrix vshape; // Dof x VDim
+   mutable DenseMatrix vshape; // Dof x Dim
 #endif
   /// Container for all DofToQuad objects created by the FiniteElement.
   /** Multiple DofToQuad objects may be needed when different quadrature rules
@@ -256,7 +253,7 @@ protected:

 public:
   /// Enumeration for range_type and deriv_range_type
-   enum RangeType { SCALAR, VECTOR };
+   enum RangeType { UNKNOWN_RANGE_TYPE = -1, SCALAR, VECTOR };

   /** @brief Enumeration for MapType: defines how reference functions are
       mapped to physical space.
@@ -270,6 +267,8 @@ public:
   */
   enum MapType
   {
+      UNKNOWN_MAP_TYPE = -1, /**< Used to distinguish an unset MapType variable
+                                  from the known values below. */
      VALUE,     /**< For scalar fields; preserves point values
                          \f$ u(x) = \hat u(\hat x) \f$ */
      INTEGRAL,  /**< For scalar fields; preserves volume integrals
@@ -348,7 +347,6 @@ public:
       H_DIV, H_CURL}. */
   int GetMapType() const { return map_type; }

-
   /** @brief Returns the FiniteElement::DerivType of the element describing the
       spatial derivative method implemented, one of {NONE, GRAD,
       DIV, CURL}. */
@@ -455,8 +453,8 @@ public:
       part of the Hessian of one shape function.
       The order in 2D is {u_xx, u_xy, u_yy}.
       The size (#dof x (#dim (#dim+1)/2) of @a Hessian must be set in advance.*/
-   virtual void CalcHessian (const IntegrationPoint &ip,
-                             DenseMatrix &Hessian) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;

   /** @brief Evaluate the Hessian of all shape functions of a scalar finite
       element in reference space at the given point @a ip. */
@@ -577,6 +575,7 @@ public:
   /** See the documentation for DofToQuad for more details. */
   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                         DofToQuad::Mode mode) const;
+
   /// Deconstruct the FiniteElement
   virtual ~FiniteElement();

@@ -623,16 +622,11 @@ public:
   }
 };

-
 /** @brief Class for finite elements with basis functions
    that return scalar values. */
 class ScalarFiniteElement : public FiniteElement
 {
 protected:
-#ifndef MFEM_THREAD_SAFE
-   mutable Vector c_shape;
-#endif
-
   static const ScalarFiniteElement &CheckScalarFE(const FiniteElement &fe)
   {
      MFEM_VERIFY(fe.GetRangeType() == SCALAR,
@@ -640,10 +634,6 @@ protected:
      return static_cast<const ScalarFiniteElement &>(fe);
   }

-   const DofToQuad &GetTensorDofToQuad(const class TensorBasisElement &tb,
-                                       const IntegrationRule &ir,
-                                       DofToQuad::Mode mode) const;
-
 public:
   /** @brief Construct ScalarFiniteElement with given
       @param D    Reference space dimension
@@ -654,13 +644,8 @@ public:
    */
   ScalarFiniteElement(int D, Geometry::Type G, int Do, int O,
                       int F = FunctionSpace::Pk)
-#ifdef MFEM_THREAD_SAFE
      : FiniteElement(D, G, Do, O, F)
   { deriv_type = GRAD; deriv_range_type = VECTOR; deriv_map_type = H_CURL; }
-#else
-      : FiniteElement(D, G, Do, O, F), c_shape(dof)
-   { deriv_type = GRAD; deriv_range_type = VECTOR; deriv_map_type = H_CURL; }
-#endif

   /** @brief Set the FiniteElement::MapType of the element to either VALUE or
       INTEGRAL. Also sets the FiniteElement::DerivType to GRAD if the
@@ -672,7 +657,6 @@ public:
      deriv_type = (M == VALUE) ? GRAD : NONE;
   }

-
   /** @brief Get the matrix @a I that defines nodal interpolation
       @a between this element and the refined element @a fine_fe. */
   void NodalLocalInterpolation(ElementTransformation &Trans,
@@ -693,15 +677,11 @@ public:
   /** If the "fine" elements cannot represent all basis functions of the
       "coarse" element, then boundary values from different sub-elements are
       generally different. */
-   void ScalarLocalRestriction(ElementTransformation &Trans,
-                               DenseMatrix &R,
-                               const ScalarFiniteElement &coarse_fe) const;
-
-   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                         DofToQuad::Mode mode) const;
+   void ScalarLocalL2Restriction(ElementTransformation &Trans,
+                                 DenseMatrix &R,
+                                 const ScalarFiniteElement &coarse_fe) const;
 };

-
 /// Class for standard nodal finite elements.
 class NodalFiniteElement : public ScalarFiniteElement
 {
@@ -723,38 +703,38 @@ public:
                      int F = FunctionSpace::Pk)
      : ScalarFiniteElement(D, G, Do, O, F) { }

-   virtual void GetLocalInterpolation(ElementTransformation &Trans,
-                                      DenseMatrix &I) const
+   void GetLocalInterpolation(ElementTransformation &Trans,
+                              DenseMatrix &I) const override
   { NodalLocalInterpolation(Trans, I, *this); }

-   virtual void GetLocalRestriction(ElementTransformation &Trans,
-                                    DenseMatrix &R) const;
+   void GetLocalRestriction(ElementTransformation &Trans,
+                            DenseMatrix &R) const override;

-   virtual void GetTransferMatrix(const FiniteElement &fe,
-                                  ElementTransformation &Trans,
-                                  DenseMatrix &I) const
+   void GetTransferMatrix(const FiniteElement &fe,
+                          ElementTransformation &Trans,
+                          DenseMatrix &I) const override
   { CheckScalarFE(fe).NodalLocalInterpolation(Trans, I, *this); }

-   virtual void Project (Coefficient &coeff,
-                         ElementTransformation &Trans, Vector &dofs) const;
+   void Project(Coefficient &coeff,
+                ElementTransformation &Trans, Vector &dofs) const override;

-   virtual void Project (VectorCoefficient &vc,
-                         ElementTransformation &Trans, Vector &dofs) const;
+   void Project(VectorCoefficient &vc,
+                ElementTransformation &Trans, Vector &dofs) const override;

   // (mc.height x mc.width) @ DOFs -> (Dof x mc.width x mc.height) in dofs
-   virtual void ProjectMatrixCoefficient(
-      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;
+   void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const override;

-   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
-                        DenseMatrix &I) const;
+   void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                DenseMatrix &I) const override;

-   virtual void ProjectGrad(const FiniteElement &fe,
-                            ElementTransformation &Trans,
-                            DenseMatrix &grad) const;
+   void ProjectGrad(const FiniteElement &fe,
+                    ElementTransformation &Trans,
+                    DenseMatrix &grad) const override;

-   virtual void ProjectDiv(const FiniteElement &fe,
-                           ElementTransformation &Trans,
-                           DenseMatrix &div) const;
+   void ProjectDiv(const FiniteElement &fe,
+                   ElementTransformation &Trans,
+                   DenseMatrix &div) const override;

   /** @brief Get an Array<int> that maps lexicographically ordered indices to
       the indices of the respective nodes/dofs/basis functions.
@@ -788,12 +768,12 @@ class VectorFiniteElement : public FiniteElement
   // Hide the scalar functions CalcShape and CalcDShape.
 private:
   /// Overrides the scalar CalcShape function to print an error.
-   virtual void CalcShape(const IntegrationPoint &ip,
-                          Vector &shape) const;
+   void CalcShape(const IntegrationPoint &ip,
+                  Vector &shape) const override;

   /// Overrides the scalar CalcDShape function to print an error.
-   virtual void CalcDShape(const IntegrationPoint &ip,
-                           DenseMatrix &dshape) const;
+   void CalcDShape(const IntegrationPoint &ip,
+                   DenseMatrix &dshape) const override;

 protected:
   bool is_nodal;
@@ -952,11 +932,10 @@ protected:
   }

 public:
-   VectorFiniteElement (int D, Geometry::Type G, int Do, int O, int M,
-                        int F = FunctionSpace::Pk);
+   VectorFiniteElement(int D, Geometry::Type G, int Do, int O, int M,
+                       int F = FunctionSpace::Pk);
 };

-
 /// @brief Class for computing 1D special polynomials and their associated basis
 /// functions
 class Poly_1D
@@ -1091,6 +1070,11 @@ public:
   // { CalcLegendre(p, x, u); }
   { CalcChebyshev(p, x, u); }

+   /** @brief Evaluate the values of a hierarchical 1D basis at point x
+       hierarchical = k-th basis function is degree k polynomial */
+   static void CalcBasis(const int p, const double x, Vector &u)
+   { CalcBasis(p, x, u.GetData()); }
+
   /// Evaluate the values and derivatives of a hierarchical 1D basis at point @a x
   static void CalcBasis(const int p, const double x, double *u, double *d)
   // { CalcMono(p, x, u, d); }
@@ -1098,6 +1082,11 @@ public:
   // { CalcLegendre(p, x, u, d); }
   { CalcChebyshev(p, x, u, d); }

+   /** @brief Evaluate the values and derivatives of a hierarchical 1D basis at
+       point @a x. */
+   static void CalcBasis(const int p, const double x, Vector &u, Vector &d)
+   { CalcBasis(p, x, u.GetData(), d.GetData()); }
+
   /// Evaluate the values, derivatives and second derivatives of a hierarchical 1D basis at point x
   static void CalcBasis(const int p, const double x, double *u, double *d,
                         double *dd)
@@ -1106,6 +1095,12 @@ public:
   // { CalcLegendre(p, x, u, d); }
   { CalcChebyshev(p, x, u, d, dd); }

+   /** @brief Evaluate the values, derivatives and second derivatives of a
+       hierarchical 1D basis at point @a x. */
+   static void CalcBasis(const int p, const double x, Vector &u, Vector &d,
+                         Vector &dd)
+   { CalcBasis(p, x, u.GetData(), d.GetData(), dd.GetData()); }
+
   /// Evaluate a representation of a Delta function at point x
   static double CalcDelta(const int p, const double x)
   { return pow(x, (double) p); }
@@ -1135,12 +1130,24 @@ public:
   static void CalcBernstein(const int p, const double x, double *u)
   { CalcBinomTerms(p, x, 1. - x, u); }

+   /** @brief Compute the values of the Bernstein basis functions of order
+       @a p at coordinate @a x and store the results in the already allocated
+       @a u array. */
+   static void CalcBernstein(const int p, const double x, Vector &u)
+   { CalcBernstein(p, x, u.GetData()); }
+
   /** @brief Compute the values and derivatives of the Bernstein basis functions
       of order @a p at coordinate @a x and store the results in the already allocated
       @a u and @a d arrays. */
   static void CalcBernstein(const int p, const double x, double *u, double *d)
   { CalcBinomTerms(p, x, 1. - x, u, d); }

+   /** @brief Compute the values and derivatives of the Bernstein basis
+       functions of order @a p at coordinate @a x and store the results in the
+       already allocated @a u and @a d arrays. */
+   static void CalcBernstein(const int p, const double x, Vector &u, Vector &d)
+   { CalcBernstein(p, x, u.GetData(), d.GetData()); }
+
   static void CalcLegendre(const int p, const double x, double *u);
   static void CalcLegendre(const int p, const double x, double *u, double *d);

@@ -1149,7 +1156,6 @@ public:

 extern Poly_1D poly1d;

-
 /// An element defined as an ND tensor product of 1D elements on a segment,
 /// square, or cube
 class TensorBasisElement
@@ -1173,7 +1179,7 @@ public:

   int GetBasisType() const { return b_type; }

-   const Poly_1D::Basis& GetBasis1D() const { return basis1d; }
+   const Poly_1D::Basis &GetBasis1D() const { return basis1d; }

   /** @brief Get an Array<int> that maps lexicographically ordered indices to
       the indices of the respective nodes/dofs/basis functions. If the dofs are
@@ -1205,6 +1211,11 @@ public:
         default: MFEM_ABORT("invalid dimension: " << dim); return -1;
      }
   }
+
+   static const DofToQuad &GetTensorDofToQuad(
+      const FiniteElement &fe, const IntegrationRule &ir,
+      DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
+      Array<DofToQuad*> &dof2quad_array);
 };

 class NodalTensorFiniteElement : public NodalFiniteElement,
@@ -1215,18 +1226,18 @@ public:
                            const DofMapType dmtype);

   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const
+                                 DofToQuad::Mode mode) const override
   {
      return (mode == DofToQuad::FULL) ?
-             ScalarFiniteElement::GetDofToQuad(ir, mode) :
-             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
+             FiniteElement::GetDofToQuad(ir, mode) :
+             GetTensorDofToQuad(*this, ir, mode, basis1d, true, dof2quad_array);
   }

-   virtual void SetMapType(const int map_type_);
+   void SetMapType(const int map_type_) override;

-   virtual void GetTransferMatrix(const FiniteElement &fe,
-                                  ElementTransformation &Trans,
-                                  DenseMatrix &I) const
+   void GetTransferMatrix(const FiniteElement &fe,
+                          ElementTransformation &Trans,
+                          DenseMatrix &I) const override
   {
      if (basis1d.IsIntegratedType())
      {
@@ -1246,7 +1257,7 @@ private:
   mutable Array<DofToQuad*> dof2quad_array_open;

 protected:
-   Poly_1D::Basis &cbasis1d, &obasis1d;
+   Poly_1D::Basis &obasis1d;

 public:
   VectorTensorFiniteElement(const int dims, const int d, const int p,
@@ -1259,16 +1270,22 @@ public:
                             const DofMapType dmtype);

   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const;
+                                 DofToQuad::Mode mode) const override
+   {
+      MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
+      return GetTensorDofToQuad(*this, ir, mode, basis1d, true,
+                                dof2quad_array);
+   }

   const DofToQuad &GetDofToQuadOpen(const IntegrationRule &ir,
-                                     DofToQuad::Mode mode) const;
+                                     DofToQuad::Mode mode) const
+   {
+      MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
+      return GetTensorDofToQuad(*this, ir, mode, obasis1d, false,
+                                dof2quad_array_open);
+   }

-   const DofToQuad &GetTensorDofToQuad(const IntegrationRule &ir,
-                                       DofToQuad::Mode mode,
-                                       const bool closed) const;
-
-   ~VectorTensorFiniteElement();
+   virtual ~VectorTensorFiniteElement();
 };

 void InvertLinearTrans(ElementTransformation &trans,
@@ -32,6 +32,11 @@ public:
   virtual void CalcDShape(const IntegrationPoint &ip,
                           DenseMatrix &dshape) const;
   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { ScalarLocalL2Restriction(Trans, R, *this); }
+
 };


@@ -55,6 +60,11 @@ public:
                            ElementTransformation &Trans,
                            DenseMatrix &curl) const
   { ProjectCurl_2D(fe, Trans, curl); }
+
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { ScalarLocalL2Restriction(Trans, R, *this); }
+
   using FiniteElement::Project;
   virtual void ProjectDiv(const FiniteElement &fe,
                           ElementTransformation &Trans,
@@ -80,6 +90,11 @@ public:
   virtual void CalcDShape(const IntegrationPoint &ip,
                           DenseMatrix &dshape) const;
   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { ScalarLocalL2Restriction(Trans, R, *this); }
+
   using FiniteElement::Project;
   virtual void ProjectDiv(const FiniteElement &fe,
                           ElementTransformation &Trans,
@@ -111,6 +126,11 @@ public:
                            ElementTransformation &Trans,
                            DenseMatrix &curl) const
   { ProjectCurl_2D(fe, Trans, curl); }
+
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { ScalarLocalL2Restriction(Trans, R, *this); }
+
 };


@@ -133,6 +153,11 @@ public:
   virtual void CalcDShape(const IntegrationPoint &ip,
                           DenseMatrix &dshape) const;
   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { ScalarLocalL2Restriction(Trans, R, *this); }
+
 };


@@ -314,14 +314,16 @@ void ND_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(p + 1), shape_ox(p), shape_cy(p + 1), shape_oy(p);
   Vector shape_cz(p + 1), shape_oz(p);
-   Vector dshape_cx, dshape_cy, dshape_cz;
 #endif

   if (obasis1d.IsIntegratedType())
   {
-      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
-      cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
+#ifdef MFEM_THREAD_SAFE
+      Vector dshape_cx(p + 1), dshape_cy(p + 1), dshape_cz(p + 1);
+#endif
+      basis1d.Eval(ip.x, shape_cx, dshape_cx);
+      basis1d.Eval(ip.y, shape_cy, dshape_cy);
+      basis1d.Eval(ip.z, shape_cz, dshape_cz);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
@@ -329,9 +331,9 @@ void ND_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
   }
   else
   {
-      cbasis1d.Eval(ip.x, shape_cx);
-      cbasis1d.Eval(ip.y, shape_cy);
-      cbasis1d.Eval(ip.z, shape_cz);
+      basis1d.Eval(ip.x, shape_cx);
+      basis1d.Eval(ip.y, shape_cy);
+      basis1d.Eval(ip.z, shape_cz);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
      obasis1d.Eval(ip.z, shape_oz);
@@ -405,9 +407,9 @@ void ND_HexahedronElement::CalcCurlShape(const IntegrationPoint &ip,
   Vector dshape_cx(p + 1), dshape_cy(p + 1), dshape_cz(p + 1);
 #endif

-   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
-   cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
+   basis1d.Eval(ip.x, shape_cx, dshape_cx);
+   basis1d.Eval(ip.y, shape_cy, dshape_cy);
+   basis1d.Eval(ip.z, shape_cz, dshape_cz);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -656,21 +658,23 @@ void ND_QuadrilateralElement::CalcVShape(const IntegrationPoint &ip,

 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(p + 1), shape_ox(p), shape_cy(p + 1), shape_oy(p);
-   Vector dshape_cx, dshape_cy;
 #endif

   if (obasis1d.IsIntegratedType())
   {
-      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+#ifdef MFEM_THREAD_SAFE
+      Vector dshape_cx(p + 1), dshape_cy(p + 1);
+#endif
+      basis1d.Eval(ip.x, shape_cx, dshape_cx);
+      basis1d.Eval(ip.y, shape_cy, dshape_cy);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
   }
   else
   {
-      cbasis1d.Eval(ip.x, shape_cx);
-      cbasis1d.Eval(ip.y, shape_cy);
+      basis1d.Eval(ip.x, shape_cx);
+      basis1d.Eval(ip.y, shape_cy);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
   }
@@ -720,8 +724,8 @@ void ND_QuadrilateralElement::CalcCurlShape(const IntegrationPoint &ip,
   Vector dshape_cx(p + 1), dshape_cy(p + 1);
 #endif

-   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+   basis1d.Eval(ip.x, shape_cx, dshape_cx);
+   basis1d.Eval(ip.y, shape_cy, dshape_cy);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -13,6 +13,7 @@

 #include "fe_pos.hpp"
 #include "../bilininteg.hpp"
+#include "../lininteg.hpp"
 #include "../coefficient.hpp"

 namespace mfem
@@ -212,7 +213,7 @@ void BiQuadPos2DFiniteElement::GetLocalInterpolation(
 void BiQuadPos2DFiniteElement::Project(
   Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {
-   double *d = dofs;
+   double *d = dofs.GetData();

   for (int i = 0; i < 9; i++)
   {
@@ -382,8 +383,8 @@ void H1Pos_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   Vector shape_x(p+1), shape_y(p+1);
 #endif

-   Poly_1D::CalcBernstein(p, ip.x, shape_x.GetData() );
-   Poly_1D::CalcBernstein(p, ip.y, shape_y.GetData() );
+   Poly_1D::CalcBernstein(p, ip.x, shape_x);
+   Poly_1D::CalcBernstein(p, ip.y, shape_y);

   // Reorder so that vertices are at the beginning of the list
   for (int o = 0, j = 0; j <= p; j++)
@@ -402,8 +403,8 @@ void H1Pos_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
   Vector shape_x(p+1), shape_y(p+1), dshape_x(p+1), dshape_y(p+1);
 #endif

-   Poly_1D::CalcBernstein(p, ip.x, shape_x.GetData(), dshape_x.GetData() );
-   Poly_1D::CalcBernstein(p, ip.y, shape_y.GetData(), dshape_y.GetData() );
+   Poly_1D::CalcBernstein(p, ip.x, shape_x, dshape_x);
+   Poly_1D::CalcBernstein(p, ip.y, shape_y, dshape_y);

   // Reorder so that vertices are at the beginning of the list
   for (int o = 0, j = 0; j <= p; j++)
@@ -40,7 +40,7 @@ public:

   virtual void GetLocalRestriction(ElementTransformation &Trans,
                                    DenseMatrix &R) const
-   { ScalarLocalRestriction(Trans, R, *this); }
+   { ScalarLocalL2Restriction(Trans, R, *this); }

   virtual void GetTransferMatrix(const FiniteElement &fe,
                                  ElementTransformation &Trans,
@@ -73,8 +73,8 @@ public:
                                 DofToQuad::Mode mode) const
   {
      return (mode == DofToQuad::FULL) ?
-             ScalarFiniteElement::GetDofToQuad(ir, mode) :
-             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
+             FiniteElement::GetDofToQuad(ir, mode) :
+             GetTensorDofToQuad(*this, ir, mode, basis1d, true, dof2quad_array);
   }
 };

@@ -145,21 +145,23 @@ void RT_QuadrilateralElement::CalcVShape(const IntegrationPoint &ip,

 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(pp1 + 1), shape_ox(pp1), shape_cy(pp1 + 1), shape_oy(pp1);
-   Vector dshape_cx, dshape_cy;
 #endif

   if (obasis1d.IsIntegratedType())
   {
-      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+#ifdef MFEM_THREAD_SAFE
+      Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1);
+#endif
+      basis1d.Eval(ip.x, shape_cx, dshape_cx);
+      basis1d.Eval(ip.y, shape_cy, dshape_cy);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
   }
   else
   {
-      cbasis1d.Eval(ip.x, shape_cx);
-      cbasis1d.Eval(ip.y, shape_cy);
+      basis1d.Eval(ip.x, shape_cx);
+      basis1d.Eval(ip.y, shape_cy);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
   }
@@ -207,8 +209,8 @@ void RT_QuadrilateralElement::CalcDivShape(const IntegrationPoint &ip,
   Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1);
 #endif

-   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+   basis1d.Eval(ip.x, shape_cx, dshape_cx);
+   basis1d.Eval(ip.y, shape_cy, dshape_cy);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -473,14 +475,16 @@ void RT_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(pp1 + 1), shape_ox(pp1), shape_cy(pp1 + 1), shape_oy(pp1);
   Vector shape_cz(pp1 + 1), shape_oz(pp1);
-   Vector dshape_cx, dshape_cy, dshape_cz;
 #endif

   if (obasis1d.IsIntegratedType())
   {
-      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
-      cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
+#ifdef MFEM_THREAD_SAFE
+      Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1), dshape_cz(pp1 + 1);
+#endif
+      basis1d.Eval(ip.x, shape_cx, dshape_cx);
+      basis1d.Eval(ip.y, shape_cy, dshape_cy);
+      basis1d.Eval(ip.z, shape_cz, dshape_cz);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
@@ -488,9 +492,9 @@ void RT_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
   }
   else
   {
-      cbasis1d.Eval(ip.x, shape_cx);
-      cbasis1d.Eval(ip.y, shape_cy);
-      cbasis1d.Eval(ip.z, shape_cz);
+      basis1d.Eval(ip.x, shape_cx);
+      basis1d.Eval(ip.y, shape_cy);
+      basis1d.Eval(ip.z, shape_cz);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
      obasis1d.Eval(ip.z, shape_oz);
@@ -564,9 +568,9 @@ void RT_HexahedronElement::CalcDivShape(const IntegrationPoint &ip,
   Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1), dshape_cz(pp1 + 1);
 #endif

-   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
-   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
-   cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
+   basis1d.Eval(ip.x, shape_cx, dshape_cx);
+   basis1d.Eval(ip.y, shape_cy, dshape_cy);
+   basis1d.Eval(ip.z, shape_cz, dshape_cz);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -22,6 +22,71 @@ namespace mfem

 using namespace std;

+const FiniteElement *
+FiniteElementCollection::FiniteElementForDim(int dim) const
+{
+   ErrorMode save_error_mode = error_mode;
+   error_mode = RETURN_NULL;
+   const FiniteElement *fe = nullptr;
+   for (int g = Geometry::DimStart[dim]; g < Geometry::DimStart[dim+1]; g++)
+   {
+      fe = FiniteElementForGeometry((Geometry::Type)g);
+      if (fe != nullptr) { break; }
+   }
+   error_mode = save_error_mode;
+   return fe;
+}
+
+int FiniteElementCollection::GetRangeType(int dim) const
+{
+   const FiniteElement *fe = FiniteElementForDim(dim);
+   if (fe)
+   {
+      return fe->GetRangeType();
+   }
+   return FiniteElement::UNKNOWN_RANGE_TYPE;
+}
+
+int FiniteElementCollection::GetDerivRangeType(int dim) const
+{
+   const FiniteElement *fe = FiniteElementForDim(dim);
+   if (fe)
+   {
+      return fe->GetDerivRangeType();
+   }
+   return FiniteElement::UNKNOWN_RANGE_TYPE;
+}
+
+int FiniteElementCollection::GetMapType(int dim) const
+{
+   const FiniteElement *fe = FiniteElementForDim(dim);
+   if (fe)
+   {
+      return fe->GetMapType();
+   }
+   return FiniteElement::UNKNOWN_MAP_TYPE;
+}
+
+int FiniteElementCollection::GetDerivType(int dim) const
+{
+   const FiniteElement *fe = FiniteElementForDim(dim);
+   if (fe)
+   {
+      return fe->GetDerivType();
+   }
+   return FiniteElement::NONE;
+}
+
+int FiniteElementCollection::GetDerivMapType(int dim) const
+{
+   const FiniteElement *fe = FiniteElementForDim(dim);
+   if (fe)
+   {
+      return fe->GetDerivMapType();
+   }
+   return FiniteElement::UNKNOWN_MAP_TYPE;
+}
+
 int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
 {
   switch (geom)
@@ -579,6 +644,7 @@ LinearFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -622,6 +688,7 @@ QuadraticFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -662,6 +729,7 @@ QuadraticPosFECollection::FiniteElementForGeometry(
      case Geometry::SEGMENT:     return &SegmentFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticPosFECollection: unknown geometry type.");
   }
   return NULL; // Make some compilers happy
@@ -702,6 +770,7 @@ CubicFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("CubicFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -768,6 +837,7 @@ CrouzeixRaviartFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("CrouzeixRaviartFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -805,6 +875,7 @@ RT0_2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT0_2DFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -847,6 +918,7 @@ RT1_2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT1_2DFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -888,6 +960,7 @@ RT2_2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT2_2DFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -929,6 +1002,7 @@ Const2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("Const2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -964,6 +1038,7 @@ LinearDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearDiscont2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -999,6 +1074,7 @@ GaussLinearDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("GaussLinearDiscont2DFECollection:"
                     " unknown geometry type.");
   }
@@ -1033,6 +1109,7 @@ P1OnQuadFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
 {
   if (GeomType != Geometry::SQUARE)
   {
+      if (error_mode == RETURN_NULL) { return nullptr; }
      mfem_error ("P1OnQuadFECollection: unknown geometry type.");
   }
   return &QuadrilateralFE;
@@ -1067,6 +1144,7 @@ QuadraticDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticDiscont2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1102,6 +1180,7 @@ QuadraticPosDiscont2DFECollection::FiniteElementForGeometry(
   {
      case Geometry::SQUARE:  return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticPosDiscont2DFECollection: unknown geometry type.");
   }
   return NULL; // Make some compilers happy
@@ -1132,6 +1211,7 @@ const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("GaussQuadraticDiscont2DFECollection:"
                     " unknown geometry type.");
   }
@@ -1170,6 +1250,7 @@ CubicDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("CubicDiscont2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1207,6 +1288,7 @@ LinearNonConf3DFECollection::FiniteElementForGeometry(
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearNonConf3DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1247,6 +1329,7 @@ Const3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("Const3DFECollection: unknown geometry type.");
   }
   return &TetrahedronFE; // Make some compilers happy
@@ -1288,6 +1371,7 @@ LinearDiscont3DFECollection::FiniteElementForGeometry(
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
   }
   return &TetrahedronFE; // Make some compilers happy
@@ -1327,6 +1411,7 @@ QuadraticDiscont3DFECollection::FiniteElementForGeometry(
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticDiscont3DFECollection: unknown geometry type.");
   }
   return &TetrahedronFE; // Make some compilers happy
@@ -1368,6 +1453,7 @@ RefinedLinearFECollection::FiniteElementForGeometry(
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RefinedLinearFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -1408,6 +1494,7 @@ ND1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("ND1_3DFECollection: unknown geometry type.");
   }
   return &HexahedronFE; // Make some compilers happy
@@ -1457,6 +1544,7 @@ RT0_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT0_3DFECollection: unknown geometry type.");
   }
   return &HexahedronFE; // Make some compilers happy
@@ -1506,6 +1594,7 @@ RT1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return &QuadrilateralFE;
      case Geometry::CUBE:        return &HexahedronFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT1_3DFECollection: unknown geometry type.");
   }
   return &HexahedronFE; // Make some compilers happy
@@ -1867,6 +1956,7 @@ H1_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
+      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("H1 Pyramid basis functions are not yet supported "
                 "for order > 1.");
      return NULL;
@@ -2247,6 +2337,7 @@ L2_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
+      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("L2 Pyramid basis functions are not yet supported "
                 "for order > 0.");
      return NULL;
@@ -2502,6 +2593,7 @@ RT_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
+      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("RT Pyramid basis functions are not yet supported "
                 "for order > 0.");
      return NULL;
@@ -2787,6 +2879,7 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
+      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("ND Pyramid basis functions are not yet supported "
                 "for order > 1.");
      return NULL;
@@ -3389,6 +3482,7 @@ NURBSFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return QuadrilateralFE;
      case Geometry::CUBE:        return ParallelepipedFE;
      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("NURBSFECollection: unknown geometry type.");
   }
   return SegmentFE; // Make some compilers happy
@@ -51,6 +51,14 @@ public:
   virtual const FiniteElement *
   FiniteElementForGeometry(Geometry::Type GeomType) const = 0;

+   /** @brief Returns the first non-NULL FiniteElement for the given dimension
+
+       @note Repeatedly calls FiniteElementForGeometry in the order defined in
+       the Geometry::Type enumeration.
+   */
+   virtual const FiniteElement *
+   FiniteElementForDim(int dim) const;
+
   virtual int DofForGeometry(Geometry::Type GeomType) const = 0;

   /** @brief Returns an array, say p, that maps a local permuted index i to a
@@ -66,6 +74,17 @@ public:

   virtual int GetContType() const = 0;

+   /** @note The following methods provide the same information as the
+       corresponding methods of the FiniteElement base class.
+       @{
+   */
+   virtual int GetRangeType(int dim) const;
+   virtual int GetDerivRangeType(int dim) const;
+   virtual int GetMapType(int dim) const;
+   virtual int GetDerivType(int dim) const;
+   virtual int GetDerivMapType(int dim) const;
+   /** @} */
+
   int HasFaceDofs(Geometry::Type geom, int p) const;

   virtual const FiniteElement *TraceFiniteElementForGeometry(
@@ -214,6 +233,19 @@ protected:
   void InitVarOrder(int p) const;

   mutable Array<FiniteElementCollection*> var_orders;
+
+   /// How to treat errors in FiniteElementForGeometry() calls.
+   enum ErrorMode
+   {
+      RETURN_NULL,      ///< Return NULL on errors
+      RAISE_MFEM_ERROR  /**< Raise an MFEM error (default in base class).
+                             Sub-classes can ignore this and return NULL. */
+   };
+
+   /// How to treat errors in FiniteElementForGeometry() calls.
+   /** The typical error in derived classes is that no FiniteElement is defined
+       for the given Geometry, or the input is not a valid Geometry. */
+   mutable ErrorMode error_mode = RAISE_MFEM_ERROR;
 };

 /// Arbitrary order H1-conforming (continuous) finite elements.
@@ -1199,8 +1199,6 @@ void FiniteElementSpace::BuildConformingInterpolation() const
      MakeVDimMatrix(*cR);
      if (cR_hp) { MakeVDimMatrix(*cR_hp); }
   }
-
-   cP->EnsureMultTranspose();
 }

 void FiniteElementSpace::MakeVDimMatrix(SparseMatrix &mat) const
@@ -1960,7 +1958,7 @@ FiniteElementSpace::DerefinementOperator::DerefinementOperator(
         DenseMatrix &lM = localM[g](mi[s]);
         DenseMatrix &lR = localR[g](lR_offset+s);
         MultAtB(lP, lM, lR); // lR = lP^T lM
-         AddMult(lR, lP, lPtMP); // lPtMP += lP^T lM lP
+         mfem::AddMult(lR, lP, lPtMP); // lPtMP += lP^T lM lP
      }
      DenseMatrixInverse lPtMP_inv(lPtMP);
      for (int s = 0; s < nm; s++)
@@ -2007,7 +2005,7 @@ void FiniteElementSpace::DerefinementOperator
         x.GetSubVector(f_vdofs, loc_x);
         loc_x_mat.UseExternalData(loc_x.GetData(), f_vdofs.Size()/fine_vdim,
                                   fine_vdim);
-         AddMult(lR, loc_x_mat, loc_y_mat);
+         mfem::AddMult(lR, loc_x_mat, loc_y_mat);
      }
      y.SetSubVector(c_vdofs, loc_y);
   }
@@ -2058,10 +2056,7 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
      GetLocalDerefinementMatrices(elem_geoms[i], localR[elem_geoms[i]]);
   }

-   SparseMatrix *R = (elem_geoms.Size() != 1)
-                     ? new SparseMatrix(ndofs*vdim, old_ndofs*vdim) // variable row size
-                     : new SparseMatrix(ndofs*vdim, old_ndofs*vdim,
-                                        localR[elem_geoms[0]].SizeI());
+   SparseMatrix *R = new SparseMatrix(ndofs*vdim, old_ndofs*vdim);

   Array<int> mark(R->Height());
   mark = 0;
@@ -2071,6 +2066,7 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,

   MFEM_ASSERT(dtrans.embeddings.Size() == old_elem_dof->Size(), "");

+   bool is_dg = FEColl()->GetContType() == FiniteElementCollection::DISCONTINUOUS;
   int num_marked = 0;
   for (int k = 0; k < dtrans.embeddings.Size(); k++)
   {
@@ -2093,10 +2089,11 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
            int r = DofToVDof(dofs[i], vd);
            int m = (r >= 0) ? r : (-1 - r);

-            if (!mark[m])
+            if (is_dg || !mark[m])
            {
               lR.GetRow(i, row);
               R->SetRow(r, old_vdofs, row);
+
               mark[m] = 1;
               num_marked++;
            }
@@ -2104,8 +2101,11 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
      }
   }

-   MFEM_VERIFY(num_marked == R->Height(),
-               "internal error: not all rows of R were set.");
+   if (!is_dg)
+   {
+      MFEM_VERIFY(num_marked == R->Height(),
+                  "internal error: not all rows of R were set.");
+   }

   R->Finalize(); // no-op if fixed width
   return R;
@@ -3147,7 +3147,7 @@ const FiniteElement *FiniteElementSpace::GetFaceElement(int i) const
         break;
      case 3:
      default:
-         fe = fec->FiniteElementForGeometry(mesh->GetFaceBaseGeometry(i));
+         fe = fec->FiniteElementForGeometry(mesh->GetFaceGeometry(i));
   }

   if (NURBSext)
@@ -531,7 +531,12 @@ public:
       Operator returned by GetElementRestriction().

       All elements will use the same IntegrationRule, @a ir as the target
-       quadrature points. */
+       quadrature points.
+
+       @note The returned pointer is shared. A good practice, before using it,
+       is to set all its properties to their expected values, as other parts of
+       the code may also change them. That is, it's good to call
+       SetOutputLayout() and DisableTensorProducts() before interpolating. */
   const QuadratureInterpolator *GetQuadratureInterpolator(
      const IntegrationRule &ir) const;

@@ -542,12 +547,22 @@ public:
       Operator returned by GetElementRestriction().

       The target quadrature points in the elements are described by the given
-       QuadratureSpace, @a qs. */
+       QuadratureSpace, @a qs.
+
+       @note The returned pointer is shared. A good practice, before using it,
+       is to set all its properties to their expected values, as other parts of
+       the code may also change them. That is, it's good to call
+       SetOutputLayout() and DisableTensorProducts() before interpolating. */
   const QuadratureInterpolator *GetQuadratureInterpolator(
      const QuadratureSpace &qs) const;

   /** @brief Return a FaceQuadratureInterpolator that interpolates E-vectors to
-       quadrature point values and/or derivatives (Q-vectors). */
+       quadrature point values and/or derivatives (Q-vectors).
+
+       @note The returned pointer is shared. A good practice, before using it,
+       is to set all its properties to their expected values, as other parts of
+       the code may also change them. That is, it's good to call
+       SetOutputLayout() and DisableTensorProducts() before interpolating. */
   const FaceQuadratureInterpolator *GetFaceQuadratureInterpolator(
      const IntegrationRule &ir, FaceType type) const;

@@ -38,7 +38,7 @@ public:
   /// Construct an empty finite element space hierarchy. This is useful if the
   /// hierarchy is constructed by coarsening a fine space, rather than refining
   /// a coarse space.
-   FiniteElementSpaceHierarchy() { }
+   FiniteElementSpaceHierarchy() = default;

   /// @brief Constructs a space hierarchy with the given mesh and space on the
   /// coarsest level.
@@ -91,6 +91,7 @@ public:
 class ParFiniteElementSpaceHierarchy : public FiniteElementSpaceHierarchy
 {
 public:
+   ParFiniteElementSpaceHierarchy() = default;
   /// @brief Constructs a parallel space hierarchy with the given mesh and spaces
   /// on level zero.
   /** The ownership of the mesh and space may be transferred to the
@@ -1920,7 +1920,7 @@ DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
                                             FMS_NODAL_GAUSS_CLOSED, 1);
      err |= FmsDataCollectionAddField(*dc, "Coords", &fcoords);
      err |= FmsFieldSet(fcoords, fdcoords, mmesh->SpaceDimension(), FMS_BY_NODES,
-                         FMS_DOUBLE, mverts);
+                         FMS_DOUBLE, mverts.HostRead());
      err |= FmsComponentSetCoordinates(volume, fcoords);
   }

@@ -189,6 +189,8 @@ void GridFunction::Update()
   {
      SetSize(fes->GetVSize());
   }
+
+   if (t_vec.Size() > 0) { SetTrueVector(); }
 }

 void GridFunction::SetSpace(FiniteElementSpace *f)
@@ -414,13 +416,24 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const

   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
      Vector shape(dof);
-      for (k = 0; k < n; k++)
+      if (FElem->GetMapType() == FiniteElement::VALUE)
      {
-         FElem->CalcShape(ElemVert->IntPoint(k), shape);
-         nval[k] = shape * ((const double *)loc_data + dof * vdim);
+         for (k = 0; k < n; k++)
+         {
+            FElem->CalcShape(ElemVert->IntPoint(k), shape);
+            nval[k] = shape * (&loc_data[dof * vdim]);
+         }
+      }
+      else
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(i);
+         for (k = 0; k < n; k++)
+         {
+            Tr->SetIntPoint(&ElemVert->IntPoint(k));
+            FElem->CalcPhysShape(*Tr, shape);
+            nval[k] = shape * (&loc_data[dof * vdim]);
+         }
      }
   }
   else
@@ -493,7 +506,7 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
      val.SetSize(vdim);
      for (int k = 0; k < vdim; k++)
      {
-         val(k) = shape * ((const double *)loc_data + dof * k);
+         val(k) = shape * (&loc_data[dof * k]);
      }
   }
   else
@@ -1025,7 +1038,7 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
      val.SetSize(vdim);
      for (int k = 0; k < vdim; k++)
      {
-         val(k) = shape * ((const double *)loc_data + dof * k);
+         val(k) = shape * (&loc_data[dof * k]);
      }
   }
   else
@@ -1076,7 +1089,7 @@ void GridFunction::GetVectorValues(ElementTransformation &T,

         for (int k = 0; k < vdim; k++)
         {
-            vals(k,j) = shape * ((const double *)loc_data + dof * k);
+            vals(k,j) = shape * (&loc_data[dof * k]);
         }
      }
   }
@@ -1179,8 +1192,7 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
         orig_fe->CalcShape(ip, shape);
         for (d = 0; d < vdim; d++)
         {
-            loc_values(d*dof+j) =
-               shape * ((const double *)orig_loc_values + d * odof) ;
+            loc_values(d*dof+j) = shape * (&orig_loc_values[d * odof]);
         }
      }
      if (doftrans)
@@ -1222,8 +1234,7 @@ void GridFunction::GetBdrValuesFrom(const GridFunction &orig_func)
         orig_fe->CalcShape(ip, shape);
         for (d = 0; d < vdim; d++)
         {
-            loc_values(d*dof+j) =
-               shape * ((const double *)orig_loc_values + d * odof);
+            loc_values(d*dof+j) = shape * (&orig_loc_values[d * odof]);
         }
      }
      SetSubVector(vdofs, loc_values);
@@ -1430,21 +1441,13 @@ void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
   }
 }

-
 void GridFunction::GetVectorGradientHat(
   ElementTransformation &T, DenseMatrix &gh) const
 {
-   int elNo = T.ElementNo;
-   const FiniteElement *FElem = fes->GetFE(elNo);
+   const FiniteElement *FElem = fes->GetFE(T.ElementNo);
   int dim = FElem->GetDim(), dof = FElem->GetDof();
-   Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(elNo, vdofs);
   Vector loc_data;
-   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
+   GetElementDofValues(T.ElementNo, loc_data);
   // assuming scalar FE
   int vdim = fes->GetVDim();
   DenseMatrix dshape(dof, dim);
@@ -1649,6 +1652,7 @@ void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
         const FiniteElement *fe = fes->GetFE(T.ElementNo);
         MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
                     "invalid FE map type");
+         MFEM_ASSERT(fes->GetVDim() == 1, "Defined for scalar functions.");
         int spaceDim = fes->GetMesh()->SpaceDimension();
         int dim = fe->GetDim(), dof = fe->GetDof();
         DenseMatrix dshape(dof, dim);
@@ -1717,13 +1721,8 @@ void GridFunction::GetGradients(ElementTransformation &tr,
   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
   DenseMatrix dshape(fe->GetDof(), fe->GetDim());
   Vector lval, gh(fe->GetDim()), gcol;
-   Array<int> dofs;
-   DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
-   GetSubVector(dofs, lval);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(lval);
-   }
+
+   GetElementDofValues(tr.ElementNo, lval);
   grad.SetSize(fe->GetDim(), ir.GetNPoints());
   for (int i = 0; i < ir.GetNPoints(); i++)
   {
@@ -4090,23 +4089,23 @@ void TensorProductLegendre(int dim,                // input
   }
   else
   {
-      // Bounding box is not reorientated no need to change orientation
+      // Bounding box is not reoriented no need to change orientation
      x = x_in;
   }

   // Map x to [0, 1] to use CalcLegendre since it uses shifted Legendre Polynomials.
   double x1 = (x(0) - xmin(0))/(xmax(0)-xmin(0)), x2, x3;
   Vector poly_x(order+1), poly_y(order+1), poly_z(order+1);
-   poly1d.CalcLegendre(order, x1, poly_x);
+   poly1d.CalcLegendre(order, x1, poly_x.GetData());
   if (dim > 1)
   {
      x2 = (x(1)-xmin(1))/(xmax(1)-xmin(1));
-      poly1d.CalcLegendre(order, x2, poly_y);
+      poly1d.CalcLegendre(order, x2, poly_y.GetData());
   }
   if (dim == 3)
   {
      x3 = (x(2)-xmin(2))/(xmax(2)-xmin(2));
-      poly1d.CalcLegendre(order, x3, poly_z);
+      poly1d.CalcLegendre(order, x3, poly_z.GetData());
   }

   int basis_dimension = static_cast<int>(pow(order+1,dim));
@@ -48,8 +48,6 @@ protected:

   void SaveSTLTri(std::ostream &out, double p1[], double p2[], double p3[]);

-   void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const;
-
   // Project the delta coefficient without scaling and return the (local)
   // integral of the projection.
   void ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
@@ -129,19 +127,21 @@ public:
   int CurlDim() const;

   /// Read only access to the (optional) internal true-dof Vector.
-   /** Note that the returned Vector may be empty, if not previously allocated
-       or set. */
-   const Vector &GetTrueVector() const { return t_vec; }
+   const Vector &GetTrueVector() const
+   {
+      MFEM_VERIFY(t_vec.Size() > 0, "SetTrueVector() before GetTrueVector()");
+      return t_vec;
+   }
   /// Read and write access to the (optional) internal true-dof Vector.
-   /** Note that the returned Vector may be empty, if not previously allocated
-       or set. */
-   Vector &GetTrueVector() { return t_vec; }
+   /** Note that @a t_vec is set if it is not allocated or set already.*/
+   Vector &GetTrueVector()
+   { if (t_vec.Size() == 0) { SetTrueVector(); } return t_vec; }

   /// Extract the true-dofs from the GridFunction.
   void GetTrueDofs(Vector &tv) const;

   /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.
-   void SetTrueVector() { GetTrueDofs(GetTrueVector()); }
+   void SetTrueVector() { GetTrueDofs(t_vec); }

   /// Set the GridFunction from the given true-dof vector.
   virtual void SetFromTrueDofs(const Vector &tv);
@@ -327,17 +327,34 @@ public:

   void GetCurl(ElementTransformation &tr, Vector &curl) const;

+   /** @brief Gradient of a scalar function at a quadrature point.
+
+       @note It is assumed that the IntegrationPoint of interest has been
+       specified by ElementTransformation::SetIntPoint() before calling
+       GetGradient().
+
+       @note Can be used from a ParGridFunction when @a tr is an
+       ElementTransformation of a face-neighbor element and face-neighbor data
+       has been exchanged. */
   void GetGradient(ElementTransformation &tr, Vector &grad) const;

+   /// Extension of GetGradient(...) for a collection of IntegrationPoints.
   void GetGradients(ElementTransformation &tr, const IntegrationRule &ir,
                     DenseMatrix &grad) const;

+   /// Extension of GetGradient(...) for a collection of IntegrationPoints.
   void GetGradients(const int elem, const IntegrationRule &ir,
                     DenseMatrix &grad) const
   { GetGradients(*fes->GetElementTransformation(elem), ir, grad); }

+   /** @brief Compute the vector gradient with respect to the physical element
+       variable. */
   void GetVectorGradient(ElementTransformation &tr, DenseMatrix &grad) const;

+   /** @brief Compute the vector gradient with respect to the reference element
+       variable. */
+   void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const;
+
   /** Compute \f$ (\int_{\Omega} (*this) \psi_i)/(\int_{\Omega} \psi_i) \f$,
       where \f$ \psi_i \f$ are the basis functions for the FE space of avgs.
       Both FE spaces should be scalar and on the same mesh. */
@@ -37,7 +37,7 @@ FindPointsGSLIB::FindPointsGSLIB()
     fec_map_lin(NULL),
     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC)
+     avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
   mesh_split.SetSize(4);
   ir_split.SetSize(4);
@@ -84,7 +84,7 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
     fec_map_lin(NULL),
     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC)
+     avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
   mesh_split.SetSize(4);
   ir_split.SetSize(4);
@@ -168,7 +168,8 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
   setupflag = true;
 }

-void FindPointsGSLIB::FindPoints(const Vector &point_pos)
+void FindPointsGSLIB::FindPoints(const Vector &point_pos,
+                                 int point_pos_ordering)
 {
   MFEM_VERIFY(setupflag, "Use FindPointsGSLIB::Setup before finding points.");
   points_cnt = point_pos.Size() / dim;
@@ -178,14 +179,27 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
   gsl_ref.SetSize(points_cnt * dim);
   gsl_dist.SetSize(points_cnt);

+   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
+   {
+      for (int d = 0; d < dim; d++)
+      {
+         if (point_pos_ordering == Ordering::byNODES)
+         {
+            xv_base[d] = point_pos.GetData() + d*points_cnt;
+            xv_stride[d] = sizeof(double);
+         }
+         else
+         {
+            xv_base[d] = point_pos.GetData() + d;
+            xv_stride[d] = dim*sizeof(double);
+         }
+      }
+   };
   if (dim == 2)
   {
      const double *xv_base[2];
-      xv_base[0] = point_pos.GetData();
-      xv_base[1] = point_pos.GetData() + points_cnt;
      unsigned xv_stride[2];
-      xv_stride[0] = sizeof(double);
-      xv_stride[1] = sizeof(double);
+      xvFill(xv_base, xv_stride, dim);
      findpts_2(gsl_code.GetData(), sizeof(unsigned int),
                gsl_proc.GetData(), sizeof(unsigned int),
                gsl_elem.GetData(), sizeof(unsigned int),
@@ -193,16 +207,11 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
                gsl_dist.GetData(), sizeof(double),
                xv_base, xv_stride, points_cnt, fdata2D);
   }
-   else
+   else  // dim == 3
   {
      const double *xv_base[3];
-      xv_base[0] = point_pos.GetData();
-      xv_base[1] = point_pos.GetData() + points_cnt;
-      xv_base[2] = point_pos.GetData() + 2*points_cnt;
      unsigned xv_stride[3];
-      xv_stride[0] = sizeof(double);
-      xv_stride[1] = sizeof(double);
-      xv_stride[2] = sizeof(double);
+      xvFill(xv_base, xv_stride, dim);
      findpts_3(gsl_code.GetData(), sizeof(unsigned int),
                gsl_proc.GetData(), sizeof(unsigned int),
                gsl_elem.GetData(), sizeof(unsigned int),
@@ -214,10 +223,12 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
   // Set the element number and reference position to 0 for points not found
   for (int i = 0; i < points_cnt; i++)
   {
-      if (gsl_code[i] == 2)
+      if (gsl_code[i] == 2 ||
+          (gsl_code[i] == 1 && gsl_dist(i) > bdr_tol))
      {
         gsl_elem[i] = 0;
         for (int d = 0; d < dim; d++) { gsl_ref(i*dim + d) = -1.; }
+         gsl_code[i] = 2;
      }
   }

@@ -227,27 +238,29 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
 }

 void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
-                                 const double bb_t, const double newt_tol,
-                                 const int npt_max)
+                                 int point_pos_ordering, const double bb_t,
+                                 const double newt_tol, const int npt_max)
 {
   if (!setupflag || (mesh != &m) )
   {
      Setup(m, bb_t, newt_tol, npt_max);
   }
-   FindPoints(point_pos);
+   FindPoints(point_pos, point_pos_ordering);
 }

 void FindPointsGSLIB::Interpolate(const Vector &point_pos,
-                                  const GridFunction &field_in, Vector &field_out)
+                                  const GridFunction &field_in, Vector &field_out,
+                                  int point_pos_ordering)
 {
-   FindPoints(point_pos);
+   FindPoints(point_pos, point_pos_ordering);
   Interpolate(field_in, field_out);
 }

 void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
-                                  const GridFunction &field_in, Vector &field_out)
+                                  const GridFunction &field_in, Vector &field_out,
+                                  int point_pos_ordering)
 {
-   FindPoints(m, point_pos);
+   FindPoints(m, point_pos, point_pos_ordering);
   Interpolate(field_in, field_out);
 }

@@ -560,7 +573,7 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
   const int pts_el = std::pow(dof_1D, dim);
   const int pts_cnt = NE_split_total * pts_el;
   node_vals.SetSize(vdim * pts_cnt);
-   node_vals *= 0;
+   node_vals = 0.0;

   int gsl_mesh_pt_index = 0;

@@ -860,7 +873,9 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
      {
         for (int i = 0; i < indl2.Size(); i++)
         {
-            int idx = indl2[i] + j*points_cnt;
+            int idx = field_in.FESpace()->GetOrdering() == Ordering::byNODES ?
+                      indl2[i] + j*points_cnt:
+                      indl2[i]*ncomp + j;
            field_out(idx) = field_out_l2(idx);
         }
      }
@@ -885,7 +900,17 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
   {
      const int dataptrin  = i*points_fld,
                dataptrout = i*points_cnt;
-      field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
+      if (field_in.FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
+      }
+      else
+      {
+         for (int j = 0; j < points_fld; j++)
+         {
+            field_in_scalar(j) = field_in(i + j*ncomp);
+         }
+      }
      GetNodalValues(&field_in_scalar, node_vals);

      if (dim==2)
@@ -907,6 +932,17 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
                        points_cnt, node_vals.GetData(), fdata3D);
      }
   }
+   if (field_in.FESpace()->GetOrdering() == Ordering::byVDIM)
+   {
+      Vector field_out_temp = field_out;
+      for (int i = 0; i < ncomp; i++)
+      {
+         for (int j = 0; j < points_cnt; j++)
+         {
+            field_out(i + j*ncomp) = field_out_temp(j + i*points_cnt);
+         }
+      }
+   }
 }

 void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
@@ -929,9 +965,19 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
         if (dim == 3) { ip.z = gsl_mfem_ref(index*dim + 2); }
         Vector localval(ncomp);
         field_in.GetVectorValue(gsl_mfem_elem[index], ip, localval);
-         for (int i = 0; i < ncomp; i++)
+         if (field_in.FESpace()->GetOrdering() == Ordering::byNODES)
         {
-            field_out(index + i*npt) = localval(i);
+            for (int i = 0; i < ncomp; i++)
+            {
+               field_out(index + i*npt) = localval(i);
+            }
+         }
+         else //byVDIM
+         {
+            for (int i = 0; i < ncomp; i++)
+            {
+               field_out(index*ncomp + i) = localval(i);
+            }
         }
      }
   }
@@ -1042,9 +1088,11 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,

            sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
            sdpt = (struct send_pt *)sendpt->ptr;
-            for (int index = 0; index < sendpt->n; index++)
+            for (int index = 0; index < static_cast<int>(sendpt->n); index++)
            {
-               int idx = sdpt->index + j*nptorig;
+               int idx = field_in.FESpace()->GetOrdering() == Ordering::byNODES ?
+                         sdpt->index + j*nptorig :
+                         sdpt->index*ncomp + j;
               field_out(idx) = sdpt->ival;
               ++sdpt;
            }
@@ -1107,7 +1155,7 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
   distfint.SetSize(pts_cnt);
   if (!gfmax)
   {
-      distfint = 0.;
+      distfint = 0.0;
   }
   else
   {
@@ -1139,7 +1187,8 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
 }

 void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
-                                        Array<unsigned int> &point_id)
+                                        Array<unsigned int> &point_id,
+                                        int point_pos_ordering)
 {
   MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before "
               "finding points.");
@@ -1153,14 +1202,27 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
   gsl_ref.SetSize(points_cnt * dim);
   gsl_dist.SetSize(points_cnt);

+   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
+   {
+      for (int d = 0; d < dim; d++)
+      {
+         if (point_pos_ordering == Ordering::byNODES)
+         {
+            xv_base[d] = point_pos.GetData() + d*points_cnt;
+            xv_stride[d] = sizeof(double);
+         }
+         else
+         {
+            xv_base[d] = point_pos.GetData() + d;
+            xv_stride[d] = dim*sizeof(double);
+         }
+      }
+   };
   if (dim == 2)
   {
      const double *xv_base[2];
-      xv_base[0] = point_pos.GetData();
-      xv_base[1] = point_pos.GetData() + points_cnt;
      unsigned xv_stride[2];
-      xv_stride[0] = sizeof(double);
-      xv_stride[1] = sizeof(double);
+      xvFill(xv_base, xv_stride, dim);
      findptsms_2(gsl_code.GetData(), sizeof(unsigned int),
                  gsl_proc.GetData(), sizeof(unsigned int),
                  gsl_elem.GetData(), sizeof(unsigned int),
@@ -1170,16 +1232,11 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
                  point_id.GetData(), sizeof(unsigned int), &match,
                  points_cnt, fdata2D);
   }
-   else
+   else  // dim == 3
   {
      const double *xv_base[3];
-      xv_base[0] = point_pos.GetData();
-      xv_base[1] = point_pos.GetData() + points_cnt;
-      xv_base[2] = point_pos.GetData() + 2*points_cnt;
      unsigned xv_stride[3];
-      xv_stride[0] = sizeof(double);
-      xv_stride[1] = sizeof(double);
-      xv_stride[2] = sizeof(double);
+      xvFill(xv_base, xv_stride, dim);
      findptsms_3(gsl_code.GetData(), sizeof(unsigned int),
                  gsl_proc.GetData(), sizeof(unsigned int),
                  gsl_elem.GetData(), sizeof(unsigned int),
@@ -1193,10 +1250,12 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
   // Set the element number and reference position to 0 for points not found
   for (int i = 0; i < points_cnt; i++)
   {
-      if (gsl_code[i] == 2)
+      if (gsl_code[i] == 2 ||
+          (gsl_code[i] == 1 && gsl_dist(i) > bdr_tol))
      {
         gsl_elem[i] = 0;
         for (int d = 0; d < dim; d++) { gsl_ref(i*dim + d) = -1.; }
+         gsl_code[i] = 2;
      }
   }

@@ -1208,9 +1267,10 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
 void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
                                         Array<unsigned int> &point_id,
                                         const GridFunction &field_in,
-                                         Vector &field_out)
+                                         Vector &field_out,
+                                         int point_pos_ordering)
 {
-   FindPoints(point_pos, point_id);
+   FindPoints(point_pos, point_id, point_pos_ordering);
   Interpolate(field_in, field_out);
 }

--- a/Show More
+++ b/Show More