Merge master in lor-gpu

LOR H1/ND/RT explicit specializations
Add lor_h1 batched kernel for order 1
2022-10-23 09:54:23 -07:00 · 2022-09-15 07:28:55 -07:00 · 2022-09-13 17:26:27 -07:00 · 2022-09-13 12:56:41 -07:00 · 2022-09-08 16:42:02 -07:00 · 2022-09-08 16:33:48 -07:00
231 changed files with 5438 additions and 11545 deletions
@@ -62,7 +62,7 @@ jobs:

      - name: GHCR Login
        if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
@@ -94,7 +94,7 @@ jobs:
    # This external action allows to interrupt a workflow already running on
    # the same branch to save resource
    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
      with:
        access_token: ${{ github.token }}

@@ -102,7 +102,7 @@ jobs:
    # /home/runner/work/mfem/mfem/mfem
    # Note: Done now to access "install-hypre" and "install-metis" actions.
    - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
      with:
        path: ${{ env.MFEM_TOP_DIR }}
        # Fetch the complete history for codecov to access commits ID
@@ -115,25 +115,25 @@ jobs:
      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install lcov

-    # Keep the following section in case we need it again in the future,
-    # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-    # - name: Set up Homebrew
-    #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-    #   uses: Homebrew/actions/setup-homebrew@master
+    - name: Set up Homebrew
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
+      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
+        export MAKE_CXX_FLAG="MPICXX=mpic++"

-    - name: get lcov (MacOS)
+    - name: get MPI (MacOS)
      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
@@ -141,14 +141,14 @@ jobs:

    - name: get MPI (Windows)
      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-      uses: mpi4py/setup-mpi@v1.1.4
+      uses: mpi4py/setup-mpi@v1.0.3

    # Get Hypre through cache, or build it.
    # Install will only run on cache miss.
    - name: cache hypre
      id: hypre-cache
      if: matrix.mpi == 'par'
-      uses: actions/cache@v3
+      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
@@ -176,7 +176,7 @@ jobs:
    - name: cache metis
      id: metis-cache
      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-      uses: actions/cache@v3
+      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
@@ -190,13 +190,12 @@ jobs:

    - name: cache vcpkg (Windows)
      id: vcpkg-cache
-      if: matrix.os == 'windows-latest'
      uses: actions/cache@v3
      with:
        path: vcpkg_cache
        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1

-    - name: prepare vcpkg binary cache location (Windows)
+    - name: prepare binary cache location
      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
      run: |
        mkdir -p vcpkg_cache
@@ -231,7 +230,11 @@ jobs:
      run: |
        cd ${{ env.MFEM_TOP_DIR }}  && make check

-    # Note: 'tests' include the unit tests
+    - name: unit tests
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }} && make unittest
+
    - name: tests
      if: matrix.build-system == 'make' && matrix.target == 'opt'
      run: |
@@ -244,7 +247,7 @@ jobs:
        cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
      shell: bash

-    - name: cmake unit tests (Ubuntu)
+    - name: cmake unit tests (Ubuntu 20.04)
      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
      run: |
        CTEST_CONFIG="Release"
@@ -262,7 +265,7 @@ jobs:
    # Code coverage (process and upload reports)
    - name: codecov
      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.2
+      uses: mfem/github-actions/upload-coverage@v2.0
      with:
        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
        project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -35,22 +35,23 @@ jobs:

    steps:
    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
      with:
        access_token: ${{ github.token }}

    - name: checkout MFEM
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
      with:
        path: mfem

    - name: Get MPI (Linux)
      run: |
        sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: Cache Hypre Install
      id: hypre-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
@@ -65,7 +66,7 @@ jobs:

    - name: Cache Metis Install
      id: metis-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
@@ -34,12 +34,12 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
      with:
        access_token: ${{ github.token }}

    - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2

    - name: copyright check
      id: copyright
@@ -84,7 +84,7 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2

    - name: get astyle
      run: |
@@ -101,14 +101,11 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2

    - name: get doxygen and graphviz
      run: |
        sudo apt-get install doxygen graphviz
-
-    - name: update doxygen config file
-      run: |
        cd doc
        doxygen -u CodeDocumentation.conf.in

@@ -126,7 +123,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
    - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
      with:
        fetch-depth: 0

@@ -18,6 +18,11 @@ CMakeFiles/
 # Backup files
 *~

+*.sqlite
+*.nsys-rep
+*.qdstrm
+*.csv
+
 # Default install location
 /mfem/

@@ -131,12 +136,6 @@ examples/hiop/ex9-mesh.*
 examples/hiop/ex9-init.*
 examples/hiop/ex9-final.*

-examples/ipopt/exContactBlockTL
-examples/ipopt/exContactBlockTL.mesh
-examples/ipopt/exContactBlockTL-mesh.*
-examples/ipopt/exContactBlockTL-init.*
-examples/ipopt/exContactBlockTL-final.*
-
 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
 examples/petsc/mesh.*
@@ -209,7 +208,6 @@ miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
 miniapps/meshing/trimmer
-miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
@@ -221,12 +219,9 @@ miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
 miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
-miniapps/meshing/mesh-explorer-visit*
-miniapps/meshing/mesh-explorer-paraview/
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
-miniapps/meshing/reflected.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh
@@ -286,7 +281,6 @@ miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
 miniapps/tools/check-tmop-metric
-miniapps/tools/tmop-metric-magnitude

 miniapps/toys/automata
 miniapps/toys/life
@@ -340,6 +334,7 @@ tests/benchmarks/bench_ceed
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals
+tests/benchmarks/bench_lor

 # Test script output
 tests/scripts/*.err
@@ -93,7 +93,7 @@ report_baseline:
          git pull && \
          git add ${rundir} && \
          git commit -m "${msg}" && \
-          ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+          git push origin master
        else
          for file in ${rundir}/*; do
            echo "------------------------------"
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Try to push to the remote 5 times. If the push fails, and the local and remote
-# have diverged, then pull from the remote to merge changes, and try pushing
-# again. If some other failure happens
-for i in {1..5}; do
-   git push origin master && exit 0
-   # Wait for 20 seconds in case someone else is pushing to the remote
-   # concurrently
-   sleep 20
-   # Fetch any updates from the remote
-   git remote update
-   # Get the latest commit on the local branch
-   LOCAL=$(git rev-parse @)
-   # Get the latest commit on the remote
-   REMOTE=$(git rev-parse @{u})
-   # Get the common ancestor
-   BASE=$(git merge-base @ @{u})
-   # Have the local and remote diverged?
-   if [[ $LOCAL != $REMOTE && $LOCAL != $BASE && $REMOTE != $BASE ]]; then
-      git pull
-      if [[ $? == 0 ]]; then
-         continue
-      else
-         exit 1 # Something else went wrong trying to pull
-      fi
-   fi
-done
-
-exit 1 # Did not succeed in 5 attempts
@@ -32,7 +32,7 @@ if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
  git pull && \
  git add ${rundir} && \
  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+  git push origin master
 else
  for file in ${rundir}/*; do
    echo "------------------------------"
@@ -29,7 +29,7 @@ if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
  git pull && \
  git add ${rundir} && \
  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+  git push origin master
 else
  for file in ${rundir}/*; do
    echo "------------------------------"
@@ -8,71 +8,6 @@
                               https://mfem.org


-Version 4.5.1 (development)
-===========================
- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
-  as the coarsening operator for non-conforming AMR meshes.
-
-Meshing improvements
--------------------
- Added support for pyramids in non-conforming meshes. Currently only isotropic
-  refinement is supported in this case.
-
- Updated logic in FindPointsGSLIB to ignore points found near (but outside) the
-  domain boundary.
-
- Added support for pyramids in Gmsh meshes.
-
- Fixed a bug in TMOP metric 301.
-
- Added an option to auto-balance compound TMOP metrics.
-
-Discretization improvements
---------------------------
- TBD
-
-Linear and nonlinear solvers
----------------------------
- Added a fast normalization-based distance solver, see the Distance miniapp
-  in the miniapps/shifted/ directory.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new meshing miniapp, reflector, which reflects a high-order or NURBS
-  hexahedral mesh about a plane.
-
- The mesh-explorer miniapp can now save mesh files in the VisIt or ParaView
-  formats using the corresponding DataCollection objects. See option 'D' in the
-  main menu.
-
-Integrations, testing and documentation
---------------------------------------
- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
-  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
-
- The following integrations have updated minimum version requirements:
-  * RAJA >= 2022.10.3
-
-Miscellaneous
-------------
- VisItDataCollection now correctly handles collection names containing
-  underscores.
-
- VisItDataCollection::SetPadDigits() no longer alters the number of digits
-  used to represent the MPI rank because VisIt seems to require 6 digits.
-  This parameter can still be explicitly overridden with
-  VisItDataCollection::SetPadDigitsRank().
-
-API changes
-----------
- The implicit cast methods of class Vector to 'double *' and 'const double *'
-  have been deprecated and generate deprecation warnings if used. They will be
-  removed in a future release.
-
- The methods Mesh::GetFaceBaseGeometry and Mesh::GetFaceGeometryType have been
-  deprecated, and Mesh::GetFaceGeometry (which provides identical functionality)
-  should be used instead.
-
 Version 4.5, released on October 22, 2022
 =========================================

@@ -116,9 +51,6 @@ Discretization improvements
 - Added a class CoefficientVector for efficient access of variable coefficient
  values at quadrature points (in particular for GPU/device kernels).

- Added support for GridFunction::GetGradients() and
-  GriFunction::GetVectorGradient() on face-neighbor elements.
-
 - Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
  spatial Gaussian white noise.

@@ -10,9 +10,7 @@
 # CONTRIBUTING.md for details.

 # The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-# Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
-# Version 3.8 or newer is required for direct CUDA support.
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.1)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
  "Path to optional user configuration file.")

@@ -53,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.5.1)
+set(${PROJECT_NAME}_VERSION 4.5.0)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -68,7 +66,8 @@ set(XSDK_ENABLE_C OFF)
 set(XSDK_ENABLE_Fortran OFF)

 # Check if we need to enable C or Fortran.
-if (MFEM_USE_CONDUIT OR
+if (CMAKE_VERSION VERSION_LESS 3.2 OR
+    MFEM_USE_CONDUIT OR
    MFEM_USE_SIDRE OR
    MFEM_USE_PETSC)
   # This seems to be needed by:
@@ -82,13 +81,11 @@ if (MFEM_USE_STRUMPACK)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
-if ((MFEM_USE_SUNDIALS OR
-     MFEM_USE_STRUMPACK OR
-     MFEM_USE_GINKGO OR
-     MFEM_USE_RAJA OR
-     MFEM_USE_UMPIRE) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
+# SUNDIALS >= 6.4.0 requires C++14:
+if (MFEM_USE_SUNDIALS AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14)
+endif()
+if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
  set(CMAKE_CXX_STANDARD 14)
 endif()

@@ -106,12 +103,17 @@ if (MFEM_USE_CUDA)
   if (MFEM_USE_HIP)
      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
   endif()
+   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
+   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
   if (NOT CMAKE_CUDA_HOST_COMPILER)
      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
   endif()
   enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
+   set(CMAKE_CUDA_STANDARD 11)
+   if (MFEM_USE_GINKGO)
+     set(CMAKE_CUDA_STANDARD 14)
+   endif()
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
   set(CMAKE_CUDA_EXTENSIONS OFF)
   set(CUDA_FLAGS "--expt-extended-lambda")
@@ -134,7 +136,8 @@ if (MFEM_USE_CUDA)
      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
   endif()
-   set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS})
+   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
+       "CUDA flags set for MFEM" FORCE)
   set(CUSPARSE_FOUND TRUE)
   set(CUSPARSE_LIBRARIES "cusparse")
   set(CUBLAS_FOUND TRUE)
@@ -196,26 +199,6 @@ if (MFEM_USE_HIP)
  find_package(HIPSPARSE REQUIRED)
 endif()

-# OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
-  endif()
-  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
-  if (OPENMP_FOUND)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    if (MFEM_USE_CUDA)
-      set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-    endif()
-  endif()
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
@@ -281,6 +264,20 @@ if (MFEM_USE_LAPACK)
  find_package(LAPACK REQUIRED)
 endif()

+# OpenMP
+if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
+  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
+    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
+  endif()
+  find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
+  if(APPLE)
+    # On macOS, the compiler needs additional help to find the <omp.h> header.
+    # See issue #2642 for more information.
+    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
+  endif(APPLE)
+endif()
+
 # SuiteSparse (before SUNDIALS which may depend on KLU)
 if (MFEM_USE_SUITESPARSE)
  find_package(SuiteSparse REQUIRED
@@ -299,6 +296,11 @@ if (MFEM_USE_SUNDIALS)
  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

+# Mesquite
+if (MFEM_USE_MESQUITE)
+  find_package(Mesquite REQUIRED)
+endif()
+
 # SuperLU_DIST can only be enabled in parallel
 if (MFEM_USE_SUPERLU)
  if (MFEM_USE_MPI)
@@ -404,15 +406,6 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

-# IpOpt optimizer
-if (MFEM_USE_IPOPT)
-  find_package(IPOPT REQUIRED)
-  message(
-      STATUS
-      "IPOPT_INCLUDE_DIRS=${IPOPT_INCLUDE_DIRS}, IPOPT_LIBRARIES=${IPOPT_LIBRARIES}, IPOPT_DIR=${IPOPT_DIR}")
-  # find_package updates IPOPT_FOUND, IPOPT_INCLUDE_DIRS, IPOPT_LIBRARIES
-endif()
-
 # CoDiPack package
 if (MFEM_USE_CODIPACK)
  find_package(CODIPACK REQUIRED)
@@ -495,10 +488,6 @@ endif()
 #   an ALIAS target is missing?
 # Call Stack (most recent call first):
 #   CMakeLists.txt:474 (mfem_add_library)
-#
-# NOTE: We need to figure out which TPL library adds the dependency on
-#       "Threads::Threads" and call the next line only when that TPL library is
-#       enabled.  -V. Dobrev
 find_package(Threads REQUIRED)

 # List all possible libraries in order of dependencies.
@@ -506,9 +495,9 @@ find_package(Threads REQUIRED)
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
-    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
-    NETCDF MPFR PUMI HIOP IPOPT POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
+set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist METIS SuiteSparse SUNDIALS
+    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
+    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

@@ -522,12 +511,18 @@ foreach(TPL IN LISTS MFEM_TPLS)
    list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
  endif()
 endforeach(TPL)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")

+if (OPENMP_FOUND)
+  message(STATUS "MFEM: using package OpenMP")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
+endif()
+
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -706,8 +701,6 @@ add_subdirectory(doc)
 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
 set(INSTALL_INCLUDE_DIR include
  CACHE PATH "Relative path for installing header files.")
-set(INSTALL_BIN_DIR bin
-  CACHE PATH "Relative path for installing the binaries.")
 set(INSTALL_LIB_DIR lib
  CACHE PATH "Relative path for installing the library.")
 # other options: "share/mfem/cmake", "lib/mfem/cmake"
@@ -726,9 +719,7 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Development)
 # Install the library
 install(TARGETS ${PROJECT_NAME}
  EXPORT ${PROJECT_NAME_UC}Targets
-  RUNTIME DESTINATION ${INSTALL_BIN_DIR}
-  LIBRARY DESTINATION ${INSTALL_LIB_DIR}
-  ARCHIVE DESTINATION ${INSTALL_LIB_DIR})
+  DESTINATION ${INSTALL_LIB_DIR})

 # Install the master headers
 foreach(Header mfem.hpp mfem-performance.hpp)
@@ -112,7 +112,6 @@ The MFEM source code has the following structure:
  │   ├── caliper
  │   ├── ginkgo
  │   ├── hiop
-  │   ├── ipopt
  │   ├── jupyter
  │   ├── moonolith
  │   ├── petsc
@@ -337,6 +337,10 @@ MFEM_USE_SUNDIALS = YES/NO
   library. When enabled, this option uses the SUNDIALS_* library options,
   see below.

+MFEM_USE_MESQUITE = YES/NO
+   Enable MFEM functionality based on the Mesquite library. When enabled, this
+   option uses the MESQUITE_* library options, see below.
+
 MFEM_USE_SUITESPARSE = YES/NO
   Enable MFEM functionality based on the SuiteSparse library. Currently, this
   option adds the classes UMFPackSolver and KLUSolver (both sparse serial
@@ -471,9 +475,6 @@ MFEM_USE_HIOP = YES/NO
   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
   HPC solver for nonlinear optimization problems.

-MFEM_USE_IPOPT = YES/NO
-   Enable the usage of Ipopt in MFEM.
-
 MFEM_USE_CODIPACK = YES/NO
  Enable automatic differentiation using the CoDiPack library.
  www.scicomp.uni-kl.de/codi/
@@ -635,6 +636,11 @@ The specific libraries and their options are:
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.

+- Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
+  URL: http://trilinos.org/oldsite/packages/mesquite
+  Options: MESQUITE_OPT, MESQUITE_LIB.
+  The Mesquite support is deprecated and will be removed in the future.
+
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
  URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
  Options: SUITESPARSE_OPT, SUITESPARSE_LIB.
@@ -741,11 +747,6 @@ The specific libraries and their options are:
  Options: HIOP_OPT, HIOP_LIB.
  Versions: HIOP >= 0.4.6.

- Ipopt (optional), used when MFEM_USE_IPOPT = YES.
-  URL: https://github.com/coin-or/Ipopt
-  Options: IPOPT_OPT, IPOPT_LIB.
-  Versions: IPOPT >= 3.14
-
 - CoDiPack (optional), used with MFEM_USE_CODIPACK = YES
  URL: https://www.scicomp.uni-kl.de/codi/
  Options: CODIPACK_OPT
@@ -801,10 +802,10 @@ The specific libraries and their options are:
  Versions: libCEED >= 0.10.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
+  Beginning with MFEM v4.3, only RAJA v0.14.0+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 2022.10.3.
+  Versions: RAJA >= 0.14.0.

 - Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
  URL: https://bitbucket.org/zulianp/par_moonolith
@@ -968,6 +969,7 @@ MFEM_USE_LEGACY_OPENMP
 MFEM_USE_OPENMP
 MFEM_USE_MEMALLOC
 MFEM_TIMER_TYPE - Set automatically, can be overwritten.
+MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_MUMPS
@@ -980,7 +982,6 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
-MFEM_USE_IPOPT
 MFEM_USE_CODIPACK
 MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
@@ -1033,6 +1034,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - HYPRE
 - METIS - The option MFEM_USE_METIS_5 is auto-detected.
 - ParMETIS
+ - MESQUITE
 - SuiteSparse
 - SuperLUDist, STRUMPACK
 - Ginkgo
@@ -1044,7 +1046,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - POSIXCLOCKS
 - PUMI
 - HIOP
- - IPOPT
 - CoDiPack
 - OCCA
 - RAJA
@@ -212,6 +212,10 @@ IF (DEFINED TPL_ENABLE_SUNDIALS)
  SET(MFEM_USE_SUNDIALS ${TPL_ENABLE_SUNDIALS} CACHE BOOL "Enable SUNDIALS usage" FORCE)
 ENDIF()

+IF (DEFINED TPL_ENABLE_MESQUITE)
+  SET(MFEM_USE_MESQUITE ${TPL_ENABLE_MESQUITE} CACHE BOOL "Enable MESQUITE usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_SUITESPARSE)
  SET(MFEM_USE_SUITESPARSE ${TPL_ENABLE_SUITESPARSE} CACHE BOOL "Enable SuiteSparse usage" FORCE)
 ENDIF()
@@ -29,6 +29,7 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
@@ -36,7 +37,6 @@ set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
-set(MFEM_USE_IPOPT @MFEM_USE_IPOPT@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
@@ -77,6 +77,9 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC

+// Enable MFEM functionality based on the Mesquite library.
+#cmakedefine MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE

@@ -131,9 +134,6 @@
 // Enable MFEM functionality based on the HiOp library
 #cmakedefine MFEM_USE_HIOP

-// Enable MFEM functionality based on the Ipopt library
-#cmakedefine MFEM_USE_IPOPT
-
 // Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
@@ -16,7 +16,7 @@

 include(MfemCmakeUtilities)
 mfem_find_package(Algoim ALGOIM ALGOIM_DIR
-        "include;src" "algoim_quad.hpp"
+       	"include" "algoim_quad.hpp"
       	"" ""
-        "Paths to headers required by Algoim."
-        "Libraries required by Algoim.")
+	"Paths to headers required by Algoim."
+	"Libraries required by Algoim.")
@@ -16,22 +16,7 @@

 include(MfemCmakeUtilities)
 mfem_find_package(Caliper CALIPER CALIPER_DIR
-      "include" "caliper/cali.h"
-      "lib" "caliper"
-      "Paths to headers required by Caliper."
-      "Libraries required by Caliper.")
-
-# Append adiak path/lib if the user provided ADIAK_DIR
-if(ADIAK_DIR AND EXISTS ${ADIAK_DIR})
-    find_package(adiak NO_DEFAULT_PATH REQUIRED PATHS ${ADIAK_DIR}/lib/cmake/adiak ${ADIAK_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${adiak_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${adiak_LIBRARIES})
-endif()
-
-# Append gotcha path/lib if the user provided GOTCHA_DIR
-if(GOTCHA_DIR AND EXISTS ${GOTCHA_DIR})
-    find_package(gotcha NO_DEFAULT_PATH REQUIRED PATHS ${GOTCHA_DIR}/lib/cmake/gotcha ${GOTCHA_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${gotcha_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${gotcha_LIBRARIES})
-endif()
-
+       	"include" "caliper/cali.h"
+       	"lib" "caliper"
+  	"Paths to headers required by Caliper."
+       	"Libraries required by Caliper.")
@@ -14,21 +14,9 @@
 #   - HDF5_LIBRARIES - The HDF5 libraries
 #   - HDF5_INCLUDE_DIRS - The HDF5 include directories

-# NOTE: Using this FindHDF5.cmake instead of the CMake provided version may lead
-#       to issues with some TPL libraries that depend (or may depend) on HDF5.
-#       For this reason, we should consider removing this file, or at least
-#       making it use the CMake provided version by default and apply the logic
-#       below only when specifically requested by a user.  -V. Dobrev
-
 # First Check for HDF5_DIR
 if(NOT HDF5_DIR)
-  message(FATAL_ERROR
-    "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
-endif()
-
-if (NOT HDF5_FIND_QUIETLY)
-  message(STATUS "Looking for HDF5 ...")
-  message(STATUS "   in HDF5_DIR = ${HDF5_DIR}")
+    MESSAGE(FATAL_ERROR "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
 endif()

 # Find includes
@@ -62,9 +50,8 @@ include(FindPackageHandleStandardArgs)

 # Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
 # listed variables are TRUE
-find_package_handle_standard_args(HDF5
-                                  " *** HDF5 not found. Please set HDF5_DIR."
-                                  HDF5_LIBRARIES
+find_package_handle_standard_args(HDF5  DEFAULT_MSG
                                  HDF5_INCLUDE_DIRS
                                  __HDF5_LIBRARY
-                                  __HDF5_HL_LIBRARY)
+                                  __HDF5_HL_LIBRARY
+                                  HDF5_LIBRARIES )
@@ -9,15 +9,12 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# Sets the following variables:
-#   - IPOPT_FOUND
-#   - IPOPT_INCLUDE_DIRS
-#   - IPOPT_LIBRARIES
+# Defines the following variables:
+#   - MESQUITE_FOUND
+#   - MESQUITE_LIBRARIES
+#   - MESQUITE_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-mfem_find_package(IPOPT IPOPT IPOPT_DIR
-  "include" "IpTNLP.hpp"
-  "lib" "ipopt"
-  "Paths to headers required by IPOPT."
-  "Libraries required by IPOPT.")
-
+mfem_find_package(Mesquite MESQUITE MESQUITE_DIR
+  "include" "Mesquite_all_headers.hpp" "lib" "mesquite"
+  "Paths to headers required by Mesquite." "Libraries required by Mesquite.")
@@ -17,24 +17,18 @@
 include(MfemCmakeUtilities)

 # FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-# (MFEM's FindHDF5.cmake does not need HDF5_ROOT)
-# set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-
+set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
 # We need to guard against the case where HDF5 was already found but without
 # the HL extensions (in which case mfem_find_package will treat the package
 # as already having been found), so we reset the variable to force FindHDF5.cmake
 # to be called for a second time
 set(HDF5_FOUND OFF)
 enable_language(C) # FindHDF5.cmake uses the C compiler
-
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
  "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-
 # The HL extension libraries are in a separate variable and must precede
 # the "regular" hdf5 library, as hdf5_hl depends on hdf5
 # The netcdf library will always be the first element of NETCDF_LIBRARIES
 # and we need to insert after that library but before the hdf5 library, so
 # position 1 is used
-# (MFEM's FindHDF5.cmake does not set HDF5_C_LIBRARY_hdf5_hl and the HL library
-# is already added to NETCDF_LIBRARIES)
-# list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
@@ -14,27 +14,17 @@
 #   - RAJA_LIBRARIES
 #   - RAJA_INCLUDE_DIRS

-if (RAJA_FOUND)
-   return()
+include(MfemCmakeUtilities)
+mfem_find_package(RAJA RAJA RAJA_DIR "include" "RAJA/RAJA.hpp" "lib" "RAJA"
+  "Paths to headers required by RAJA." "Libraries required by RAJA.")
+
+if (NOT RAJA_CONFIG_CMAKE)
+   set(RAJA_CONFIG_CMAKE "${RAJA_DIR}/share/raja/cmake/raja-config.cmake")
 endif()
-message(STATUS "Looking for RAJA ...")
-if (RAJA_DIR)
-   message(STATUS "   in RAJA_DIR = ${RAJA_DIR}")
-   find_package(RAJA CONFIG NO_DEFAULT_PATH PATHS "${RAJA_DIR}")
-endif()
-if (NOT RAJA_FOUND)
-   message(STATUS "   in standard CMake locations")
-   find_package(RAJA CONFIG)
-endif()
-if (RAJA_FOUND)
-   set(RAJA_LIBRARIES "RAJA" CACHE STRING "RAJA imported target." FORCE)
-   set(RAJA_INCLUDE_DIRS "" CACHE STRING "RAJA include dirs (not used)" FORCE)
-   message(STATUS
-      "Found RAJA target: ${RAJA_LIBRARIES} (version: ${RAJA_VERSION})")
-else()
-   set(msg STATUS)
-   if (RAJA_FIND_REQUIRED)
-      set(msg FATAL_ERROR)
+if (EXISTS "${RAJA_CONFIG_CMAKE}")
+   include("${RAJA_CONFIG_CMAKE}")
+   if (ENABLE_CUDA AND NOT MFEM_USE_CUDA)
+      message(FATAL_ERROR
+         "RAJA is built with CUDA: MFEM_USE_CUDA=YES is required")
   endif()
-   message(${msg} "RAJA not found. Please set RAJA_DIR to the RAJA prefix.")
 endif()
@@ -14,23 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS

-if (NOT umpire_DIR AND UMPIRE_DIR)
-  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
-endif()
-message(STATUS "Looking for UMPIRE ...")
-message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
-message(STATUS "      umpire_DIR = ${umpire_DIR}")
-find_package(umpire CONFIG)
+find_package(umpire REQUIRED CONFIG)
 set(UMPIRE_FOUND ${umpire_FOUND})
 set(UMPIRE_LIBRARIES "umpire")
-if (UMPIRE_FOUND)
-  message(STATUS
-    "Found UMPIRE target: ${UMPIRE_LIBRARIES} (version: ${umpire_VERSION})")
-else()
-  set(msg STATUS)
-  if (UMPIRE_FIND_REQUIRED)
-    set(msg FATAL_ERROR)
-  endif()
-  message(${msg}
-    "UMPIRE not found. Please set UMPIRE_DIR to the install prefix.")
-endif()
@@ -46,10 +46,6 @@ endfunction()
 # Wrapper for add_executable
 macro(mfem_add_executable NAME)
  add_executable(${NAME} ${ARGN})
-  if (MFEM_USE_CUDA)
-    set_target_properties(${NAME} PROPERTIES
-      CUDA_RESOLVE_DEVICE_SYMBOLS ON)
-  endif()
 endmacro()

 # Wrapper for add_library
@@ -162,12 +158,27 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)

  # Append the additional libraries and options
  if (LIBRARIES_LIST)
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${LIBRARIES_LIST})
+    endif()
  endif()
  if (EXTRA_OPTIONS_LIST)
    string(REPLACE ";" " " EXTRA_OPTIONS_STRING "${EXTRA_OPTIONS_LIST}")
    message(STATUS "${MFEM_EXE_NAME}: add flags \"${EXTRA_OPTIONS_STRING}\"")
-    target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    else()
+      get_target_property(THIS_COMPILE_FLAGS ${MFEM_EXE_NAME} COMPILE_FLAGS)
+      if (THIS_COMPILE_FLAGS)
+        set(THIS_COMPILE_FLAGS "${THIS_COMPILE_FLAGS} ${EXTRA_OPTIONS_STRING}")
+      else()
+        set(THIS_COMPILE_FLAGS "${EXTRA_OPTIONS_STRING}")
+      endif()
+      set_target_properties(${MFEM_EXE_NAME}
+        PROPERTIES COMPILE_FLAGS ${THIS_COMPILE_FLAGS})
+    endif()
  endif()
  if (EXTRA_DEFINES_LIST)
    target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
@@ -176,15 +187,17 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
  # Handle the MPI separately
  if (MFEM_USE_MPI)
    # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${MPI_CXX_LIBRARIES})
+    endif()

    if (MPI_CXX_INCLUDE_PATH)
      target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
    endif()
    if (MPI_CXX_COMPILE_FLAGS)
-      separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-        "${MPI_CXX_COMPILE_FLAGS}")
-      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_FLAGS})
    endif()

    if (MPI_CXX_LINK_FLAGS)
@@ -865,11 +878,11 @@ function(mfem_export_mk_files)
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_IPOPT MFEM_USE_GSLIB MFEM_USE_CUDA
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
@@ -85,6 +85,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS

+// Enable MFEM functionality based on the Mesquite library.
+// #define MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE

@@ -141,9 +144,6 @@
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP

-// Enable MFEM functionality based on the IPOPT library.
-// #define MFEM_USE_IPOPT
-
 // Enable MFEM functionality based on the GSLIB library
 // #define MFEM_USE_GSLIB

@@ -29,6 +29,7 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
@@ -46,7 +47,6 @@ MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
-MFEM_USE_IPOPT          = @MFEM_USE_IPOPT@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
@@ -30,6 +30,7 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
@@ -48,7 +49,6 @@ option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
-option(MFEM_USE_IPOPT "Enable Ipopt" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
 option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
@@ -124,6 +124,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")

+set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
+    "Path to the Mesquite library.")
+
 set(SuiteSparse_DIR "${MFEM_DIR}/../SuiteSparse" CACHE PATH
    "Path to the SuiteSparse library.")
 set(SuiteSparse_REQUIRED_PACKAGES "BLAS" "METIS"
@@ -185,7 +188,6 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")

-set(HDF5_DIR "/usr" CACHE PATH "Path to the HDF5 library.")
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
 set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
    "Additional packages required by NetCDF.")
@@ -221,10 +223,6 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
    "Directory where HiOp is installed")
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
    "Packages that HiOp depends on.")
-set(IPOPT_DIR "${MFEM_DIR}/../ipopt/install" CACHE STRING
-    "Directory where IpOpt is installed")
-set(IPOPT_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
-    "Packages that IpOpt depends on.")

 set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
@@ -232,12 +230,17 @@ set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")

 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
+# If RAJA is built with external CAMP:
+# set(RAJA_REQUIRED_PACKAGES "camp"
+#     CACHE STRING "Packages that RAJA depends on.")
+# set(camp_DIR "${MFEM_DIR}/../camp/lib/cmake/camp"
+#     CACHE PATH "Path to CAMP CMake files.")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
 set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
 set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
-set(Algoim_REQUIRED_PACKAGES "Blitz" CACHE STRING
+set(ALGOIM_REQUIRED_PACKAGES "BLITZ" CACHE STRING
    "Packages that ALGOIM depends on.")

 set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
@@ -131,6 +131,7 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
@@ -148,7 +149,6 @@ MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
-MFEM_USE_IPOPT         = NO
 MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
 MFEM_USE_HIP           = NO
@@ -270,6 +270,11 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES

+# MESQUITE library configuration
+MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
+MESQUITE_OPT = -I$(MESQUITE_DIR)/include
+MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
+
 # SuiteSparse library configuration
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
@@ -319,9 +324,6 @@ MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\

 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
-ifeq ($(MFEM_USE_STRUMPACK),YES)
-   BASE_FLAGS = -std=c++14
-endif
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 # If STRUMPACK was build with OpenMP support, the following may be need:
 # STRUMPACK_OPT += $(OPENMP_OPT)
@@ -448,11 +450,6 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)

-# IPOPT
-IPOPT_DIR = @MFEM_DIR@/../ipopt/install
-IPOPT_OPT = -I$(IPOPT_DIR)/include
-IPOPT_LIB = -L$(IPOPT_DIR)/lib -lipopt $(LAPACK_LIB)
-
 # CoDiPack
 CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
 CODIPACK_OPT = -I$(CODIPACK_DIR)
@@ -479,17 +476,7 @@ OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 # CALIPER library configuration
 CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 $(XLINKER)-rpath,$(CALIPER_DIR)/lib -L$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib -lcaliper
-
-ifdef ADIAK_DIR
-   CALIPER_OPT += -I$(ADIAK_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(ADIAK_DIR)/lib64 $(XLINKER)-rpath,$(ADIAK_DIR)/lib -L$(ADIAK_DIR)/lib64 -L$(ADIAK_DIR)/lib -ladiak
-endif
-ifdef GOTCHA_DIR
-   CALIPER_OPT += -I$(GOTCHA_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
-endif
-
+CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper

 # BLITZ library configuration
 BLITZ_DIR = @MFEM_DIR@/../blitz
@@ -512,29 +499,20 @@ CEED_OPT = -I$(CEED_DIR)/include
 CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed

 # RAJA library configuration
-ifeq ($(MFEM_USE_RAJA),YES)
-   BASE_FLAGS = -std=c++14
-endif
 RAJA_DIR = @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
   RAJA_OPT += -I$(CUB_DIR)
 endif
-
-CAMP_LIB = -lcamp
 ifdef CAMP_DIR
   RAJA_OPT += -I$(CAMP_DIR)/include
-   CAMP_LIB = $(XLINKER)-rpath,$(CAMP_DIR)/lib -L$(CAMP_DIR)/lib -lcamp
 endif
-RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA $(CAMP_LIB)
+RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA

 # UMPIRE library configuration
-ifeq ($(MFEM_USE_UMPIRE),YES)
-   BASE_FLAGS = -std=c++14
-endif
 UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
-UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire $(CAMP_LIB)
+UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

 # MKL CPardiso library configuration
 MKL_CPARDISO_DIR ?=
@@ -58,10 +58,6 @@ groups_serial=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp"'
-'"ipopt"
-   "IpOpt examples:"
-   "examples/ipopt"
-   "ex10.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -219,10 +215,6 @@ groups_all=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp ex9p.cpp"'
-'"ipopt"
-   "IpOpt examples:"
-   "examples/ipopt"
-   "ex10.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.5.1
+PROJECT_NUMBER         = v4.5.0

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -763,55 +763,36 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/config \
                         @MFEM_SOURCE_DIR@/general \
                         @MFEM_SOURCE_DIR@/linalg \
-                         @MFEM_SOURCE_DIR@/linalg/simd \
                         @MFEM_SOURCE_DIR@/mesh \
-                         @MFEM_SOURCE_DIR@/mesh/submesh \
                         @MFEM_SOURCE_DIR@/fem \
-                         @MFEM_SOURCE_DIR@/fem/ceed \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/convection \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/diffusion \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/mass \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/nlconvection \
-                         @MFEM_SOURCE_DIR@/fem/ceed/interface \
-                         @MFEM_SOURCE_DIR@/fem/ceed/solvers \
+                         @MFEM_SOURCE_DIR@/fem/moonolith \
                         @MFEM_SOURCE_DIR@/fem/fe \
                         @MFEM_SOURCE_DIR@/fem/lor \
-                         @MFEM_SOURCE_DIR@/fem/moonolith \
-                         @MFEM_SOURCE_DIR@/fem/qinterp \
-                         @MFEM_SOURCE_DIR@/fem/tmop \
                         @MFEM_SOURCE_DIR@/examples \
-                         @MFEM_SOURCE_DIR@/examples/amgx \
                         @MFEM_SOURCE_DIR@/examples/caliper \
+                         @MFEM_SOURCE_DIR@/examples/amgx \
                         @MFEM_SOURCE_DIR@/examples/ginkgo \
-                         @MFEM_SOURCE_DIR@/examples/hiop \
-                         @MFEM_SOURCE_DIR@/examples/ipopt \
                         @MFEM_SOURCE_DIR@/examples/moonolith \
+                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/petsc \
                         @MFEM_SOURCE_DIR@/examples/pumi \
                         @MFEM_SOURCE_DIR@/examples/sundials \
                         @MFEM_SOURCE_DIR@/examples/superlu \
                         @MFEM_SOURCE_DIR@/miniapps/adjoint \
-                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/materials \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/operators \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/preconditioners \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
                         @MFEM_SOURCE_DIR@/miniapps/mtop \
-                         @MFEM_SOURCE_DIR@/miniapps/multidomain \
+                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
-                         @MFEM_SOURCE_DIR@/miniapps/parelag \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
                         @MFEM_SOURCE_DIR@/miniapps/shifted \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/parelag

 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -178,11 +178,6 @@ if (MFEM_USE_HIOP)
  add_subdirectory(hiop)
 endif()

-# Include the examples/ipopt directory if IpOpt is enabled
-if (MFEM_USE_IPOPT)
-  add_subdirectory(ipopt)
-endif()
-
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -33,13 +33,13 @@ add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})
 if (MFEM_ENABLE_TESTING)
  foreach(SRC_FILE ${CALIPER_EXE_SRCS})
    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${PREFIX}${SRC_FILENAME})
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

    set(THIS_TEST_OPTIONS "-no-vis")

    if (NOT (${TEST_NAME} MATCHES ".*p$"))
      add_test(NAME ${TEST_NAME}_ser
-        COMMAND $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS})
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
    else()
      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
@@ -1,810 +0,0 @@
-//                                Contact example
-//
-// Compile with: make contact
-//
-// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
-// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include "nodepair.hpp"
-
-using namespace std;
-using namespace mfem;
-
-bool ifequalarray(const Array<int> a1, const Array<int> a2)
-{
-   if (a1.Size()!=a2.Size())
-   {
-      return false;
-   }
-   for (int i=0; i<a1.Size(); i++)
-   {
-      if (a1[i] != a2[i])
-      {
-         return false;
-      }
-   }
-   return true;
-}
-
-void FindSurfaceToProject(Mesh& mesh, const int elem, int& cbdrface)
-{
-   Array<int> attr;
-   attr.Append(2);
-   Array<int> faces;
-   Array<int> ori;
-   std::vector<Array<int> > facesVertices;
-   std::vector<int > faceid;
-   mesh.GetElementFaces(elem, faces, ori);
-   int face = -1;
-   for (int i=0; i<faces.Size(); i++)
-   {
-      face = faces[i];
-      Array<int> faceVert;
-      if (!mesh.FaceIsInterior(face)) // if on the boundary
-      {
-         mesh.GetFaceVertices(face, faceVert);
-         faceVert.Sort();
-         facesVertices.push_back(faceVert);
-         faceid.push_back(face);
-      }
-   }
-   int bdrface = facesVertices.size();
-
-   Array<int> bdryFaces;
-   // This shoulnd't need to be rebuilt
-   std::vector<Array<int> > bdryVerts;
-   for (int b=0; b<mesh.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh.GetBdrAttribute(b)) >= 0)  // found the contact surface
-      {
-         bdryFaces.Append(b);
-         Array<int> vert;
-         mesh.GetBdrElementVertices(b, vert);
-         vert.Sort();
-         bdryVerts.push_back(vert);
-      }
-   }
-
-   int bdrvert = bdryVerts.size();
-   cbdrface = -1;  // the face number of the contact surface element
-   int count_cbdrface = 0;  // the number of matching surfaces, used for checks
-
-   for (int i=0; i<bdrface; i++)
-   {
-      for (int j=0; j<bdrvert; j++)
-      {
-         if (ifequalarray(facesVertices[i], bdryVerts[j]))
-         {
-            cbdrface = faceid[i];
-            count_cbdrface += 1;
-         }
-      }
-   }
-   MFEM_VERIFY(count_cbdrface == 1,"projection surface not found");
-
-};
-
-Vector GetNormalVector(Mesh & mesh, const int elem, const double *ref,
-                       int & refFace, int & refNormal, bool & interior)
-{
-   ElementTransformation *trans = mesh.GetElementTransformation(elem);
-   const int dim = mesh.Dimension();
-   const int spaceDim = trans->GetSpaceDim();
-
-   MFEM_VERIFY(spaceDim == 3, "");
-
-   Vector n(spaceDim);
-
-   IntegrationPoint ip;
-   ip.Set(ref, dim);
-
-   trans->SetIntPoint(&ip);
-   //CalcOrtho(trans->Jacobian(), n);  // Works only for face transformations
-   const DenseMatrix jac = trans->Jacobian();
-
-   int dimNormal = -1;
-   int normalSide = -1;
-
-   const double tol = 1.0e-8;
-   for (int i=0; i<dim; ++i)
-   {
-      const double d0 = std::abs(ref[i]);
-      const double d1 = std::abs(ref[i] - 1.0);
-
-      const double d = std::min(d0, d1);
-      // TODO: this works only for hexahedral meshes!
-
-      if (d < tol)
-      {
-         MFEM_VERIFY(dimNormal == -1, "");
-         dimNormal = i;
-
-         if (d0 < tol)
-         {
-            normalSide = 0;
-         }
-         else
-         {
-            normalSide = 1;
-         }
-      }
-   }
-   // closest point on the boundary
-   if (dimNormal < 0 || normalSide < 0) // node is inside the element
-   {
-      interior = 1;
-      Vector n(3);
-      n = 0.0;
-      return n;
-   }
-
-   MFEM_VERIFY(dimNormal >= 0 && normalSide >= 0, "");
-   refNormal = dimNormal;
-
-   MFEM_VERIFY(dim == 3, "");
-
-   {
-      // Find the reference face
-      if (dimNormal == 0)
-      {
-         refFace = (normalSide == 1) ? 2 : 4;
-      }
-      else if (dimNormal == 1)
-      {
-         refFace = (normalSide == 1) ? 3 : 1;
-      }
-      else
-      {
-         refFace = (normalSide == 1) ? 5 : 0;
-      }
-   }
-
-   std::vector<Vector> tang(2);
-
-   int tangDir[2] = {-1, -1};
-   {
-      int t = 0;
-      for (int i=0; i<dim; ++i)
-      {
-         if (i != dimNormal)
-         {
-            tangDir[t] = i;
-            t++;
-         }
-      }
-
-      MFEM_VERIFY(t == 2, "");
-   }
-
-   for (int i=0; i<2; ++i)
-   {
-      tang[i].SetSize(3);
-
-      Vector tangRef(3);
-      tangRef = 0.0;
-      tangRef[tangDir[i]] = 1.0;
-
-      jac.Mult(tangRef, tang[i]);
-   }
-
-   Vector c(3);  // Cross product
-
-   c[0] = (tang[0][1] * tang[1][2]) - (tang[0][2] * tang[1][1]);
-   c[1] = (tang[0][2] * tang[1][0]) - (tang[0][0] * tang[1][2]);
-   c[2] = (tang[0][0] * tang[1][1]) - (tang[0][1] * tang[1][0]);
-
-   c /= c.Norml2();
-
-   Vector nref(3);
-   nref = 0.0;
-   nref[dimNormal] = 1.0;
-
-   Vector ndir(3);
-   jac.Mult(nref, ndir);
-
-   ndir /= ndir.Norml2();
-
-   const double dp = ndir * c;
-
-   // TODO: eliminate c?
-   n = c;
-   if (dp < 0.0)
-   {
-      n *= -1.0;
-   }
-   interior = 0;
-   return n;
-}
-
-// WARNING: global variable, just for this little example.
-std::array<std::array<int, 3>, 8> HEX_VERT =
-{
-   {  {0,0,0},
-      {1,0,0},
-      {1,1,0},
-      {0,1,0},
-      {0,0,1},
-      {1,0,1},
-      {1,1,1},
-      {0,1,1}
-   }
-};
-
-int GetHexVertex(int cdim, int c, int fa, int fb, Vector & refCrd)
-{
-   int ref[3];
-   ref[cdim] = c;
-   ref[cdim == 0 ? 1 : 0] = fa;
-   ref[cdim == 2 ? 1 : 2] = fb;
-
-   for (int i=0; i<3; ++i) { refCrd[i] = ref[i]; }
-
-   int refv = -1;
-
-   for (int i=0; i<8; ++i)
-   {
-      bool match = true;
-      for (int j=0; j<3; ++j)
-      {
-         if (ref[j] != HEX_VERT[i][j]) { match = false; }
-      }
-
-      if (match) { refv = i; }
-   }
-
-   MFEM_VERIFY(refv >= 0, "");
-
-   return refv;
-}
-
-// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
-// where X is the list of x-coordinates for all points and so on.
-// conn: connectivity of the target surface elements
-// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
-void FindPointsInMesh(Mesh & mesh, Vector const& xyz, Array<int>& conn,
-                      Vector& xi)
-{
-   const int dim = mesh.Dimension();
-   const int np = xyz.Size() / dim;
-
-   MFEM_VERIFY(np * dim == xyz.Size(), "");
-
-   mesh.EnsureNodes();
-
-   //FindPointsGSLIB finder(MPI_COMM_WORLD);
-   FindPointsGSLIB finder;
-
-   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
-
-   const double bb_t = 0.5;
-   finder.Setup(mesh, bb_t);
-
-   finder.FindPoints(xyz);
-
-   /// Return code for each point searched by FindPoints: inside element (0), on
-   /// element boundary (1), or not found (2).
-   Array<unsigned int> codes = finder.GetCode();
-
-   /// Return element number for each point found by FindPoints.
-   Array<unsigned int> elems = finder.GetElem();
-
-   /// Return reference coordinates for each point found by FindPoints.
-   Vector refcrd = finder.GetReferencePosition();
-
-   /// Return distance between the sought and the found point in physical space,
-   /// for each point found by FindPoints.
-   Vector dist = finder.GetDist();
-
-   MFEM_VERIFY(dist.Size() == np, "");
-   MFEM_VERIFY(refcrd.Size() == np * dim, "");
-   MFEM_VERIFY(elems.Size() == np, "");
-   MFEM_VERIFY(codes.Size() == np, "");
-
-   bool allfound = true;
-   for (auto code : codes)
-      if (code == 2) { allfound = false; }
-
-   MFEM_VERIFY(allfound, "A point was not found");
-
-   cout << "Maximum distance of projected points: " << dist.Max() << endl;
-
-   // extract information
-   for (int i=0; i<np; ++i)
-   {
-      /*cout << "Point " << i << ": (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << xyz[i + (j*np)];
-         if (j == dim-1) {cout << ")" << endl;}
-         else{cout << ", ";}
-      }*/
-      //cout << "  element: " << elems[i] << endl;
-      //cout << "  element " << elems[i] << " vertices:" << endl;
-      //Array<int> vert;
-      //mesh.GetElementVertices(elems[i], vert);
-      //for (auto v : vert)
-      //{
-      //   cout << "    " << v << endl;
-      //}
-
-      /*cout << "  reference coordinates: (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << refcrd[(i*dim) + j];
-         if (j == dim-1)
-         {
-            cout << ")" << endl;
-         }
-         else
-         {
-            cout << ", ";
-         }
-      }*/
-
-      int refFace, refNormal, refNormalSide;
-      bool is_interior = -1;
-      Vector normal = GetNormalVector(mesh, elems[i], refcrd.GetData() + (i*dim),
-                                      refFace, refNormal, is_interior);
-      int phyFace;
-      if (is_interior)
-      {
-         phyFace = -1; // the id of the face that has the closest point
-         FindSurfaceToProject(mesh, elems[i], phyFace);
-
-         Array<int> cbdrVert;
-         mesh.GetFaceVertices(phyFace, cbdrVert);
-         Vector xs(dim);
-         xs[0] = xyz[i + 0*np];
-         xs[1] = xyz[i + 1*np];
-         xs[2] = xyz[i + 2*np];
-         Vector xi_tmp(dim-1);
-         // get nodes!
-
-         GridFunction *nodes = mesh.GetNodes();
-         DenseMatrix coords(4,3);
-         for (int i=0; i<4; i++)
-         {
-            for (int j=0; j<3; j++)
-            {
-               coords(i,j) = (*nodes)[cbdrVert[i]*3+j];
-            }
-         }
-         SlaveToMaster(coords, xs, xi_tmp);
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = xi_tmp[j];
-         }
-         // now get get the projection to the surface
-      }
-      else
-      {
-         Vector faceRefCrd(dim-1);
-         {
-            int fd = 0;
-            for (int j=0; j<dim; ++j)
-            {
-               if (j == refNormal)
-               {
-                  refNormalSide = (refcrd[(i*dim) + j] > 0.5);
-               }
-               else
-               {
-                  faceRefCrd[fd] = refcrd[(i*dim) + j];
-                  fd++;
-               }
-            }
-
-            MFEM_VERIFY(fd == dim-1, "");
-         }
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
-         }
-         //cout << "  face reference coordinates: (";
-         /*for (int j=0; j<dim-1; ++j)
-         {
-            cout << faceRefCrd[j];
-            if (j == dim-2){cout << ")" << endl;}
-            else{cout << ", ";}
-         }*/
-      }
-      //cout << "  normal vector: ";
-      //normal.Print();
-
-      // ask, does this do anything?
-      /*
-      IntegrationPoint ip;
-      ip.Set(refcrd.GetData() + (i*dim), dim);
-      ElementTransformation *trans = mesh.GetElementTransformation(elems[i]);
-      Vector phys(trans->GetSpaceDim());
-      trans->Transform(ip, phys);
-      cout << "  physical coordinates: ";
-      phys.Print();
-      */
-
-      // Get the element face
-      Array<int> faces;
-      Array<int> ori;
-      int face;
-
-      if (is_interior)
-      {
-         face = phyFace;
-      }
-      else
-      {
-         mesh.GetElementFaces(elems[i], faces, ori);
-         face = faces[refFace];
-      }
-
-      Array<int> faceVert;
-      mesh.GetFaceVertices(face, faceVert);
-
-      //cout << "  face " << face << " vertices:" << endl;
-      //for (auto v : faceVert){ cout << "    " << v << endl;}
-
-      for (int p=0; p<4; p++)
-      {
-         conn[4*i+p] = faceVert[p];
-      }
-      /*
-      Vector ref(dim);
-
-      for (int p=0; p<2; ++p)
-         for (int q=0; q<2; ++q)
-         {
-            const int refv = GetHexVertex(refNormal, refNormalSide, p, q, ref);
-            cout << "  face reference vertex (" << p << "," << q
-                 << ") is global vertex " << vert[refv] << endl;
-
-            {
-               // Sanity check
-               ip.Set(ref.GetData(), dim);
-               trans->Transform(ip, phys);
-               for (int j=0; j<dim; ++j)
-               {
-                  phys[j] -= mesh.GetVertex(vert[refv])[j];
-               }
-          phys.Print();
-          cout<<vert[refv]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[0]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[1]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[2]<<endl;
-               MFEM_VERIFY(phys.Norml2() < 1.0e-12, "Sanity check failed");
-            }
-         }*/
-   }
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file1 = "block1.mesh";
-   const char *mesh_file2 = "block2.mesh";
-
-   Array<int> attr;
-   Array<int> m_attr;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file1, "-m1", "--mesh1",
-                  "First mesh file to use.");
-   args.AddOption(&mesh_file2, "-m2", "--mesh2",
-                  "Second mesh file to use.");
-   args.AddOption(&attr, "-at", "--attributes-surf",
-                  "Attributes of boundary faces on contact surface for mesh 2.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Mesh mesh1(mesh_file1, 1, 1);
-   Mesh mesh2(mesh_file2, 1, 1);
-
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mesh1a_sock(vishost, visport);
-      mesh1a_sock.precision(8);
-      mesh1a_sock << "mesh\n" << mesh1 << flush;
-      socketstream mesh2a_sock(vishost, visport);
-      mesh2a_sock.precision(8);
-      mesh2a_sock << "mesh\n" << mesh2 << flush;
-   }
-
-   const int dim = mesh1.Dimension();
-   MFEM_VERIFY(dim == mesh2.Dimension(), "");
-
-   // boundary attribute 2 is the potential contact surface of nodes
-   attr.Append(2);
-   // boundary attribute 2 is the potential contact surface for master surface
-   m_attr.Append(2);
-
-   //  Define a finite element space on the mesh. Here we use vector finite
-   //  elements, i.e. dim copies of a scalar finite element space. The vector
-   //  dimension is specified by the last argument of the FiniteElementSpace
-   //  constructor.
-   FiniteElementCollection *fec1;
-   FiniteElementSpace *fespace1;
-   fec1 = new H1_FECollection(1, dim);
-   fespace1 = new FiniteElementSpace(&mesh1, fec1, dim, Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh1: "
-        << fespace1->GetTrueVSize() << endl;
-   mesh1.SetNodalFESpace(fespace1);
-   GridFunction nodes0 = *mesh1.GetNodes(); // undeformed mesh1 nodal grid function
-   GridFunction *nodes1 = mesh1.GetNodes();
-
-   FiniteElementCollection *fec2 = new H1_FECollection(1, dim);
-   FiniteElementSpace *fespace2 = new FiniteElementSpace(&mesh2, fec2, dim,
-                                                         Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh2: "
-        << fespace2->GetTrueVSize() << endl;
-
-   // degrees of freedom of both meshes
-   int ndof_1 = fespace1->GetTrueVSize();
-   int ndof_2 = fespace2->GetTrueVSize();
-   int ndofs = ndof_1 + ndof_2;
-   // number of nodes for each mesh
-   int nnd_1 = mesh1.GetNV();
-   int nnd_2 = mesh2.GetNV();
-   int nnd = nnd_1 + nnd_2;
-   // Determine the list of true (i.e. conforming) essential boundary dofs.
-   // In this example, the boundary conditions are defined by marking only
-   // boundary attribute 1 from the mesh as essential and converting it to a
-   // list of true dofs.
-   Array<int> ess_tdof_list1, ess_bdr1(mesh1.bdr_attributes.Max());
-   ess_bdr1 = 0;
-   //ess_bdr1[0] = 1;
-   // Not ready to be passed on yet
-   // fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   Array<int> ess_tdof_list2, ess_bdr2(mesh2.bdr_attributes.Max());
-   ess_bdr2 = 0;
-   //ess_bdr2[0] = 1;
-
-   // Define the displacement vector x as a finite element grid function
-   // corresponding to fespace. GridFunction is a derived class of Vector.
-   GridFunction x1(fespace1);
-   x1 = 0.0;
-   GridFunction x2(fespace2);
-   x2 = 0.0;
-
-   // Generate force
-   LinearForm *b1 = new LinearForm(fespace1);
-   b1->Assemble();
-
-   LinearForm *b2 = new LinearForm(fespace2);
-   b2->Assemble();
-
-   // Set up the bilinear form a(.,.) on the finite element space
-   //  corresponding to the linear elasticity integrator with piece-wise
-   //  constants coefficient lambda and mu.
-   Vector lambda1(mesh1.attributes.Max());
-   lambda1 = 57.6923076923;
-   PWConstCoefficient lambda1_func(lambda1);
-   Vector mu1(mesh1.attributes.Max());
-   mu1 = 38.4615384615;
-   PWConstCoefficient mu1_func(mu1);
-
-   BilinearForm *a1 = new BilinearForm(fespace1);
-   a1->AddDomainIntegrator(new ElasticityIntegrator(lambda1_func,mu1_func));
-
-   Vector lambda2(mesh2.attributes.Max());
-   lambda2 = 57.6923076923;
-   PWConstCoefficient lambda2_func(lambda2);
-   Vector mu2(mesh2.attributes.Max());
-   mu2 = 38.4615384615;
-   PWConstCoefficient mu2_func(mu2);
-
-   BilinearForm *a2 = new BilinearForm(fespace2);
-   a2->AddDomainIntegrator(new ElasticityIntegrator(lambda2_func,mu2_func));
-
-   a1->Assemble();
-   SparseMatrix A1;
-   Vector B1, X1;
-   a1->FormLinearSystem(ess_tdof_list1, x1, *b1, A1, X1, B1);
-
-   a2->Assemble();
-   SparseMatrix A2;
-   Vector B2, X2;
-   a2->FormLinearSystem(ess_tdof_list2, x2, *b2, A2, X2, B2);
-
-   // Combine elasticity operator for two meshes into one.
-   // Block Matrix
-   SparseMatrix K(ndofs,ndofs);
-   for (int i=0; i<A1.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A1.GetRow(i, col_tmp, v_tmp);
-      K.SetRow(i, col_tmp, v_tmp);
-   }
-   for (int i=0; i<A2.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A2.GetRow(i, col_tmp, v_tmp);
-      for (int j=0; j<col_tmp.Size(); j++)
-      {
-         col_tmp[j] += ndof_1;
-      }
-      K.SetRow(i+ndof_1, col_tmp, v_tmp);  // mesh1 top left corner
-   }
-
-   // Construct node to segment contact constraint.
-
-   attr.Sort();
-   cout << "Boundary attributes for contact surface faces in mesh 2" << endl;
-   for (auto a : attr) { cout << a << endl; }
-
-   Array<int> bdryFaces2;  // TODO: remove this?
-
-   std::set<int> bdryVerts2;
-   for (int b=0; b<mesh2.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh2.GetBdrAttribute(b)) >= 0)
-      {
-         bdryFaces2.Append(b);
-         Array<int> vert;
-         mesh2.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            bdryVerts2.insert(v);
-         }
-      }
-   }
-
-   int npoints = bdryVerts2.size();
-   Array<int> s_conn(npoints); // connectivity of the second/slave mesh
-   Vector xyz(dim * npoints);
-   xyz = 0.0;
-
-   cout << "Boundary vertices for contact surface vertices in mesh 2" << endl;
-
-   // construct the nodal coordinates on mesh2 to be projected, including displacement
-   int count = 0;
-   for (auto v : bdryVerts2)
-   {
-      cout << v << ": " << mesh2.GetVertex(v)[0] << ", "
-           << mesh2.GetVertex(v)[1] << ", "
-           << mesh2.GetVertex(v)[2] << endl;
-
-      for (int i=0; i<dim; ++i)
-      {
-         xyz[count + (i * npoints)] = mesh2.GetVertex(v)[i] + x2[v*dim+i];
-      }
-
-      s_conn[count] = v + nnd_1; // dof1 is the master
-      count++;
-   }
-
-   MFEM_VERIFY(count == npoints, "");
-
-   // gap function
-   Vector g(npoints*dim);
-   g = -1.0;
-   // segment reference coordinates of the closest point
-   Vector m_xi(npoints*(dim-1));
-   m_xi = -1.0;
-   Vector xs(dim*npoints);
-   xs = 0.0;
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<dim; j++)
-      {
-         xs[i*dim+j] = xyz[i + (j*npoints)];
-      }
-   }
-
-   Array<int> m_conn(
-      npoints*4); // only works for linear elements that have 4 vertices!
-   DenseMatrix coordsm(npoints*4, dim);
-
-   // adding displacement to mesh1 using a fixed grid function from mesh1
-   x1 = 1e-4; // x1 order: [xyz xyz... xyz]
-   add(nodes0, x1, *nodes1);
-
-   FindPointsInMesh(mesh1, xyz, m_conn, m_xi);
-
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<4; j++)
-      {
-         for (int k=0; k<dim; k++)
-         {
-            coordsm(i*4+j,k) = mesh1.GetVertex(m_conn[i*4+j])[k]+x1[dim*m_conn[i*4+j]+k];
-         }
-      }
-   }
-   //coordsm.Print();
-   SparseMatrix M(nnd,ndofs);
-   std::vector<SparseMatrix> dM(nnd, SparseMatrix(ndofs,ndofs));
-
-   Assemble_Contact(nnd, npoints, ndofs, xs, m_xi, coordsm,
-                    s_conn, m_conn, g, M, dM);
-
-   std::set<int> dirbdryv2;
-   for (int b=0; b<mesh2.GetNBE(); ++b)
-   {
-      if (mesh2.GetBdrAttribute(b) == 1)
-      {
-         Array<int> vert;
-         mesh2.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            dirbdryv2.insert(v);
-         }
-      }
-   }
-   std::set<int> dirbdryv1;
-   for (int b=0; b<mesh1.GetNBE(); ++b)
-   {
-      if (mesh1.GetBdrAttribute(b) == 1)
-      {
-         Array<int> vert;
-         mesh1.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            dirbdryv1.insert(v);
-         }
-      }
-   }
-
-   Array<int> Dirichlet_dof;
-   Array<double> Dirichlet_val;
-
-   for (auto v : dirbdryv2)
-   {
-      for (int i=0; i<dim; ++i)
-      {
-         Dirichlet_dof.Append(v*dim + i + ndof_1);
-         Dirichlet_val.Append(0.);
-      }
-   }
-   double delta = 0.1;
-   for (auto v : dirbdryv1)
-   {
-      Dirichlet_dof.Append(v*dim + 0);
-      Dirichlet_val.Append(delta);
-      Dirichlet_dof.Append(v*dim + 1);
-      Dirichlet_val.Append(0.);
-      Dirichlet_dof.Append(v*dim + 2);
-      Dirichlet_val.Append(0.);
-   }
-
-
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mesh1_sock(vishost, visport);
-      mesh1_sock.precision(8);
-      mesh1_sock << "mesh\n" << mesh1 << flush;
-      socketstream mesh2_sock(vishost, visport);
-      mesh2_sock.precision(8);
-      mesh2_sock << "mesh\n" << mesh2 << flush;
-   }
-
-   //M.Print();
-   /*Vector eps(ndofs);
-   Vector sol(ndofs); sol = 0.;
-   for(int i=0;i<ndofs;i++) eps[i] = 1e-5 * i ;
-   for(int i=0;i<9;i++)
-   {
-     cout<<i<<endl;
-     dM[s_conn[i]].Mult(eps,sol);
-     sol.Print();
-   }
-   */
-   return 0;
-}
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
-//               ex1 -fa -d cuda
+//             * ex1 -fa -d cuda
 //               ex1 -pa -d raja-cuda
 //             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
@@ -192,14 +192,7 @@ int main(int argc, char *argv[])
   //    domain integrator.
   BilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -fa -d cuda
+//             * mpirun -np 4 ex1p -fa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
@@ -219,14 +219,7 @@ int main(int argc, char *argv[])
   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
@@ -1,60 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(IPOPT_EXAMPLES_SRCS)
-list(APPEND IPOPT_EXAMPLES_SRCS exContactBlockTL.cpp)
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_ipopt" target, see below.
-add_custom_target(test_ipopt
-  ${CMAKE_CTEST_COMMAND} -R ipopt USES_TERMINAL)
-
-# Add one executable per cpp file, adding "ipopt_" as prefix. Sets
-# "test_ipopt" as a target that depends on the given examples.
-set(PFX ipopt_)
-add_mfem_examples(IPOPT_EXAMPLES_SRCS ${PFX} "" test_ipopt)
-
-# Testing.
-# The IPOPT tests can be run separately using the target "test_ipopt"
-# which builds the examples and runs:
-#   ctest -R ipopt
-
-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 9:
-  set(EXCONTACTBTL_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-  set(EXCONTACTBTL_TEST_OPTS ${EXCONTACTBTL_COMMON_OPTS}  -r  2 )
-
-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${IPOPT_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
-
-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
@@ -1,19 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM for solving nonlinear constrained optimization problems, including
-features based on the IpOpt, a lightweight HPC solver for nonlinear optimization
-problems.
-
-To use the Ipopt features, make sure that MFEM is configured with the option
-"MFEM_USE_IPOPT = YES", see the top-level INSTALL file for details.
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.
@@ -1,103 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-9
-1   5     0      1     3     2     8    9     11    10
-1   5     2      3     5     4    10    11    13    12
-1   5     4      5     7     6    12    13    15    14
-1   5     8      9    11    10    16    17    19    18
-1   5     10    11    13    12    18    19    21    20
-1   5     12    13    15    14    20    21    23    22
-1   5     16    17    19    18    24    25    27    26
-1   5     18    19    21    20    26    27    29    28
-1   5     20    21    23    22    28    29    31    30
-
-
-
-# 0 nothing
-# 1 dirichlet bc
-# 2 contact
-boundary
-30
-0   3    1     0     2     3
-0   3    3     2     4     5
-0   3    5     4     6     7
-0   3    24    25    27    26
-0   3    26    27    29    28
-0   3    28    29    31    30
-1   3    2     0     8    10
-1   3    4     2    10    12
-1   3    6     4    12    14
-1  3    10    8    16    18
-1  3    12    10    18    20
-1  3    14    12    20    22
-1  3    18    16    24    26
-1  3    20    18    26    28
-1  3    22    20    28    30
-2  3    1     3    11    9
-2  3     3     5    13    11
-2  3     5     7    15    13
-2  3    9    11    19    17
-2  3    11    13    21    19
-2  3    13    15    23    21
-2  3    17    19    27    25
-2  3    19    21    29    27
-2  3    21    23    31    29
-0  3     8     0     1    9
-0  3    16     8    9    17
-0  3    24    16    17    25
-0  3     6    14    15     7
-0  3    14    22    23    15
-0  3    22    30    31    23
-
-
-vertices
-32
-3
-   -1.0000         0         0
-         0         0         0
-   -1.0000    0.3333         0
-         0    0.3333         0
-   -1.0000    0.6667         0
-         0    0.6667         0
-   -1.0000    1.0000         0
-         0    1.0000         0
-   -1.0000         0    0.3333
-         0         0    0.3333
-   -1.0000    0.3333    0.3333
-         0    0.3333    0.3333
-   -1.0000    0.6667    0.3333
-         0    0.6667    0.3333
-   -1.0000    1.0000    0.3333
-         0    1.0000    0.3333
-   -1.0000         0    0.6667
-         0         0    0.6667
-   -1.0000    0.3333    0.6667
-         0    0.3333    0.6667
-   -1.0000    0.6667    0.6667
-         0    0.6667    0.6667
-   -1.0000    1.0000    0.6667
-         0    1.0000    0.6667
-   -1.0000         0    1.0000
-         0         0    1.0000
-   -1.0000    0.3333    1.0000
-         0    0.3333    1.0000
-   -1.0000    0.6667    1.0000
-         0    0.6667    1.0000
-   -1.0000    1.0000    1.0000
-         0    1.0000    1.0000
@@ -1,68 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-# 1 nothing
-elements
-4
-1   5     0     1     3     2     6     7    9     8
-1   5     2     3     5     4     8     9    11    10
-1   5     6     7     9     8    12    13    15    14
-1   5     8     9    11    10    14    15    17    16
-
-# 0 nothing
-# 1 dirichlet bc
-# 2 contact
-boundary
-16
-0   3     1     0     2     3
-0   3     3     2     4     5
-0   3    12    13    15    14
-0   3    14    15    17    16
-2   3     2     0     6     8
-2   3     4     2     8    10
-2   3     8     6    12    14
-2   3    10     8    14    16
-1   3     1     3     9     7
-1   3     3     5    11    9
-1   3     7    9    15    13
-1   3     9    11    17    15
-0   3     6     0     1     7
-0   3    12     6     7    13
-0   3     4    10    11     5
-0   3    10    16    17    11
-
-vertices
-18
-3
-         0    0.2464    0.2464
-    0.5071    0.2464    0.2464
-         0    0.5000    0.2464
-    0.5071    0.5000    0.2464
-         0    0.7536    0.2464
-    0.5071    0.7536    0.2464
-         0    0.2464    0.5000
-    0.5071    0.2464    0.5000
-         0    0.5000    0.5000
-    0.5071    0.5000    0.5000
-         0    0.7536    0.5000
-    0.5071    0.7536    0.5000
-         0    0.2464    0.7536
-    0.5071    0.2464    0.7536
-         0    0.5000    0.7536
-    0.5071    0.5000    0.7536
-         0    0.7536    0.7536
-    0.5071    0.7536    0.7536
@@ -1,742 +0,0 @@
-//                                Contact example
-//
-// Compile with: make contact
-//
-// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
-// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include "nodepair.hpp"
-
-using namespace std;
-using namespace mfem;
-
-bool ifequalarray(const Array<int> a1, const Array<int> a2)
-{
-   if (a1.Size()!=a2.Size())
-   {
-      return false;
-   }
-   for (int i=0; i<a1.Size(); i++)
-   {
-      if (a1[i] != a2[i])
-      {
-         return false;
-      }
-   }
-   return true;
-}
-
-void FindSurfaceToProject(Mesh& mesh, const int elem, int& cbdrface)
-{
-   Array<int> attr;
-   attr.Append(2);
-   Array<int> faces;
-   Array<int> ori;
-   std::vector<Array<int> > facesVertices;
-   std::vector<int > faceid;
-   mesh.GetElementFaces(elem, faces, ori);
-   int face = -1;
-   for (int i=0; i<faces.Size(); i++)
-   {
-      face = faces[i];
-      Array<int> faceVert;
-      if (!mesh.FaceIsInterior(face)) // if on the boundary
-      {
-         mesh.GetFaceVertices(face, faceVert);
-         faceVert.Sort();
-         facesVertices.push_back(faceVert);
-         faceid.push_back(face);
-      }
-   }
-   int bdrface = facesVertices.size();
-
-   Array<int> bdryFaces;
-   // This shoulnd't need to be rebuilt
-   std::vector<Array<int> > bdryVerts;
-   for (int b=0; b<mesh.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh.GetBdrAttribute(b)) >= 0)  // found the contact surface
-      {
-         bdryFaces.Append(b);
-         Array<int> vert;
-         mesh.GetBdrElementVertices(b, vert);
-         vert.Sort();
-         bdryVerts.push_back(vert);
-      }
-   }
-
-   int bdrvert = bdryVerts.size();
-   cbdrface = -1;  // the face number of the contact surface element
-   int count_cbdrface = 0;  // the number of matching surfaces, used for checks
-
-   for (int i=0; i<bdrface; i++)
-   {
-      for (int j=0; j<bdrvert; j++)
-      {
-         if (ifequalarray(facesVertices[i], bdryVerts[j]))
-         {
-            cbdrface = faceid[i];
-            count_cbdrface += 1;
-         }
-      }
-   }
-   MFEM_VERIFY(count_cbdrface == 1,"projection surface not found");
-
-};
-
-Vector GetNormalVector(Mesh & mesh, const int elem, const double *ref,
-                       int & refFace, int & refNormal, bool & interior)
-{
-   ElementTransformation *trans = mesh.GetElementTransformation(elem);
-   const int dim = mesh.Dimension();
-   const int spaceDim = trans->GetSpaceDim();
-
-   MFEM_VERIFY(spaceDim == 3, "");
-
-   Vector n(spaceDim);
-
-   IntegrationPoint ip;
-   ip.Set(ref, dim);
-
-   trans->SetIntPoint(&ip);
-   //CalcOrtho(trans->Jacobian(), n);  // Works only for face transformations
-   const DenseMatrix jac = trans->Jacobian();
-
-   int dimNormal = -1;
-   int normalSide = -1;
-
-   const double tol = 1.0e-8;
-   for (int i=0; i<dim; ++i)
-   {
-      const double d0 = std::abs(ref[i]);
-      const double d1 = std::abs(ref[i] - 1.0);
-
-      const double d = std::min(d0, d1);
-      // TODO: this works only for hexahedral meshes!
-
-      if (d < tol)
-      {
-         MFEM_VERIFY(dimNormal == -1, "");
-         dimNormal = i;
-
-         if (d0 < tol)
-         {
-            normalSide = 0;
-         }
-         else
-         {
-            normalSide = 1;
-         }
-      }
-   }
-   // closest point on the boundary
-   if (dimNormal < 0 || normalSide < 0) // node is inside the element
-   {
-      interior = 1;
-      Vector n(3);
-      n = 0.0;
-      return n;
-   }
-
-   MFEM_VERIFY(dimNormal >= 0 && normalSide >= 0, "");
-   refNormal = dimNormal;
-
-   MFEM_VERIFY(dim == 3, "");
-
-   {
-      // Find the reference face
-      if (dimNormal == 0)
-      {
-         refFace = (normalSide == 1) ? 2 : 4;
-      }
-      else if (dimNormal == 1)
-      {
-         refFace = (normalSide == 1) ? 3 : 1;
-      }
-      else
-      {
-         refFace = (normalSide == 1) ? 5 : 0;
-      }
-   }
-
-   std::vector<Vector> tang(2);
-
-   int tangDir[2] = {-1, -1};
-   {
-      int t = 0;
-      for (int i=0; i<dim; ++i)
-      {
-         if (i != dimNormal)
-         {
-            tangDir[t] = i;
-            t++;
-         }
-      }
-
-      MFEM_VERIFY(t == 2, "");
-   }
-
-   for (int i=0; i<2; ++i)
-   {
-      tang[i].SetSize(3);
-
-      Vector tangRef(3);
-      tangRef = 0.0;
-      tangRef[tangDir[i]] = 1.0;
-
-      jac.Mult(tangRef, tang[i]);
-   }
-
-   Vector c(3);  // Cross product
-
-   c[0] = (tang[0][1] * tang[1][2]) - (tang[0][2] * tang[1][1]);
-   c[1] = (tang[0][2] * tang[1][0]) - (tang[0][0] * tang[1][2]);
-   c[2] = (tang[0][0] * tang[1][1]) - (tang[0][1] * tang[1][0]);
-
-   c /= c.Norml2();
-
-   Vector nref(3);
-   nref = 0.0;
-   nref[dimNormal] = 1.0;
-
-   Vector ndir(3);
-   jac.Mult(nref, ndir);
-
-   ndir /= ndir.Norml2();
-
-   const double dp = ndir * c;
-
-   // TODO: eliminate c?
-   n = c;
-   if (dp < 0.0)
-   {
-      n *= -1.0;
-   }
-   interior = 0;
-   return n;
-}
-
-// WARNING: global variable, just for this little example.
-std::array<std::array<int, 3>, 8> HEX_VERT =
-{
-   {  {0,0,0},
-      {1,0,0},
-      {1,1,0},
-      {0,1,0},
-      {0,0,1},
-      {1,0,1},
-      {1,1,1},
-      {0,1,1}
-   }
-};
-
-int GetHexVertex(int cdim, int c, int fa, int fb, Vector & refCrd)
-{
-   int ref[3];
-   ref[cdim] = c;
-   ref[cdim == 0 ? 1 : 0] = fa;
-   ref[cdim == 2 ? 1 : 2] = fb;
-
-   for (int i=0; i<3; ++i) { refCrd[i] = ref[i]; }
-
-   int refv = -1;
-
-   for (int i=0; i<8; ++i)
-   {
-      bool match = true;
-      for (int j=0; j<3; ++j)
-      {
-         if (ref[j] != HEX_VERT[i][j]) { match = false; }
-      }
-
-      if (match) { refv = i; }
-   }
-
-   MFEM_VERIFY(refv >= 0, "");
-
-   return refv;
-}
-
-// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
-// where X is the list of x-coordinates for all points and so on.
-// conn: connectivity of the target surface elements
-// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
-void FindPointsInMesh(Mesh & mesh, Vector const& xyz, Array<int>& conn,
-                      Vector& xi)
-{
-   const int dim = mesh.Dimension();
-   const int np = xyz.Size() / dim;
-
-   MFEM_VERIFY(np * dim == xyz.Size(), "");
-
-   mesh.EnsureNodes();
-
-   //FindPointsGSLIB finder(MPI_COMM_WORLD);
-   FindPointsGSLIB finder;
-
-   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
-
-   const double bb_t = 0.5;
-   finder.Setup(mesh, bb_t);
-
-   finder.FindPoints(xyz);
-
-   /// Return code for each point searched by FindPoints: inside element (0), on
-   /// element boundary (1), or not found (2).
-   Array<unsigned int> codes = finder.GetCode();
-
-   /// Return element number for each point found by FindPoints.
-   Array<unsigned int> elems = finder.GetElem();
-
-   /// Return reference coordinates for each point found by FindPoints.
-   Vector refcrd = finder.GetReferencePosition();
-
-   /// Return distance between the sought and the found point in physical space,
-   /// for each point found by FindPoints.
-   Vector dist = finder.GetDist();
-
-   MFEM_VERIFY(dist.Size() == np, "");
-   MFEM_VERIFY(refcrd.Size() == np * dim, "");
-   MFEM_VERIFY(elems.Size() == np, "");
-   MFEM_VERIFY(codes.Size() == np, "");
-
-   bool allfound = true;
-   for (auto code : codes)
-      if (code == 2) { allfound = false; }
-
-   MFEM_VERIFY(allfound, "A point was not found");
-
-   cout << "Maximum distance of projected points: " << dist.Max() << endl;
-
-   // extract information
-   for (int i=0; i<np; ++i)
-   {
-      cout << "Point " << i << ": (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << xyz[i + (j*np)];
-         if (j == dim-1) {cout << ")" << endl;}
-         else {cout << ", ";}
-      }
-      //cout << "  element: " << elems[i] << endl;
-      //cout << "  element " << elems[i] << " vertices:" << endl;
-      //Array<int> vert;
-      //mesh.GetElementVertices(elems[i], vert);
-      //for (auto v : vert)
-      //{
-      //   cout << "    " << v << endl;
-      //}
-
-      /*cout << "  reference coordinates: (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << refcrd[(i*dim) + j];
-         if (j == dim-1)
-         {
-            cout << ")" << endl;
-         }
-         else
-         {
-            cout << ", ";
-         }
-      }*/
-
-      int refFace, refNormal, refNormalSide;
-      bool is_interior = -1;
-      Vector normal = GetNormalVector(mesh, elems[i], refcrd.GetData() + (i*dim),
-                                      refFace, refNormal, is_interior);
-      int phyFace;
-      if (is_interior)
-      {
-         phyFace = -1; // the id of the face that has the closest point
-         FindSurfaceToProject(mesh, elems[i], phyFace);
-
-         Array<int> cbdrVert;
-         mesh.GetFaceVertices(phyFace, cbdrVert);
-         Vector xs(dim);
-         xs[0] = xyz[i + 0*np];
-         xs[1] = xyz[i + 1*np];
-         xs[2] = xyz[i + 2*np];
-         Vector xi_tmp(dim-1);
-         // get nodes!
-
-         GridFunction *nodes = mesh.GetNodes();
-         DenseMatrix coords(4,3);
-         for (int i=0; i<4; i++)
-         {
-            for (int j=0; j<3; j++)
-            {
-               coords(i,j) = (*nodes)[cbdrVert[i]*3+j];
-            }
-         }
-         SlaveToMaster(coords, xs, xi_tmp);
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = xi_tmp[j];
-         }
-         // now get get the projection to the surface
-      }
-      else
-      {
-         Vector faceRefCrd(dim-1);
-         {
-            int fd = 0;
-            for (int j=0; j<dim; ++j)
-            {
-               if (j == refNormal)
-               {
-                  refNormalSide = (refcrd[(i*dim) + j] > 0.5);
-               }
-               else
-               {
-                  faceRefCrd[fd] = refcrd[(i*dim) + j];
-                  fd++;
-               }
-            }
-
-            MFEM_VERIFY(fd == dim-1, "");
-         }
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
-         }
-         //cout << "  face reference coordinates: (";
-         for (int j=0; j<dim-1; ++j)
-         {
-            cout << faceRefCrd[j];
-            if (j == dim-2) {cout << ")" << endl;}
-            else {cout << ", ";}
-         }
-      }
-      //cout << "  normal vector: ";
-      //normal.Print();
-
-      // ask, does this do anything?
-      /*
-      IntegrationPoint ip;
-      ip.Set(refcrd.GetData() + (i*dim), dim);
-      ElementTransformation *trans = mesh.GetElementTransformation(elems[i]);
-      Vector phys(trans->GetSpaceDim());
-      trans->Transform(ip, phys);
-      cout << "  physical coordinates: ";
-      phys.Print();
-      */
-
-      // Get the element face
-      Array<int> faces;
-      Array<int> ori;
-      int face;
-
-      if (is_interior)
-      {
-         face = phyFace;
-      }
-      else
-      {
-         mesh.GetElementFaces(elems[i], faces, ori);
-         face = faces[refFace];
-      }
-
-      Array<int> faceVert;
-      mesh.GetFaceVertices(face, faceVert);
-
-      //cout << "  face " << face << " vertices:" << endl;
-      //for (auto v : faceVert){ cout << "    " << v << endl;}
-
-      for (int p=0; p<4; p++)
-      {
-         conn[4*i+p] = faceVert[p];
-      }
-      /*
-      Vector ref(dim);
-
-      for (int p=0; p<2; ++p)
-         for (int q=0; q<2; ++q)
-         {
-            const int refv = GetHexVertex(refNormal, refNormalSide, p, q, ref);
-            cout << "  face reference vertex (" << p << "," << q
-                 << ") is global vertex " << vert[refv] << endl;
-
-            {
-               // Sanity check
-               ip.Set(ref.GetData(), dim);
-               trans->Transform(ip, phys);
-               for (int j=0; j<dim; ++j)
-               {
-                  phys[j] -= mesh.GetVertex(vert[refv])[j];
-               }
-          phys.Print();
-          cout<<vert[refv]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[0]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[1]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[2]<<endl;
-               MFEM_VERIFY(phys.Norml2() < 1.0e-12, "Sanity check failed");
-            }
-         }*/
-   }
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file1 = "block1.mesh";
-   const char *mesh_file2 = "block2.mesh";
-
-   Array<int> attr;
-   Array<int> m_attr;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file1, "-m1", "--mesh1",
-                  "First mesh file to use.");
-   args.AddOption(&mesh_file2, "-m2", "--mesh2",
-                  "Second mesh file to use.");
-   args.AddOption(&attr, "-at", "--attributes-surf",
-                  "Attributes of boundary faces on contact surface for mesh 2.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Mesh mesh1(mesh_file1, 1, 1);
-   Mesh mesh2(mesh_file2, 1, 1);
-
-   const int dim = mesh1.Dimension();
-   MFEM_VERIFY(dim == mesh2.Dimension(), "");
-
-   // boundary attribute 2 is the potential contact surface of nodes
-   attr.Append(2);
-   // boundary attribute 2 is the potential contact surface for master surface
-   m_attr.Append(2);
-
-   //  Define a finite element space on the mesh. Here we use vector finite
-   //  elements, i.e. dim copies of a scalar finite element space. The vector
-   //  dimension is specified by the last argument of the FiniteElementSpace
-   //  constructor.
-   FiniteElementCollection *fec1;
-   FiniteElementSpace *fespace1;
-   fec1 = new H1_FECollection(1, dim);
-   fespace1 = new FiniteElementSpace(&mesh1, fec1, dim, Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh1: "
-        << fespace1->GetTrueVSize() << endl;
-   mesh1.SetNodalFESpace(fespace1);
-   GridFunction nodes0 = *mesh1.GetNodes(); // undeformed mesh1 nodal grid function
-   GridFunction *nodes1 = mesh1.GetNodes();
-
-   FiniteElementCollection *fec2 = new H1_FECollection(1, dim);
-   FiniteElementSpace *fespace2 = new FiniteElementSpace(&mesh2, fec2, dim,
-                                                         Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh2: "
-        << fespace2->GetTrueVSize() << endl;
-
-   // degrees of freedom of both meshes
-   int ndof_1 = fespace1->GetTrueVSize();
-   int ndof_2 = fespace2->GetTrueVSize();
-   int ndofs = ndof_1 + ndof_2;
-   // number of nodes for each mesh
-   int nnd_1 = mesh1.GetNV();
-   int nnd_2 = mesh2.GetNV();
-   int nnd = nnd_1 + nnd_2;
-   // Determine the list of true (i.e. conforming) essential boundary dofs.
-   // In this example, the boundary conditions are defined by marking only
-   // boundary attribute 1 from the mesh as essential and converting it to a
-   // list of true dofs.
-   Array<int> ess_tdof_list1, ess_bdr1(mesh1.bdr_attributes.Max());
-   cout<<mesh1.bdr_attributes.Max()<<endl;
-   ess_bdr1 = 0;
-   //ess_bdr1[0] = 1;
-   // Not ready to be passed on yet
-   // fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   Array<int> ess_tdof_list2, ess_bdr2(mesh2.bdr_attributes.Max());
-   ess_bdr2 = 0;
-   //ess_bdr2[0] = 1;
-
-   // Define the displacement vector x as a finite element grid function
-   // corresponding to fespace. GridFunction is a derived class of Vector.
-   GridFunction x1(fespace1);
-   x1 = 0.0;
-   GridFunction x2(fespace2);
-   x2 = 0.0;
-
-   // Generate force
-   LinearForm *b1 = new LinearForm(fespace1);
-   b1->Assemble();
-
-   LinearForm *b2 = new LinearForm(fespace2);
-   b2->Assemble();
-
-   // Set up the bilinear form a(.,.) on the finite element space
-   //  corresponding to the linear elasticity integrator with piece-wise
-   //  constants coefficient lambda and mu.
-   Vector lambda1(mesh1.attributes.Max());
-   lambda1 = 57.6923076923;
-   PWConstCoefficient lambda1_func(lambda1);
-   Vector mu1(mesh1.attributes.Max());
-   mu1 = 38.4615384615;
-   PWConstCoefficient mu1_func(mu1);
-
-   BilinearForm *a1 = new BilinearForm(fespace1);
-   a1->AddDomainIntegrator(new ElasticityIntegrator(lambda1_func,mu1_func));
-
-   Vector lambda2(mesh2.attributes.Max());
-   lambda2 = 57.6923076923;
-   PWConstCoefficient lambda2_func(lambda2);
-   Vector mu2(mesh2.attributes.Max());
-   mu2 = 38.4615384615;
-   PWConstCoefficient mu2_func(mu2);
-
-   BilinearForm *a2 = new BilinearForm(fespace2);
-   a2->AddDomainIntegrator(new ElasticityIntegrator(lambda2_func,mu2_func));
-
-   a1->Assemble();
-   SparseMatrix A1;
-   Vector B1, X1;
-   a1->FormLinearSystem(ess_tdof_list1, x1, *b1, A1, X1, B1);
-
-   a2->Assemble();
-   SparseMatrix A2;
-   Vector B2, X2;
-   a2->FormLinearSystem(ess_tdof_list2, x2, *b2, A2, X2, B2);
-
-   // Combine elasticity operator for two meshes into one.
-   // Block Matrix
-   SparseMatrix K(ndofs,ndofs);
-   for (int i=0; i<A1.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A1.GetRow(i, col_tmp, v_tmp);
-      K.SetRow(i, col_tmp, v_tmp);
-   }
-   for (int i=0; i<A2.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A2.GetRow(i, col_tmp, v_tmp);
-      for (int j=0; j<col_tmp.Size(); j++)
-      {
-         col_tmp[j] += ndof_1;
-      }
-      K.SetRow(i+ndof_1, col_tmp, v_tmp);  // mesh1 top left corner
-   }
-
-   // Construct node to segment contact constraint.
-
-   attr.Sort();
-   cout << "Boundary attributes for contact surface faces in mesh 2" << endl;
-   for (auto a : attr)
-   {
-      cout << a << endl;
-   }
-
-   Array<int> bdryFaces2;  // TODO: remove this?
-
-   std::set<int> bdryVerts2;
-   for (int b=0; b<mesh2.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh2.GetBdrAttribute(b)) >= 0)
-      {
-         bdryFaces2.Append(b);
-         Array<int> vert;
-         mesh2.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            bdryVerts2.insert(v);
-         }
-      }
-   }
-
-   int npoints = bdryVerts2.size();
-   Array<int> s_conn(npoints); // connectivity of the second/slave mesh
-   Vector xyz(dim * npoints);
-   xyz = 0.0;
-
-   cout << "Boundary vertices for contact surface vertices in mesh 2" << endl;
-
-   // construct the nodal coordinates on mesh2 to be projected, including displacement
-   int count = 0;
-   for (auto v : bdryVerts2)
-   {
-      cout << v << ": " << mesh2.GetVertex(v)[0] << ", "
-           << mesh2.GetVertex(v)[1] << ", "
-           << mesh2.GetVertex(v)[2] << endl;
-
-      for (int i=0; i<dim; ++i)
-      {
-         xyz[count + (i * npoints)] = mesh2.GetVertex(v)[i] + x2[v*dim+i];
-      }
-
-      s_conn[count] = v + nnd_1; // dof1 is the master
-      count++;
-   }
-
-   MFEM_VERIFY(count == npoints, "");
-
-   // gap function
-   Vector g(npoints*dim);
-   g = -1.0;
-   // segment reference coordinates of the closest point
-   Vector m_xi(npoints*(dim-1));
-   m_xi = -1.0;
-   Vector xs(dim*npoints);
-   xs = 0.0;
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<dim; j++)
-      {
-         xs[i*dim+j] = xyz[i + (j*npoints)];
-      }
-   }
-
-   Array<int> m_conn(
-      npoints*4); // only works for linear elements that have 4 vertices!
-   DenseMatrix coordsm(npoints*4, dim);
-
-   // adding displacement to mesh1 using a fixed grid function from mesh1
-   x1 = 1e-4; // x1 order: [xyz xyz... xyz]
-   add(nodes0, x1, *nodes1);
-
-   FindPointsInMesh(mesh1, xyz, m_conn, m_xi);
-
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<4; j++)
-      {
-         for (int k=0; k<dim; k++)
-         {
-            coordsm(i*4+j,k) = mesh1.GetVertex(m_conn[i*4+j])[k]+x1[dim*m_conn[i*4+j]+k];
-         }
-      }
-   }
-   //coordsm.Print();
-   SparseMatrix M(nnd,ndofs);
-   std::vector<SparseMatrix> dM(nnd, SparseMatrix(ndofs,ndofs));
-
-   Assemble_Contact(nnd, npoints, ndofs, xs, m_xi, coordsm,
-                    s_conn, m_conn, g, M, dM);
-
-   //M.Print();
-   /*Vector eps(ndofs);
-   Vector sol(ndofs); sol = 0.;
-   for(int i=0;i<ndofs;i++) eps[i] = 1e-5 * i ;
-   for(int i=0;i<9;i++)
-   {
-     cout<<i<<endl;
-     dM[s_conn[i]].Mult(eps,sol);
-     sol.Print();
-   }
-   */
-   return 0;
-}
@@ -1,230 +0,0 @@
-//                                Contact example
-//
-// Compile with: make exContactBlockTL
-//
-// Sample runs:  ./exContactBlockTL -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
-// Sample runs:  ./exContactBlockTL -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
-
-#ifndef EXCONTACTBLOCKTL_HPP
-#define EXCONTACTBLOCKTL_HPP
-
-#include "mfem.hpp"
-#include "IpTNLP.hpp"
-
-using namespace std;
-using namespace mfem;
-using namespace Ipopt;
-
-
-class ExContactBlockTL: public TNLP
-{
-public:
-   /** default constructor */
-   ExContactBlockTL(int argc, char *argv[]);
-
-   /** default destructor */
-   virtual ~ExContactBlockTL();
-
-   /**@name Overloaded from TNLP */
-   /** Method to return some info about the nlp */
-   virtual bool get_nlp_info(
-      Index&          n,
-      Index&          m,
-      Index&          nnz_jac_g,
-      Index&          nnz_h_lag,
-      IndexStyleEnum& index_style
-   );
-
-   /** Method to return the bounds for my problem */
-   virtual bool get_bounds_info(
-      Index   n,
-      Number* x_l,
-      Number* x_u,
-      Index   m,
-      Number* g_l,
-      Number* g_u
-   );
-
-   /** Method to return the starting point for the algorithm */
-   virtual bool get_starting_point(
-      Index   n,
-      bool    init_x,
-      Number* x,
-      bool    init_z,
-      Number* z_L,
-      Number* z_U,
-      Index   m,
-      bool    init_lambda,
-      Number* lambda
-   );
-
-   /** Method to return the objective value */
-   virtual bool eval_f(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Number&       obj_value
-   );
-
-   /** Method to return the gradient of the objective */
-   virtual bool eval_grad_f(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Number*       grad_f
-   );
-
-   /** Method to return the constraint residuals */
-   virtual bool eval_g(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Index         m,
-      Number*       cons
-   );
-
-   /** Method to return:
-    *   1) The structure of the Jacobian (if "values" is NULL)
-    *   2) The values of the Jacobian (if "values" is not NULL)
-    */
-   virtual bool eval_jac_g(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Index         m,
-      Index         nele_jac,
-      Index*        iRow,
-      Index*        jCol,
-      Number*       values
-   );
-
-   /** Method to return:
-    *   1) The structure of the Hessian of the Lagrangian (if "values" is NULL)
-    *   2) The values of the Hessian of the Lagrangian (if "values" is not NULL)
-    */
-   virtual bool eval_h(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Number        obj_factor,
-      Index         m,
-      const Number* lambda,
-      bool          new_lambda,
-      Index         nele_hess,
-      Index*        iRow,
-      Index*        jCol,
-      Number*       values
-   );
-
-   /** This method is called when the algorithm is complete so the TNLP can store/write the solution */
-   virtual void finalize_solution(
-      SolverReturn               status,
-      Index                      n,
-      const Number*              x,
-      const Number*              z_L,
-      const Number*              z_U,
-      Index                      m,
-      const Number*              g,
-      const Number*              lambda,
-      Number                     obj_value,
-      const IpoptData*           ip_data,
-      IpoptCalculatedQuantities* ip_cq
-   );
-
-private:
-   void update_g();
-   void update_jac();
-   void update_hess();
-
-private:
-   /**@name Methods to block default compiler methods.
-    *
-    * The compiler automatically generates the following three methods.
-    *  Since the default compiler implementation is generally not what
-    *  you want (for all but the most simple classes), we usually
-    *  put the declarations of these methods in the private section
-    *  and never implement them. This prevents the compiler from
-    *  implementing an incorrect "default" behavior without us
-    *  knowing. (See Scott Meyers book, "Effective C++")
-    */
-   ExContactBlockTL(
-      const ExContactBlockTL&
-   );
-
-   ExContactBlockTL& operator=(
-      const ExContactBlockTL&
-   );
-
-   Array<int> attr;
-   Array<int> m_attr;
-   Array<int> s_conn; // connectivity of the second/slave mesh
-   std::string mesh_file1;
-   std::string mesh_file2;
-   Mesh* mesh1;
-   Mesh* mesh2;
-   FiniteElementCollection* fec1;
-   FiniteElementCollection* fec2;
-   FiniteElementSpace* fespace1;
-   FiniteElementSpace* fespace2;
-   Array<int> ess_tdof_list1;
-   Array<int> ess_tdof_list2;
-   GridFunction nodes0;
-   GridFunction* nodes1;
-   GridFunction* nodes2;
-   GridFunction* x1;
-   GridFunction* x2;
-   LinearForm* b1;
-   LinearForm* b2;
-   PWConstCoefficient* lambda1_func;
-   PWConstCoefficient* lambda2_func;
-   PWConstCoefficient* mu1_func;
-   PWConstCoefficient* mu2_func;
-   BilinearForm* a1;
-   BilinearForm* a2;
-
-   mfem::Vector lambda1;
-   mfem::Vector lambda2;
-   mfem::Vector mu1;
-   mfem::Vector mu2;
-   mfem::Vector xyz;
-
-   std::set<int> bdryVerts2;
-
-   int dim;
-   // degrees of freedom of both meshes
-   int ndof_1;
-   int ndof_2;
-   int ndofs;
-   // number of nodes for each mesh
-   int nnd_1;
-   int nnd_2;
-   int nnd;
-
-   int npoints;
-
-   SparseMatrix A1;
-   mfem::Vector B1, X1;
-   SparseMatrix A2;
-   mfem::Vector B2, X2;
-
-   SparseMatrix* K;
-   mfem::Vector g;
-   mfem::Vector m_xi;
-   mfem::Vector xs;
-
-   Array<int> m_conn; // only works for linear elements that have 4 vertices!
-   DenseMatrix* coordsm;
-   SparseMatrix* M;
-
-   std::vector<SparseMatrix>* dM;
-
-   Array<int> Dirichlet_dof;
-   Array<double> Dirichlet_val;
-
-public: 
-   Mesh * GetMesh1() {return mesh1;}
-   Mesh * GetMesh2() {return mesh2;}
-
-};
-
-#endif
@@ -1,68 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ipopt/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = exContactBlockTL
-EXAMPLES = $(SEQ_EXAMPLES)
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_IPOPT),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with IPOPT)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f exContactBlockTL.mesh exContactBlockTL-mesh.* exContactBlockTL-init.* exContactBlockTL-final.* ExampleContactBlockTL*
@@ -1,888 +0,0 @@
-
-using namespace std;
-using namespace mfem;
-
-void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]);
-   dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);
-   dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]);
-   dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);
-   dNdxi(1,3) = 0.25*(1-xi[0]);
-}
-
-
-void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
-                     DenseMatrix& dN2dxi)
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(2,4); dNdxi = 0.0;
-   dN2dxi.SetSize(3,4);
-   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
-
-   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
-   dN2dxi(1,3) = -0.25;
-}
-
-// returns the vector and matrix form of the shape functions and its derivative
-void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
-                       DenseMatrix& ddNdxi)
-{
-   N.SetSize(3,12); N = 0.0;
-   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
-   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
-   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
-   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
-   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
-   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
-   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
-
-   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
-   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
-   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
-   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
-   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
-   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
-
-   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
-   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
-   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
-   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
-   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
-   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
-   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
-
-   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
-   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
-   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
-   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
-   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
-   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
-}
-
-
-void cross(const Vector a, const Vector b, Vector& c)
-{
-   assert(a.Size()==3);
-   c.SetSize(3);
-   c[0] = a[1]*b[2] - a[2]*b[1];
-   c[1] = -a[0]*b[2] + b[0]*a[2];
-   c[2] = a[0]*b[1] - a[1]*b[0];
-
-}
-// a outer b
-void outer(const Vector a, const Vector b, DenseMatrix& c)
-{
-   int m = a.Size();
-   int n = b.Size();
-   assert(c.Height()==m);
-   assert(c.Width() ==n);
-   for (int i=0; i<m; i++)
-   {
-      for (int j=0; j<n; j++)
-      {
-         c(i,j) = a[i]*b[j];
-      }
-   }
-}
-// dphidxi 2*4
-// coords 4*3
-void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
-                   Vector& normal, double& nnorm)
-{
-
-   DenseMatrix dxdxi(2,3);
-   Mult(dphidxi, coords, dxdxi);
-   Vector dxdxi1(3);
-   Vector dxdxi2(3);
-
-   dxdxi.GetRow(0,dxdxi1);
-   dxdxi.GetRow(1,dxdxi2);
-
-   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
-   // VectorCrossProductCoefficient::Eval has hard-coded cross product
-   nnorm = normal.Norml2( );
-   normal /=  nnorm;
-}
-
-void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
-{
-   bool converged = false;
-   bool pt_on_elem = false;
-   int dim = 3;
-   xi.SetSize(dim-1);
-   xi = 0.0;
-   double r = 1e10;
-   int max_iter = 15;
-   double off_el_xi = 1e-2;
-   double proj_newton_tol = 1e-13;
-   double proj_max_gap = 0.5;
-   Vector gap_v(dim);
-   // warm start from linear solution
-
-   for (int it=0; it<max_iter; it++)
-   {
-      //cout<<it<<endl;
-      Vector m_N(4);
-      m_N = 0.;
-      DenseMatrix m_dN(2,4);
-      m_dN = 0.;
-      DenseMatrix m_dN2(3,4);
-      m_dN2 = 0.;
-      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-      Vector x_c(dim);
-      m_coords.MultTranspose(m_N, x_c);
-
-      gap_v = s_x;
-      gap_v -= x_c;
-
-      DenseMatrix m_dx(2,3);
-      m_dx = 0.;
-      Mult(m_dN, m_coords, m_dx);
-
-      Vector r(dim-1);
-      r = 0.0;
-      m_dx.Mult(gap_v, r);
-
-      if (r.Normlinf() < proj_newton_tol)
-      {
-         converged = true;
-         break;
-      }
-
-      DenseMatrix drdxi(dim-1,dim-1);
-      drdxi = 0.;
-      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
-      drdxi *= -1.0;
-
-      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-      Mult(m_dN2,m_coords, m_dx2);
-
-      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
-      for (int d=0; d<3; d++)
-      {
-         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-         drdxi.Add(gap_v[d], Mtemp);
-      }
-
-      //cond_num = rcond(drdxi);  condition number?
-      //drdxi.TestInversion();
-      DenseMatrixInverse drdxi_inv(drdxi);
-      Vector xi_tmp(dim-1);
-
-      drdxi_inv.Mult(r,xi_tmp);
-      xi -= xi_tmp;
-   }
-   if (!converged)
-   {
-      xi = 0.0;
-   }
-   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
-
-   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
-   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
-   {
-      pt_on_elem = true;
-   }
-
-   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
-   MFEM_VERIFY(converged == true, "projection didn't converge");
-}
-
-
-
-// m_coords is expected to be 4 * 3
-void  ComputeGapJacobian(const Vector x_s, const Vector xi,
-                         const DenseMatrix m_coords,
-                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N, x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   double nnorm = 0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   gap = gap_v * normal; // gap function value, dot product between vectors
-
-   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
-
-   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
-   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
-
-   Vector m_dxrow1(3);
-   m_dx.GetRow(0, m_dxrow1);
-   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
-   dr_dx_res1 *= -1.0;
-
-   Vector m_dxrow2(3);
-   m_dx.GetRow(1, m_dxrow2);
-   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
-   dr_dx_res2 *= -1.0;
-
-   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
-   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
-
-   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
-   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
-   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
-   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
-
-   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
-   dr_dx_res2 += dr_dx_res2_tmp;
-
-
-   DenseMatrix K_dxidx1(2,2); // 2*2
-   K_dxidx1 = 0.;
-   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   // how to get 2nd order? multidimensional matrix?
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= K_dxidx1;
-   K_dxidx += K_dxidx2;
-
-   // resize the vectors and matrices
-   Vector dxidx(24); dxidx = 0.0;
-   Vector drdx_r(24); drdx_r = 0.0;
-
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         drdx_r[4*j+i] = dr_dx_res1(i,j);
-         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
-
-      }
-   }
-   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
-   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
-   DenseMatrix drdx_K(24,24); drdx_K = 0.;
-   for (int i =0; i<12; i++)
-   {
-      drdx_K(i,i) = K_dxidx(0,0);
-      drdx_K(i,12+i) = K_dxidx(0,1);
-      drdx_K(12+i,i) = K_dxidx(1,0);
-      drdx_K(12+i,12+i) = K_dxidx(1,1);
-   }
-
-   DenseMatrixInverse drdxK_inv(drdx_K);
-   drdxK_inv.Mult(drdx_r,dxidx);
-   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
-   dxidx *= -1.0;
-
-
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-
-   Vector dxidxs(6); dxidxs = 0.0;
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
-
-   dgdxm.SetSize(12); dgdxm = 0.;
-   DenseMatrix dgdxm_tmp(4,3);
-   outer(m_N, normal,dgdxm_tmp);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
-      }
-   }
-   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
-
-   dgdxs.SetSize(3);
-   dgdxs += normal;
-   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
-};
-
-void ComputeGapHessian(const Vector x_s, const Vector xi,
-                       const DenseMatrix m_coords,
-                       DenseMatrix& dg2dx)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   int dim = 3;
-   int num_dofs1 = dim;
-   int num_dofs2 = 4*dim;
-   int num_dofs = num_dofs1 + num_dofs2;
-   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N,x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-   Mult(m_dN2,m_coords, m_dx2);
-   double nnorm = 0.0;
-   Vector normal(3); normal = 0.0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   double gap = gap_v * normal; // gap function value, dot product between vectors
-
-   DenseMatrix M(2,2); M = 0.0;
-   MultABt(m_dx, m_dx, M);
-
-   DenseMatrix f(2, num_dofs2); f = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-      M.Add(-gap_v[d], Mtemp);
-
-      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
-      DenseMatrix ftmp(2,4);
-      outer(m_dxcol, m_N, ftmp);
-      ftmp *= -1;
-      ftmp.Add( gap_v[d], m_dN);   // 2*4
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         f(0,d+j*3) = ftmp(0,j);
-         f(1,d+j*3) = ftmp(1,j);
-      }
-   }
-   //fprintf('hess dxidxm\n');
-   DenseMatrixInverse Minv(M);
-   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
-   Minv.Mult(f, dxidxm);
-   //LinearSolve??
-   //dxidxm = M\f;
-
-   DenseMatrix nde2(2,2); nde2 = 0.0;
-   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
-      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
-      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
-
-      nde2 += ndetmp;
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
-         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
-      }
-
-   }
-
-   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
-   Ndn += Nndx2;
-   AddMult(nde2, dxidxm, Ndn);
-
-
-   DenseMatrix M2(2,2); M2 = 0.0;
-   MultABt(m_dx, m_dx, M2);
-   DenseMatrixInverse M2inv(M2);
-   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
-   DenseMatrix m_con(2,2); m_con = 0.0;
-
-   M2inv.Mult(diag2, m_con);
-
-   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
-
-   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
-   MultAtB(Ndn, m_con, dg2dxm_tmp);
-   Mult(dg2dxm_tmp, Ndn, dg2dxm);
-   dg2dxm *= gap;
-
-   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
-   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   dg2dxm_tmp = 0.0;
-   MultAtB(dxidxm, nde2, dg2dxm_tmp);
-
-   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
-
-   dg2dxm_tmp2 = 0.0;
-   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= M2;
-   K_dxidx += K_dxidx2;
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-   Vector dxidxs(6);
-
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
-   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
-   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
-
-   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
-   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
-
-   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
-   dxidxs_row2 = 0.0;
-   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
-   Vector mdx2_row3(3); mdx2_row3 = 0.0;
-   dxidxs_m.GetRow(0,dxidxs_row1);
-   dxidxs_m.GetRow(1,dxidxs_row2);
-   m_dx2.GetRow(0,mdx2_row1);
-   m_dx2.GetRow(1,mdx2_row2);
-   m_dx2.GetRow(2,mdx2_row3);
-
-   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
-   outer(mdx2_row1, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row2, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-
-   outer(mdx2_row2, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row3, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-
-   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
-
-   for (int d=0; d<3; d++)
-   {
-      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
-      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
-      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
-
-      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
-      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
-
-      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
-      m_dx.GetRow(0, m_dxrow);
-      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
-      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
-      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
-
-      dtaodxs_tmp2 += dtaodxs_tmp;
-      dtaodxs.SetCol(d, dtaodxs_tmp2);
-   }
-
-   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
-   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
-   outer(normal, normal, dndxs_tmp);
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
-
-   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
-   MultAtB(m_dx, dxidxs_m, dgvdxs);
-   dgvdxs *= -1;
-   for (int d=0; d<3; d++)
-   {
-      dgvdxs(d,d) += 1.0;
-   }
-   //dxidxs: 2*3
-
-   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
-   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
-   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
-   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
-   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
-   dg2dxs += dg2dxs_tmp2;
-   dg2dxs_tmp2 = 0.0;
-   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
-   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
-
-   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
-   BasisVectorDerivs(xi, Ne, Be, dBe);
-
-   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
-   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
-
-   Vector m_coords_v(12);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         m_coords_v[i*3+j] = m_coords(i,j);
-      }
-   }
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix dBe_tmp(3,12);
-      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
-
-      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
-
-      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
-      dBe_tmp = 0.0;
-      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
-
-   }
-
-   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
-
-   for (int d=0; d<12; d++)
-   {
-      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
-      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
-      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
-      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
-      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
-      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
-
-      cross(tmp1, m_dxrow2, dtaodxm_tmp);
-      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
-      dtaodxm_tmp += dtaodxm_tmp2;
-
-      dtaodxm.SetCol(d, dtaodxm_tmp);
-   }
-
-   DenseMatrix dndxm(3,12); dndxm = 0.0;
-   dndxm += dtaodxm;
-   dndxm *= 1.0/nnorm;
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
-
-   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
-   dgvdxm -= Ne;
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
-      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
-
-   }
-
-   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
-   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
-   MultAtB(dgvdxs, dndxm, dg2dxsxm);
-
-   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
-   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
-
-   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
-   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
-
-   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
-
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
-      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
-      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
-   }
-
-   dg2dxsxm += dgvdxsxmn;
-
-   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
-   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
-   MultAtB(dgvdxm, dndxs, dg2dxmxs);
-   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
-   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
-
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
-
-   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
-   dgvdxmxsn_tmp *= -1.0;
-   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-      Be_tmp.Transpose(); // Be is now 12*3
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
-      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
-      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
-
-   }
-
-   dg2dxmxs += dgvdxmxsn;
-
-   dg2dx.CopyMN(dg2dxs, 0, 0);
-   dg2dx.CopyMN(dg2dxm, 3, 3);
-   dg2dx.CopyMN(dg2dxsxm, 0, 3);
-   dg2dx.CopyMN(dg2dxmxs, 3, 0);
-
-};
-
-
-
-void NodeSegConPairs(const Vector x1, const Vector xi2,
-                     const DenseMatrix coords2,
-                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
-{
-   double gap = 0.0;
-   Vector normal(3); normal = 0.0;
-   Vector dgdxm(12); dgdxm = 0.0;
-   Vector dgdxs(3); dgdxs = 0.0;
-
-   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
-   node_g = gap;
-
-   node_dg.SetSize(12+3);
-   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
-   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
-
-   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   ComputeGapHessian(x1, xi2, coords2, dg2dx);
-
-   node_dg2.SetSize(15,15);
-   node_dg2 = dg2dx;
-
-   /*
-   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
-
-   v1 = 1:3;
-   v2 = 1:12;
-   %v1 = ones(1,3)
-   %v2 = ones(1,12)
-   v2 = reshape(v2,4,3);
-   x1n1 = x1 + 0.01*v1;
-   coords2n1 = coords2 + 0.001*v2;
-      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
-      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
-   x1n2 = x1 - 0.01*v1;
-   coords2n2 = coords2 - 0.001*v2;
-      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
-      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
-   fprintf('fd\n');
-      %gapv1-gapv2
-   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
-
-   %dgdxsn1-dgdxsn2
-   fprintf('code\n');
-   v2n = v2';
-   %dg2dx(1:3,1:3)*0.04*ones(3,1)
-   temp = zeros(12,3);
-   for i = 1:4
-   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
-   temp((i-1)*3+1:i*3,:) = temp1';
-   end
-   temp2 = zeros(3,12);
-   for i = 1:4
-   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
-   temp2(:,(i-1)*3+1:i*3) = temp3';
-   end
-   %dg2dx
-   %dg2dx(4:end,1:3) = temp;
-   %dg2dx(1:3,4:end) = temp2;
-   %dgvdxm * 0.002*v2n(:)
-   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
-   %dg2dx(4:end,1:3)
-   end*/
-
-};
-
-
-// coordsm : (npoints*4, 3) use what class?
-// m_conn: (npoints*4)
-void Assemble_Contact(const int m, const int npoints, const int ndofs,
-                      const Vector x_s,
-                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
-                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
-                      std::vector<SparseMatrix>& dM)
-{
-   int n = ndofs;
-   int ndim = 3;
-
-   g.SetSize(m);
-   g = 0.0;
-
-   //SparseMatrix M(m, n);  // M needs to be the correct size
-
-   //dM.resize(m); // needs to clear?
-
-   double g_tmp = 0.;
-   Vector dg(4*ndim+ndim);
-   dg = 0.;
-   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
-   dg2 = 0.;
-
-   for (int i=0; i<npoints; i++)
-   {
-      Vector x1(ndim);
-      x1[0] = x_s[i*ndim];
-      x1[1] = x_s[i*ndim+1];
-      x1[2] = x_s[i*ndim+2];
-
-      Vector xi2(ndim-1);
-      xi2[0] = xi[i*(ndim-1)];
-      xi2[1] = xi[i*(ndim-1)+1];
-
-      DenseMatrix coords2(4,3);
-      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
-
-      //how to get coords2?
-      dg = 0.0;
-      dg2 = 0.;
-      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
-      //x1.Print();
-      //xi2.Print();
-      //coords2.Print();
-      g[s_conn[i]] = g_tmp; // should be unique
-      Array<int> m_conn_i(4);
-      m_conn.GetSubArray(4*i, 4, m_conn_i);
-
-      Array<int> node_conn(5);
-      node_conn[0] = s_conn[i];
-      for (int j=0; j<4; j++)
-      {
-         node_conn[j+1] = m_conn_i[j];
-      }
-
-      Array<int>  M_i_tmp(1);
-      M_i_tmp[0] = s_conn[i];
-
-      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
-      Array<int> j_idx(5*ndim); j_idx = 0;
-      for (int j=0; j< 5; j++)
-      {
-         for (int k=0; k<ndim; k++)
-         {
-            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
-         }
-      }
-      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
-      M_v_tmp.SetRow(0, dg);
-
-      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
-
-      Array<int> dM_i(ndim*(4+1));
-      Array<int> dM_j(ndim*(4+1));
-
-      for (int j=0; j< ndim*(4+1); j++)
-      {
-         dM_i[j] = j_idx[j];
-         dM_j[j] = j_idx[j];
-      }
-      //dg2.Print();
-      //dM[s_conn[i]].Print();
-      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
-   }
-};
-
@@ -46,9 +46,6 @@ endif
 ifeq ($(MFEM_USE_HIOP),YES)
   SUBDIRS += hiop
 endif
-ifeq ($(MFEM_USE_IPOPT),YES)
-   SUBDIRS += ipopt
-endif
 ifeq ($(MFEM_USE_PETSC),YES)
   SUBDIRS += petsc
 endif
@@ -1,888 +0,0 @@
-
-using namespace std;
-using namespace mfem;
-
-void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]);
-   dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);
-   dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]);
-   dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);
-   dNdxi(1,3) = 0.25*(1-xi[0]);
-}
-
-
-void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
-                     DenseMatrix& dN2dxi)
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(2,4); dNdxi = 0.0;
-   dN2dxi.SetSize(3,4);
-   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
-
-   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
-   dN2dxi(1,3) = -0.25;
-}
-
-// returns the vector and matrix form of the shape functions and its derivative
-void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
-                       DenseMatrix& ddNdxi)
-{
-   N.SetSize(3,12); N = 0.0;
-   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
-   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
-   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
-   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
-   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
-   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
-   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
-
-   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
-   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
-   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
-   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
-   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
-   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
-
-   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
-   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
-   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
-   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
-   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
-   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
-   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
-
-   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
-   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
-   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
-   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
-   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
-   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
-}
-
-
-void cross(const Vector a, const Vector b, Vector& c)
-{
-   assert(a.Size()==3);
-   c.SetSize(3);
-   c[0] = a[1]*b[2] - a[2]*b[1];
-   c[1] = -a[0]*b[2] + b[0]*a[2];
-   c[2] = a[0]*b[1] - a[1]*b[0];
-
-}
-// a outer b
-void outer(const Vector a, const Vector b, DenseMatrix& c)
-{
-   int m = a.Size();
-   int n = b.Size();
-   assert(c.Height()==m);
-   assert(c.Width() ==n);
-   for (int i=0; i<m; i++)
-   {
-      for (int j=0; j<n; j++)
-      {
-         c(i,j) = a[i]*b[j];
-      }
-   }
-}
-// dphidxi 2*4
-// coords 4*3
-void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
-                   Vector& normal, double& nnorm)
-{
-
-   DenseMatrix dxdxi(2,3);
-   Mult(dphidxi, coords, dxdxi);
-   Vector dxdxi1(3);
-   Vector dxdxi2(3);
-
-   dxdxi.GetRow(0,dxdxi1);
-   dxdxi.GetRow(1,dxdxi2);
-
-   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
-   // VectorCrossProductCoefficient::Eval has hard-coded cross product
-   nnorm = normal.Norml2( );
-   normal /=  nnorm;
-}
-
-void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
-{
-   bool converged = false;
-   bool pt_on_elem = false;
-   int dim = 3;
-   xi.SetSize(dim-1);
-   xi = 0.0;
-   double r = 1e10;
-   int max_iter = 15;
-   double off_el_xi = 1e-2;
-   double proj_newton_tol = 1e-13;
-   double proj_max_gap = 0.5;
-   Vector gap_v(dim);
-   // warm start from linear solution
-
-   for (int it=0; it<max_iter; it++)
-   {
-      //cout<<it<<endl;
-      Vector m_N(4);
-      m_N = 0.;
-      DenseMatrix m_dN(2,4);
-      m_dN = 0.;
-      DenseMatrix m_dN2(3,4);
-      m_dN2 = 0.;
-      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-      Vector x_c(dim);
-      m_coords.MultTranspose(m_N, x_c);
-
-      gap_v = s_x;
-      gap_v -= x_c;
-
-      DenseMatrix m_dx(2,3);
-      m_dx = 0.;
-      Mult(m_dN, m_coords, m_dx);
-
-      Vector r(dim-1);
-      r = 0.0;
-      m_dx.Mult(gap_v, r);
-
-      if (r.Normlinf() < proj_newton_tol)
-      {
-         converged = true;
-         break;
-      }
-
-      DenseMatrix drdxi(dim-1,dim-1);
-      drdxi = 0.;
-      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
-      drdxi *= -1.0;
-
-      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-      Mult(m_dN2,m_coords, m_dx2);
-
-      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
-      for (int d=0; d<3; d++)
-      {
-         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-         drdxi.Add(gap_v[d], Mtemp);
-      }
-
-      //cond_num = rcond(drdxi);  condition number?
-      //drdxi.TestInversion();
-      DenseMatrixInverse drdxi_inv(drdxi);
-      Vector xi_tmp(dim-1);
-
-      drdxi_inv.Mult(r,xi_tmp);
-      xi -= xi_tmp;
-   }
-   if (!converged)
-   {
-      xi = 0.0;
-   }
-   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
-
-   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
-   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
-   {
-      pt_on_elem = true;
-   }
-
-   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
-   MFEM_VERIFY(converged == true, "projection didn't converge");
-}
-
-
-
-// m_coords is expected to be 4 * 3
-void  ComputeGapJacobian(const Vector x_s, const Vector xi,
-                         const DenseMatrix m_coords,
-                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N, x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   double nnorm = 0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   gap = gap_v * normal; // gap function value, dot product between vectors
-
-   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
-
-   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
-   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
-
-   Vector m_dxrow1(3);
-   m_dx.GetRow(0, m_dxrow1);
-   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
-   dr_dx_res1 *= -1.0;
-
-   Vector m_dxrow2(3);
-   m_dx.GetRow(1, m_dxrow2);
-   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
-   dr_dx_res2 *= -1.0;
-
-   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
-   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
-
-   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
-   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
-   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
-   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
-
-   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
-   dr_dx_res2 += dr_dx_res2_tmp;
-
-
-   DenseMatrix K_dxidx1(2,2); // 2*2
-   K_dxidx1 = 0.;
-   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   // how to get 2nd order? multidimensional matrix?
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= K_dxidx1;
-   K_dxidx += K_dxidx2;
-
-   // resize the vectors and matrices
-   Vector dxidx(24); dxidx = 0.0;
-   Vector drdx_r(24); drdx_r = 0.0;
-
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         drdx_r[4*j+i] = dr_dx_res1(i,j);
-         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
-
-      }
-   }
-   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
-   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
-   DenseMatrix drdx_K(24,24); drdx_K = 0.;
-   for (int i =0; i<12; i++)
-   {
-      drdx_K(i,i) = K_dxidx(0,0);
-      drdx_K(i,12+i) = K_dxidx(0,1);
-      drdx_K(12+i,i) = K_dxidx(1,0);
-      drdx_K(12+i,12+i) = K_dxidx(1,1);
-   }
-
-   DenseMatrixInverse drdxK_inv(drdx_K);
-   drdxK_inv.Mult(drdx_r,dxidx);
-   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
-   dxidx *= -1.0;
-
-
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-
-   Vector dxidxs(6); dxidxs = 0.0;
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
-
-   dgdxm.SetSize(12); dgdxm = 0.;
-   DenseMatrix dgdxm_tmp(4,3);
-   outer(m_N, normal,dgdxm_tmp);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
-      }
-   }
-   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
-
-   dgdxs.SetSize(3);
-   dgdxs += normal;
-   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
-};
-
-void ComputeGapHessian(const Vector x_s, const Vector xi,
-                       const DenseMatrix m_coords,
-                       DenseMatrix& dg2dx)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   int dim = 3;
-   int num_dofs1 = dim;
-   int num_dofs2 = 4*dim;
-   int num_dofs = num_dofs1 + num_dofs2;
-   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N,x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-   Mult(m_dN2,m_coords, m_dx2);
-   double nnorm = 0.0;
-   Vector normal(3); normal = 0.0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   double gap = gap_v * normal; // gap function value, dot product between vectors
-
-   DenseMatrix M(2,2); M = 0.0;
-   MultABt(m_dx, m_dx, M);
-
-   DenseMatrix f(2, num_dofs2); f = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-      M.Add(-gap_v[d], Mtemp);
-
-      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
-      DenseMatrix ftmp(2,4);
-      outer(m_dxcol, m_N, ftmp);
-      ftmp *= -1;
-      ftmp.Add( gap_v[d], m_dN);   // 2*4
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         f(0,d+j*3) = ftmp(0,j);
-         f(1,d+j*3) = ftmp(1,j);
-      }
-   }
-   //fprintf('hess dxidxm\n');
-   DenseMatrixInverse Minv(M);
-   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
-   Minv.Mult(f, dxidxm);
-   //LinearSolve??
-   //dxidxm = M\f;
-
-   DenseMatrix nde2(2,2); nde2 = 0.0;
-   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
-      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
-      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
-
-      nde2 += ndetmp;
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
-         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
-      }
-
-   }
-
-   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
-   Ndn += Nndx2;
-   AddMult(nde2, dxidxm, Ndn);
-
-
-   DenseMatrix M2(2,2); M2 = 0.0;
-   MultABt(m_dx, m_dx, M2);
-   DenseMatrixInverse M2inv(M2);
-   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
-   DenseMatrix m_con(2,2); m_con = 0.0;
-
-   M2inv.Mult(diag2, m_con);
-
-   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
-
-   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
-   MultAtB(Ndn, m_con, dg2dxm_tmp);
-   Mult(dg2dxm_tmp, Ndn, dg2dxm);
-   dg2dxm *= gap;
-
-   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
-   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   dg2dxm_tmp = 0.0;
-   MultAtB(dxidxm, nde2, dg2dxm_tmp);
-
-   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
-
-   dg2dxm_tmp2 = 0.0;
-   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= M2;
-   K_dxidx += K_dxidx2;
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-   Vector dxidxs(6);
-
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
-   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
-   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
-
-   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
-   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
-
-   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
-   dxidxs_row2 = 0.0;
-   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
-   Vector mdx2_row3(3); mdx2_row3 = 0.0;
-   dxidxs_m.GetRow(0,dxidxs_row1);
-   dxidxs_m.GetRow(1,dxidxs_row2);
-   m_dx2.GetRow(0,mdx2_row1);
-   m_dx2.GetRow(1,mdx2_row2);
-   m_dx2.GetRow(2,mdx2_row3);
-
-   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
-   outer(mdx2_row1, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row2, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-
-   outer(mdx2_row2, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row3, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-
-   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
-
-   for (int d=0; d<3; d++)
-   {
-      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
-      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
-      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
-
-      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
-      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
-
-      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
-      m_dx.GetRow(0, m_dxrow);
-      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
-      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
-      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
-
-      dtaodxs_tmp2 += dtaodxs_tmp;
-      dtaodxs.SetCol(d, dtaodxs_tmp2);
-   }
-
-   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
-   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
-   outer(normal, normal, dndxs_tmp);
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
-
-   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
-   MultAtB(m_dx, dxidxs_m, dgvdxs);
-   dgvdxs *= -1;
-   for (int d=0; d<3; d++)
-   {
-      dgvdxs(d,d) += 1.0;
-   }
-   //dxidxs: 2*3
-
-   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
-   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
-   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
-   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
-   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
-   dg2dxs += dg2dxs_tmp2;
-   dg2dxs_tmp2 = 0.0;
-   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
-   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
-
-   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
-   BasisVectorDerivs(xi, Ne, Be, dBe);
-
-   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
-   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
-
-   Vector m_coords_v(12);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         m_coords_v[i*3+j] = m_coords(i,j);
-      }
-   }
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix dBe_tmp(3,12);
-      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
-
-      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
-
-      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
-      dBe_tmp = 0.0;
-      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
-
-   }
-
-   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
-
-   for (int d=0; d<12; d++)
-   {
-      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
-      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
-      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
-      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
-      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
-      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
-
-      cross(tmp1, m_dxrow2, dtaodxm_tmp);
-      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
-      dtaodxm_tmp += dtaodxm_tmp2;
-
-      dtaodxm.SetCol(d, dtaodxm_tmp);
-   }
-
-   DenseMatrix dndxm(3,12); dndxm = 0.0;
-   dndxm += dtaodxm;
-   dndxm *= 1.0/nnorm;
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
-
-   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
-   dgvdxm -= Ne;
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
-      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
-
-   }
-
-   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
-   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
-   MultAtB(dgvdxs, dndxm, dg2dxsxm);
-
-   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
-   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
-
-   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
-   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
-
-   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
-
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
-      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
-      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
-   }
-
-   dg2dxsxm += dgvdxsxmn;
-
-   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
-   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
-   MultAtB(dgvdxm, dndxs, dg2dxmxs);
-   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
-   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
-
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
-
-   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
-   dgvdxmxsn_tmp *= -1.0;
-   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-      Be_tmp.Transpose(); // Be is now 12*3
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
-      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
-      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
-
-   }
-
-   dg2dxmxs += dgvdxmxsn;
-
-   dg2dx.CopyMN(dg2dxs, 0, 0);
-   dg2dx.CopyMN(dg2dxm, 3, 3);
-   dg2dx.CopyMN(dg2dxsxm, 0, 3);
-   dg2dx.CopyMN(dg2dxmxs, 3, 0);
-
-};
-
-
-
-void NodeSegConPairs(const Vector x1, const Vector xi2,
-                     const DenseMatrix coords2,
-                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
-{
-   double gap = 0.0;
-   Vector normal(3); normal = 0.0;
-   Vector dgdxm(12); dgdxm = 0.0;
-   Vector dgdxs(3); dgdxs = 0.0;
-
-   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
-   node_g = gap;
-
-   node_dg.SetSize(12+3);
-   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
-   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
-
-   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   ComputeGapHessian(x1, xi2, coords2, dg2dx);
-
-   node_dg2.SetSize(15,15);
-   node_dg2 = dg2dx;
-
-   /*
-   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
-
-   v1 = 1:3;
-   v2 = 1:12;
-   %v1 = ones(1,3)
-   %v2 = ones(1,12)
-   v2 = reshape(v2,4,3);
-   x1n1 = x1 + 0.01*v1;
-   coords2n1 = coords2 + 0.001*v2;
-      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
-      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
-   x1n2 = x1 - 0.01*v1;
-   coords2n2 = coords2 - 0.001*v2;
-      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
-      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
-   fprintf('fd\n');
-      %gapv1-gapv2
-   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
-
-   %dgdxsn1-dgdxsn2
-   fprintf('code\n');
-   v2n = v2';
-   %dg2dx(1:3,1:3)*0.04*ones(3,1)
-   temp = zeros(12,3);
-   for i = 1:4
-   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
-   temp((i-1)*3+1:i*3,:) = temp1';
-   end
-   temp2 = zeros(3,12);
-   for i = 1:4
-   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
-   temp2(:,(i-1)*3+1:i*3) = temp3';
-   end
-   %dg2dx
-   %dg2dx(4:end,1:3) = temp;
-   %dg2dx(1:3,4:end) = temp2;
-   %dgvdxm * 0.002*v2n(:)
-   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
-   %dg2dx(4:end,1:3)
-   end*/
-
-};
-
-
-// coordsm : (npoints*4, 3) use what class?
-// m_conn: (npoints*4)
-void Assemble_Contact(const int m, const int npoints, const int ndofs,
-                      const Vector x_s,
-                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
-                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
-                      std::vector<SparseMatrix>& dM)
-{
-   int n = ndofs;
-   int ndim = 3;
-
-   g.SetSize(m);
-   g = 0.0;
-
-   //SparseMatrix M(m, n);  // M needs to be the correct size
-
-   //dM.resize(m); // needs to clear?
-
-   double g_tmp = 0.;
-   Vector dg(4*ndim+ndim);
-   dg = 0.;
-   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
-   dg2 = 0.;
-
-   for (int i=0; i<npoints; i++)
-   {
-      Vector x1(ndim);
-      x1[0] = x_s[i*ndim];
-      x1[1] = x_s[i*ndim+1];
-      x1[2] = x_s[i*ndim+2];
-
-      Vector xi2(ndim-1);
-      xi2[0] = xi[i*(ndim-1)];
-      xi2[1] = xi[i*(ndim-1)+1];
-
-      DenseMatrix coords2(4,3);
-      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
-
-      //how to get coords2?
-      dg = 0.0;
-      dg2 = 0.;
-      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
-      //x1.Print();
-      //xi2.Print();
-      //coords2.Print();
-      g[s_conn[i]] = g_tmp; // should be unique
-      Array<int> m_conn_i(4);
-      m_conn.GetSubArray(4*i, 4, m_conn_i);
-
-      Array<int> node_conn(5);
-      node_conn[0] = s_conn[i];
-      for (int j=0; j<4; j++)
-      {
-         node_conn[j+1] = m_conn_i[j];
-      }
-
-      Array<int>  M_i_tmp(1);
-      M_i_tmp[0] = s_conn[i];
-
-      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
-      Array<int> j_idx(5*ndim); j_idx = 0;
-      for (int j=0; j< 5; j++)
-      {
-         for (int k=0; k<ndim; k++)
-         {
-            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
-         }
-      }
-      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
-      M_v_tmp.SetRow(0, dg);
-
-      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
-
-      Array<int> dM_i(ndim*(4+1));
-      Array<int> dM_j(ndim*(4+1));
-
-      for (int j=0; j< ndim*(4+1); j++)
-      {
-         dM_i[j] = j_idx[j];
-         dM_j[j] = j_idx[j];
-      }
-      //dg2.Print();
-      //dM[s_conn[i]].Print();
-      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
-   }
-};
-
@@ -14,5 +14,4 @@
 -mg_levels_esteig_ksp_type cg
 -mg_levels_esteig_ksp_max_it 10
 -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05
-pc_gamg_use_sa_esteig 0
 -mg_levels_pc_type sor
@@ -124,7 +124,6 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
      case AssemblyLevel::LEGACY:
         break;
      case AssemblyLevel::FULL:
-         SetDiagonalPolicy( DIAG_ONE ); // Only diagonal policy supported on device
         ext = new FABilinearFormExtension(this);
         break;
      case AssemblyLevel::ELEMENT:
@@ -80,9 +80,6 @@ protected:
   /** @brief Extension for supporting Full Assembly (FA), Element Assembly (EA),
       Partial Assembly (PA), or Matrix Free assembly (MF). */
   BilinearFormExtension *ext;
-   /** Indicates if the sparse matrix is sorted after assembly when using
-       Full Assembly (FA). */
-   bool sort_sparse_matrix = false;

   /** @brief Indicates the Mesh::sequence corresponding to the current state of
       the BilinearForm. */
@@ -184,21 +181,6 @@ public:
       If used, this method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

-   /** @brief Force the sparse matrix column indices to be sorted when using
-       AssemblyLevel::FULL.
-
-       When assembling on device the assembly algorithm uses atomic operations
-       to insert values in the sparse matrix, which can result in different
-       column index orderings across runs. Calling this method with @a enable_it
-       set to @a true forces a sorting algorithm to be called at the end of the
-       assembly procedure to ensure sorted column indices (and therefore
-       deterministic results).
-   */
-   void EnableSparseMatrixSorting(bool enable_it)
-   {
-      sort_sparse_matrix = enable_it;
-   }
-
   /// Returns the assembly level
   AssemblyLevel GetAssemblyLevel() const { return assembly; }

@@ -18,6 +18,8 @@
 #include "pgridfunc.hpp"
 #include "ceed/interface/util.hpp"

+#include "../general/nvtx.hpp"
+
 namespace mfem
 {

@@ -289,6 +291,10 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)

 void PABilinearFormExtension::Assemble()
 {
+#undef MFEM_NVTX_COLOR
+#define MFEM_NVTX_COLOR NavyBlue
+   NVTX("HO Assemble");
+
   SetupRestrictionOperators(L2FaceValues::DoubleValued);

   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
@@ -383,6 +389,10 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,

 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
+#undef MFEM_NVTX_COLOR
+#define MFEM_NVTX_COLOR MediumSpringGreen
+   NVTX("HO Apply");
+
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();

   const int iSz = integrators.Size();
@@ -529,7 +539,7 @@ void EABilinearFormExtension::Assemble()
   }

   faceDofs = trial_fes ->
-              GetTraceElement(0, trial_fes->GetMesh()->GetFaceGeometry(0)) ->
+              GetTraceElement(0, trial_fes->GetMesh()->GetFaceBaseGeometry(0)) ->
              GetDof();

   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
@@ -955,10 +965,6 @@ void FABilinearFormExtension::Assemble()
      }
      a->mat = mat;
   }
-   if ( a->sort_sparse_matrix )
-   {
-      a->mat->SortColumnIndices();
-   }
 }


@@ -208,6 +208,10 @@ public:
                                            Vector &x, Vector &b,
                                            OperatorHandle &A, Vector &X, Vector &B) = 0;

+   virtual void AddMult(const Vector &x, Vector &y, const double c=1.0) const = 0;
+   virtual void AddMultTranspose(const Vector &x, Vector &y,
+                                 const double c=1.0) const = 0;
+
   virtual void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const = 0;

   virtual void Update() = 0;
@@ -283,7 +287,7 @@ public:
   /// Partial assembly of all internal integrators
   void Assemble();

-   void AddMult(const Vector &x, Vector &y, const double c=1.0) const;
+   void AddMult(const Vector &x, Vector &y, const double c) const;

   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;

@@ -2338,7 +2338,7 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
   if (spaceH1)
   {
      dshape.SetSize(trial_dof,dim);
-      curlshape.SetSize(trial_dof,dim);
+      curlshape.SetSize(dim*trial_dof,1);
      dimc = dim;
   }
   else
@@ -2367,7 +2367,7 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
      if (spaceH1)
      {
         trial_fe.CalcPhysDShape(Trans, dshape);
-         dshape.GradToVectorCurl2D(curlshape);
+         dshape.GradToCurl(curlshape);
      }
      else
      {
@@ -3059,8 +3059,8 @@ public:

 /** Integrator for the DG form:

-        - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
-        + kappa < {h^{-1} Q} [u], [v] >
+    - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
+    + kappa < {h^{-1} Q} [u], [v] >,

    where Q is a scalar or matrix diffusion coefficient and u, v are the trial
    and test spaces, respectively. The parameters sigma and kappa determine the
@@ -145,7 +145,7 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   // Assumes tensor-product elements
   Mesh *mesh = fes.GetMesh();
   const FiniteElement &el =
-      *fes.GetTraceElement(0, fes.GetMesh()->GetFaceGeometry(0));
+      *fes.GetTraceElement(0, fes.GetMesh()->GetFaceBaseGeometry(0));
   FaceElementTransformations &T0 =
      *fes.GetMesh()->GetFaceElementTransformations(0);
   const IntegrationRule *ir = IntRule?
@@ -1686,7 +1686,7 @@ static void PADiffusionApply(const int dim,
         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+            // default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }

@@ -1704,7 +1704,14 @@ static void PADiffusionApply(const int dim,
         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x33: return SmemPADiffusionApply3D<3,3>(NE,symm,B,G,D,X,Y);
+         case 0x44: return SmemPADiffusionApply3D<4,4>(NE,symm,B,G,D,X,Y);
+         case 0x55: return SmemPADiffusionApply3D<5,5>(NE,symm,B,G,D,X,Y);
+         case 0x66: return SmemPADiffusionApply3D<6,6>(NE,symm,B,G,D,X,Y);
+         case 0x77: return SmemPADiffusionApply3D<7,7>(NE,symm,B,G,D,X,Y);
+         case 0x88: return SmemPADiffusionApply3D<8,8>(NE,symm,B,G,D,X,Y);
+         case 0x99: return SmemPADiffusionApply3D<9,9>(NE,symm,B,G,D,X,Y);
+            // default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
   MFEM_ABORT("Unknown kernel: 0x"<<std::hex << id << std::dec);
@@ -56,7 +56,8 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
+                                    GeometricFactors::JACOBIANS, mt);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
@@ -73,7 +74,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
      const bool const_c = coeff.Size() == 1;
      const bool by_val = map_type == FiniteElement::VALUE;
      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
-      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,NE);
+      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
                     Reshape(coeff.Read(), Q1D,Q1D,NE);
      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
@@ -83,7 +84,11 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
         {
            MFEM_FOREACH_THREAD(qy,y,Q1D)
            {
-               const double detJ = J(qx,qy,e);
+               const double J11 = J(qx,qy,0,0,e);
+               const double J12 = J(qx,qy,1,0,e);
+               const double J21 = J(qx,qy,0,1,e);
+               const double J22 = J(qx,qy,1,1,e);
+               const double detJ = (J11*J22)-(J21*J12);
               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
            }
@@ -97,7 +102,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
      const bool const_c = coeff.Size() == 1;
      const bool by_val = map_type == FiniteElement::VALUE;
      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
-      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,Q1D,NE);
+      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
@@ -109,7 +114,18 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
            {
               MFEM_FOREACH_THREAD(qz,z,Q1D)
               {
-                  const double detJ = J(qx,qy,qz,e);
+                  const double J11 = J(qx,qy,qz,0,0,e);
+                  const double J21 = J(qx,qy,qz,1,0,e);
+                  const double J31 = J(qx,qy,qz,2,0,e);
+                  const double J12 = J(qx,qy,qz,0,1,e);
+                  const double J22 = J(qx,qy,qz,1,1,e);
+                  const double J32 = J(qx,qy,qz,2,1,e);
+                  const double J13 = J(qx,qy,qz,0,2,e);
+                  const double J23 = J(qx,qy,qz,1,2,e);
+                  const double J33 = J(qx,qy,qz,2,2,e);
+                  const double detJ = J11 * (J22 * J33 - J32 * J23) -
+                  /* */               J21 * (J12 * J33 - J32 * J13) +
+                  /* */               J31 * (J12 * J23 - J22 * J13);
                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
               }
@@ -75,7 +75,7 @@ void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
      Vector ea_data_ext_tmp(ea_data_ext.Size());
      bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
      auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
      auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
      auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
@@ -102,7 +102,7 @@ void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
   {
      bfi->AssembleEAInteriorFaces(fes, ea_data_int, ea_data_ext, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
      auto A_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
      auto A_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
      MFEM_FORALL(f, nf,
@@ -146,7 +146,7 @@ void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
      Vector ea_data_bdr_tmp(ea_data_bdr.Size());
      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
      auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
      auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
      MFEM_FORALL(f, nf,
@@ -165,7 +165,7 @@ void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
   {
      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr, false);
      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceGeometry(0))->GetDof();
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
      auto A_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
      MFEM_FORALL(f, nf,
      {
@@ -69,11 +69,9 @@ void Operator::Mult(const mfem::Vector &x, mfem::Vector &y) const
 #endif
 }

-void Operator::AddMult(const mfem::Vector &x, mfem::Vector &y,
-                       const double a) const
+void Operator::AddMult(const mfem::Vector &x, mfem::Vector &y) const
 {
 #ifdef MFEM_USE_CEED
-   MFEM_VERIFY(a == 1.0, "General coefficient case is not yet supported!");
   const CeedScalar *x_ptr;
   CeedScalar *y_ptr;
   CeedMemType mem;
@@ -38,8 +38,7 @@ public:
   Operator(CeedOperator op);
 #endif
   void Mult(const mfem::Vector &x, mfem::Vector &y) const override;
-   void AddMult(const mfem::Vector &x, mfem::Vector &y,
-                const double a = 1.0) const override;
+   void AddMult(const mfem::Vector &x, mfem::Vector &y) const;
   void GetDiagonal(mfem::Vector &diag) const;
   using mfem::Operator::SetupRAP;
   virtual ~Operator()
@@ -47,7 +47,6 @@ void InitRestrictionWithIndices(const FiniteElementSpace &fes,

 /** @brief Initialize a strided CeedElemRestriction

-    @param[in] fes Input finite element space.
    @param[in] nelem is the number of elements.
    @param[in] nqpts is the total number of quadrature points.
    @param[in] qdatasize is the number of data per quadrature point.
@@ -140,11 +140,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   CeedOperator *subops;
   if (isComposite)
   {
-#if CEED_VERSION_GE(0, 10, 2)
-      ierr = CeedCompositeOperatorGetSubList(oper, &subops); CeedChk(ierr);
-#else
      ierr = CeedOperatorGetSubList(oper, &subops); CeedChk(ierr);
-#endif
      ierr = CeedOperatorGetQFunction(subops[0], &qf); CeedChk(ierr);
   }
   else
@@ -275,14 +275,9 @@ CeedOperator CoarsenCeedCompositeOperator(
                                      &op_coarse); PCeedChk(ierr);

   int nsub;
-   CeedOperator *subops;
-#if CEED_VERSION_GE(0, 10, 2)
-   ierr = CeedCompositeOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
-   ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
-#else
   ierr = CeedOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
+   CeedOperator *subops;
   ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
-#endif
   for (int isub=0; isub<nsub; ++isub)
   {
      CeedOperator subop = subops[isub];
@@ -310,13 +310,8 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
   {
      CeedInt numsub;
      CeedOperator *subops;
-#if CEED_VERSION_GE(0, 10, 2)
-      CeedCompositeOperatorGetNumSub(op, &numsub);
-      ierr = CeedCompositeOperatorGetSubList(op, &subops); CeedChk(ierr);
-#else
      CeedOperatorGetNumSub(op, &numsub);
      ierr = CeedOperatorGetSubList(op, &subops); CeedChk(ierr);
-#endif
      for (int i = 0; i < numsub; ++i)
      {
         ierr = CeedSingleOperatorFullAssemble(subops[i], out); CeedChk(ierr);
@@ -66,7 +66,6 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin, CeedBasis* basisout,
    @param[in] coarse_er         CeedElemRestriction for coarse operator
                                 (see CeedATPMGElemRestriction)
    @param[out] coarse_basis_out CeedBasis for coarser operator
-    @param[out] basis_ctof_out   CeedBasis describing interpolation from coarse to fine
    @param[out] out              coarsened CeedOperator
 */
 int CeedATPMGOperator(CeedOperator oper, int order_reduction,
@@ -1319,6 +1319,7 @@ public:
   virtual void Eval(DenseSymmetricMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip) = 0;

+   using MatrixCoefficient::Eval;
   /** @brief Evaluate the matrix coefficient in the element described by @a T
       at the point @a ip, storing the result as a dense matrix @a K. */
   /** This function allows the use of SymmetricMatrixCoefficient in situations
@@ -1347,6 +1348,7 @@ public:
   ///Construct using matrix @a m for the constant.
   SymmetricMatrixConstantCoefficient(const DenseSymmetricMatrix &m)
      : SymmetricMatrixCoefficient(m.Height()), mat(m) { }
+   using MatrixCoefficient::Eval;
   using SymmetricMatrixCoefficient::Eval;
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
@@ -1398,6 +1400,7 @@ public:
   /// Set the time for internally stored coefficients
   void SetTime(double t);

+   using MatrixCoefficient::Eval;
   using SymmetricMatrixCoefficient::Eval;
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &K, ElementTransformation &T,
@@ -110,8 +110,8 @@ DataCollection::DataCollection(const std::string& collection_name, Mesh *mesh_)
   precision = precision_default;
   pad_digits_cycle = pad_digits_rank = pad_digits_default;
   format = SERIAL_FORMAT; // use serial mesh format
-   compression = 0;
-   error = No_Error;
+   compression = false;
+   error = NO_ERROR;
 }

 void DataCollection::SetMesh(Mesh *new_mesh)
@@ -494,7 +494,7 @@ void VisItDataCollection::Load(int cycle_)
 {
   DeleteAll();
   time_step = 0.0;
-   error = No_Error;
+   error = NO_ERROR;
   cycle = cycle_;
   std::string root_name = prefix_path + name + "_" +
                           to_padded_string(cycle, pad_digits_cycle) +
@@ -724,7 +724,7 @@ void VisItDataCollection::ParseVisItRootString(const std::string& json)

   // Set the DataCollection::name using the mesh path
   std::string path = mesh.get("path").get<std::string>();
-   size_t right_sep = path.rfind('_');
+   size_t right_sep = path.find('_');
   if (right_sep == std::string::npos)
   {
      error = READ_ERROR;
@@ -767,13 +767,10 @@ ParaViewDataCollection::ParaViewDataCollection(const std::string&
     high_order_output(false),
     restart_mode(false)
 {
-   cycle = 0; // always include a valid cycle index in file names
-
-   compression_level = -1;  // default zlib compression level, equivalent to 6
 #ifdef MFEM_USE_ZLIB
-   compression = true; // if we have zlib, enable compression
+   compression = -1; // default zlib compression level, equivalent to 6
 #else
-   compression = false; // otherwise, disable compression
+   compression = 0;
 #endif
 }

@@ -922,7 +919,7 @@ void ParaViewDataCollection::Save()
   {
      const std::string &field_name = qfield.first;
      std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
-      qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel());
+      qfield.second->SaveVTU(os, pv_data_format, compression);
   }

   // MPI rank 0 also creates a "PVTU" file that points to all of the separately
@@ -1036,13 +1033,13 @@ void ParaViewDataCollection::WritePVTUFooter(std::ostream &os,
 void ParaViewDataCollection::SaveDataVTU(std::ostream &os, int ref)
 {
   os << "<VTKFile type=\"UnstructuredGrid\"";
-   if (GetCompressionLevel() != 0)
+   if (compression != 0)
   {
      os << " compressor=\"vtkZLibDataCompressor\"";
   }
   os << " version=\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
   os << "<UnstructuredGrid>\n";
-   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel());
+   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,compression);

   // dump out the grid functions as point data
   os << "<PointData >\n";
@@ -1106,7 +1103,7 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,

   if (IsBinaryFormat())
   {
-      WriteVTKEncodedCompressed(os,buf.data(),buf.size(),GetCompressionLevel());
+      WriteVTKEncodedCompressed(os,buf.data(),buf.size(),compression);
      os << '\n';
   }
   os << "</DataArray>" << std::endl;
@@ -1131,13 +1128,18 @@ void ParaViewDataCollection::SetCompressionLevel(int compression_level_)
 {
   MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9,
               "Compression level must be between -1 and 9 (inclusive).");
-   compression_level = compression_level_;
-   compression = compression_level_ != 0;
+   compression = compression_level_;
 }

 void ParaViewDataCollection::SetCompression(bool compression_)
 {
-   compression = compression_;
+   // If we are enabling compression, and it was disabled previously, use the
+   // default compression level. Otherwise, leave the compression level
+   // unchanged.
+   if (compression_ && compression == 0)
+   {
+      SetCompressionLevel(-1);
+   }
 }

 void ParaViewDataCollection::UseRestartMode(bool restart_mode_)
@@ -1169,9 +1171,4 @@ const char *ParaViewDataCollection::GetDataTypeString() const
   }
 }

-int ParaViewDataCollection::GetCompressionLevel() const
-{
-   return compression ? compression_level : 0;
-}
-
 }  // end namespace MFEM
@@ -338,12 +338,11 @@ public:
   /// Set the precision (number of digits) used for the text output of doubles
   void SetPrecision(int prec) { precision = prec; }
   /// Set the number of digits used for both the cycle and the MPI rank
-   virtual void SetPadDigits(int digits)
-   { pad_digits_cycle=pad_digits_rank = digits; }
+   void SetPadDigits(int digits) { pad_digits_cycle=pad_digits_rank = digits; }
   /// Set the number of digits used for the cycle
-   virtual void SetPadDigitsCycle(int digits) { pad_digits_cycle = digits; }
+   void SetPadDigitsCycle(int digits) { pad_digits_cycle = digits; }
   /// Set the number of digits used for the MPI rank in filenames
-   virtual void SetPadDigitsRank(int digits) { pad_digits_rank = digits; }
+   void SetPadDigitsRank(int digits) { pad_digits_rank = digits; }
   /// Set the desired output mesh and data format.
   /** See the enumeration #Format for valid options. Derived classes can define
       their own format enumerations and override this method to perform input
@@ -378,24 +377,12 @@ public:
   virtual ~DataCollection();

   /// Errors returned by Error()
-   enum
-   {
-      // Workaround for use with headers that define NO_ERROR as a macro,
-      // e.g. winerror.h (which is included by Windows.h):
-#ifndef NO_ERROR
-      NO_ERROR = 0,
-#endif
-      // Use the following identifier if NO_ERROR is defined as a macro,
-      // e.g. winerror.h (which is included by Windows.h):
-      No_Error    = 0,
-      READ_ERROR  = 1,
-      WRITE_ERROR = 2
-   };
+   enum { NO_ERROR = 0, READ_ERROR = 1, WRITE_ERROR = 2 };

   /// Get the current error state
   int Error() const { return error; }
   /// Reset the error state
-   void ResetError(int err_state = No_Error) { error = err_state; }
+   void ResetError(int err_state = NO_ERROR) { error = err_state; }

 #ifdef MFEM_USE_MPI
   friend class ParMesh;
@@ -454,29 +441,21 @@ public:
 #endif

   /// Set/change the mesh associated with the collection
-   virtual void SetMesh(Mesh *new_mesh) override;
+   virtual void SetMesh(Mesh *new_mesh);

 #ifdef MFEM_USE_MPI
   /// Set/change the mesh associated with the collection.
-   virtual void SetMesh(MPI_Comm comm, Mesh *new_mesh) override;
+   virtual void SetMesh(MPI_Comm comm, Mesh *new_mesh);
 #endif

   /// Add a grid function to the collection and update the root file
-   virtual void RegisterField(const std::string& field_name,
-                              GridFunction *gf) override;
+   virtual void RegisterField(const std::string& field_name, GridFunction *gf);

   /// Add a quadrature function to the collection and update the root file.
   /** Visualization of quadrature function is not supported in VisIt(3.12).
       A patch has been sent to VisIt developers in June 2020. */
   virtual void RegisterQField(const std::string& q_field_name,
-                               QuadratureFunction *qf) override;
-
-   /// Set the number of digits used for both the cycle and the MPI rank
-   /// @note VisIt seems to require 6 pad digits for the MPI rank. Therefore,
-   /// this function uses this default value. This behavior can be overridden
-   /// by calling SetPadDigitsCycle() and SetPadDigitsRank() instead.
-   virtual void SetPadDigits(int digits) override
-   { pad_digits_cycle=digits; pad_digits_rank=6; }
+                               QuadratureFunction *qf);

   /// Set VisIt parameter: default levels of detail for the MultiresControl
   void SetLevelsOfDetail(int levels_of_detail);
@@ -489,13 +468,13 @@ public:
   void DeleteAll();

   /// Save the collection and a VisIt root file
-   virtual void Save() override;
+   virtual void Save();

   /// Save a VisIt root file for the collection
   void SaveRootFile();

   /// Load the collection based on its VisIt data (described in its root file)
-   virtual void Load(int cycle_ = 0) override;
+   virtual void Load(int cycle_ = 0);

   /// We will delete the mesh and fields if we own them
   virtual ~VisItDataCollection() {}
@@ -507,7 +486,6 @@ class ParaViewDataCollection : public DataCollection
 {
 private:
   int levels_of_detail;
-   int compression_level;
   std::fstream pvd_stream;
   VTKFormat pv_data_format;
   bool high_order_output;
@@ -520,9 +498,6 @@ protected:
   void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
   const char *GetDataFormatString() const;
   const char *GetDataTypeString() const;
-   /// @brief If compression is enabled, return the compression level, otherwise
-   /// return 0.
-   int GetCompressionLevel() const;

   std::string GenerateCollectionPath();
   std::string GenerateVTUFileName(const std::string &prefix, int rank);
@@ -541,7 +516,7 @@ public:
                          mfem::Mesh *mesh_ = NULL);

   /// Set refinement levels - every element is uniformly split based on
-   /// levels_of_detail_. The initial value is 1.
+   /// levels_of_detail_
   void SetLevelsOfDetail(int levels_of_detail_);

   /// Save the collection - the directory name is constructed based on the
@@ -552,27 +527,18 @@ public:
   /// VTKFormat::ASCII, VTKFormat::BINARY, and VTKFormat::BINARY32.
   /// The ASCII and BINARY options output double precision data, whereas the
   /// BINARY32 option outputs single precision data.
-   ///
-   /// The initial format is VTKFormat::BINARY.
   void SetDataFormat(VTKFormat fmt);

-   /// @brief Set the zlib compression level.
-   ///
-   /// 0 indicates no compression, -1 indicates the default compression level.
-   /// Otherwise, specify a number between 1 and 9, 1 being the fastest, and 9
-   /// being the best compression. Compression only takes effect if the output
-   /// format is BINARY or BINARY32. MFEM must be compiled with MFEM_USE_ZLIB =
-   /// YES.
-   ///
-   /// The initial compression level is 0 if MFEM is compiled with MFEM_USE_ZLIB
-   /// turned off, and -1 otherwise.
-   ///
-   /// Any nonzero compression level will enable compression.
+   /// Set the zlib compression level. 0 indicates no compression, -1 indicates
+   /// the default compression level. Otherwise, specify a number between 1 and
+   /// 9, 1 being the fastest, and 9 being the best compression. Compression
+   /// only takes effect if the output format is BINARY or BINARY32. MFEM must
+   /// be compiled with MFEM_USE_ZLIB = YES.
   void SetCompressionLevel(int compression_level_);

   /// Enable or disable zlib compression. If the input is true, use the default
   /// zlib compression level (unless the compression level has previously been
-   /// set by calling SetCompressionLevel()).
+   /// set by calling SetCompressionLevel).
   void SetCompression(bool compression_) override;

   /// Returns true if the output format is BINARY or BINARY32, false if ASCII.
@@ -585,8 +551,6 @@ public:
   /// Enable or disable restart mode. If restart is enabled, new writes will
   /// preserve timestep metadata for any solutions prior to the currently
   /// defined time.
-   ///
-   /// Initially, restart mode is disabled.
   void UseRestartMode(bool restart_mode_);

   /// Load the collection - not implemented in the ParaView writer
@@ -339,11 +339,8 @@ ND_TriDofTransformation::TransformDual(double *v) const
 void
 ND_TriDofTransformation::InvTransformDual(double *v) const
 {
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face

   double data[2];
   Vector v2(data, 2);
@@ -435,11 +432,8 @@ ND_TetDofTransformation::TransformDual(double *v) const
 void
 ND_TetDofTransformation::InvTransformDual(double *v) const
 {
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face

   double data[2];
   Vector v2(data, 2);
@@ -36,19 +36,19 @@ FiniteElement::FiniteElement(int D, Geometry::Type G,
 #endif
 }

-void FiniteElement::CalcVShape(
+void FiniteElement::CalcVShape (
   const IntegrationPoint &ip, DenseMatrix &shape) const
 {
   MFEM_ABORT("method is not implemented for this class");
 }

-void FiniteElement::CalcVShape(
+void FiniteElement::CalcVShape (
   ElementTransformation &Trans, DenseMatrix &shape) const
 {
   MFEM_ABORT("method is not implemented for this class");
 }

-void FiniteElement::CalcDivShape(
+void FiniteElement::CalcDivShape (
   const IntegrationPoint &ip, Vector &divshape) const
 {
   MFEM_ABORT("method is not implemented for this class");
@@ -97,14 +97,14 @@ void FiniteElement::GetFaceDofs(int face, int **dofs, int *ndofs) const
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::CalcHessian(const IntegrationPoint &ip,
-                                DenseMatrix &h) const
+void FiniteElement::CalcHessian (const IntegrationPoint &ip,
+                                 DenseMatrix &h) const
 {
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::GetLocalInterpolation(ElementTransformation &Trans,
-                                          DenseMatrix &I) const
+void FiniteElement::GetLocalInterpolation (ElementTransformation &Trans,
+                                           DenseMatrix &I) const
 {
   MFEM_ABORT("method is not overloaded");
 }
@@ -122,13 +122,13 @@ void FiniteElement::GetTransferMatrix(const FiniteElement &fe,
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::Project(
+void FiniteElement::Project (
   Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {
   MFEM_ABORT("method is not overloaded");
 }

-void FiniteElement::Project(
+void FiniteElement::Project (
   VectorCoefficient &vc, ElementTransformation &Trans, Vector &dofs) const
 {
   MFEM_ABORT("method is not overloaded");
@@ -137,7 +137,7 @@ void FiniteElement::Project(
 void FiniteElement::ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
                                     Vector &dofs) const
 {
-   mfem_error("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
+   mfem_error ("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
 }

 void FiniteElement::ProjectMatrixCoefficient(
@@ -239,6 +239,7 @@ void FiniteElement::CalcPhysLaplacian(ElementTransformation &Trans,
   }
 }

+
 // Assume a linear mapping
 void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
                                         Vector &Laplacian) const
@@ -249,7 +250,7 @@ void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
   DenseMatrix Gij(dim,dim);
   Vector scale(size);

-   CalcHessian(Trans.GetIntPoint(), hess);
+   CalcHessian (Trans.GetIntPoint(), hess);
   MultAAt(Trans.InverseJacobian(), Gij);

   if (dim == 3)
@@ -282,6 +283,7 @@ void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
         Laplacian[nd] += hess(nd,ii)*scale[ii];
      }
   }
+
 }

 void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
@@ -361,128 +363,11 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
   Mult( hess, lhm, Hessian);
 }

-const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
-                                             DofToQuad::Mode mode) const
+const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &,
+                                             DofToQuad::Mode) const
 {
-   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-#ifdef MFEM_THREAD_SAFE
-   DenseMatrix vshape(dof, dim);
-#endif
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = dof;
-   d2q->nqpt = nqpt;
-   if (range_type == SCALAR)
-   {
-      d2q->B.SetSize(nqpt*dof);
-      d2q->Bt.SetSize(dof*nqpt);
-
-      Vector shape;
-      vshape.GetColumnReference(0, shape);
-      for (int i = 0; i < nqpt; i++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcShape(ip, shape);
-         for (int j = 0; j < dof; j++)
-         {
-            d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
-         }
-      }
-   }
-   else
-   {
-      d2q->B.SetSize(nqpt*dim*dof);
-      d2q->Bt.SetSize(dof*nqpt*dim);
-
-      for (int i = 0; i < nqpt; i++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcVShape(ip, vshape);
-         for (int d = 0; d < dim; d++)
-         {
-            for (int j = 0; j < dof; j++)
-            {
-               d2q->B[i+nqpt*(d+dim*j)] = d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
-            }
-         }
-      }
-   }
-   switch (deriv_type)
-   {
-      case GRAD:
-      {
-         d2q->G.SetSize(nqpt*dim*dof);
-         d2q->Gt.SetSize(dof*nqpt*dim);
-
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDShape(ip, vshape);
-            for (int d = 0; d < dim; d++)
-            {
-               for (int j = 0; j < dof; j++)
-               {
-                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
-               }
-            }
-         }
-         break;
-      }
-      case DIV:
-      {
-         d2q->G.SetSize(nqpt*dof);
-         d2q->Gt.SetSize(dof*nqpt);
-
-         Vector divshape;
-         vshape.GetColumnReference(0, divshape);
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDivShape(ip, divshape);
-            for (int j = 0; j < dof; j++)
-            {
-               d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
-            }
-         }
-         break;
-      }
-      case CURL:
-      {
-         d2q->G.SetSize(nqpt*cdim*dof);
-         d2q->Gt.SetSize(dof*nqpt*cdim);
-
-         DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcCurlShape(ip, curlshape);
-            for (int d = 0; d < cdim; d++)
-            {
-               for (int j = 0; j < dof; j++)
-               {
-                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
-               }
-            }
-         }
-         break;
-      }
-      case NONE:
-      default:
-         MFEM_ABORT("invalid finite element derivative type");
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
+   MFEM_ABORT("method is not implemented for this element");
+   return *dof2quad_array[0]; // suppress a warning
 }

 FiniteElement::~FiniteElement()
@@ -494,19 +379,16 @@ FiniteElement::~FiniteElement()
 }


-void ScalarFiniteElement::NodalLocalInterpolation(
+void ScalarFiniteElement::NodalLocalInterpolation (
   ElementTransformation &Trans, DenseMatrix &I,
   const ScalarFiniteElement &fine_fe) const
 {
   double v[Geometry::MaxDim];
-   Vector vv(v, dim);
+   Vector vv (v, dim);
   IntegrationPoint f_ip;

 #ifdef MFEM_THREAD_SAFE
-   Vector shape(dof);
-#else
-   Vector shape;
-   vshape.GetColumnReference(0, shape);
+   Vector c_shape(dof);
 #endif

   MFEM_ASSERT(map_type == fine_fe.GetMapType(), "");
@@ -516,10 +398,10 @@ void ScalarFiniteElement::NodalLocalInterpolation(
   {
      Trans.Transform(fine_fe.Nodes.IntPoint(i), vv);
      f_ip.Set(v, dim);
-      CalcShape(f_ip, shape);
+      CalcShape(f_ip, c_shape);
      for (int j = 0; j < dof; j++)
      {
-         if (fabs(I(i,j) = shape(j)) < 1.0e-12)
+         if (fabs(I(i,j) = c_shape(j)) < 1.0e-12)
         {
            I(i,j) = 0.0;
         }
@@ -540,7 +422,7 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
   // General "interpolation", defined by L2 projection

   double v[Geometry::MaxDim];
-   Vector vv(v, dim);
+   Vector vv (v, dim);
   IntegrationPoint f_ip;

   const int fs = fine_fe.GetDof(), cs = this->GetDof();
@@ -574,13 +456,14 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
   }
 }

-void ScalarFiniteElement::ScalarLocalL2Restriction(
+void ScalarFiniteElement::ScalarLocalRestriction(
   ElementTransformation &Trans, DenseMatrix &R,
   const ScalarFiniteElement &coarse_fe) const
 {
   // General "restriction", defined by L2 projection
   double v[Geometry::MaxDim];
-   Vector vv(v, dim);
+   Vector vv (v, dim);
+   IntegrationPoint f_ip;

   const int cs = coarse_fe.GetDof(), fs = this->GetDof();
   R.SetSize(cs, fs);
@@ -589,27 +472,16 @@ void ScalarFiniteElement::ScalarLocalL2Restriction(
   const int ir_order = GetOrder() + coarse_fe.GetOrder();
   const IntegrationRule &ir = IntRules.Get(coarse_fe.GetGeomType(), ir_order);

-   // integrate coarse_mass in the coarse space
   for (int i = 0; i < ir.GetNPoints(); i++)
   {
-      const IntegrationPoint &c_ip = ir.IntPoint(i);
-      coarse_fe.CalcShape(c_ip, coarse_shape);
-      AddMult_a_VVt(c_ip.weight, coarse_shape, coarse_mass);
-   }
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      this->CalcShape(ip, fine_shape);
+      Trans.Transform(ip, vv);
+      f_ip.Set(v, dim);
+      coarse_fe.CalcShape(f_ip, coarse_shape);

-   // integrate coarse_fine_mass in the fine space
-   Trans.SetIntPoint(&Geometries.GetCenter(geom_type));
-   for (int i = 0; i < ir.GetNPoints(); i++)
-   {
-      const IntegrationPoint &f_ip = ir.IntPoint(i);
-      this->CalcShape(f_ip, fine_shape);
-      Trans.Transform(f_ip, vv);
-
-      IntegrationPoint c_ip;
-      c_ip.Set(v, dim);
-      coarse_fe.CalcShape(c_ip, coarse_shape);
-      AddMult_a_VWt(f_ip.weight*Trans.Weight(), coarse_shape, fine_shape,
-                    coarse_fine_mass);
+      AddMult_a_VVt(ip.weight, coarse_shape, coarse_mass);
+      AddMult_a_VWt(ip.weight, coarse_shape, fine_shape, coarse_fine_mass);
   }

   DenseMatrixInverse coarse_mass_inv(coarse_mass);
@@ -622,6 +494,95 @@ void ScalarFiniteElement::ScalarLocalL2Restriction(
      R *= 1.0 / Trans.Weight();
   }
 }
+const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
+                                                   DofToQuad::Mode mode) const
+{
+   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");
+
+   for (int i = 0; i < dof2quad_array.Size(); i++)
+   {
+      const DofToQuad &d2q = *dof2quad_array[i];
+      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+   }
+
+   DofToQuad *d2q = new DofToQuad;
+   const int nqpt = ir.GetNPoints();
+   d2q->FE = this;
+   d2q->IntRule = &ir;
+   d2q->mode = mode;
+   d2q->ndof = dof;
+   d2q->nqpt = nqpt;
+   d2q->B.SetSize(nqpt*dof);
+   d2q->Bt.SetSize(dof*nqpt);
+   d2q->G.SetSize(nqpt*dim*dof);
+   d2q->Gt.SetSize(dof*nqpt*dim);
+#ifdef MFEM_THREAD_SAFE
+   Vector c_shape(dof);
+   DenseMatrix vshape(dof, dim);
+#endif
+   for (int i = 0; i < nqpt; i++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      CalcShape(ip, c_shape);
+      for (int j = 0; j < dof; j++)
+      {
+         d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = c_shape(j);
+      }
+      CalcDShape(ip, vshape);
+      for (int d = 0; d < dim; d++)
+      {
+         for (int j = 0; j < dof; j++)
+         {
+            d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j,d);
+         }
+      }
+   }
+   dof2quad_array.Append(d2q);
+   return *d2q;
+}
+
+// protected method
+const DofToQuad &ScalarFiniteElement::GetTensorDofToQuad(
+   const TensorBasisElement &tb,
+   const IntegrationRule &ir, DofToQuad::Mode mode) const
+{
+   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
+
+   for (int i = 0; i < dof2quad_array.Size(); i++)
+   {
+      const DofToQuad &d2q = *dof2quad_array[i];
+      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+   }
+
+   DofToQuad *d2q = new DofToQuad;
+   const Poly_1D::Basis &basis_1d = tb.GetBasis1D();
+   const int ndof = order + 1;
+   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/dim) + 0.5);
+   d2q->FE = this;
+   d2q->IntRule = &ir;
+   d2q->mode = mode;
+   d2q->ndof = ndof;
+   d2q->nqpt = nqpt;
+   d2q->B.SetSize(nqpt*ndof);
+   d2q->Bt.SetSize(ndof*nqpt);
+   d2q->G.SetSize(nqpt*ndof);
+   d2q->Gt.SetSize(ndof*nqpt);
+   Vector val(ndof), grad(ndof);
+   for (int i = 0; i < nqpt; i++)
+   {
+      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
+      // of the 1D rule.
+      basis_1d.Eval(ir.IntPoint(i).x, val, grad);
+      for (int j = 0; j < ndof; j++)
+      {
+         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
+         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+      }
+   }
+   dof2quad_array.Append(d2q);
+   return *d2q;
+}
+

 void NodalFiniteElement::ProjectCurl_2D(
   const FiniteElement &fe, ElementTransformation &Trans,
@@ -657,7 +618,7 @@ void InvertLinearTrans(ElementTransformation &trans,

   double store[3];
   Vector v(store, x.Size());
-   pt.Get(store, x.Size());
+   pt.Get(v, x.Size());
   v -= x;

   trans.InverseJacobian().Mult(v, x);
@@ -670,10 +631,7 @@ void NodalFiniteElement::GetLocalRestriction(ElementTransformation &Trans,
   Vector pt(&ipt.x, dim);

 #ifdef MFEM_THREAD_SAFE
-   Vector shape(dof);
-#else
-   Vector shape;
-   vshape.GetColumnReference(0, shape);
+   Vector c_shape(dof);
 #endif

   Trans.SetIntPoint(&Nodes[0]);
@@ -683,8 +641,8 @@ void NodalFiniteElement::GetLocalRestriction(ElementTransformation &Trans,
      InvertLinearTrans(Trans, Nodes[j], pt);
      if (Geometries.CheckPoint(geom_type, ipt)) // do we need an epsilon here?
      {
-         CalcShape(ipt, shape);
-         R.SetRow(j, shape);
+         CalcShape(ipt, c_shape);
+         R.SetRow(j, c_shape);
      }
      else
      {
@@ -695,7 +653,7 @@ void NodalFiniteElement::GetLocalRestriction(ElementTransformation &Trans,
   R.Threshold(1e-12);
 }

-void NodalFiniteElement::Project(
+void NodalFiniteElement::Project (
   Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {
   for (int i = 0; i < dof; i++)
@@ -704,7 +662,7 @@ void NodalFiniteElement::Project(
      // some coefficients expect that Trans.IntPoint is the same
      // as the second argument of Eval
      Trans.SetIntPoint(&ip);
-      dofs(i) = coeff.Eval(Trans, ip);
+      dofs(i) = coeff.Eval (Trans, ip);
      if (map_type == INTEGRAL)
      {
         dofs(i) *= Trans.Weight();
@@ -712,7 +670,7 @@ void NodalFiniteElement::Project(
   }
 }

-void NodalFiniteElement::Project(
+void NodalFiniteElement::Project (
   VectorCoefficient &vc, ElementTransformation &Trans, Vector &dofs) const
 {
   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
@@ -891,18 +849,18 @@ VectorFiniteElement::VectorFiniteElement(int D, Geometry::Type G,
   }
 }

-void VectorFiniteElement::CalcShape(
+void VectorFiniteElement::CalcShape (
   const IntegrationPoint &ip, Vector &shape ) const
 {
-   mfem_error("Error: Cannot use scalar CalcShape(...) function with\n"
-              "   VectorFiniteElements!");
+   mfem_error ("Error: Cannot use scalar CalcShape(...) function with\n"
+               "   VectorFiniteElements!");
 }

-void VectorFiniteElement::CalcDShape(
+void VectorFiniteElement::CalcDShape (
   const IntegrationPoint &ip, DenseMatrix &dshape ) const
 {
-   mfem_error("Error: Cannot use scalar CalcDShape(...) function with\n"
-              "   VectorFiniteElements!");
+   mfem_error ("Error: Cannot use scalar CalcDShape(...) function with\n"
+               "   VectorFiniteElements!");
 }

 void VectorFiniteElement::SetDerivMembers()
@@ -942,7 +900,7 @@ void VectorFiniteElement::SetDerivMembers()
   }
 }

-void VectorFiniteElement::CalcVShape_RT(
+void VectorFiniteElement::CalcVShape_RT (
   ElementTransformation &Trans, DenseMatrix &shape) const
 {
   MFEM_ASSERT(map_type == H_DIV, "");
@@ -954,7 +912,7 @@ void VectorFiniteElement::CalcVShape_RT(
   shape *= (1.0 / Trans.Weight());
 }

-void VectorFiniteElement::CalcVShape_ND(
+void VectorFiniteElement::CalcVShape_ND (
   ElementTransformation &Trans, DenseMatrix &shape) const
 {
   MFEM_ASSERT(map_type == H_CURL, "");
@@ -996,8 +954,8 @@ void VectorFiniteElement::Project_RT(
   {
      Trans.SetIntPoint(&Nodes.IntPoint(k));
      // dof_k = nk^t adj(J) xk
-      dofs(k) = Trans.AdjugateJacobian().InnerProduct(
-                   &vc[k*sdim], nk + d2n[k]*dim);
+      Vector vk(vc.GetData()+k*sdim, sdim);
+      dofs(k) = Trans.AdjugateJacobian().InnerProduct(vk, nk + d2n[k]*dim);
      if (!square_J) { dofs(k) /= Trans.Weight(); }
   }
 }
@@ -1213,8 +1171,9 @@ void VectorFiniteElement::Project_ND(
   for (int k = 0; k < dof; k++)
   {
      Trans.SetIntPoint(&Nodes.IntPoint(k));
+      Vector vk(vc.GetData()+k*dim, dim);
      // dof_k = xk^t J tk
-      dofs(k) = Trans.Jacobian().InnerProduct(tk + d2t[k]*dim, &vc[k*dim]);
+      dofs(k) = Trans.Jacobian().InnerProduct(tk + d2t[k]*dim, vk);
   }
 }

@@ -1361,7 +1320,7 @@ void VectorFiniteElement::LocalL2Projection_RT(
      double w = ip.weight;
      this->CalcVShape(ip, fine_shape);
      Trans.Transform(ip, v);
-      tr_ip.Set(v.GetData(), dim);
+      tr_ip.Set(v, dim);
      cfe.CalcVShape(tr_ip, coarse_shape);

      AddMult_a_AAt(w, fine_shape, fine_mass);
@@ -1448,7 +1407,7 @@ void VectorFiniteElement::LocalL2Projection_ND(
      const IntegrationPoint &ip = ir.IntPoint(i);
      this->CalcVShape(ip, fine_shape);
      Trans.Transform(ip, v);
-      tr_ip.Set(v.GetData(), dim);
+      tr_ip.Set(v, dim);
      cfe.CalcVShape(tr_ip, coarse_shape);

      AddMult_a_AAt(ip.weight, fine_shape, fine_mass);
@@ -2444,46 +2403,6 @@ TensorBasisElement::TensorBasisElement(const int dims, const int p,
   }
 }

-const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
-   const FiniteElement &fe, const IntegrationRule &ir,
-   DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
-   Array<DofToQuad*> &dof2quad_array)
-{
-   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
-   d2q->FE = &fe;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}

 NodalTensorFiniteElement::NodalTensorFiniteElement(const int dims,
                                                   const int p,
@@ -2518,7 +2437,8 @@ VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                                                     const DofMapType dmtype)
   : VectorFiniteElement(dims, GetTensorProductGeometry(dims), d,
                         p, M, FunctionSpace::Qk),
-     TensorBasisElement(dims, p, VerifyNodal(VerifyClosed(cbtype)), dmtype),
+     TensorBasisElement(dims, p, VerifyNodal(cbtype), dmtype),
+     cbasis1d(poly1d.GetBasis(p, VerifyClosed(cbtype))),
     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(obtype)))
 {
   MFEM_VERIFY(dims > 1, "Constructor for VectorTensorFiniteElement with both "
@@ -2533,13 +2453,93 @@ VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                                                     const DofMapType dmtype)
   : VectorFiniteElement(dims, GetTensorProductGeometry(dims), d,
                         p, M, FunctionSpace::Pk),
-     TensorBasisElement(dims, p, VerifyOpen(obtype), dmtype),
+     TensorBasisElement(dims, p, obtype, dmtype),
+     cbasis1d(poly1d.GetBasis(p, VerifyOpen(obtype))),
     obasis1d(poly1d.GetBasis(p, VerifyOpen(obtype)))
 {
   MFEM_VERIFY(dims == 1, "Constructor for VectorTensorFiniteElement without "
               "closed basis is only valid for 1D elements.");
 }

+const DofToQuad &VectorTensorFiniteElement::GetDofToQuad(
+   const IntegrationRule &ir,
+   DofToQuad::Mode mode) const
+{
+   MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
+
+   return GetTensorDofToQuad(ir, mode, true);
+}
+
+const DofToQuad &VectorTensorFiniteElement::GetDofToQuadOpen(
+   const IntegrationRule &ir,
+   DofToQuad::Mode mode) const
+{
+   MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
+
+   return GetTensorDofToQuad(ir, mode, false);
+}
+
+const DofToQuad &VectorTensorFiniteElement::GetTensorDofToQuad(
+   const IntegrationRule &ir,
+   DofToQuad::Mode mode,
+   const bool closed) const
+{
+   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
+
+   for (int i = 0;
+        i < (closed ? dof2quad_array.Size() : dof2quad_array_open.Size());
+        i++)
+   {
+      const DofToQuad &d2q = closed ? *dof2quad_array[i] : *dof2quad_array_open[i];
+      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+   }
+
+   DofToQuad *d2q = new DofToQuad;
+   const int ndof = closed ? order + 1 : order;
+   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/dim) + 0.5);
+   d2q->FE = this;
+   d2q->IntRule = &ir;
+   d2q->mode = mode;
+   d2q->ndof = ndof;
+   d2q->nqpt = nqpt;
+   d2q->B.SetSize(nqpt*ndof);
+   d2q->Bt.SetSize(ndof*nqpt);
+   d2q->G.SetSize(nqpt*ndof);
+   d2q->Gt.SetSize(ndof*nqpt);
+   Vector val(ndof), grad(ndof);
+   for (int i = 0; i < nqpt; i++)
+   {
+      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
+      // of the 1D rule.
+
+      if (closed)
+      {
+         cbasis1d.Eval(ir.IntPoint(i).x, val, grad);
+      }
+      else
+      {
+         obasis1d.Eval(ir.IntPoint(i).x, val, grad);
+      }
+
+      for (int j = 0; j < ndof; j++)
+      {
+         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
+         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+      }
+   }
+
+   if (closed)
+   {
+      dof2quad_array.Append(d2q);
+   }
+   else
+   {
+      dof2quad_array_open.Append(d2q);
+   }
+
+   return *d2q;
+}
+
 VectorTensorFiniteElement::~VectorTensorFiniteElement()
 {
   for (int i = 0; i < dof2quad_array_open.Size(); i++)
@@ -127,6 +127,7 @@ public:
   }
 };

+
 /** @brief Structure representing the matrices/tensors needed to evaluate (in
    reference space) the values, gradients, divergences, or curls of a
    FiniteElement at a the quadrature points of a given IntegrationRule. */
@@ -156,7 +157,8 @@ public:
          dimensions using 1D number of quadrature points and degrees of
          freedom. */
      /** When representing a vector-valued FiniteElement, two DofToQuad objects
-          are used to describe the "closed" and "open" 1D basis functions. */
+          are used to describe the "closed" and "open" 1D basis functions
+          (TODO). */
      TENSOR
   };

@@ -174,7 +176,7 @@ public:
   /// Basis functions evaluated at quadrature points.
   /** The storage layout is column-major with dimensions:
       - #nqpt x #ndof, for scalar elements, or
-       - #nqpt x dim x #ndof, for vector elements,
+       - #nqpt x dim x #ndof, for vector elements, (TODO)

       where

@@ -185,15 +187,15 @@ public:
   /// Transpose of #B.
   /** The storage layout is column-major with dimensions:
       - #ndof x #nqpt, for scalar elements, or
-       - #ndof x #nqpt x dim, for vector elements. */
+       - #ndof x #nqpt x dim, for vector elements (TODO). */
   Array<double> Bt;

   /** @brief Gradients/divergences/curls of basis functions evaluated at
       quadrature points. */
   /** The storage layout is column-major with dimensions:
       - #nqpt x dim x #ndof, for scalar elements, or
-       - #nqpt x #ndof, for H(div) vector elements, or
-       - #nqpt x cdim x #ndof, for H(curl) vector elements,
+       - #nqpt x #ndof, for H(div) vector elements (TODO), or
+       - #nqpt x cdim x #ndof, for H(curl) vector elements (TODO),

       where

@@ -206,11 +208,12 @@ public:
   /// Transpose of #G.
   /** The storage layout is column-major with dimensions:
       - #ndof x #nqpt x dim, for scalar elements, or
-       - #ndof x #nqpt, for H(div) vector elements, or
-       - #ndof x #nqpt x cdim, for H(curl) vector elements. */
+       - #ndof x #nqpt, for H(div) vector elements (TODO), or
+       - #ndof x #nqpt x cdim, for H(curl) vector elements (TODO). */
   Array<double> Gt;
 };

+
 /// Describes the function space on each element
 class FunctionSpace
 {
@@ -244,7 +247,7 @@ protected:
   mutable int orders[Geometry::MaxDim]; ///< Anisotropic orders
   IntegrationRule Nodes;
 #ifndef MFEM_THREAD_SAFE
-   mutable DenseMatrix vshape; // Dof x Dim
+   mutable DenseMatrix vshape; // Dof x VDim
 #endif
   /// Container for all DofToQuad objects created by the FiniteElement.
   /** Multiple DofToQuad objects may be needed when different quadrature rules
@@ -253,7 +256,7 @@ protected:

 public:
   /// Enumeration for range_type and deriv_range_type
-   enum RangeType { UNKNOWN_RANGE_TYPE = -1, SCALAR, VECTOR };
+   enum RangeType { SCALAR, VECTOR };

   /** @brief Enumeration for MapType: defines how reference functions are
       mapped to physical space.
@@ -267,8 +270,6 @@ public:
   */
   enum MapType
   {
-      UNKNOWN_MAP_TYPE = -1, /**< Used to distinguish an unset MapType variable
-                                  from the known values below. */
      VALUE,     /**< For scalar fields; preserves point values
                          \f$ u(x) = \hat u(\hat x) \f$ */
      INTEGRAL,  /**< For scalar fields; preserves volume integrals
@@ -347,6 +348,7 @@ public:
       H_DIV, H_CURL}. */
   int GetMapType() const { return map_type; }

+
   /** @brief Returns the FiniteElement::DerivType of the element describing the
       spatial derivative method implemented, one of {NONE, GRAD,
       DIV, CURL}. */
@@ -453,8 +455,8 @@ public:
       part of the Hessian of one shape function.
       The order in 2D is {u_xx, u_xy, u_yy}.
       The size (#dof x (#dim (#dim+1)/2) of @a Hessian must be set in advance.*/
-   virtual void CalcHessian(const IntegrationPoint &ip,
-                            DenseMatrix &Hessian) const;
+   virtual void CalcHessian (const IntegrationPoint &ip,
+                             DenseMatrix &Hessian) const;

   /** @brief Evaluate the Hessian of all shape functions of a scalar finite
       element in reference space at the given point @a ip. */
@@ -575,7 +577,6 @@ public:
   /** See the documentation for DofToQuad for more details. */
   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                         DofToQuad::Mode mode) const;
-
   /// Deconstruct the FiniteElement
   virtual ~FiniteElement();

@@ -622,11 +623,16 @@ public:
   }
 };

+
 /** @brief Class for finite elements with basis functions
    that return scalar values. */
 class ScalarFiniteElement : public FiniteElement
 {
 protected:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector c_shape;
+#endif
+
   static const ScalarFiniteElement &CheckScalarFE(const FiniteElement &fe)
   {
      MFEM_VERIFY(fe.GetRangeType() == SCALAR,
@@ -634,6 +640,10 @@ protected:
      return static_cast<const ScalarFiniteElement &>(fe);
   }

+   const DofToQuad &GetTensorDofToQuad(const class TensorBasisElement &tb,
+                                       const IntegrationRule &ir,
+                                       DofToQuad::Mode mode) const;
+
 public:
   /** @brief Construct ScalarFiniteElement with given
       @param D    Reference space dimension
@@ -644,8 +654,13 @@ public:
    */
   ScalarFiniteElement(int D, Geometry::Type G, int Do, int O,
                       int F = FunctionSpace::Pk)
+#ifdef MFEM_THREAD_SAFE
      : FiniteElement(D, G, Do, O, F)
   { deriv_type = GRAD; deriv_range_type = VECTOR; deriv_map_type = H_CURL; }
+#else
+      : FiniteElement(D, G, Do, O, F), c_shape(dof)
+   { deriv_type = GRAD; deriv_range_type = VECTOR; deriv_map_type = H_CURL; }
+#endif

   /** @brief Set the FiniteElement::MapType of the element to either VALUE or
       INTEGRAL. Also sets the FiniteElement::DerivType to GRAD if the
@@ -657,6 +672,7 @@ public:
      deriv_type = (M == VALUE) ? GRAD : NONE;
   }

+
   /** @brief Get the matrix @a I that defines nodal interpolation
       @a between this element and the refined element @a fine_fe. */
   void NodalLocalInterpolation(ElementTransformation &Trans,
@@ -677,11 +693,15 @@ public:
   /** If the "fine" elements cannot represent all basis functions of the
       "coarse" element, then boundary values from different sub-elements are
       generally different. */
-   void ScalarLocalL2Restriction(ElementTransformation &Trans,
-                                 DenseMatrix &R,
-                                 const ScalarFiniteElement &coarse_fe) const;
+   void ScalarLocalRestriction(ElementTransformation &Trans,
+                               DenseMatrix &R,
+                               const ScalarFiniteElement &coarse_fe) const;
+
+   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
+                                         DofToQuad::Mode mode) const;
 };

+
 /// Class for standard nodal finite elements.
 class NodalFiniteElement : public ScalarFiniteElement
 {
@@ -703,38 +723,38 @@ public:
                      int F = FunctionSpace::Pk)
      : ScalarFiniteElement(D, G, Do, O, F) { }

-   void GetLocalInterpolation(ElementTransformation &Trans,
-                              DenseMatrix &I) const override
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
   { NodalLocalInterpolation(Trans, I, *this); }

-   void GetLocalRestriction(ElementTransformation &Trans,
-                            DenseMatrix &R) const override;
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const;

-   void GetTransferMatrix(const FiniteElement &fe,
-                          ElementTransformation &Trans,
-                          DenseMatrix &I) const override
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
   { CheckScalarFE(fe).NodalLocalInterpolation(Trans, I, *this); }

-   void Project(Coefficient &coeff,
-                ElementTransformation &Trans, Vector &dofs) const override;
+   virtual void Project (Coefficient &coeff,
+                         ElementTransformation &Trans, Vector &dofs) const;

-   void Project(VectorCoefficient &vc,
-                ElementTransformation &Trans, Vector &dofs) const override;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;

   // (mc.height x mc.width) @ DOFs -> (Dof x mc.width x mc.height) in dofs
-   void ProjectMatrixCoefficient(
-      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const override;
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;

-   void Project(const FiniteElement &fe, ElementTransformation &Trans,
-                DenseMatrix &I) const override;
+   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                        DenseMatrix &I) const;

-   void ProjectGrad(const FiniteElement &fe,
-                    ElementTransformation &Trans,
-                    DenseMatrix &grad) const override;
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;

-   void ProjectDiv(const FiniteElement &fe,
-                   ElementTransformation &Trans,
-                   DenseMatrix &div) const override;
+   virtual void ProjectDiv(const FiniteElement &fe,
+                           ElementTransformation &Trans,
+                           DenseMatrix &div) const;

   /** @brief Get an Array<int> that maps lexicographically ordered indices to
       the indices of the respective nodes/dofs/basis functions.
@@ -768,12 +788,12 @@ class VectorFiniteElement : public FiniteElement
   // Hide the scalar functions CalcShape and CalcDShape.
 private:
   /// Overrides the scalar CalcShape function to print an error.
-   void CalcShape(const IntegrationPoint &ip,
-                  Vector &shape) const override;
+   virtual void CalcShape(const IntegrationPoint &ip,
+                          Vector &shape) const;

   /// Overrides the scalar CalcDShape function to print an error.
-   void CalcDShape(const IntegrationPoint &ip,
-                   DenseMatrix &dshape) const override;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;

 protected:
   bool is_nodal;
@@ -932,10 +952,11 @@ protected:
   }

 public:
-   VectorFiniteElement(int D, Geometry::Type G, int Do, int O, int M,
-                       int F = FunctionSpace::Pk);
+   VectorFiniteElement (int D, Geometry::Type G, int Do, int O, int M,
+                        int F = FunctionSpace::Pk);
 };

+
 /// @brief Class for computing 1D special polynomials and their associated basis
 /// functions
 class Poly_1D
@@ -1070,11 +1091,6 @@ public:
   // { CalcLegendre(p, x, u); }
   { CalcChebyshev(p, x, u); }

-   /** @brief Evaluate the values of a hierarchical 1D basis at point x
-       hierarchical = k-th basis function is degree k polynomial */
-   static void CalcBasis(const int p, const double x, Vector &u)
-   { CalcBasis(p, x, u.GetData()); }
-
   /// Evaluate the values and derivatives of a hierarchical 1D basis at point @a x
   static void CalcBasis(const int p, const double x, double *u, double *d)
   // { CalcMono(p, x, u, d); }
@@ -1082,11 +1098,6 @@ public:
   // { CalcLegendre(p, x, u, d); }
   { CalcChebyshev(p, x, u, d); }

-   /** @brief Evaluate the values and derivatives of a hierarchical 1D basis at
-       point @a x. */
-   static void CalcBasis(const int p, const double x, Vector &u, Vector &d)
-   { CalcBasis(p, x, u.GetData(), d.GetData()); }
-
   /// Evaluate the values, derivatives and second derivatives of a hierarchical 1D basis at point x
   static void CalcBasis(const int p, const double x, double *u, double *d,
                         double *dd)
@@ -1095,12 +1106,6 @@ public:
   // { CalcLegendre(p, x, u, d); }
   { CalcChebyshev(p, x, u, d, dd); }

-   /** @brief Evaluate the values, derivatives and second derivatives of a
-       hierarchical 1D basis at point @a x. */
-   static void CalcBasis(const int p, const double x, Vector &u, Vector &d,
-                         Vector &dd)
-   { CalcBasis(p, x, u.GetData(), d.GetData(), dd.GetData()); }
-
   /// Evaluate a representation of a Delta function at point x
   static double CalcDelta(const int p, const double x)
   { return pow(x, (double) p); }
@@ -1130,24 +1135,12 @@ public:
   static void CalcBernstein(const int p, const double x, double *u)
   { CalcBinomTerms(p, x, 1. - x, u); }

-   /** @brief Compute the values of the Bernstein basis functions of order
-       @a p at coordinate @a x and store the results in the already allocated
-       @a u array. */
-   static void CalcBernstein(const int p, const double x, Vector &u)
-   { CalcBernstein(p, x, u.GetData()); }
-
   /** @brief Compute the values and derivatives of the Bernstein basis functions
       of order @a p at coordinate @a x and store the results in the already allocated
       @a u and @a d arrays. */
   static void CalcBernstein(const int p, const double x, double *u, double *d)
   { CalcBinomTerms(p, x, 1. - x, u, d); }

-   /** @brief Compute the values and derivatives of the Bernstein basis
-       functions of order @a p at coordinate @a x and store the results in the
-       already allocated @a u and @a d arrays. */
-   static void CalcBernstein(const int p, const double x, Vector &u, Vector &d)
-   { CalcBernstein(p, x, u.GetData(), d.GetData()); }
-
   static void CalcLegendre(const int p, const double x, double *u);
   static void CalcLegendre(const int p, const double x, double *u, double *d);

@@ -1156,6 +1149,7 @@ public:

 extern Poly_1D poly1d;

+
 /// An element defined as an ND tensor product of 1D elements on a segment,
 /// square, or cube
 class TensorBasisElement
@@ -1179,7 +1173,7 @@ public:

   int GetBasisType() const { return b_type; }

-   const Poly_1D::Basis &GetBasis1D() const { return basis1d; }
+   const Poly_1D::Basis& GetBasis1D() const { return basis1d; }

   /** @brief Get an Array<int> that maps lexicographically ordered indices to
       the indices of the respective nodes/dofs/basis functions. If the dofs are
@@ -1211,11 +1205,6 @@ public:
         default: MFEM_ABORT("invalid dimension: " << dim); return -1;
      }
   }
-
-   static const DofToQuad &GetTensorDofToQuad(
-      const FiniteElement &fe, const IntegrationRule &ir,
-      DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
-      Array<DofToQuad*> &dof2quad_array);
 };

 class NodalTensorFiniteElement : public NodalFiniteElement,
@@ -1226,18 +1215,18 @@ public:
                            const DofMapType dmtype);

   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const override
+                                 DofToQuad::Mode mode) const
   {
      return (mode == DofToQuad::FULL) ?
-             FiniteElement::GetDofToQuad(ir, mode) :
-             GetTensorDofToQuad(*this, ir, mode, basis1d, true, dof2quad_array);
+             ScalarFiniteElement::GetDofToQuad(ir, mode) :
+             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
   }

-   void SetMapType(const int map_type_) override;
+   virtual void SetMapType(const int map_type_);

-   void GetTransferMatrix(const FiniteElement &fe,
-                          ElementTransformation &Trans,
-                          DenseMatrix &I) const override
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
   {
      if (basis1d.IsIntegratedType())
      {
@@ -1257,7 +1246,7 @@ private:
   mutable Array<DofToQuad*> dof2quad_array_open;

 protected:
-   Poly_1D::Basis &obasis1d;
+   Poly_1D::Basis &cbasis1d, &obasis1d;

 public:
   VectorTensorFiniteElement(const int dims, const int d, const int p,
@@ -1270,22 +1259,16 @@ public:
                             const DofMapType dmtype);

   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const override
-   {
-      MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
-      return GetTensorDofToQuad(*this, ir, mode, basis1d, true,
-                                dof2quad_array);
-   }
+                                 DofToQuad::Mode mode) const;

   const DofToQuad &GetDofToQuadOpen(const IntegrationRule &ir,
-                                     DofToQuad::Mode mode) const
-   {
-      MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
-      return GetTensorDofToQuad(*this, ir, mode, obasis1d, false,
-                                dof2quad_array_open);
-   }
+                                     DofToQuad::Mode mode) const;

-   virtual ~VectorTensorFiniteElement();
+   const DofToQuad &GetTensorDofToQuad(const IntegrationRule &ir,
+                                       DofToQuad::Mode mode,
+                                       const bool closed) const;
+
+   ~VectorTensorFiniteElement();
 };

 void InvertLinearTrans(ElementTransformation &trans,
@@ -32,11 +32,6 @@ public:
   virtual void CalcDShape(const IntegrationPoint &ip,
                           DenseMatrix &dshape) const;
   virtual void ProjectDelta(int vertex, Vector &dofs) const;
-
-   virtual void GetLocalRestriction(ElementTransformation &Trans,
-                                    DenseMatrix &R) const
-   { ScalarLocalL2Restriction(Trans, R, *this); }
-
 };


@@ -60,11 +55,6 @@ public:
                            ElementTransformation &Trans,
                            DenseMatrix &curl) const
   { ProjectCurl_2D(fe, Trans, curl); }
-
-   virtual void GetLocalRestriction(ElementTransformation &Trans,
-                                    DenseMatrix &R) const
-   { ScalarLocalL2Restriction(Trans, R, *this); }
-
   using FiniteElement::Project;
   virtual void ProjectDiv(const FiniteElement &fe,
                           ElementTransformation &Trans,
@@ -90,11 +80,6 @@ public:
   virtual void CalcDShape(const IntegrationPoint &ip,
                           DenseMatrix &dshape) const;
   virtual void ProjectDelta(int vertex, Vector &dofs) const;
-
-   virtual void GetLocalRestriction(ElementTransformation &Trans,
-                                    DenseMatrix &R) const
-   { ScalarLocalL2Restriction(Trans, R, *this); }
-
   using FiniteElement::Project;
   virtual void ProjectDiv(const FiniteElement &fe,
                           ElementTransformation &Trans,
@@ -126,11 +111,6 @@ public:
                            ElementTransformation &Trans,
                            DenseMatrix &curl) const
   { ProjectCurl_2D(fe, Trans, curl); }
-
-   virtual void GetLocalRestriction(ElementTransformation &Trans,
-                                    DenseMatrix &R) const
-   { ScalarLocalL2Restriction(Trans, R, *this); }
-
 };


@@ -153,11 +133,6 @@ public:
   virtual void CalcDShape(const IntegrationPoint &ip,
                           DenseMatrix &dshape) const;
   virtual void ProjectDelta(int vertex, Vector &dofs) const;
-
-   virtual void GetLocalRestriction(ElementTransformation &Trans,
-                                    DenseMatrix &R) const
-   { ScalarLocalL2Restriction(Trans, R, *this); }
-
 };


@@ -314,16 +314,14 @@ void ND_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(p + 1), shape_ox(p), shape_cy(p + 1), shape_oy(p);
   Vector shape_cz(p + 1), shape_oz(p);
+   Vector dshape_cx(p + 1), dshape_cy(p + 1), dshape_cz(p + 1);
 #endif

   if (obasis1d.IsIntegratedType())
   {
-#ifdef MFEM_THREAD_SAFE
-      Vector dshape_cx(p + 1), dshape_cy(p + 1), dshape_cz(p + 1);
-#endif
-      basis1d.Eval(ip.x, shape_cx, dshape_cx);
-      basis1d.Eval(ip.y, shape_cy, dshape_cy);
-      basis1d.Eval(ip.z, shape_cz, dshape_cz);
+      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+      cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
@@ -331,9 +329,9 @@ void ND_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
   }
   else
   {
-      basis1d.Eval(ip.x, shape_cx);
-      basis1d.Eval(ip.y, shape_cy);
-      basis1d.Eval(ip.z, shape_cz);
+      cbasis1d.Eval(ip.x, shape_cx);
+      cbasis1d.Eval(ip.y, shape_cy);
+      cbasis1d.Eval(ip.z, shape_cz);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
      obasis1d.Eval(ip.z, shape_oz);
@@ -407,9 +405,9 @@ void ND_HexahedronElement::CalcCurlShape(const IntegrationPoint &ip,
   Vector dshape_cx(p + 1), dshape_cy(p + 1), dshape_cz(p + 1);
 #endif

-   basis1d.Eval(ip.x, shape_cx, dshape_cx);
-   basis1d.Eval(ip.y, shape_cy, dshape_cy);
-   basis1d.Eval(ip.z, shape_cz, dshape_cz);
+   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+   cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -658,23 +656,21 @@ void ND_QuadrilateralElement::CalcVShape(const IntegrationPoint &ip,

 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(p + 1), shape_ox(p), shape_cy(p + 1), shape_oy(p);
+   Vector dshape_cx(p + 1), dshape_cy(p + 1);
 #endif

   if (obasis1d.IsIntegratedType())
   {
-#ifdef MFEM_THREAD_SAFE
-      Vector dshape_cx(p + 1), dshape_cy(p + 1);
-#endif
-      basis1d.Eval(ip.x, shape_cx, dshape_cx);
-      basis1d.Eval(ip.y, shape_cy, dshape_cy);
+      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
   }
   else
   {
-      basis1d.Eval(ip.x, shape_cx);
-      basis1d.Eval(ip.y, shape_cy);
+      cbasis1d.Eval(ip.x, shape_cx);
+      cbasis1d.Eval(ip.y, shape_cy);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
   }
@@ -724,8 +720,8 @@ void ND_QuadrilateralElement::CalcCurlShape(const IntegrationPoint &ip,
   Vector dshape_cx(p + 1), dshape_cy(p + 1);
 #endif

-   basis1d.Eval(ip.x, shape_cx, dshape_cx);
-   basis1d.Eval(ip.y, shape_cy, dshape_cy);
+   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -13,7 +13,6 @@

 #include "fe_pos.hpp"
 #include "../bilininteg.hpp"
-#include "../lininteg.hpp"
 #include "../coefficient.hpp"

 namespace mfem
@@ -213,7 +212,7 @@ void BiQuadPos2DFiniteElement::GetLocalInterpolation(
 void BiQuadPos2DFiniteElement::Project(
   Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {
-   double *d = dofs.GetData();
+   double *d = dofs;

   for (int i = 0; i < 9; i++)
   {
@@ -383,8 +382,8 @@ void H1Pos_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   Vector shape_x(p+1), shape_y(p+1);
 #endif

-   Poly_1D::CalcBernstein(p, ip.x, shape_x);
-   Poly_1D::CalcBernstein(p, ip.y, shape_y);
+   Poly_1D::CalcBernstein(p, ip.x, shape_x.GetData() );
+   Poly_1D::CalcBernstein(p, ip.y, shape_y.GetData() );

   // Reorder so that vertices are at the beginning of the list
   for (int o = 0, j = 0; j <= p; j++)
@@ -403,8 +402,8 @@ void H1Pos_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
   Vector shape_x(p+1), shape_y(p+1), dshape_x(p+1), dshape_y(p+1);
 #endif

-   Poly_1D::CalcBernstein(p, ip.x, shape_x, dshape_x);
-   Poly_1D::CalcBernstein(p, ip.y, shape_y, dshape_y);
+   Poly_1D::CalcBernstein(p, ip.x, shape_x.GetData(), dshape_x.GetData() );
+   Poly_1D::CalcBernstein(p, ip.y, shape_y.GetData(), dshape_y.GetData() );

   // Reorder so that vertices are at the beginning of the list
   for (int o = 0, j = 0; j <= p; j++)
@@ -40,7 +40,7 @@ public:

   virtual void GetLocalRestriction(ElementTransformation &Trans,
                                    DenseMatrix &R) const
-   { ScalarLocalL2Restriction(Trans, R, *this); }
+   { ScalarLocalRestriction(Trans, R, *this); }

   virtual void GetTransferMatrix(const FiniteElement &fe,
                                  ElementTransformation &Trans,
@@ -73,8 +73,8 @@ public:
                                 DofToQuad::Mode mode) const
   {
      return (mode == DofToQuad::FULL) ?
-             FiniteElement::GetDofToQuad(ir, mode) :
-             GetTensorDofToQuad(*this, ir, mode, basis1d, true, dof2quad_array);
+             ScalarFiniteElement::GetDofToQuad(ir, mode) :
+             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
   }
 };

@@ -145,23 +145,21 @@ void RT_QuadrilateralElement::CalcVShape(const IntegrationPoint &ip,

 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(pp1 + 1), shape_ox(pp1), shape_cy(pp1 + 1), shape_oy(pp1);
+   Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1);
 #endif

   if (obasis1d.IsIntegratedType())
   {
-#ifdef MFEM_THREAD_SAFE
-      Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1);
-#endif
-      basis1d.Eval(ip.x, shape_cx, dshape_cx);
-      basis1d.Eval(ip.y, shape_cy, dshape_cy);
+      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
   }
   else
   {
-      basis1d.Eval(ip.x, shape_cx);
-      basis1d.Eval(ip.y, shape_cy);
+      cbasis1d.Eval(ip.x, shape_cx);
+      cbasis1d.Eval(ip.y, shape_cy);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
   }
@@ -209,8 +207,8 @@ void RT_QuadrilateralElement::CalcDivShape(const IntegrationPoint &ip,
   Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1);
 #endif

-   basis1d.Eval(ip.x, shape_cx, dshape_cx);
-   basis1d.Eval(ip.y, shape_cy, dshape_cy);
+   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -475,16 +473,14 @@ void RT_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
 #ifdef MFEM_THREAD_SAFE
   Vector shape_cx(pp1 + 1), shape_ox(pp1), shape_cy(pp1 + 1), shape_oy(pp1);
   Vector shape_cz(pp1 + 1), shape_oz(pp1);
+   Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1), dshape_cz(pp1 + 1);
 #endif

   if (obasis1d.IsIntegratedType())
   {
-#ifdef MFEM_THREAD_SAFE
-      Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1), dshape_cz(pp1 + 1);
-#endif
-      basis1d.Eval(ip.x, shape_cx, dshape_cx);
-      basis1d.Eval(ip.y, shape_cy, dshape_cy);
-      basis1d.Eval(ip.z, shape_cz, dshape_cz);
+      cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+      cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+      cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
      obasis1d.ScaleIntegrated(false);
      obasis1d.EvalIntegrated(dshape_cx, shape_ox);
      obasis1d.EvalIntegrated(dshape_cy, shape_oy);
@@ -492,9 +488,9 @@ void RT_HexahedronElement::CalcVShape(const IntegrationPoint &ip,
   }
   else
   {
-      basis1d.Eval(ip.x, shape_cx);
-      basis1d.Eval(ip.y, shape_cy);
-      basis1d.Eval(ip.z, shape_cz);
+      cbasis1d.Eval(ip.x, shape_cx);
+      cbasis1d.Eval(ip.y, shape_cy);
+      cbasis1d.Eval(ip.z, shape_cz);
      obasis1d.Eval(ip.x, shape_ox);
      obasis1d.Eval(ip.y, shape_oy);
      obasis1d.Eval(ip.z, shape_oz);
@@ -568,9 +564,9 @@ void RT_HexahedronElement::CalcDivShape(const IntegrationPoint &ip,
   Vector dshape_cx(pp1 + 1), dshape_cy(pp1 + 1), dshape_cz(pp1 + 1);
 #endif

-   basis1d.Eval(ip.x, shape_cx, dshape_cx);
-   basis1d.Eval(ip.y, shape_cy, dshape_cy);
-   basis1d.Eval(ip.z, shape_cz, dshape_cz);
+   cbasis1d.Eval(ip.x, shape_cx, dshape_cx);
+   cbasis1d.Eval(ip.y, shape_cy, dshape_cy);
+   cbasis1d.Eval(ip.z, shape_cz, dshape_cz);
   if (obasis1d.IsIntegratedType())
   {
      obasis1d.ScaleIntegrated(false);
@@ -22,71 +22,6 @@ namespace mfem

 using namespace std;

-const FiniteElement *
-FiniteElementCollection::FiniteElementForDim(int dim) const
-{
-   ErrorMode save_error_mode = error_mode;
-   error_mode = RETURN_NULL;
-   const FiniteElement *fe = nullptr;
-   for (int g = Geometry::DimStart[dim]; g < Geometry::DimStart[dim+1]; g++)
-   {
-      fe = FiniteElementForGeometry((Geometry::Type)g);
-      if (fe != nullptr) { break; }
-   }
-   error_mode = save_error_mode;
-   return fe;
-}
-
-int FiniteElementCollection::GetRangeType(int dim) const
-{
-   const FiniteElement *fe = FiniteElementForDim(dim);
-   if (fe)
-   {
-      return fe->GetRangeType();
-   }
-   return FiniteElement::UNKNOWN_RANGE_TYPE;
-}
-
-int FiniteElementCollection::GetDerivRangeType(int dim) const
-{
-   const FiniteElement *fe = FiniteElementForDim(dim);
-   if (fe)
-   {
-      return fe->GetDerivRangeType();
-   }
-   return FiniteElement::UNKNOWN_RANGE_TYPE;
-}
-
-int FiniteElementCollection::GetMapType(int dim) const
-{
-   const FiniteElement *fe = FiniteElementForDim(dim);
-   if (fe)
-   {
-      return fe->GetMapType();
-   }
-   return FiniteElement::UNKNOWN_MAP_TYPE;
-}
-
-int FiniteElementCollection::GetDerivType(int dim) const
-{
-   const FiniteElement *fe = FiniteElementForDim(dim);
-   if (fe)
-   {
-      return fe->GetDerivType();
-   }
-   return FiniteElement::NONE;
-}
-
-int FiniteElementCollection::GetDerivMapType(int dim) const
-{
-   const FiniteElement *fe = FiniteElementForDim(dim);
-   if (fe)
-   {
-      return fe->GetDerivMapType();
-   }
-   return FiniteElement::UNKNOWN_MAP_TYPE;
-}
-
 int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
 {
   switch (geom)
@@ -644,7 +579,6 @@ LinearFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -688,7 +622,6 @@ QuadraticFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -729,7 +662,6 @@ QuadraticPosFECollection::FiniteElementForGeometry(
      case Geometry::SEGMENT:     return &SegmentFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticPosFECollection: unknown geometry type.");
   }
   return NULL; // Make some compilers happy
@@ -770,7 +702,6 @@ CubicFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("CubicFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -837,7 +768,6 @@ CrouzeixRaviartFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("CrouzeixRaviartFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -875,7 +805,6 @@ RT0_2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT0_2DFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -918,7 +847,6 @@ RT1_2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT1_2DFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -960,7 +888,6 @@ RT2_2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT2_2DFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -1002,7 +929,6 @@ Const2DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("Const2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1038,7 +964,6 @@ LinearDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearDiscont2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1074,7 +999,6 @@ GaussLinearDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("GaussLinearDiscont2DFECollection:"
                     " unknown geometry type.");
   }
@@ -1109,7 +1033,6 @@ P1OnQuadFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
 {
   if (GeomType != Geometry::SQUARE)
   {
-      if (error_mode == RETURN_NULL) { return nullptr; }
      mfem_error ("P1OnQuadFECollection: unknown geometry type.");
   }
   return &QuadrilateralFE;
@@ -1144,7 +1067,6 @@ QuadraticDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticDiscont2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1180,7 +1102,6 @@ QuadraticPosDiscont2DFECollection::FiniteElementForGeometry(
   {
      case Geometry::SQUARE:  return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticPosDiscont2DFECollection: unknown geometry type.");
   }
   return NULL; // Make some compilers happy
@@ -1211,7 +1132,6 @@ const
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("GaussQuadraticDiscont2DFECollection:"
                     " unknown geometry type.");
   }
@@ -1250,7 +1170,6 @@ CubicDiscont2DFECollection::FiniteElementForGeometry(
      case Geometry::TRIANGLE:    return &TriangleFE;
      case Geometry::SQUARE:      return &QuadrilateralFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("CubicDiscont2DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1288,7 +1207,6 @@ LinearNonConf3DFECollection::FiniteElementForGeometry(
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearNonConf3DFECollection: unknown geometry type.");
   }
   return &TriangleFE; // Make some compilers happy
@@ -1329,7 +1247,6 @@ Const3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("Const3DFECollection: unknown geometry type.");
   }
   return &TetrahedronFE; // Make some compilers happy
@@ -1371,7 +1288,6 @@ LinearDiscont3DFECollection::FiniteElementForGeometry(
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
   }
   return &TetrahedronFE; // Make some compilers happy
@@ -1411,7 +1327,6 @@ QuadraticDiscont3DFECollection::FiniteElementForGeometry(
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("QuadraticDiscont3DFECollection: unknown geometry type.");
   }
   return &TetrahedronFE; // Make some compilers happy
@@ -1453,7 +1368,6 @@ RefinedLinearFECollection::FiniteElementForGeometry(
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RefinedLinearFECollection: unknown geometry type.");
   }
   return &SegmentFE; // Make some compilers happy
@@ -1494,7 +1408,6 @@ ND1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("ND1_3DFECollection: unknown geometry type.");
   }
   return &HexahedronFE; // Make some compilers happy
@@ -1544,7 +1457,6 @@ RT0_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::PYRAMID:     return &PyramidFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT0_3DFECollection: unknown geometry type.");
   }
   return &HexahedronFE; // Make some compilers happy
@@ -1594,7 +1506,6 @@ RT1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return &QuadrilateralFE;
      case Geometry::CUBE:        return &HexahedronFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("RT1_3DFECollection: unknown geometry type.");
   }
   return &HexahedronFE; // Make some compilers happy
@@ -1956,7 +1867,6 @@ H1_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
-      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("H1 Pyramid basis functions are not yet supported "
                 "for order > 1.");
      return NULL;
@@ -2337,7 +2247,6 @@ L2_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
-      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("L2 Pyramid basis functions are not yet supported "
                 "for order > 0.");
      return NULL;
@@ -2593,7 +2502,6 @@ RT_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
-      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("RT Pyramid basis functions are not yet supported "
                 "for order > 0.");
      return NULL;
@@ -2879,7 +2787,6 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
   else
   {
-      if (error_mode == RETURN_NULL) { return nullptr; }
      MFEM_ABORT("ND Pyramid basis functions are not yet supported "
                 "for order > 1.");
      return NULL;
@@ -3482,7 +3389,6 @@ NURBSFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return QuadrilateralFE;
      case Geometry::CUBE:        return ParallelepipedFE;
      default:
-         if (error_mode == RETURN_NULL) { return nullptr; }
         mfem_error ("NURBSFECollection: unknown geometry type.");
   }
   return SegmentFE; // Make some compilers happy
@@ -51,14 +51,6 @@ public:
   virtual const FiniteElement *
   FiniteElementForGeometry(Geometry::Type GeomType) const = 0;

-   /** @brief Returns the first non-NULL FiniteElement for the given dimension
-
-       @note Repeatedly calls FiniteElementForGeometry in the order defined in
-       the Geometry::Type enumeration.
-   */
-   virtual const FiniteElement *
-   FiniteElementForDim(int dim) const;
-
   virtual int DofForGeometry(Geometry::Type GeomType) const = 0;

   /** @brief Returns an array, say p, that maps a local permuted index i to a
@@ -74,17 +66,6 @@ public:

   virtual int GetContType() const = 0;

-   /** @note The following methods provide the same information as the
-       corresponding methods of the FiniteElement base class.
-       @{
-   */
-   virtual int GetRangeType(int dim) const;
-   virtual int GetDerivRangeType(int dim) const;
-   virtual int GetMapType(int dim) const;
-   virtual int GetDerivType(int dim) const;
-   virtual int GetDerivMapType(int dim) const;
-   /** @} */
-
   int HasFaceDofs(Geometry::Type geom, int p) const;

   virtual const FiniteElement *TraceFiniteElementForGeometry(
@@ -233,19 +214,6 @@ protected:
   void InitVarOrder(int p) const;

   mutable Array<FiniteElementCollection*> var_orders;
-
-   /// How to treat errors in FiniteElementForGeometry() calls.
-   enum ErrorMode
-   {
-      RETURN_NULL,      ///< Return NULL on errors
-      RAISE_MFEM_ERROR  /**< Raise an MFEM error (default in base class).
-                             Sub-classes can ignore this and return NULL. */
-   };
-
-   /// How to treat errors in FiniteElementForGeometry() calls.
-   /** The typical error in derived classes is that no FiniteElement is defined
-       for the given Geometry, or the input is not a valid Geometry. */
-   mutable ErrorMode error_mode = RAISE_MFEM_ERROR;
 };

 /// Arbitrary order H1-conforming (continuous) finite elements.
@@ -1958,7 +1958,7 @@ FiniteElementSpace::DerefinementOperator::DerefinementOperator(
         DenseMatrix &lM = localM[g](mi[s]);
         DenseMatrix &lR = localR[g](lR_offset+s);
         MultAtB(lP, lM, lR); // lR = lP^T lM
-         mfem::AddMult(lR, lP, lPtMP); // lPtMP += lP^T lM lP
+         AddMult(lR, lP, lPtMP); // lPtMP += lP^T lM lP
      }
      DenseMatrixInverse lPtMP_inv(lPtMP);
      for (int s = 0; s < nm; s++)
@@ -2005,7 +2005,7 @@ void FiniteElementSpace::DerefinementOperator
         x.GetSubVector(f_vdofs, loc_x);
         loc_x_mat.UseExternalData(loc_x.GetData(), f_vdofs.Size()/fine_vdim,
                                   fine_vdim);
-         mfem::AddMult(lR, loc_x_mat, loc_y_mat);
+         AddMult(lR, loc_x_mat, loc_y_mat);
      }
      y.SetSubVector(c_vdofs, loc_y);
   }
@@ -2056,7 +2056,10 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
      GetLocalDerefinementMatrices(elem_geoms[i], localR[elem_geoms[i]]);
   }

-   SparseMatrix *R = new SparseMatrix(ndofs*vdim, old_ndofs*vdim);
+   SparseMatrix *R = (elem_geoms.Size() != 1)
+                     ? new SparseMatrix(ndofs*vdim, old_ndofs*vdim) // variable row size
+                     : new SparseMatrix(ndofs*vdim, old_ndofs*vdim,
+                                        localR[elem_geoms[0]].SizeI());

   Array<int> mark(R->Height());
   mark = 0;
@@ -2066,7 +2069,6 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,

   MFEM_ASSERT(dtrans.embeddings.Size() == old_elem_dof->Size(), "");

-   bool is_dg = FEColl()->GetContType() == FiniteElementCollection::DISCONTINUOUS;
   int num_marked = 0;
   for (int k = 0; k < dtrans.embeddings.Size(); k++)
   {
@@ -2089,11 +2091,10 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
            int r = DofToVDof(dofs[i], vd);
            int m = (r >= 0) ? r : (-1 - r);

-            if (is_dg || !mark[m])
+            if (!mark[m])
            {
               lR.GetRow(i, row);
               R->SetRow(r, old_vdofs, row);
-
               mark[m] = 1;
               num_marked++;
            }
@@ -2101,11 +2102,8 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
      }
   }

-   if (!is_dg)
-   {
-      MFEM_VERIFY(num_marked == R->Height(),
-                  "internal error: not all rows of R were set.");
-   }
+   MFEM_VERIFY(num_marked == R->Height(),
+               "internal error: not all rows of R were set.");

   R->Finalize(); // no-op if fixed width
   return R;
@@ -3147,7 +3145,7 @@ const FiniteElement *FiniteElementSpace::GetFaceElement(int i) const
         break;
      case 3:
      default:
-         fe = fec->FiniteElementForGeometry(mesh->GetFaceGeometry(i));
+         fe = fec->FiniteElementForGeometry(mesh->GetFaceBaseGeometry(i));
   }

   if (NURBSext)
@@ -38,7 +38,7 @@ public:
   /// Construct an empty finite element space hierarchy. This is useful if the
   /// hierarchy is constructed by coarsening a fine space, rather than refining
   /// a coarse space.
-   FiniteElementSpaceHierarchy() = default;
+   FiniteElementSpaceHierarchy() { }

   /// @brief Constructs a space hierarchy with the given mesh and space on the
   /// coarsest level.
@@ -91,7 +91,6 @@ public:
 class ParFiniteElementSpaceHierarchy : public FiniteElementSpaceHierarchy
 {
 public:
-   ParFiniteElementSpaceHierarchy() = default;
   /// @brief Constructs a parallel space hierarchy with the given mesh and spaces
   /// on level zero.
   /** The ownership of the mesh and space may be transferred to the
@@ -1920,7 +1920,7 @@ DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
                                             FMS_NODAL_GAUSS_CLOSED, 1);
      err |= FmsDataCollectionAddField(*dc, "Coords", &fcoords);
      err |= FmsFieldSet(fcoords, fdcoords, mmesh->SpaceDimension(), FMS_BY_NODES,
-                         FMS_DOUBLE, mverts.HostRead());
+                         FMS_DOUBLE, mverts);
      err |= FmsComponentSetCoordinates(volume, fcoords);
   }

@@ -416,24 +416,13 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const

   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
+      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
+                  "invalid FE map type");
      Vector shape(dof);
-      if (FElem->GetMapType() == FiniteElement::VALUE)
+      for (k = 0; k < n; k++)
      {
-         for (k = 0; k < n; k++)
-         {
-            FElem->CalcShape(ElemVert->IntPoint(k), shape);
-            nval[k] = shape * (&loc_data[dof * vdim]);
-         }
-      }
-      else
-      {
-         ElementTransformation *Tr = fes->GetElementTransformation(i);
-         for (k = 0; k < n; k++)
-         {
-            Tr->SetIntPoint(&ElemVert->IntPoint(k));
-            FElem->CalcPhysShape(*Tr, shape);
-            nval[k] = shape * (&loc_data[dof * vdim]);
-         }
+         FElem->CalcShape(ElemVert->IntPoint(k), shape);
+         nval[k] = shape * ((const double *)loc_data + dof * vdim);
      }
   }
   else
@@ -506,7 +495,7 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
      val.SetSize(vdim);
      for (int k = 0; k < vdim; k++)
      {
-         val(k) = shape * (&loc_data[dof * k]);
+         val(k) = shape * ((const double *)loc_data + dof * k);
      }
   }
   else
@@ -1038,7 +1027,7 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
      val.SetSize(vdim);
      for (int k = 0; k < vdim; k++)
      {
-         val(k) = shape * (&loc_data[dof * k]);
+         val(k) = shape * ((const double *)loc_data + dof * k);
      }
   }
   else
@@ -1089,7 +1078,7 @@ void GridFunction::GetVectorValues(ElementTransformation &T,

         for (int k = 0; k < vdim; k++)
         {
-            vals(k,j) = shape * (&loc_data[dof * k]);
+            vals(k,j) = shape * ((const double *)loc_data + dof * k);
         }
      }
   }
@@ -1192,7 +1181,8 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
         orig_fe->CalcShape(ip, shape);
         for (d = 0; d < vdim; d++)
         {
-            loc_values(d*dof+j) = shape * (&orig_loc_values[d * odof]);
+            loc_values(d*dof+j) =
+               shape * ((const double *)orig_loc_values + d * odof) ;
         }
      }
      if (doftrans)
@@ -1234,7 +1224,8 @@ void GridFunction::GetBdrValuesFrom(const GridFunction &orig_func)
         orig_fe->CalcShape(ip, shape);
         for (d = 0; d < vdim; d++)
         {
-            loc_values(d*dof+j) = shape * (&orig_loc_values[d * odof]);
+            loc_values(d*dof+j) =
+               shape * ((const double *)orig_loc_values + d * odof);
         }
      }
      SetSubVector(vdofs, loc_values);
@@ -1441,13 +1432,21 @@ void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
   }
 }

+
 void GridFunction::GetVectorGradientHat(
   ElementTransformation &T, DenseMatrix &gh) const
 {
-   const FiniteElement *FElem = fes->GetFE(T.ElementNo);
+   int elNo = T.ElementNo;
+   const FiniteElement *FElem = fes->GetFE(elNo);
   int dim = FElem->GetDim(), dof = FElem->GetDof();
+   Array<int> vdofs;
+   DofTransformation * doftrans = fes->GetElementVDofs(elNo, vdofs);
   Vector loc_data;
-   GetElementDofValues(T.ElementNo, loc_data);
+   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
   // assuming scalar FE
   int vdim = fes->GetVDim();
   DenseMatrix dshape(dof, dim);
@@ -1652,7 +1651,6 @@ void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
         const FiniteElement *fe = fes->GetFE(T.ElementNo);
         MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
                     "invalid FE map type");
-         MFEM_ASSERT(fes->GetVDim() == 1, "Defined for scalar functions.");
         int spaceDim = fes->GetMesh()->SpaceDimension();
         int dim = fe->GetDim(), dof = fe->GetDof();
         DenseMatrix dshape(dof, dim);
@@ -1721,8 +1719,13 @@ void GridFunction::GetGradients(ElementTransformation &tr,
   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
   DenseMatrix dshape(fe->GetDof(), fe->GetDim());
   Vector lval, gh(fe->GetDim()), gcol;
-
-   GetElementDofValues(tr.ElementNo, lval);
+   Array<int> dofs;
+   DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
+   GetSubVector(dofs, lval);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(lval);
+   }
   grad.SetSize(fe->GetDim(), ir.GetNPoints());
   for (int i = 0; i < ir.GetNPoints(); i++)
   {
@@ -4096,16 +4099,16 @@ void TensorProductLegendre(int dim,                // input
   // Map x to [0, 1] to use CalcLegendre since it uses shifted Legendre Polynomials.
   double x1 = (x(0) - xmin(0))/(xmax(0)-xmin(0)), x2, x3;
   Vector poly_x(order+1), poly_y(order+1), poly_z(order+1);
-   poly1d.CalcLegendre(order, x1, poly_x.GetData());
+   poly1d.CalcLegendre(order, x1, poly_x);
   if (dim > 1)
   {
      x2 = (x(1)-xmin(1))/(xmax(1)-xmin(1));
-      poly1d.CalcLegendre(order, x2, poly_y.GetData());
+      poly1d.CalcLegendre(order, x2, poly_y);
   }
   if (dim == 3)
   {
      x3 = (x(2)-xmin(2))/(xmax(2)-xmin(2));
-      poly1d.CalcLegendre(order, x3, poly_z.GetData());
+      poly1d.CalcLegendre(order, x3, poly_z);
   }

   int basis_dimension = static_cast<int>(pow(order+1,dim));
@@ -48,6 +48,8 @@ protected:

   void SaveSTLTri(std::ostream &out, double p1[], double p2[], double p3[]);

+   void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const;
+
   // Project the delta coefficient without scaling and return the (local)
   // integral of the projection.
   void ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
@@ -327,34 +329,17 @@ public:

   void GetCurl(ElementTransformation &tr, Vector &curl) const;

-   /** @brief Gradient of a scalar function at a quadrature point.
-
-       @note It is assumed that the IntegrationPoint of interest has been
-       specified by ElementTransformation::SetIntPoint() before calling
-       GetGradient().
-
-       @note Can be used from a ParGridFunction when @a tr is an
-       ElementTransformation of a face-neighbor element and face-neighbor data
-       has been exchanged. */
   void GetGradient(ElementTransformation &tr, Vector &grad) const;

-   /// Extension of GetGradient(...) for a collection of IntegrationPoints.
   void GetGradients(ElementTransformation &tr, const IntegrationRule &ir,
                     DenseMatrix &grad) const;

-   /// Extension of GetGradient(...) for a collection of IntegrationPoints.
   void GetGradients(const int elem, const IntegrationRule &ir,
                     DenseMatrix &grad) const
   { GetGradients(*fes->GetElementTransformation(elem), ir, grad); }

-   /** @brief Compute the vector gradient with respect to the physical element
-       variable. */
   void GetVectorGradient(ElementTransformation &tr, DenseMatrix &grad) const;

-   /** @brief Compute the vector gradient with respect to the reference element
-       variable. */
-   void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const;
-
   /** Compute \f$ (\int_{\Omega} (*this) \psi_i)/(\int_{\Omega} \psi_i) \f$,
       where \f$ \psi_i \f$ are the basis functions for the FE space of avgs.
       Both FE spaces should be scalar and on the same mesh. */
@@ -37,7 +37,7 @@ FindPointsGSLIB::FindPointsGSLIB()
     fec_map_lin(NULL),
     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
+     avgtype(AvgType::ARITHMETIC)
 {
   mesh_split.SetSize(4);
   ir_split.SetSize(4);
@@ -84,7 +84,7 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
     fec_map_lin(NULL),
     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
+     avgtype(AvgType::ARITHMETIC)
 {
   mesh_split.SetSize(4);
   ir_split.SetSize(4);
@@ -179,27 +179,24 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
   gsl_ref.SetSize(points_cnt * dim);
   gsl_dist.SetSize(points_cnt);

-   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
+   const double *xv_base[dim];
+   unsigned xv_stride[dim];
+   for (int d = 0; d < dim; d++)
   {
-      for (int d = 0; d < dim; d++)
+      if (point_pos_ordering == Ordering::byNODES)
      {
-         if (point_pos_ordering == Ordering::byNODES)
-         {
-            xv_base[d] = point_pos.GetData() + d*points_cnt;
-            xv_stride[d] = sizeof(double);
-         }
-         else
-         {
-            xv_base[d] = point_pos.GetData() + d;
-            xv_stride[d] = dim*sizeof(double);
-         }
+         xv_base[d] = point_pos.GetData() + d*points_cnt;
+         xv_stride[d] = sizeof(double);
      }
-   };
+      else
+      {
+         xv_base[d] = point_pos.GetData() + d;
+         xv_stride[d] = dim*sizeof(double);
+      }
+   }
+
   if (dim == 2)
   {
-      const double *xv_base[2];
-      unsigned xv_stride[2];
-      xvFill(xv_base, xv_stride, dim);
      findpts_2(gsl_code.GetData(), sizeof(unsigned int),
                gsl_proc.GetData(), sizeof(unsigned int),
                gsl_elem.GetData(), sizeof(unsigned int),
@@ -207,11 +204,8 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
                gsl_dist.GetData(), sizeof(double),
                xv_base, xv_stride, points_cnt, fdata2D);
   }
-   else  // dim == 3
+   else
   {
-      const double *xv_base[3];
-      unsigned xv_stride[3];
-      xvFill(xv_base, xv_stride, dim);
      findpts_3(gsl_code.GetData(), sizeof(unsigned int),
                gsl_proc.GetData(), sizeof(unsigned int),
                gsl_elem.GetData(), sizeof(unsigned int),
@@ -223,12 +217,10 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
   // Set the element number and reference position to 0 for points not found
   for (int i = 0; i < points_cnt; i++)
   {
-      if (gsl_code[i] == 2 ||
-          (gsl_code[i] == 1 && gsl_dist(i) > bdr_tol))
+      if (gsl_code[i] == 2)
      {
         gsl_elem[i] = 0;
         for (int d = 0; d < dim; d++) { gsl_ref(i*dim + d) = -1.; }
-         gsl_code[i] = 2;
      }
   }

@@ -573,7 +565,7 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
   const int pts_el = std::pow(dof_1D, dim);
   const int pts_cnt = NE_split_total * pts_el;
   node_vals.SetSize(vdim * pts_cnt);
-   node_vals = 0.0;
+   node_vals *= 0;

   int gsl_mesh_pt_index = 0;

@@ -1088,7 +1080,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,

            sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
            sdpt = (struct send_pt *)sendpt->ptr;
-            for (int index = 0; index < static_cast<int>(sendpt->n); index++)
+            for (int index = 0; index < sendpt->n; index++)
            {
               int idx = field_in.FESpace()->GetOrdering() == Ordering::byNODES ?
                         sdpt->index + j*nptorig :
@@ -1155,7 +1147,7 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
   distfint.SetSize(pts_cnt);
   if (!gfmax)
   {
-      distfint = 0.0;
+      distfint = 0.;
   }
   else
   {
@@ -1202,27 +1194,24 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
   gsl_ref.SetSize(points_cnt * dim);
   gsl_dist.SetSize(points_cnt);

-   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
+   const double *xv_base[dim];
+   unsigned xv_stride[dim];
+   for (int d = 0; d < dim; d++)
   {
-      for (int d = 0; d < dim; d++)
+      if (point_pos_ordering == Ordering::byNODES)
      {
-         if (point_pos_ordering == Ordering::byNODES)
-         {
-            xv_base[d] = point_pos.GetData() + d*points_cnt;
-            xv_stride[d] = sizeof(double);
-         }
-         else
-         {
-            xv_base[d] = point_pos.GetData() + d;
-            xv_stride[d] = dim*sizeof(double);
-         }
+         xv_base[d] = point_pos.GetData() + d*points_cnt;
+         xv_stride[d] = sizeof(double);
      }
-   };
+      else
+      {
+         xv_base[d] = point_pos.GetData() + d;
+         xv_stride[d] = dim*sizeof(double);
+      }
+   }
+
   if (dim == 2)
   {
-      const double *xv_base[2];
-      unsigned xv_stride[2];
-      xvFill(xv_base, xv_stride, dim);
      findptsms_2(gsl_code.GetData(), sizeof(unsigned int),
                  gsl_proc.GetData(), sizeof(unsigned int),
                  gsl_elem.GetData(), sizeof(unsigned int),
@@ -1232,11 +1221,8 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
                  point_id.GetData(), sizeof(unsigned int), &match,
                  points_cnt, fdata2D);
   }
-   else  // dim == 3
+   else
   {
-      const double *xv_base[3];
-      unsigned xv_stride[3];
-      xvFill(xv_base, xv_stride, dim);
      findptsms_3(gsl_code.GetData(), sizeof(unsigned int),
                  gsl_proc.GetData(), sizeof(unsigned int),
                  gsl_elem.GetData(), sizeof(unsigned int),
@@ -1250,12 +1236,10 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
   // Set the element number and reference position to 0 for points not found
   for (int i = 0; i < points_cnt; i++)
   {
-      if (gsl_code[i] == 2 ||
-          (gsl_code[i] == 1 && gsl_dist(i) > bdr_tol))
+      if (gsl_code[i] == 2)
      {
         gsl_elem[i] = 0;
         for (int d = 0; d < dim; d++) { gsl_ref(i*dim + d) = -1.; }
-         gsl_code[i] = 2;
      }
   }

@@ -38,11 +38,6 @@ namespace mfem
 *     coordinates inside the element that each point is located in. gslib also
 *     returns a code that indicates whether the point was found inside an
 *     element, on element border, or not found in the domain.
- *     For points returned as found on `element border`, the point is either
- *     on an element edge/face or near the domain boundary, and gslib also
- *     returns a distance to the border. Points near (but outside) the domain
- *     boundary must then be marked as not found using the distance returned
- *     by gslib.
 *
 *  3. Interpolate - Interpolates any grid function at the points found using 2.
 *
@@ -75,8 +70,6 @@ protected:
   Array<int> split_element_map;
   Array<int> split_element_index;
   int        NE_split_total;
-   // Tolerance to ignore points just outside elements at the boundary.
-   double     bdr_tol;

   /// Use GSLIB for communication and interpolation
   virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out);
@@ -188,14 +181,6 @@ public:
      default_interp_value = interp_value_;
   }

-   /// Set the tolerance for detecting points outside the 'curvilinear' boundary
-   /// that gslib may return as found on the boundary. Points found on boundary
-   /// with distance greater than @ bdr_tol are marked as not found.
-   virtual void SetDistanceToleranceForPointsFoundOnBoundary(double bdr_tol_)
-   {
-      bdr_tol = bdr_tol_;
-   }
-
   /** Cleans up memory allocated internally by gslib.
       Note that in parallel, this must be called before MPI_Finalize(), as it
       calls MPI_Comm_free() for internal gslib communicators. */
@@ -1032,12 +1032,12 @@ void WhiteGaussianNoiseDomainLFIntegrator::AssembleRHSElementVect
      massinteg.AssembleElementMatrix(el, Tr, *M);
      CholeskyFactors chol(M->Data());
      chol.Factor(M->Height());
-      chol.LMult(n,1,elvect.GetData());
+      chol.LMult(n,1,elvect);
   }
   else
   {
      CholeskyFactors chol(L[iel]->Data());
-      chol.LMult(n,1,elvect.GetData());
+      chol.LMult(n,1,elvect);
   }
 }

@@ -365,8 +365,11 @@ FiniteElementSpace &LORBase::GetFESpace() const

 void LORBase::AssembleSystem(BilinearForm &a_ho, const Array<int> &ess_dofs)
 {
-   A.Clear();
-   delete a;
+   if (a)
+   {
+      A.Clear();
+      delete a;
+   }
   if (BatchedLORAssembly::FormIsSupported(a_ho))
   {
      // Skip forming the space
@@ -13,6 +13,9 @@
 #include "../../general/forall.hpp"
 #include "../../fem/pbilinearform.hpp"

+#define MFEM_NVTX_COLOR DeepSkyBlue
+#include "../../general/nvtx.hpp"
+
 namespace mfem
 {

@@ -78,8 +81,10 @@ void BatchedLOR_ADS::Form3DFaceToEdge(Array<int> &face2edge)
   }
 }

-void BatchedLOR_ADS::FormCurlMatrix()
+void BatchedLOR_ADS::FormCurlMatrixLocal()
 {
+   NVTX("Discrete Curl");
+
   // The curl matrix maps from LOR edges to LOR faces. Given a quadrilateral
   // face (defined by its four edges) f_i = (e_j1, e_j2, e_j3, e_j4), the
   // matrix has nonzeros A(i, jk), so there are always exactly four nonzeros
@@ -87,10 +92,9 @@ void BatchedLOR_ADS::FormCurlMatrix()
   const int nface_dof = face_fes.GetNDofs();
   const int nedge_dof = edge_fes.GetNDofs();

-   SparseMatrix C_local;
   C_local.OverrideSize(nface_dof, nedge_dof);
-
-   C_local.GetMemoryI().New(nedge_dof+1, Device::GetDeviceMemoryType());
+   EnsureCapacity(C_local.GetMemoryI(), nedge_dof+1,
+                  Device::GetDeviceMemoryType());
   // Each row always has four nonzeros
   const int nnz = 4*nedge_dof;
   auto I = C_local.WriteI();
@@ -120,8 +124,8 @@ void BatchedLOR_ADS::FormCurlMatrix()
   const auto f2e = Reshape(face2edge.Read(), 4, nface_per_el);

   // Fill J and data
-   C_local.GetMemoryJ().New(nnz, Device::GetDeviceMemoryType());
-   C_local.GetMemoryData().New(nnz, Device::GetDeviceMemoryType());
+   EnsureCapacity(C_local.GetMemoryJ(), nnz, Device::GetDeviceMemoryType());
+   EnsureCapacity(C_local.GetMemoryData(), nnz, Device::GetDeviceMemoryType());

   auto J = C_local.WriteJ();
   auto V = C_local.WriteData();
@@ -146,6 +150,11 @@ void BatchedLOR_ADS::FormCurlMatrix()
         V[i*4 + k] = sgn*sgn_f*sgn_e;
      }
   });
+}
+
+void BatchedLOR_ADS::FormCurlMatrix()
+{
+   FormCurlMatrixLocal();

   // Create a block diagonal parallel matrix
   OperatorHandle C_diag(Operator::Hypre_ParCSR);
@@ -179,6 +188,8 @@ void BatchedLOR_ADS::FormCurlMatrix()
   }
   C->CopyRowStarts();
   C->CopyColStarts();
+
+   C_local.Clear();
 }

 HypreParMatrix *BatchedLOR_ADS::StealCurlMatrix()
@@ -36,7 +36,9 @@ protected:
   ND_FECollection edge_fec; ///< The associated Nedelec collection.
   ParFiniteElementSpace edge_fes; ///< The associated Nedelec space.
   BatchedLOR_AMS ams; ///< The associated AMS object.
-   HypreParMatrix *C; ///< The discrete curl matrix.
+   HypreParMatrix *C = nullptr; ///< The discrete curl matrix.
+
+   SparseMatrix C_local;

   /// Form the local elementwise discrete curl matrix.
   void Form3DFaceToEdge(Array<int> &face2edge);
@@ -64,6 +66,9 @@ public:

   /// Form the discrete curl matrix (not part of the public API).
   void FormCurlMatrix();
+
+   void FormCurlMatrixLocal();
+
   ~BatchedLOR_ADS();
 };

@@ -13,6 +13,9 @@
 #include "../../general/forall.hpp"
 #include "../../fem/pbilinearform.hpp"

+#define MFEM_NVTX_COLOR DeepSkyBlue
+#include "../../general/nvtx.hpp"
+
 namespace mfem
 {

@@ -136,8 +139,9 @@ void BatchedLOR_AMS::Form3DEdgeToVertex(Array<int> &edge2vert)
   }
 }

-void BatchedLOR_AMS::FormGradientMatrix()
+void BatchedLOR_AMS::FormGradientMatrixLocal()
 {
+   NVTX("Discrete Gradient");
   // The gradient matrix maps from LOR vertices to LOR edges. Given an edge
   // (defined by its two vertices) e_i = (v_j1, v_j2), the matrix has nonzeros
   // A(i, j1) = -1 and A(i, j2) = 1, so there are always exactly two nonzeros
@@ -145,10 +149,10 @@ void BatchedLOR_AMS::FormGradientMatrix()
   const int nedge_dof = edge_fes.GetNDofs();
   const int nvert_dof = vert_fes.GetNDofs();

-   SparseMatrix G_local;
   G_local.OverrideSize(nedge_dof, nvert_dof);
+   EnsureCapacity(G_local.GetMemoryI(), nedge_dof+1,
+                  Device::GetDeviceMemoryType());

-   G_local.GetMemoryI().New(nedge_dof+1, Device::GetDeviceMemoryType());
   // Each row always has two nonzeros
   const int nnz = 2*nedge_dof;
   auto I = G_local.WriteI();
@@ -180,8 +184,8 @@ void BatchedLOR_AMS::FormGradientMatrix()
   const auto e2v = Reshape(edge2vertex.Read(), 2, nedge_per_el);

   // Fill J and data
-   G_local.GetMemoryJ().New(nnz, Device::GetDeviceMemoryType());
-   G_local.GetMemoryData().New(nnz, Device::GetDeviceMemoryType());
+   EnsureCapacity(G_local.GetMemoryJ(), nnz, Device::GetDeviceMemoryType());
+   EnsureCapacity(G_local.GetMemoryData(), nnz, Device::GetDeviceMemoryType());

   auto J = G_local.WriteJ();
   auto V = G_local.WriteData();
@@ -204,6 +208,11 @@ void BatchedLOR_AMS::FormGradientMatrix()
      V[i*2 + 0] = -sgn;
      V[i*2 + 1] = sgn;
   });
+}
+
+void BatchedLOR_AMS::FormGradientMatrix()
+{
+   FormGradientMatrixLocal();

   // Create a block diagonal parallel matrix
   OperatorHandle G_diag(Operator::Hypre_ParCSR);
@@ -237,6 +246,8 @@ void BatchedLOR_AMS::FormGradientMatrix()
   }
   G->CopyRowStarts();
   G->CopyColStarts();
+
+   G_local.Clear();
 }

 template <typename T>
@@ -281,7 +292,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const MemoryClass mc = GetHypreMemoryClass();
   bool dev = (mc == MemoryClass::DEVICE);

-   xyz_tvec = new Vector(ntdofs*dim);
+   if (xyz_tvec == nullptr) { xyz_tvec = new Vector(ntdofs*dim); }

   auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, dim);
   const auto xyz_e =
@@ -302,21 +313,24 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   });

   // Make x, y, z HypreParVectors point to T-vector data
-   HYPRE_BigInt glob_size = vert_fes.GlobalTrueVSize();
-   HYPRE_BigInt *cols = vert_fes.GetTrueDofOffsets();
+   if (x == nullptr)
+   {
+      HYPRE_BigInt glob_size = vert_fes.GlobalTrueVSize();
+      HYPRE_BigInt *cols = vert_fes.GetTrueDofOffsets();

-   double *d_x_ptr = xyz_tv + 0*ntdofs;
-   x = new HypreParVector(vert_fes.GetComm(), glob_size, d_x_ptr, cols, dev);
-   double *d_y_ptr = xyz_tv + 1*ntdofs;
-   y = new HypreParVector(vert_fes.GetComm(), glob_size, d_y_ptr, cols, dev);
-   if (dim == 3)
-   {
-      double *d_z_ptr = xyz_tv + 2*ntdofs;
-      z = new HypreParVector(vert_fes.GetComm(), glob_size, d_z_ptr, cols, dev);
-   }
-   else
-   {
-      z = NULL;
+      double *d_x_ptr = xyz_tv + 0*ntdofs;
+      x = new HypreParVector(vert_fes.GetComm(), glob_size, d_x_ptr, cols, dev);
+      double *d_y_ptr = xyz_tv + 1*ntdofs;
+      y = new HypreParVector(vert_fes.GetComm(), glob_size, d_y_ptr, cols, dev);
+      if (dim == 3)
+      {
+         double *d_z_ptr = xyz_tv + 2*ntdofs;
+         z = new HypreParVector(vert_fes.GetComm(), glob_size, d_z_ptr, cols, dev);
+      }
+      else
+      {
+         z = NULL;
+      }
   }
 }

@@ -33,12 +33,14 @@ protected:
   const int order; ///< Polynomial degree.
   H1_FECollection vert_fec; ///< The corresponding H1 collection.
   ParFiniteElementSpace vert_fes; ///< The corresponding H1 space.
-   Vector *xyz_tvec; ///< Mesh vertex coordinates in true-vector format.
-   HypreParMatrix *G; ///< Discrete gradient matrix.
+   Vector *xyz_tvec = nullptr; ///< Mesh vertex coordinates in true-vector format.
+   HypreParMatrix *G = nullptr; ///< Discrete gradient matrix.
+
+   SparseMatrix G_local;

   /// @name Mesh coordinate vectors in HypreParVector format
   ///@{
-   HypreParVector *x, *y, *z;
+   HypreParVector *x = nullptr, *y = nullptr, *z = nullptr;
   ///@}

   /// @name Construct the local (elementwise) discrete gradient
@@ -89,6 +91,9 @@ public:

   /// Construct the discrete gradient matrix (not part of the public API).
   void FormGradientMatrix();
+
+   void FormGradientMatrixLocal();
+
   ~BatchedLOR_AMS();
 };

@@ -15,6 +15,8 @@
 #include <climits>
 #include "../pbilinearform.hpp"

+#include "../../general/nvtx.hpp"
+
 // Specializations
 #include "lor_h1.hpp"
 #include "lor_nd.hpp"
@@ -70,8 +72,13 @@ bool BatchedLORAssembly::FormIsSupported(BilinearForm &a)
 }

 void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
-                                                  Vector &X_vert)
+                                                  Vector &X_vert,
+                                                  Vector *evec)
 {
+#undef MFEM_NVTX_COLOR
+#define MFEM_NVTX_COLOR DeepSkyBlue
+   NVTX("LOR Coordinates");
+
   Mesh &mesh_ho = *fes_ho.GetMesh();
   mesh_ho.EnsureNodes();

@@ -87,9 +94,22 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
   const Operator *nodal_restriction =
      nodal_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);

+   Vector *tmp_evec = nullptr;
+   Vector *nodal_evec;
+
+   if (evec)
+   {
+      nodal_evec = evec;
+      nodal_evec->SetSize(nodal_restriction->Height());
+   }
+   else
+   {
+      tmp_evec = new Vector(nodal_restriction->Height());
+      nodal_evec = tmp_evec;
+   }
+
   // Map from nodal L-vector to E-vector
-   Vector nodal_evec(nodal_restriction->Height());
-   nodal_restriction->Mult(*nodal_gf, nodal_evec);
+   nodal_restriction->Mult(*nodal_gf, *nodal_evec);

   IntegrationRule ir = GetCollocatedIntRule(fes_ho);

@@ -98,7 +118,9 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
   const QuadratureInterpolator *quad_interp =
      nodal_fes->GetQuadratureInterpolator(ir);
   quad_interp->SetOutputLayout(QVectorLayout::byVDIM);
-   quad_interp->Values(nodal_evec, X_vert);
+   quad_interp->Values(*nodal_evec, X_vert);
+
+   delete tmp_evec;
 }

 // The following two functions (GetMinElt and GetAndIncrementNnzIndex) are
@@ -372,11 +394,11 @@ void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const

   A_mat->OverrideSize(nvdof, nvdof);

-   A_mat->GetMemoryI().New(nvdof+1, Device::GetDeviceMemoryType());
+   EnsureCapacity(A_mat->GetMemoryI(), nvdof+1, Device::GetDeviceMemoryType());
   int nnz = FillI(*A_mat);

-   A_mat->GetMemoryJ().New(nnz, Device::GetDeviceMemoryType());
-   A_mat->GetMemoryData().New(nnz, Device::GetDeviceMemoryType());
+   EnsureCapacity(A_mat->GetMemoryJ(), nnz, Device::GetDeviceMemoryType());
+   EnsureCapacity(A_mat->GetMemoryData(), nnz, Device::GetDeviceMemoryType());
   FillJAndData(*A_mat);
 }

@@ -455,7 +477,6 @@ void BatchedLORAssembly::ParAssemble(
   BilinearForm &a, const Array<int> &ess_dofs, OperatorHandle &A)
 {
   // Assemble the system matrix local to this partition
-   OperatorHandle A_local;
   AssembleWithoutBC(a, A_local);

   ParBilinearForm *pa =
@@ -471,6 +492,9 @@ void BatchedLORAssembly::ParAssemble(
 void BatchedLORAssembly::Assemble(
   BilinearForm &a, const Array<int> ess_dofs, OperatorHandle &A)
 {
+#undef MFEM_NVTX_COLOR
+#define MFEM_NVTX_COLOR NavyBlue
+   NVTX("LOR Assemble");
 #ifdef MFEM_USE_MPI
   if (dynamic_cast<ParFiniteElementSpace*>(&fes_ho))
   {
@@ -53,6 +53,8 @@ protected:
   /// nonzero).
   Array<int> sparse_mapping;

+   OperatorHandle A_local; // Cache this
+
 public:
   /// Construct the batched assembly object corresponding to @a fes_ho_.
   BatchedLORAssembly(FiniteElementSpace &fes_ho_);
@@ -68,12 +70,12 @@ public:

   /// Compute the vertices of the LOR mesh and place the result in @a X_vert.
   static void FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
-                                        Vector &X_vert);
+                                        Vector &X_vert,
+                                        Vector *evec = nullptr);

   /// Return the vertices of the LOR mesh in E-vector format
   const Vector &GetLORVertexCoordinates() { return X_vert; }

-protected:
   /// After assembling the "sparse IJ" format, convert it to CSR.
   void SparseIJToCSR(OperatorHandle &A) const;

@@ -117,12 +119,12 @@ public:
 /// If the capacity of @a mem is not large enough, delete it and allocate new
 /// memory with size @a capacity.
 template <typename T>
-void EnsureCapacity(Memory<T> &mem, int capacity)
+void EnsureCapacity(Memory<T> &mem, int capacity, MemoryType mt)
 {
   if (mem.Capacity() < capacity)
   {
      mem.Delete();
-      mem.New(capacity, mem.GetMemoryType());
+      mem.New(capacity, mt);
   }
 }

@@ -205,6 +205,49 @@ void BatchedLOR_H1::Assemble2D()
   }
 }

+template<int ORDER>
+static void SparseMapping3D(Array<int> &sparse_mapping)
+{
+   static constexpr int nnz_per_row = 27;
+   static constexpr int nd1d = ORDER + 1;
+   static constexpr int ndof_per_el = nd1d*nd1d*nd1d;
+
+   sparse_mapping.SetSize(nnz_per_row*ndof_per_el);
+   sparse_mapping = -1;
+   auto map = Reshape(sparse_mapping.HostReadWrite(), nnz_per_row, ndof_per_el);
+
+   for (int iz=0; iz<nd1d; ++iz)
+   {
+      const int jz_begin = (iz > 0) ? iz - 1 : 0;
+      const int jz_end = (iz < ORDER) ? iz + 1 : ORDER;
+      for (int iy=0; iy<nd1d; ++iy)
+      {
+         const int jy_begin = (iy > 0) ? iy - 1 : 0;
+         const int jy_end = (iy < ORDER) ? iy + 1 : ORDER;
+         for (int ix=0; ix<nd1d; ++ix)
+         {
+            const int jx_begin = (ix > 0) ? ix - 1 : 0;
+            const int jx_end = (ix < ORDER) ? ix + 1 : ORDER;
+
+            const int ii_el = ix + nd1d*(iy + nd1d*iz);
+
+            for (int jz=jz_begin; jz<=jz_end; ++jz)
+            {
+               for (int jy=jy_begin; jy<=jy_end; ++jy)
+               {
+                  for (int jx=jx_begin; jx<=jx_end; ++jx)
+                  {
+                     const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
+                     const int jj_el = jx + nd1d*(jy + nd1d*jz);
+                     map(jj_off, ii_el) = jj_el;
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
 template <int ORDER>
 void BatchedLOR_H1::Assemble3D()
 {
@@ -330,21 +373,16 @@ void BatchedLOR_H1::Assemble3D()
                  }
               }

-               //MFEM_UNROLL(2)
               for (int iqx=0; iqx<2; ++iqx)
               {
-                  //MFEM_UNROLL(2)
                  for (int jz=0; jz<2; ++jz)
                  {
                     // Note loop starts at iz=jz here, taking advantage of
                     // symmetries.
-                     //MFEM_UNROLL(2)
                     for (int iz=jz; iz<2; ++iz)
                     {
-                        //MFEM_UNROLL(2)
                        for (int iqy=0; iqy<2; ++iqy)
                        {
-                           //MFEM_UNROLL(2)
                           for (int iqz=0; iqz<2; ++iqz)
                           {
                              const double mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
@@ -379,10 +417,8 @@ void BatchedLOR_H1::Assemble3D()
                              double wdetJ = Q(6,iqz,iqy,iqx);
                              mass_A(iqy,iz,jz,iqx) += mq*wdetJ*biz*bjz;
                           }
-                           //MFEM_UNROLL(2)
                           for (int jy=0; jy<2; ++jy)
                           {
-                              //MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
                                 const double biy = (iy == iqy) ? 1.0 : 0.0;
@@ -405,16 +441,12 @@ void BatchedLOR_H1::Assemble3D()
                              }
                           }
                        }
-                        //MFEM_UNROLL(2)
                        for (int jy=0; jy<2; ++jy)
                        {
-                           //MFEM_UNROLL(2)
                           for (int jx=0; jx<2; ++jx)
                           {
-                              //MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
-                                 //MFEM_UNROLL(2)
                                 for (int ix=0; ix<2; ++ix)
                                 {
                                    const double bix = (ix == iqx) ? 1.0 : 0.0;
@@ -482,40 +514,176 @@ void BatchedLOR_H1::Assemble3D()
         }
      }
   });
+   SparseMapping3D<ORDER>(sparse_mapping);
+}

-   sparse_mapping.SetSize(nnz_per_row*ndof_per_el);
-   sparse_mapping = -1;
-   auto map = Reshape(sparse_mapping.HostReadWrite(), nnz_per_row, ndof_per_el);
-   for (int iz=0; iz<nd1d; ++iz)
+template <>
+void BatchedLOR_H1::Assemble3D<1>()
+{
+   static constexpr int nv = 8;
+   static constexpr int nd1d = 2;
+   static constexpr int ndof_per_el = 8;
+   static constexpr int nnz_per_row = 27;
+   static constexpr int sz_local_mat = nv*nv;
+
+   const int nel_ho = fes_ho.GetNE();
+
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1, 1)
+                   : Reshape(c1.Read(), nd1d, nd1d, nd1d, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1, 1)
+                   : Reshape(c2.Read(), nd1d, nd1d, nd1d, nel_ho);
+
+   sparse_ij.SetSize(nel_ho*ndof_per_el*nnz_per_row);
+   auto V = Reshape(sparse_ij.Write(), nnz_per_row, nd1d, nd1d, nd1d, nel_ho);
+
+   const auto X = X_vert.Read();
+
+   MFEM_FORALL_3D(iel_ho, nel_ho, 8, 4, 1,
   {
-      const int jz_begin = (iz > 0) ? iz - 1 : 0;
-      const int jz_end = (iz < ORDER) ? iz + 1 : ORDER;
-      for (int iy=0; iy<nd1d; ++iy)
+      static constexpr int e[8] = {0,1,3,2,4,5,7,6};
+      MFEM_SHARED double vx[8], vy[8], vz[8];
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED double local_mat_[sz_local_mat];
+      DeviceTensor<4> local_mat(local_mat_, 2,2,2, nv);
+
+      if (tidz == 0)
      {
-         const int jy_begin = (iy > 0) ? iy - 1 : 0;
-         const int jy_end = (iy < ORDER) ? iy + 1 : ORDER;
-         for (int ix=0; ix<nd1d; ++ix)
+         MFEM_FOREACH_THREAD(xyz,x,8)
         {
-            const int jx_begin = (ix > 0) ? ix - 1 : 0;
-            const int jx_end = (ix < ORDER) ? ix + 1 : ORDER;
-
-            const int ii_el = ix + nd1d*(iy + nd1d*iz);
-
-            for (int jz=jz_begin; jz<=jz_end; ++jz)
+            const int z = xyz%2, y = (xyz/2)%2, x = xyz/2/2;
+            MFEM_FOREACH_THREAD(j,y,nnz_per_row)
            {
-               for (int jy=jy_begin; jy<=jy_end; ++jy)
+               if (j < 8) { local_mat(z,y,x,j) = 0.0; }
+               V(j,x,y,z,iel_ho) = 0.0;
+               if (j == 0)
               {
-                  for (int jx=jx_begin; jx<=jx_end; ++jx)
-                  {
-                     const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
-                     const int jj_el = jx + nd1d*(jy + nd1d*jz);
-                     map(jj_off, ii_el) = jj_el;
-                  }
+                  const int i = x + 2*y + 4*z;
+                  const int ei = 3*(e[i] + 8*iel_ho);
+                  vx[i] = X[ei + 0];
+                  vy[i] = X[ei + 1];
+                  vz[i] = X[ei + 2];
               }
            }
         }
      }
-   }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(xyz,x,8)
+      {
+         const int qz = xyz%2, qy = (xyz/2)%2, qx = xyz/2/2;
+         static constexpr double w = 1.0/8.0;
+
+         double J_[3*3];
+         DeviceTensor<2> J(J_, 3,3);
+         Jacobian3D(qx,qy,qz, vx,vy,vz, J);
+         const double detJ = Det3D(J);
+         const double w_detJ = w/detJ;
+
+         // adj(J)
+         double A_[3*3];
+         DeviceTensor<2> A(A_, 3, 3);
+         Adjugate3D(J, A);
+
+         const double J11 = w_detJ*(A(0,0)*A(0,0)+A(0,1)*A(0,1)+A(0,2)*A(0,2)); // 1,1
+         const double J21 = w_detJ*(A(0,0)*A(1,0)+A(0,1)*A(1,1)+A(0,2)*A(1,2)); // 2,1
+         const double J31 = w_detJ*(A(0,0)*A(2,0)+A(0,1)*A(2,1)+A(0,2)*A(2,2)); // 3,1
+         const double J12 = J21;
+         const double J22 = w_detJ*(A(1,0)*A(1,0)+A(1,1)*A(1,1)+A(1,2)*A(1,2)); // 2,2
+         const double J32 = w_detJ*(A(1,0)*A(2,0)+A(1,1)*A(2,1)+A(1,2)*A(2,2)); // 3,2
+         const double J13 = J31;
+         const double J23 = J32;
+         const double J33 = w_detJ*(A(2,0)*A(2,0)+A(2,1)*A(2,1)+A(2,2)*A(2,2)); // 3,3
+         const double wdetJ =  w*detJ;
+
+         const double mq = const_mq ? MQ(0,0,0,0) : MQ(qx,qy,qz, iel_ho);
+         const double dq = const_dq ? DQ(0,0,0,0) : DQ(qx,qy,qz, iel_ho);
+
+         MFEM_FOREACH_THREAD(xyz,y,8)
+         {
+            const int jz = xyz%2, jy = (xyz/2)%2, jx = xyz/2/2;
+            const double bjz = (jz == qz) ? 1.0 : 0.0;
+            const double gjz = (jz == 0) ? -1.0 : 1.0;
+            const double bjy = (jy == qy) ? 1.0 : 0.0;
+            const double gjy = (jy == 0) ? -1.0 : 1.0;
+            const double bjx = (jx == qx) ? 1.0 : 0.0;
+            const double gjx = (jx == 0) ? -1.0 : 1.0;
+            const double djx = gjx*bjy*bjz;
+            const double djy = bjx*gjy*bjz;
+            const double djz = bjx*bjy*gjz;
+            const int jj_loc = jx + 2*jy + 4*jz;
+
+            MFEM_FOREACH_THREAD(xyz,z,8)
+            {
+               const int iz = xyz%2, iy = (xyz/2)%2, ix = xyz/2/2;
+               const double biz = (iz == qz) ? 1.0 : 0.0;
+               const double giz = (iz == 0) ? -1.0 : 1.0;
+               const double biy = (iy == qy) ? 1.0 : 0.0;
+               const double giy = (iy == 0) ? -1.0 : 1.0;
+               const double bix = (ix == qx) ? 1.0 : 0.0;
+               const double gix = (ix == 0) ? -1.0 : 1.0;
+               const double dix = gix*biy*biz;
+               const double diy = bix*giy*biz;
+               const double diz = bix*biy*giz;
+               const int ii_loc = ix + 2*iy + 4*iz;
+
+               // Only store the lower-triangular part of
+               // the matrix (by symmetry).
+               if (jj_loc > ii_loc) { continue; }
+
+               double grad_grad = 0.0;
+               grad_grad += dix*djx*J11;
+               grad_grad += diy*djx*J12;
+               grad_grad += diz*djx*J13;
+
+               grad_grad += dix*djy*J21;
+               grad_grad += diy*djy*J22;
+               grad_grad += diz*djy*J23;
+
+               grad_grad += dix*djz*J31;
+               grad_grad += diy*djz*J32;
+               grad_grad += diz*djz*J33;
+
+               const double basis_basis = wdetJ*bix*biy*biz*bjx*bjy*bjz;
+               const double value = dq*grad_grad + mq*basis_basis;
+               AtomicAdd(local_mat(iz,iy,ix, jj_loc), value);
+            } // i
+         } // j
+      } // q
+      MFEM_SYNC_THREAD;
+
+      // Assemble the local matrix into the macro-element sparse matrix
+      // in a format similar to coordinate format.
+      // The (I,J) arrays are implicit (not stored explicitly).
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(xyz,x,8)
+         {
+            const int iz = xyz%2, iy = (xyz/2)%2, ix = xyz/2/2;
+            const int ii_loc = ix + 2*iy + 4*iz;
+            MFEM_FOREACH_THREAD(jj_loc,y,8)
+            {
+               const int jx = jj_loc%2, jy = (jj_loc/2)%2, jz = jj_loc/2/2;
+               const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
+
+               if (jj_loc <= ii_loc)
+               {
+                  AtomicAdd(V(jj_off, ix,iy,iz, iel_ho), local_mat(iz,iy,ix, jj_loc));
+               }
+               else
+               {
+                  AtomicAdd(V(jj_off, ix,iy,iz, iel_ho), local_mat(jz,jy,jx, ii_loc));
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+   SparseMapping3D<1>(sparse_mapping);
 }

 // Explicit template instantiations
@@ -528,7 +696,7 @@ template void BatchedLOR_H1::Assemble2D<6>();
 template void BatchedLOR_H1::Assemble2D<7>();
 template void BatchedLOR_H1::Assemble2D<8>();

-template void BatchedLOR_H1::Assemble3D<1>();
+//template void BatchedLOR_H1::Assemble3D<1>(); // explicitly specialized
 template void BatchedLOR_H1::Assemble3D<2>();
 template void BatchedLOR_H1::Assemble3D<3>();
 template void BatchedLOR_H1::Assemble3D<4>();
@@ -216,6 +216,90 @@ void BatchedLOR_ND::Assemble2D()
   }
 }

+
+template<int ORDER>
+static void SparseMapping3D(Array<int> &sparse_mapping)
+{
+   static constexpr int nnz_per_row = 33;
+   static constexpr int dim = 3;
+   static constexpr int o = ORDER;
+   static constexpr int op1 = ORDER + 1;
+   static constexpr int ndof_per_el = dim*o*op1*op1;
+
+   sparse_mapping.SetSize(nnz_per_row*ndof_per_el);
+   sparse_mapping = -1;
+   auto map = Reshape(sparse_mapping.HostReadWrite(), nnz_per_row, ndof_per_el);
+
+   for (int ci=0; ci<dim; ++ci)
+   {
+      const int i_off = ci*o*op1*op1;
+      const int id0 = ci;
+      const int id1 = (ci+1)%3;
+      const int id2 = (ci+2)%3;
+
+      const int nxi = (ci == 0) ? o : op1;
+      const int nyi = (ci == 1) ? o : op1;
+
+      for (int i0=0; i0<o; ++i0)
+      {
+         for (int i1=0; i1<op1; ++i1)
+         {
+            for (int i2=0; i2<op1; ++i2)
+            {
+               int ii_lex[3];
+               ii_lex[id0] = i0;
+               ii_lex[id1] = i1;
+               ii_lex[id2] = i2;
+               const int ii_el = i_off + ii_lex[0] + ii_lex[1]*nxi + ii_lex[2]*nxi*nyi;
+
+               for (int cj_rel=0; cj_rel<dim; ++cj_rel)
+               {
+                  const int cj = (ci + cj_rel) % 3;
+                  const int j_off = cj*o*op1*op1;
+
+                  const int nxj = (cj == 0) ? o : op1;
+                  const int nyj = (cj == 1) ? o : op1;
+
+                  const int j0_begin = i0;
+                  const int j0_end = (cj_rel == 0) ? i0 : i0 + 1;
+                  const int j1_begin = (i1 > 0) ? i1-1 : i1;
+                  const int j1_end = (cj_rel == 1)
+                                     ? ((i1 < o) ? i1 : i1-1)
+                                     : ((i1 < o) ? i1+1 : i1);
+                  const int j2_begin = (i2 > 0) ? i2-1 : i2;
+                  const int j2_end = (cj_rel == 2)
+                                     ? ((i2 < o) ? i2 : i2-1)
+                                     : ((i2 < o) ? i2+1 : i2);
+
+                  for (int j0=j0_begin; j0<=j0_end; ++j0)
+                  {
+                     const int d0 = j0 - i0;
+                     for (int j1=j1_begin; j1<=j1_end; ++j1)
+                     {
+                        const int d1 = j1 - i1 + 1;
+                        for (int j2=j2_begin; j2<=j2_end; ++j2)
+                        {
+                           const int d2 = j2 - i2 + 1;
+                           int jj_lex[3];
+                           jj_lex[id0] = j0;
+                           jj_lex[id1] = j1;
+                           jj_lex[id2] = j2;
+                           const int jj_el = j_off + jj_lex[0] + jj_lex[1]*nxj + jj_lex[2]*nxj*nyj;
+                           int jj_off;
+                           if (cj_rel == 0) { jj_off = d1 + 3*d2; }
+                           else if (cj_rel == 1) { jj_off = 9 + d0 + 2*d1 + 4*d2; }
+                           else /* if (cj_rel == 2) */ { jj_off = 21 + d0 + 2*d1 + 6*d2; }
+                           map(jj_off, ii_el) = jj_el;
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
 template <int ORDER>
 void BatchedLOR_ND::Assemble3D()
 {
@@ -244,7 +328,7 @@ void BatchedLOR_ND::Assemble3D()
   sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
   auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*op1*op1, dim, nel_ho);

-   auto X = X_vert.Read();
+   const auto X = X_vert.Read();

   // Last thread dimension is lowered to avoid "too many resources" error
   MFEM_FORALL_3D(iel_ho, nel_ho, ORDER, ORDER, (ORDER>6)?4:ORDER,
@@ -488,78 +572,231 @@ void BatchedLOR_ND::Assemble3D()
         }
      }
   });
+   SparseMapping3D<ORDER>(sparse_mapping);
+}

-   sparse_mapping.SetSize(nnz_per_row*ndof_per_el);
-   sparse_mapping = -1;
-   auto map = Reshape(sparse_mapping.HostReadWrite(), nnz_per_row, ndof_per_el);
-   for (int ci=0; ci<dim; ++ci)
+template<>
+void BatchedLOR_ND::Assemble3D<1>()
+{
+   static constexpr int ne = 12; // number of edges in hexahedron
+   static constexpr int dim = 3;
+   static constexpr int o = 1;
+   static constexpr int op1 = 2;
+   static constexpr int ndof_per_el = dim*o*op1*op1;
+   static constexpr int nnz_per_row = 33;
+   static constexpr int sz_local_mat = ne*ne;
+
+   const int nel_ho = fes_ho.GetNE();
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1, 1)
+                   : Reshape(c1.Read(), op1, op1, op1, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1, 1)
+                   : Reshape(c2.Read(), op1, op1, op1, nel_ho);
+
+   sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
+   auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*op1*op1, dim, nel_ho);
+
+   const auto X = X_vert.Read();
+
+   MFEM_FORALL_3D(iel_ho, nel_ho, 8, 1, 4,
   {
-      const int i_off = ci*o*op1*op1;
-      const int id0 = ci;
-      const int id1 = (ci+1)%3;
-      const int id2 = (ci+2)%3;
-
-      const int nxi = (ci == 0) ? o : op1;
-      const int nyi = (ci == 1) ? o : op1;
-
-      for (int i0=0; i0<o; ++i0)
+      MFEM_FOREACH_THREAD(iz,z,o) // 1
      {
-         for (int i1=0; i1<op1; ++i1)
+         MFEM_FOREACH_THREAD(iy,y,op1) // 2
         {
-            for (int i2=0; i2<op1; ++i2)
+            MFEM_FOREACH_THREAD(ix,x,op1) // 2
            {
-               int ii_lex[3];
-               ii_lex[id0] = i0;
-               ii_lex[id1] = i1;
-               ii_lex[id2] = i2;
-               const int ii_el = i_off + ii_lex[0] + ii_lex[1]*nxi + ii_lex[2]*nxi*nyi;
-
-               for (int cj_rel=0; cj_rel<dim; ++cj_rel)
+               for (int c=0; c<dim; ++c)
               {
-                  const int cj = (ci + cj_rel) % 3;
-                  const int j_off = cj*o*op1*op1;
-
-                  const int nxj = (cj == 0) ? o : op1;
-                  const int nyj = (cj == 1) ? o : op1;
-
-                  const int j0_begin = i0;
-                  const int j0_end = (cj_rel == 0) ? i0 : i0 + 1;
-                  const int j1_begin = (i1 > 0) ? i1-1 : i1;
-                  const int j1_end = (cj_rel == 1)
-                                     ? ((i1 < o) ? i1 : i1-1)
-                                     : ((i1 < o) ? i1+1 : i1);
-                  const int j2_begin = (i2 > 0) ? i2-1 : i2;
-                  const int j2_end = (cj_rel == 2)
-                                     ? ((i2 < o) ? i2 : i2-1)
-                                     : ((i2 < o) ? i2+1 : i2);
-
-                  for (int j0=j0_begin; j0<=j0_end; ++j0)
+                  for (int j=0; j<nnz_per_row; ++j)
                  {
-                     const int d0 = j0 - i0;
-                     for (int j1=j1_begin; j1<=j1_end; ++j1)
-                     {
-                        const int d1 = j1 - i1 + 1;
-                        for (int j2=j2_begin; j2<=j2_end; ++j2)
-                        {
-                           const int d2 = j2 - i2 + 1;
-                           int jj_lex[3];
-                           jj_lex[id0] = j0;
-                           jj_lex[id1] = j1;
-                           jj_lex[id2] = j2;
-                           const int jj_el = j_off + jj_lex[0] + jj_lex[1]*nxj + jj_lex[2]*nxj*nyj;
-                           int jj_off;
-                           if (cj_rel == 0) { jj_off = d1 + 3*d2; }
-                           else if (cj_rel == 1) { jj_off = 9 + d0 + 2*d1 + 4*d2; }
-                           else /* if (cj_rel == 2) */ { jj_off = 21 + d0 + 2*d1 + 6*d2; }
-                           map(jj_off, ii_el) = jj_el;
-                        }
-                     }
+                     V(j,ix+iy*op1+iz*op1*op1,c,iel_ho) = 0.0;
                  }
               }
            }
         }
      }
-   }
+      MFEM_SYNC_THREAD;
+
+      MFEM_SHARED double local_mat_[sz_local_mat];
+      DeviceTensor<4> local_mat(local_mat_, 3,4, 3,4);
+      /// should be optimized
+      for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }
+
+      MFEM_SHARED double vx[8], vy[8], vz[8];
+      /// should be optimized
+      LORVertexCoordinates3D<1>(X, iel_ho, 0,0,0, vx,vy,vz);
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(xyz,x,8)
+      {
+         const int qz = xyz%2, qy = (xyz/2)%2, qx = xyz/2/2;
+         static constexpr double w = 1.0/8.0;
+
+         double J_[3*3];
+         DeviceTensor<2> J(J_, 3,3);
+         Jacobian3D(qx,qy,qz, vx,vy,vz, J);
+         const double detJ = Det3D(J);
+         const double w_detJ = w/detJ;
+
+         // adj(J)
+         double A_[3*3];
+         DeviceTensor<2> A(A_, 3,3);
+         Adjugate3D(J, A);
+
+         const double Q0 = w_detJ*(A(0,0)*A(0,0)+A(0,1)*A(0,1)+A(0,2)*A(0,2)); // 1,1
+         const double Q1 = w_detJ*(A(0,0)*A(1,0)+A(0,1)*A(1,1)+A(0,2)*A(1,2)); // 2,1
+         const double Q2 = w_detJ*(A(0,0)*A(2,0)+A(0,1)*A(2,1)+A(0,2)*A(2,2)); // 3,1
+         const double Q3 = w_detJ*(A(1,0)*A(1,0)+A(1,1)*A(1,1)+A(1,2)*A(1,2)); // 2,2
+         const double Q4 = w_detJ*(A(1,0)*A(2,0)+A(1,1)*A(2,1)+A(1,2)*A(2,2)); // 3,2
+         const double Q5 = w_detJ*(A(2,0)*A(2,0)+A(2,1)*A(2,1)+A(2,2)*A(2,2)); // 3,3
+
+         // w J^T J / det(J)
+         const double Q6  = w_detJ*(J(0,0)*J(0,0)+J(1,0)*J(1,0)+J(2,0)*J(2,0)); // 1,1
+         const double Q7  = w_detJ*(J(0,0)*J(0,1)+J(1,0)*J(1,1)+J(2,0)*J(2,1)); // 2,1
+         const double Q8  = w_detJ*(J(0,0)*J(0,2)+J(1,0)*J(1,2)+J(2,0)*J(2,2)); // 3,1
+         const double Q9  = w_detJ*(J(0,1)*J(0,1)+J(1,1)*J(1,1)+J(2,1)*J(2,1)); // 2,2
+         const double Q10 = w_detJ*(J(0,1)*J(0,2)+J(1,1)*J(1,2)+J(2,1)*J(2,2)); // 3,2
+         const double Q11 = w_detJ*(J(0,2)*J(0,2)+J(1,2)*J(1,2)+J(2,2)*J(2,2)); // 3,3
+
+         const double mq = const_mq ? MQ(0,0,0,0) : MQ(qx,qy,qz, iel_ho);
+         const double dq = const_dq ? DQ(0,0,0,0) : DQ(qx,qy,qz, iel_ho);
+
+         // Loop over x,y,z components. 0 => x, 1 => y, 2 => z
+         MFEM_FOREACH_THREAD(cj,y,dim)
+         {
+            const double jq1 = (cj == 0) ? qy : ((cj == 1) ? qz : qx);
+            const double jq2 = (cj == 0) ? qz : ((cj == 1) ? qx : qy);
+
+            const int jd_0 = cj;
+            const int jd_1 = (cj + 1)%3;
+            const int jd_2 = (cj + 2)%3;
+
+            MFEM_FOREACH_THREAD(bj,z,4) // 4 edges in each dim
+            {
+               const int bj1 = bj%2;
+               const int bj2 = bj/2;
+
+               double curl_j[3];
+               curl_j[jd_0] = 0.0;
+               curl_j[jd_1] = ((bj1 == 0) ? jq1 - 1 : -jq1)*((bj2 == 0) ? 1 : -1);
+               curl_j[jd_2] = ((bj2 == 0) ? 1 - jq2 : jq2)*((bj1 == 0) ? 1 : -1);
+
+               double basis_j[3];
+               basis_j[jd_0] = ((bj1 == 0) ? 1 - jq1 : jq1)*((bj2 == 0) ? 1 - jq2 : jq2);
+               basis_j[jd_1] = 0.0;
+               basis_j[jd_2] = 0.0;
+
+               const int jj_loc = bj + 4*cj;
+
+               for (int ci=0; ci<dim; ++ci)
+               {
+                  const double iq1 = (ci == 0) ? qy : ((ci == 1) ? qz : qx);
+                  const double iq2 = (ci == 0) ? qz : ((ci == 1) ? qx : qy);
+                  const int id_0 = ci, id_1 = (ci + 1)%3, id_2 = (ci + 2)%3;
+
+                  for (int bi=0; bi<4; ++bi)
+                  {
+                     const int bi1 = bi%2, bi2 = bi/2;
+
+                     double curl_i[3];
+                     curl_i[id_0] = 0.0;
+                     curl_i[id_1] = ((bi1 == 0) ? iq1 - 1 : -iq1)*((bi2 == 0) ? 1 : -1);
+                     curl_i[id_2] = ((bi2 == 0) ? 1 - iq2 : iq2)*((bi1 == 0) ? 1 : -1);
+
+                     double basis_i[3];
+                     basis_i[id_0] = ((bi1 == 0) ? 1 - iq1 : iq1)*((bi2 == 0) ? 1 - iq2 : iq2);
+                     basis_i[id_1] = 0.0;
+                     basis_i[id_2] = 0.0;
+
+                     const int ii_loc = bi + 4*ci;
+
+                     // Only store the lower-triangular part of
+                     // the matrix (by symmetry).
+                     if (jj_loc > ii_loc) { continue; }
+
+                     double curl_curl = 0.0;
+                     curl_curl += Q6*curl_i[0]*curl_j[0];
+                     curl_curl += Q7*(curl_i[0]*curl_j[1] + curl_i[1]*curl_j[0]);
+                     curl_curl += Q8*(curl_i[0]*curl_j[2] + curl_i[2]*curl_j[0]);
+                     curl_curl += Q9*curl_i[1]*curl_j[1];
+                     curl_curl += Q10*(curl_i[1]*curl_j[2] + curl_i[2]*curl_j[1]);
+                     curl_curl += Q11*curl_i[2]*curl_j[2];
+
+                     double basis_basis = 0.0;
+                     basis_basis += Q0*basis_i[0]*basis_j[0];
+                     basis_basis += Q1*(basis_i[0]*basis_j[1] + basis_i[1]*basis_j[0]);
+                     basis_basis += Q2*(basis_i[0]*basis_j[2] + basis_i[2]*basis_j[0]);
+                     basis_basis += Q3*basis_i[1]*basis_j[1];
+                     basis_basis += Q4*(basis_i[1]*basis_j[2] + basis_i[2]*basis_j[1]);
+                     basis_basis += Q5*basis_i[2]*basis_j[2];
+
+                     const double val = dq*curl_curl + mq*basis_basis;
+                     AtomicAdd(local_mat(ci,bi, cj,bj), val);
+                  } // bi
+               } // ci
+            } // bj
+         } // cj
+      } // q
+      MFEM_SYNC_THREAD;
+
+      // Assemble the local matrix into the macro-element sparse matrix
+      // The nonzeros of the macro-element sparse matrix are ordered as
+      // follows:
+      //
+      // The axes are ordered relative to the direction of the basis
+      // vector, e.g. for x-vectors, the axes are (x,y,z), for
+      // y-vectors the axes are (y,z,x), and for z-vectors the axes are
+      // (z,x,y).
+      //
+      // The nonzeros are then given in "rotated lexicographic"
+      // ordering, according to these axes.
+      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(ii_loc,z,ne)
+         {
+            const int ci = ii_loc/4, bi = ii_loc%4;
+            const int i0 = 0, i1 = bi%2, i2 = bi/2;
+            const int id0 = ci, id1 = (ci+1)%3, id2 = (ci+2)%3;
+
+            int ii_lex[3];
+            ii_lex[id0] = i0, ii_lex[id1] = i1, ii_lex[id2] = i2;
+
+            const int nx = (ci == 0) ? o : op1, ny = (ci == 1) ? o : op1;
+            const int ii = ii_lex[0] + (ii_lex[1])*nx + (ii_lex[2])*nx*ny;
+
+            for (int jj_loc=0; jj_loc<ne; ++jj_loc)
+            {
+               const int cj = jj_loc/4, bj = jj_loc%4;
+               // add 3 to take modulus (rather than remainder) when
+               // (cj - ci) is negative
+               const int cj_rel = (3 + cj - ci)%3;
+               const int jd0 = cj_rel, jd1 = (cj_rel+1)%3, jd2 = (cj_rel+2)%3;
+
+               int jj_rel[3];
+               jj_rel[jd0] = 0, jj_rel[jd1] = bj%2, jj_rel[jd2] = bj/2;
+
+               const int d0 = jj_rel[0] - i0;
+               const int d1 = 1 + jj_rel[1] - i1;
+               const int d2 = 1 + jj_rel[2] - i2;
+
+               const int jj_off = (cj_rel == 0) ? d1 + 3*d2 :
+                                  (cj_rel == 1) ? 9 + d0 + 2*d1 + 4*d2 :
+                                  (cj_rel == 2) ? 21 + d0 + 2*d1 + 6*d2 : -1;
+
+               // Symmetry
+               const double val = (jj_loc <= ii_loc)
+                                  ? local_mat(ci,bi, cj,bj)
+                                  : local_mat(cj,bj, ci,bi);
+               AtomicAdd(V(jj_off, ii, ci, iel_ho), val);
+            }
+         }
+      }
+   });
+   SparseMapping3D<1>(sparse_mapping);
 }

 // Explicit template instantiations
@@ -572,7 +809,7 @@ template void BatchedLOR_ND::Assemble2D<6>();
 template void BatchedLOR_ND::Assemble2D<7>();
 template void BatchedLOR_ND::Assemble2D<8>();

-template void BatchedLOR_ND::Assemble3D<1>();
+//template void BatchedLOR_ND::Assemble3D<1>();  // explicitly specialized
 template void BatchedLOR_ND::Assemble3D<2>();
 template void BatchedLOR_ND::Assemble3D<3>();
 template void BatchedLOR_ND::Assemble3D<4>();
@@ -233,6 +233,87 @@ void BatchedLOR_RT::Assemble2D()
   }
 }

+template<int ORDER>
+static void SparseMapping3D(Array<int> &sparse_mapping)
+{
+   static constexpr int nnz_per_row = 11;
+   static constexpr int dim = 3;
+   static constexpr int o = ORDER;
+   static constexpr int op1 = ORDER + 1;
+   static constexpr int ndof_per_el = dim*o*o*op1;
+
+   sparse_mapping.SetSize(nnz_per_row*ndof_per_el);
+   sparse_mapping = -1;
+   auto map = Reshape(sparse_mapping.HostReadWrite(), nnz_per_row, ndof_per_el);
+
+   for (int ci=0; ci<dim; ++ci)
+   {
+      const int i_off = ci*o*o*op1;
+      const int id0 = ci;
+      const int id1 = (ci+1)%3;
+      const int id2 = (ci+2)%3;
+
+      const int nxi = (ci == 0) ? op1 : o;
+      const int nyi = (ci == 1) ? op1 : o;
+
+      for (int i0=0; i0<op1; ++i0)
+      {
+         for (int i1=0; i1<o; ++i1)
+         {
+            for (int i2=0; i2<o; ++i2)
+            {
+               int ii_lex[3];
+               ii_lex[id0] = i0;
+               ii_lex[id1] = i1;
+               ii_lex[id2] = i2;
+               const int ii_el = i_off + ii_lex[0] + ii_lex[1]*nxi + ii_lex[2]*nxi*nyi;
+
+               for (int cj_rel=0; cj_rel<dim; ++cj_rel)
+               {
+                  const int cj = (ci + cj_rel) % 3;
+                  const int j_off = cj*o*o*op1;
+
+                  const int nxj = (cj == 0) ? op1 : o;
+                  const int nyj = (cj == 1) ? op1 : o;
+
+                  const int j0_begin = (i0 > 0) ? i0-1 : i0;
+                  const int j0_end = (cj_rel == 0)
+                                     ? ((i0 < o) ? i0+1 : i0)
+                                     : ((i0 < o) ? i0 : i0-1);
+                  const int j1_begin = i1;
+                  const int j1_end = (cj_rel == 1) ? i1+1 : i1;
+                  const int j2_begin = i2;
+                  const int j2_end = (cj_rel == 2) ? i2+1 : i2;
+
+                  for (int j0=j0_begin; j0<=j0_end; ++j0)
+                  {
+                     const int d0 = 1 + j0 - i0;
+                     for (int j1=j1_begin; j1<=j1_end; ++j1)
+                     {
+                        const int d1 = j1 - i1;
+                        for (int j2=j2_begin; j2<=j2_end; ++j2)
+                        {
+                           const int d2 = j2 - i2;
+                           int jj_lex[3];
+                           jj_lex[id0] = j0;
+                           jj_lex[id1] = j1;
+                           jj_lex[id2] = j2;
+                           const int jj_el = j_off + jj_lex[0] + jj_lex[1]*nxj + jj_lex[2]*nxj*nyj;
+                           int jj_off;
+                           if (cj_rel == 0) { jj_off = d0; }
+                           else if (cj_rel == 1) { jj_off = 3 + d0 + 2*d1; }
+                           else /* if (cj_rel == 2) */ { jj_off = 7 + d0 + 2*d2; }
+                           map(jj_off, ii_el) = jj_el;
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
 template <int ORDER>
 void BatchedLOR_RT::Assemble3D()
 {
@@ -261,7 +342,7 @@ void BatchedLOR_RT::Assemble3D()
   sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
   auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*o*op1, dim, nel_ho);

-   auto X = X_vert.Read();
+   const auto X = X_vert.Read();

   // Last thread dimension is lowered to avoid "too many resources" error
   MFEM_FORALL_3D(iel_ho, nel_ho, ORDER, ORDER, (ORDER>6)?4:ORDER,
@@ -474,76 +555,185 @@ void BatchedLOR_RT::Assemble3D()
         }
      }
   });
+   SparseMapping3D<ORDER>(sparse_mapping);
+}

-   sparse_mapping.SetSize(nnz_per_row*ndof_per_el);
-   sparse_mapping = -1;
-   auto map = Reshape(sparse_mapping.HostReadWrite(), nnz_per_row, ndof_per_el);
-   for (int ci=0; ci<dim; ++ci)
+template <>
+void BatchedLOR_RT::Assemble3D<1>()
+{
+
+   static constexpr int o = 1;
+   static constexpr int op1 = 2;
+   static constexpr int nf = 6; // number of faces in hexahedron
+   static constexpr int dim = 3;
+   static constexpr int ndof_per_el = dim*o*o*op1;
+   static constexpr int nnz_per_row = 11;
+   static constexpr int sz_local_mat = nf*nf;
+
+   const int nel_ho = fes_ho.GetNE();
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1, 1)
+                   : Reshape(c1.Read(), op1, op1, op1, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1, 1)
+                   : Reshape(c2.Read(), op1, op1, op1, nel_ho);
+
+   sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
+   auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*o*op1, dim, nel_ho);
+
+   const auto X = X_vert.Read();
+
+   MFEM_FORALL_3D(iel_ho, nel_ho, 8, 1, 1,
   {
-      const int i_off = ci*o*o*op1;
-      const int id0 = ci;
-      const int id1 = (ci+1)%3;
-      const int id2 = (ci+2)%3;
-
-      const int nxi = (ci == 0) ? op1 : o;
-      const int nyi = (ci == 1) ? op1 : o;
-
-      for (int i0=0; i0<op1; ++i0)
+      MFEM_FOREACH_THREAD(j,x,nnz_per_row)
      {
-         for (int i1=0; i1<o; ++i1)
+         for (int ix = 0; ix < op1; ++ix)
         {
-            for (int i2=0; i2<o; ++i2)
+            for (int c = 0; c < dim; ++c)
            {
-               int ii_lex[3];
-               ii_lex[id0] = i0;
-               ii_lex[id1] = i1;
-               ii_lex[id2] = i2;
-               const int ii_el = i_off + ii_lex[0] + ii_lex[1]*nxi + ii_lex[2]*nxi*nyi;
-
-               for (int cj_rel=0; cj_rel<dim; ++cj_rel)
-               {
-                  const int cj = (ci + cj_rel) % 3;
-                  const int j_off = cj*o*o*op1;
-
-                  const int nxj = (cj == 0) ? op1 : o;
-                  const int nyj = (cj == 1) ? op1 : o;
-
-                  const int j0_begin = (i0 > 0) ? i0-1 : i0;
-                  const int j0_end = (cj_rel == 0)
-                                     ? ((i0 < o) ? i0+1 : i0)
-                                     : ((i0 < o) ? i0 : i0-1);
-                  const int j1_begin = i1;
-                  const int j1_end = (cj_rel == 1) ? i1+1 : i1;
-                  const int j2_begin = i2;
-                  const int j2_end = (cj_rel == 2) ? i2+1 : i2;
-
-                  for (int j0=j0_begin; j0<=j0_end; ++j0)
-                  {
-                     const int d0 = 1 + j0 - i0;
-                     for (int j1=j1_begin; j1<=j1_end; ++j1)
-                     {
-                        const int d1 = j1 - i1;
-                        for (int j2=j2_begin; j2<=j2_end; ++j2)
-                        {
-                           const int d2 = j2 - i2;
-                           int jj_lex[3];
-                           jj_lex[id0] = j0;
-                           jj_lex[id1] = j1;
-                           jj_lex[id2] = j2;
-                           const int jj_el = j_off + jj_lex[0] + jj_lex[1]*nxj + jj_lex[2]*nxj*nyj;
-                           int jj_off;
-                           if (cj_rel == 0) { jj_off = d0; }
-                           else if (cj_rel == 1) { jj_off = 3 + d0 + 2*d1; }
-                           else /* if (cj_rel == 2) */ { jj_off = 7 + d0 + 2*d2; }
-                           map(jj_off, ii_el) = jj_el;
-                        }
-                     }
-                  }
-               }
+               V(j,ix,c,iel_ho) = 0.0;
            }
         }
      }
-   }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(xyz,x,8)
+      {
+         const int qz = xyz%2, qy = (xyz/2)%2, qx = xyz/2/2;
+         static constexpr double w = 1.0/8.0;
+
+         double local_mat_[sz_local_mat];
+         DeviceTensor<4> local_mat(local_mat_, 3,2, 3,2);
+         for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }
+
+         double vx[8], vy[8], vz[8];
+         LORVertexCoordinates3D<1>(X, iel_ho, 0,0,0, vx, vy, vz);
+
+         double J_[3*3];
+         DeviceTensor<2> J(J_, 3,3);
+         Jacobian3D(qx,qy,qz, vx,vy,vz, J);
+         const double detJ = Det3D(J);
+         const double w_detJ = w/detJ;
+
+         const double Q0 = w_detJ*(J(0,0)*J(0,0)+J(1,0)*J(1,0)+J(2,0)*J(2,0)); // 1,1
+         const double Q1 = w_detJ*(J(0,1)*J(0,0)+J(1,1)*J(1,0)+J(2,1)*J(2,0)); // 2,1
+         const double Q2 = w_detJ*(J(0,2)*J(0,0)+J(1,2)*J(1,0)+J(2,2)*J(2,0)); // 3,1
+         const double Q3 = w_detJ*(J(0,1)*J(0,1)+J(1,1)*J(1,1)+J(2,1)*J(2,1)); // 2,2
+         const double Q4 = w_detJ*(J(0,2)*J(0,1)+J(1,2)*J(1,1)+J(2,2)*J(2,1)); // 3,2
+         const double Q5 = w_detJ*(J(0,2)*J(0,2)+J(1,2)*J(1,2)+J(2,2)*J(2,2)); // 3,3
+         const double Q6 = w_detJ;
+
+         const double mq = const_mq ? MQ(0,0,0,0) : MQ(qx,qy,qz, iel_ho);
+         const double dq = const_dq ? DQ(0,0,0,0) : DQ(qx,qy,qz, iel_ho);
+
+         // Loop over x,y,z components. 0 => x, 1 => y, 2 => z
+         for (int cj=0; cj<dim; ++cj)//MFEM_FOREACH_THREAD(cj,y,dim)
+         {
+            const int jq0 = (cj == 0) ? qx : (cj == 1) ? qy : qz;
+            const int jd0 = cj, jd1 = (cj + 1)%3, jd2 = (cj + 2)%3;
+
+            for (int bj=0; bj<2; ++bj)//MFEM_FOREACH_THREAD(bj,z,2)  // 2 faces in each dim
+            {
+               const double div_j = (bj == 0) ? -1.0 : 1.0;
+
+               double basis_j[3];
+               basis_j[jd0] = (bj == jq0) ? 1.0 : 0.0;
+               basis_j[jd1] = 0.0;
+               basis_j[jd2] = 0.0;
+
+               const int jj_loc = bj + 2*cj;
+
+               for (int ci=0; ci<dim; ++ci)
+               {
+                  const double iq0 = (ci == 0) ? qx : ((ci == 1) ? qy : qz);
+                  const int id0 = ci, id1 = (ci + 1)%3, id2 = (ci + 2)%3;
+
+                  for (int bi=0; bi<2; ++bi)
+                  {
+                     const double div_i = (bi == 0) ? -1.0 : 1.0;
+                     double basis_i[3];
+                     basis_i[id0] = (bi == iq0) ? 1.0 : 0.0;
+                     basis_i[id1] = 0.0;
+                     basis_i[id2] = 0.0;
+
+                     const int ii_loc = bi + 2*ci;
+
+                     // Only store the lower-triangular part of
+                     // the matrix (by symmetry).
+                     if (jj_loc > ii_loc) { continue; }
+
+                     const double div_div = Q6*div_i*div_j;
+
+                     double basis_basis = 0.0;
+                     basis_basis += Q0*basis_i[0]*basis_j[0];
+                     basis_basis += Q1*(basis_i[0]*basis_j[1] + basis_i[1]*basis_j[0]);
+                     basis_basis += Q2*(basis_i[0]*basis_j[2] + basis_i[2]*basis_j[0]);
+                     basis_basis += Q3*basis_i[1]*basis_j[1];
+                     basis_basis += Q4*(basis_i[1]*basis_j[2] + basis_i[2]*basis_j[1]);
+                     basis_basis += Q5*basis_i[2]*basis_j[2];
+
+                     const double val = dq*div_div + mq*basis_basis;
+                     local_mat(ci,bi, cj,bj) += val;
+                  } // bi
+               } // ci
+            } // bj
+         } // cj
+
+         // Assemble the local matrix into the macro-element sparse matrix
+         // The nonzeros of the macro-element sparse matrix are ordered as
+         // follows:
+         //
+         // The axes are ordered relative to the direction of the basis
+         // vector, e.g. for x-vectors, the axes are (x,y,z), for
+         // y-vectors the axes are (y,z,x), and for z-vectors the axes are
+         // (z,x,y).
+         //
+         // The nonzeros are then given in "rotated lexicographic"
+         // ordering, according to these axes.
+         for (int ii_loc=0; ii_loc<nf; ++ii_loc)
+         {
+            const int ci = ii_loc/2, bi = ii_loc%2;
+            const int id0 = ci, id1 = (ci+1)%3, id2 = (ci+2)%3;
+            const int i0 = bi, i1 = 0, i2 = 0;
+
+            int ii_lex[3];
+            ii_lex[id0] = i0, ii_lex[id1] = i1, ii_lex[id2] = i2;
+
+            const int nx = (ci == 0) ? op1 : o;
+            const int ny = (ci == 1) ? op1 : o;
+            const int ii = ii_lex[0] + ii_lex[1]*nx + ii_lex[2]*nx*ny;
+
+            for (int jj_loc=0; jj_loc<nf; ++jj_loc)
+            {
+               const int cj = jj_loc/2, bj = jj_loc%2;
+               // add 3 to take modulus (rather than remainder) when
+               // (cj - ci) is negative
+               const int cj_rel = (3 + cj - ci)%3;
+               const int jd0 = cj_rel, jd1 = (cj_rel+1)%3, jd2 = (cj_rel+2)%3;
+
+               int jj_rel[3];
+               jj_rel[jd0] = bj, jj_rel[jd1] = 0, jj_rel[jd2] = 0;
+
+               const int d0 = jj_rel[0] - i0 + 1;
+               const int d1 = jj_rel[1] - i1;
+               const int d2 = jj_rel[2] - i2;
+
+               const int jj_off = (cj_rel == 0) ?d0 :
+                                  (cj_rel == 1) ?  3 + d0 + 2*d1 :
+                                  (cj_rel == 2) ?  7 + d0 + 2*d2 : -1;
+
+               // Symmetry
+               const double val = (jj_loc <= ii_loc)
+                                  ? local_mat(ci,bi, cj,bj)
+                                  : local_mat(cj,bj, ci,bi);
+               AtomicAdd(V(jj_off, ii, ci, iel_ho), val);
+            } // jj_loc
+         } // ii_loc
+      } // q
+   });
+   SparseMapping3D<1>(sparse_mapping);
 }

 // Explicit template instantiations
@@ -556,7 +746,7 @@ template void BatchedLOR_RT::Assemble2D<6>();
 template void BatchedLOR_RT::Assemble2D<7>();
 template void BatchedLOR_RT::Assemble2D<8>();

-template void BatchedLOR_RT::Assemble3D<1>();
+//template void BatchedLOR_RT::Assemble3D<1>(); // explicitly specialized
 template void BatchedLOR_RT::Assemble3D<2>();
 template void BatchedLOR_RT::Assemble3D<3>();
 template void BatchedLOR_RT::Assemble3D<4>();
@@ -14,26 +14,41 @@
 namespace mfem
 {

-MultigridBase::MultigridBase()
-   : cycleType(CycleType::VCYCLE), preSmoothingSteps(1), postSmoothingSteps(1),
-     nrhs(0)
+Multigrid::Multigrid()
+   : cycleType(CycleType::VCYCLE), preSmoothingSteps(1), postSmoothingSteps(1)
 {}

-MultigridBase::MultigridBase(const Array<Operator*>& operators_,
-                             const Array<Solver*>& smoothers_,
-                             const Array<bool>& ownedOperators_,
-                             const Array<bool>& ownedSmoothers_)
-   : Solver(operators_.Last()->Height(), operators_.Last()->Width()),
-     cycleType(CycleType::VCYCLE), preSmoothingSteps(1), postSmoothingSteps(1),
-     nrhs(0)
+Multigrid::Multigrid(const Array<Operator*>& operators_,
+                     const Array<Solver*>& smoothers_,
+                     const Array<Operator*>& prolongations_,
+                     const Array<bool>& ownedOperators_,
+                     const Array<bool>& ownedSmoothers_,
+                     const Array<bool>& ownedProlongations_)
+   : Solver(operators_.Last()->NumRows()), cycleType(CycleType::VCYCLE),
+     preSmoothingSteps(1), postSmoothingSteps(1),
+     X(operators_.Size()), Y(X.Size()), R(X.Size()), Z(X.Size())
 {
   operators_.Copy(operators);
   smoothers_.Copy(smoothers);
+   prolongations_.Copy(prolongations);
   ownedOperators_.Copy(ownedOperators);
   ownedSmoothers_.Copy(ownedSmoothers);
+   ownedProlongations_.Copy(ownedProlongations);
+
+   for (int level = 0; level < operators.Size(); ++level)
+   {
+      X[level] = new Vector(operators[level]->NumRows());
+      *X[level] = 0.0;
+      Y[level] = new Vector(operators[level]->NumRows());
+      *Y[level] = 0.0;
+      R[level] = new Vector(operators[level]->NumRows());
+      *R[level] = 0.0;
+      Z[level] = new Vector(operators[level]->NumRows());
+      *Z[level] = 0.0;
+   }
 }

-MultigridBase::~MultigridBase()
+Multigrid::~Multigrid()
 {
   for (int i = 0; i < operators.Size(); ++i)
   {
@@ -45,210 +60,12 @@ MultigridBase::~MultigridBase()
      {
         delete smoothers[i];
      }
-   }
-   EraseVectors();
-}
-
-void MultigridBase::InitVectors() const
-{
-   if (X.NumRows() > 0 && X.NumCols() > 0) { EraseVectors(); }
-   const int M = NumLevels();
-   X.SetSize(M, nrhs);
-   Y.SetSize(M, nrhs);
-   R.SetSize(M, nrhs);
-   Z.SetSize(M, nrhs);
-   for (int i = 0; i < X.NumRows(); ++i)
-   {
-      const int n = operators[i]->Height();
-      for (int j = 0; j < X.NumCols(); ++j)
-      {
-         X(i, j) = new Vector(n);
-         Y(i, j) = new Vector(n);
-         R(i, j) = new Vector(n);
-         Z(i, j) = new Vector(n);
-      }
-   }
-}
-
-void MultigridBase::EraseVectors() const
-{
-   for (int i = 0; i < X.NumRows(); ++i)
-   {
-      for (int j = 0; j < X.NumCols(); ++j)
-      {
-         delete X(i, j);
-         delete Y(i, j);
-         delete R(i, j);
-         delete Z(i, j);
-      }
-   }
-}
-
-void MultigridBase::AddLevel(Operator* op, Solver* smoother,
-                             bool ownOperator, bool ownSmoother)
-{
-   height = op->Height();
-   width = op->Width();
-   operators.Append(op);
-   smoothers.Append(smoother);
-   ownedOperators.Append(ownOperator);
-   ownedSmoothers.Append(ownSmoother);
-}
-
-void MultigridBase::SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
-                                 int postSmoothingSteps_)
-{
-   cycleType = cycleType_;
-   preSmoothingSteps = preSmoothingSteps_;
-   postSmoothingSteps = postSmoothingSteps_;
-}
-
-void MultigridBase::Mult(const Vector& x, Vector& y) const
-{
-   Array<const Vector*> X_(1);
-   Array<Vector*> Y_(1);
-   X_[0] = &x;
-   Y_[0] = &y;
-   ArrayMult(X_, Y_);
-}
-
-void MultigridBase::ArrayMult(const Array<const Vector*>& X_,
-                              Array<Vector*>& Y_) const
-{
-   MFEM_ASSERT(operators.Size() > 0,
-               "Multigrid solver does not have operators set!");
-   MFEM_ASSERT(X_.Size() == Y_.Size(),
-               "Number of columns mismatch in MultigridBase::Mult!");
-   if (iterative_mode)
-   {
-      MFEM_WARNING("Multigrid solver does not use iterative_mode and ignores "
-                   "the initial guess!");
+      delete X[i];
+      delete Y[i];
+      delete R[i];
+      delete Z[i];
   }

-   // Add capacity as necessary
-   nrhs = X_.Size();
-   if (X.NumCols() < nrhs) { InitVectors(); }
-
-   // Perform a single cycle
-   const int M = NumLevels();
-   for (int j = 0; j < nrhs; ++j)
-   {
-      MFEM_ASSERT(X_[j] && Y_[j], "Missing Vector in MultigridBase::Mult!");
-      *X(M - 1, j) = *X_[j];
-      *Y(M - 1, j) = 0.0;
-   }
-   Cycle(M - 1);
-   for (int j = 0; j < nrhs; ++j)
-   {
-      *Y_[j] = *Y(M - 1, j);
-   }
-}
-
-void MultigridBase::SmoothingStep(int level, bool zero, bool transpose) const
-{
-   // y = y + S (x - A y) or y = y + S^T (x - A y)
-   if (zero)
-   {
-      Array<Vector *> X_(X[level], nrhs), Y_(Y[level], nrhs);
-      GetSmootherAtLevel(level)->ArrayMult(X_, Y_);
-   }
-   else
-   {
-      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs),
-            Z_(Z[level], nrhs);
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *R_[j] = *X(level, j);
-      }
-      GetOperatorAtLevel(level)->ArrayAddMult(Y_, R_, -1.0);
-      if (transpose)
-      {
-         GetSmootherAtLevel(level)->ArrayMultTranspose(R_, Z_);
-      }
-      else
-      {
-         GetSmootherAtLevel(level)->ArrayMult(R_, Z_);
-      }
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *Y_[j] += *Z_[j];
-      }
-   }
-}
-
-void MultigridBase::Cycle(int level) const
-{
-   // Coarse solve
-   if (level == 0)
-   {
-      SmoothingStep(0, true, false);
-      return;
-   }
-
-   // Pre-smooth
-   for (int i = 0; i < preSmoothingSteps; ++i)
-   {
-      SmoothingStep(level, (cycleType == CycleType::VCYCLE && i == 0), false);
-   }
-
-   // Compute residual and restrict
-   {
-      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs),
-            X_(X[level - 1], nrhs);
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *R_[j] = *X(level, j);
-      }
-      GetOperatorAtLevel(level)->ArrayAddMult(Y_, R_, -1.0);
-      GetProlongationAtLevel(level - 1)->ArrayMultTranspose(R_, X_);
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *Y(level - 1, j) = 0.0;
-      }
-   }
-
-   // Corrections
-   Cycle(level - 1);
-   if (cycleType == CycleType::WCYCLE)
-   {
-      Cycle(level - 1);
-   }
-
-   // Prolongate and add
-   {
-      Array<Vector *> Y_(Y[level - 1], nrhs), Z_(Z[level], nrhs);
-      GetProlongationAtLevel(level - 1)->ArrayMult(Y_, Z_);
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *Y(level, j) += *Z_[j];
-      }
-   }
-
-   // Post-smooth
-   for (int i = 0; i < postSmoothingSteps; ++i)
-   {
-      SmoothingStep(level, false, true);
-   }
-}
-
-Multigrid::Multigrid()
-   : MultigridBase()
-{}
-
-Multigrid::Multigrid(const Array<Operator*>& operators_,
-                     const Array<Solver*>& smoothers_,
-                     const Array<Operator*>& prolongations_,
-                     const Array<bool>& ownedOperators_,
-                     const Array<bool>& ownedSmoothers_,
-                     const Array<bool>& ownedProlongations_)
-   : MultigridBase(operators_, smoothers_, ownedOperators_, ownedSmoothers_)
-{
-   prolongations_.Copy(prolongations);
-   ownedProlongations_.Copy(ownedProlongations);
-}
-
-Multigrid::~Multigrid()
-{
   for (int i = 0; i < prolongations.Size(); ++i)
   {
      if (ownedProlongations[i])
@@ -256,12 +73,158 @@ Multigrid::~Multigrid()
         delete prolongations[i];
      }
   }
+
+   operators.DeleteAll();
+   smoothers.DeleteAll();
+   prolongations.DeleteAll();
+   X.DeleteAll();
+   Y.DeleteAll();
+   R.DeleteAll();
+   Z.DeleteAll();
 }

-GeometricMultigrid::
-GeometricMultigrid(const FiniteElementSpaceHierarchy& fespaces_)
-   : MultigridBase(), fespaces(fespaces_)
-{}
+void Multigrid::AddLevel(Operator* opr, Solver* smoother, bool ownOperator,
+                         bool ownSmoother)
+{
+   operators.Append(opr);
+   smoothers.Append(smoother);
+   ownedOperators.Append(ownOperator);
+   ownedSmoothers.Append(ownSmoother);
+   width = opr->Width();
+   height = opr->Height();
+
+   X.Append(new Vector(height));
+   *X.Last() = 0.0;
+   Y.Append(new Vector(height));
+   *Y.Last() = 0.0;
+   R.Append(new Vector(height));
+   *R.Last() = 0.0;
+   Z.Append(new Vector(height));
+   *Z.Last() = 0.0;
+}
+
+int Multigrid::NumLevels() const { return operators.Size(); }
+
+int Multigrid::GetFinestLevelIndex() const { return NumLevels() - 1; }
+
+const Operator* Multigrid::GetOperatorAtLevel(int level) const
+{
+   return operators[level];
+}
+
+Operator* Multigrid::GetOperatorAtLevel(int level)
+{
+   return operators[level];
+}
+
+const Operator* Multigrid::GetOperatorAtFinestLevel() const
+{
+   return GetOperatorAtLevel(operators.Size() - 1);
+}
+
+Operator* Multigrid::GetOperatorAtFinestLevel()
+{
+   return GetOperatorAtLevel(operators.Size() - 1);
+}
+
+Solver* Multigrid::GetSmootherAtLevel(int level) const
+{
+   return smoothers[level];
+}
+
+Solver* Multigrid::GetSmootherAtLevel(int level)
+{
+   return smoothers[level];
+}
+
+void Multigrid::SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
+                             int postSmoothingSteps_)
+{
+   cycleType = cycleType_;
+   preSmoothingSteps = preSmoothingSteps_;
+   postSmoothingSteps = postSmoothingSteps_;
+}
+
+void Multigrid::Mult(const Vector& x, Vector& y) const
+{
+   MFEM_ASSERT(NumLevels() > 0, "");
+   *X.Last() = x;
+   *Y.Last() = 0.0;
+   Cycle(GetFinestLevelIndex());
+   y = *Y.Last();
+}
+
+void Multigrid::SetOperator(const Operator& op)
+{
+   MFEM_ABORT("SetOperator not supported in Multigrid");
+}
+
+void Multigrid::SmoothingStep(int level, bool transpose) const
+{
+   GetOperatorAtLevel(level)->Mult(*Y[level], *R[level]); // r = A x
+   subtract(*X[level], *R[level], *R[level]);             // r = b - A x
+   if (transpose)
+   {
+      GetSmootherAtLevel(level)->MultTranspose(*R[level], *Z[level]); // z = S r
+   }
+   else
+   {
+      GetSmootherAtLevel(level)->Mult(*R[level], *Z[level]); // z = S r
+   }
+   add(*Y[level], 1.0, *Z[level], *Y[level]);             // x = x + S (b - A x)
+}
+
+void Multigrid::Cycle(int level) const
+{
+   if (level == 0)
+   {
+      GetSmootherAtLevel(level)->Mult(*X[level], *Y[level]);
+      return;
+   }
+
+   for (int i = 0; i < preSmoothingSteps; i++)
+   {
+      SmoothingStep(level, false);
+   }
+
+   // Compute residual
+   GetOperatorAtLevel(level)->Mult(*Y[level], *R[level]);
+   subtract(*X[level], *R[level], *R[level]);
+
+   // Restrict residual
+   GetProlongationAtLevel(level - 1)->MultTranspose(*R[level], *X[level - 1]);
+
+   // Init zeros
+   *Y[level - 1] = 0.0;
+
+   // Corrections
+   int corrections = 1;
+   if (cycleType == CycleType::WCYCLE)
+   {
+      corrections = 2;
+   }
+   for (int correction = 0; correction < corrections; ++correction)
+   {
+      Cycle(level - 1);
+   }
+
+   // Prolongate
+   GetProlongationAtLevel(level - 1)->Mult(*Y[level - 1], *R[level]);
+
+   // Add update
+   *Y[level] += *R[level];
+
+   // Post-smooth
+   for (int i = 0; i < postSmoothingSteps; i++)
+   {
+      SmoothingStep(level, true);
+   }
+}
+
+const Operator* Multigrid::GetProlongationAtLevel(int level) const
+{
+   return prolongations[level];
+}

 GeometricMultigrid::~GeometricMultigrid()
 {
@@ -269,10 +232,15 @@ GeometricMultigrid::~GeometricMultigrid()
   {
      delete bfs[i];
   }
+
+   bfs.DeleteAll();
+
   for (int i = 0; i < essentialTrueDofs.Size(); ++i)
   {
      delete essentialTrueDofs[i];
   }
+
+   essentialTrueDofs.DeleteAll();
 }

 void GeometricMultigrid::FormFineLinearSystem(Vector& x, Vector& b,
@@ -288,4 +256,9 @@ void GeometricMultigrid::RecoverFineFEMSolution(const Vector& X,
   bfs.Last()->RecoverFEMSolution(X, b, x);
 }

+const Operator* GeometricMultigrid::GetProlongationAtLevel(int level) const
+{
+   return fespaces.GetProlongationAtLevel(level);
+}
+
 } // namespace mfem
@@ -21,8 +21,8 @@
 namespace mfem
 {

-/// Abstract base class for Multigrid solvers
-class MultigridBase : public Solver
+/// Multigrid solver class
+class Multigrid : public Solver
 {
 public:
   enum class CycleType
@@ -34,118 +34,29 @@ public:
 protected:
   Array<Operator*> operators;
   Array<Solver*> smoothers;
+   Array<Operator*> prolongations;
+
   Array<bool> ownedOperators;
   Array<bool> ownedSmoothers;
+   Array<bool> ownedProlongations;

   CycleType cycleType;
   int preSmoothingSteps;
   int postSmoothingSteps;

-   mutable Array2D<Vector*> X, Y, R, Z;
-   mutable int nrhs;
+   mutable Array<Vector*> X;
+   mutable Array<Vector*> Y;
+   mutable Array<Vector*> R;
+   mutable Array<Vector*> Z;

 public:
-   /// Constructs an empty multigrid hierarchy
-   MultigridBase();
-
-   /// Constructs a multigrid hierarchy from the given inputs
-   /** Inputs include operators and smoothers on all levels, and ownership of
-       the given operators and smoothers */
-   MultigridBase(const Array<Operator*>& operators_,
-                 const Array<Solver*>& smoothers_,
-                 const Array<bool>& ownedOperators_,
-                 const Array<bool>& ownedSmoothers_);
-
-   /// Destructor
-   virtual ~MultigridBase();
-
-   /// Adds a level to the multigrid operator hierarchy
-   /** The ownership of the operators and solvers/smoothers may be transferred
-       to the Multigrid by setting the according boolean variables */
-   void AddLevel(Operator* op, Solver* smoother, bool ownOperator,
-                 bool ownSmoother);
-
-   /// Returns the number of levels
-   int NumLevels() const { return operators.Size(); }
-
-   /// Returns the index of the finest level
-   int GetFinestLevelIndex() const { return NumLevels() - 1; }
-
-   /// Returns operator at given level
-   const Operator* GetOperatorAtLevel(int level) const
-   {
-      return operators[level];
-   }
-   Operator* GetOperatorAtLevel(int level)
-   {
-      return operators[level];
-   }
-
-   /// Returns operator at finest level
-   const Operator* GetOperatorAtFinestLevel() const
-   {
-      return GetOperatorAtLevel(GetFinestLevelIndex());
-   }
-   Operator* GetOperatorAtFinestLevel()
-   {
-      return GetOperatorAtLevel(GetFinestLevelIndex());
-   }
-
-   /// Returns smoother at given level
-   const Solver* GetSmootherAtLevel(int level) const
-   {
-      return smoothers[level];
-   }
-   Solver* GetSmootherAtLevel(int level)
-   {
-      return smoothers[level];
-   }
-
-   /// Set cycle type and number of pre- and post-smoothing steps used by Mult
-   void SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
-                     int postSmoothingSteps_);
-
-   /// Application of the multigrid as a preconditioner
-   virtual void Mult(const Vector& x, Vector& y) const override;
-   virtual void ArrayMult(const Array<const Vector*>& X_,
-                          Array<Vector*>& Y_) const override;
-
-   /// Not supported for multigrid
-   virtual void SetOperator(const Operator& op) override
-   {
-      MFEM_ABORT("SetOperator is not supported in Multigrid!");
-   }
-
-private:
-   /// Application of a multigrid cycle at particular level
-   void Cycle(int level) const;
-
-   /// Application of a pre-/post-smoothing step at particular level
-   void SmoothingStep(int level, bool zero, bool transpose) const;
-
-   /// Allocate or destroy temporary storage
-   void InitVectors() const;
-   void EraseVectors() const;
-
-   /// Returns prolongation operator at given level
-   virtual const Operator* GetProlongationAtLevel(int level) const = 0;
-};
-
-/// Multigrid solver class
-class Multigrid : public MultigridBase
-{
-protected:
-   Array<Operator*> prolongations;
-   Array<bool> ownedProlongations;
-
-public:
-   /// Constructs an empty multigrid hierarchy
+   /// Constructs an empty multigrid hierarchy.
   Multigrid();

-   /// Constructs a multigrid hierarchy from the given inputs
+   /// Constructs a multigrid hierarchy from the given inputs.
   /** Inputs include operators and smoothers on all levels, prolongation
       operators that go from coarser to finer levels, and ownership of the
-       given operators, smoothers, and prolongations */
+       given operators, smoothers, and prolongations. */
   Multigrid(const Array<Operator*>& operators_, const Array<Solver*>& smoothers_,
             const Array<Operator*>& prolongations_, const Array<bool>& ownedOperators_,
             const Array<bool>& ownedSmoothers_, const Array<bool>& ownedProlongations_);
@@ -153,16 +64,59 @@ public:
   /// Destructor
   virtual ~Multigrid();

+   /// Adds a level to the multigrid operator hierarchy.
+   /** The ownership of the operators and solvers/smoothers may be transferred
+       to the Multigrid by setting the according boolean variables. */
+   void AddLevel(Operator* opr, Solver* smoother, bool ownOperator,
+                 bool ownSmoother);
+
+   /// Returns the number of levels
+   int NumLevels() const;
+
+   /// Returns the index of the finest level
+   int GetFinestLevelIndex() const;
+
+   /// Returns operator at given level
+   const Operator* GetOperatorAtLevel(int level) const;
+
+   /// Returns operator at given level
+   Operator* GetOperatorAtLevel(int level);
+
+   /// Returns operator at finest level
+   const Operator* GetOperatorAtFinestLevel() const;
+
+   /// Returns operator at finest level
+   Operator* GetOperatorAtFinestLevel();
+
+   /// Returns smoother at given level
+   Solver* GetSmootherAtLevel(int level) const;
+
+   /// Returns smoother at given level
+   Solver* GetSmootherAtLevel(int level);
+
+   /// Set cycle type and number of pre- and post-smoothing steps used by Mult
+   void SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
+                     int postSmoothingSteps_);
+
+   /// Application of the multigrid as a preconditioner
+   virtual void Mult(const Vector& x, Vector& y) const override;
+
+   /// Not supported for multigrid
+   virtual void SetOperator(const Operator& op) override;
+
 private:
+   /// Application of a smoothing step at particular level
+   void SmoothingStep(int level, bool transpose) const;
+
+   /// Application of a multigrid cycle at particular level
+   void Cycle(int level) const;
+
   /// Returns prolongation operator at given level
-   virtual const Operator* GetProlongationAtLevel(int level) const override
-   {
-      return prolongations[level];
-   }
+   virtual const Operator* GetProlongationAtLevel(int level) const;
 };

 /// Geometric multigrid associated with a hierarchy of finite element spaces
-class GeometricMultigrid : public MultigridBase
+class GeometricMultigrid : public Multigrid
 {
 protected:
   const FiniteElementSpaceHierarchy& fespaces;
@@ -172,7 +126,8 @@ protected:
 public:
   /** Construct an empty multigrid object for the given finite element space
       hierarchy @a fespaces_ */
-   GeometricMultigrid(const FiniteElementSpaceHierarchy& fespaces_);
+   GeometricMultigrid(const FiniteElementSpaceHierarchy& fespaces_)
+      : Multigrid(), fespaces(fespaces_) { }

   /// Destructor
   virtual ~GeometricMultigrid();
@@ -187,10 +142,7 @@ public:

 private:
   /// Returns prolongation operator at given level
-   virtual const Operator* GetProlongationAtLevel(int level) const override
-   {
-      return fespaces.GetProlongationAtLevel(level);
-   }
+   virtual const Operator* GetProlongationAtLevel(int level) const override;
 };

 } // namespace mfem
@@ -16,6 +16,9 @@
 #include "fem.hpp"
 #include "../general/sort_pairs.hpp"

+#define MFEM_NVTX_COLOR Crimson
+#include "../general/nvtx.hpp"
+
 namespace mfem
 {

@@ -124,6 +127,7 @@ void ParBilinearForm::pAllocMat()
 void ParBilinearForm::ParallelRAP(SparseMatrix &loc_A, OperatorHandle &A,
                                  bool steal_loc_A)
 {
+   NVTX("RAP");
   ParFiniteElementSpace &pfespace = *ParFESpace();

   // Create a block diagonal parallel matrix
@@ -347,12 +351,14 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const double a)
 const
 {
-   const Operator *P = pfes->GetProlongationMatrix();
-   Xaux.SetSize(P->Height());
-   Yaux.SetSize(P->Height());
-   Ytmp.SetSize(P->Width());
+   if (Xaux.ParFESpace() != pfes)
+   {
+      Xaux.SetSpace(pfes);
+      Yaux.SetSpace(pfes);
+      Ytmp.SetSize(pfes->GetTrueVSize());
+   }

-   P->Mult(x, Xaux);
+   Xaux.Distribute(&x);
   if (ext)
   {
      ext->Mult(Xaux, Yaux);
@@ -364,8 +370,8 @@ const
                  " implemented");
      mat->Mult(Xaux, Yaux);
   }
-   P->MultTranspose(Yaux, Ytmp);
-   y.Add(a, Ytmp);
+   pfes->GetProlongationMatrix()->MultTranspose(Yaux, Ytmp);
+   y.Add(a,Ytmp);
 }

 void ParBilinearForm::FormLinearSystem(
@@ -31,8 +31,9 @@ class ParBilinearForm : public BilinearForm
 protected:
   ParFiniteElementSpace *pfes; ///< Points to the same object as #fes

-   /// Auxiliary vectors used in TrueAddMult(): L-, L-, and T-vector, resp.
-   mutable Vector Xaux, Yaux, Ytmp;
+   /// Auxiliary objects used in TrueAddMult().
+   mutable ParGridFunction Xaux, Yaux;
+   mutable Vector Ytmp;

   OperatorHandle p_mat, p_mat_e;

@@ -1536,7 +1536,7 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const
   // Works in tandem with GetFaceNbrFaceVDofs() defined above.

   MFEM_ASSERT(Nonconforming() && !NURBSext, "");
-   Geometry::Type face_geom = pmesh->GetFaceGeometry(i);
+   Geometry::Type face_geom = pmesh->GetFaceGeometryType(i);
   return fec->FiniteElementForGeometry(face_geom);
 }

@@ -2606,7 +2606,7 @@ int ParFiniteElementSpace
      if (dump < 10)
      {
         char fname[100];
-         snprintf(fname, 100, "dofs%02d.txt", MyRank);
+         sprintf(fname, "dofs%02d.txt", MyRank);
         std::ofstream f(fname);
         DebugDumpDOFs(f, deps, dof_group, dof_owner, finalized);
         dump++;
@@ -3025,8 +3025,6 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
   Array<char> mark(diag->Height());
   mark = 0;

-   bool is_dg = FEColl()->GetContType() == FiniteElementCollection::DISCONTINUOUS;
-
   for (int k = 0; k < dtrans.embeddings.Size(); k++)
   {
      const Embedding &emb = dtrans.embeddings[k];
@@ -3055,7 +3053,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
               int r = DofToVDof(dofs[i], vd);
               int m = (r >= 0) ? r : (-1 - r);

-               if (is_dg || !mark[m])
+               if (!mark[m])
               {
                  lR.GetRow(i, row);
                  diag->SetRow(r, old_vdofs, row);
@@ -3107,7 +3105,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
               int r = DofToVDof(dofs[i], vd);
               int m = (r >= 0) ? r : (-1 - r);

-               if (is_dg || !mark[m])
+               if (!mark[m])
               {
                  lR.GetRow(i, row);
                  MFEM_ASSERT(ldof[geom] == row.Size(), "");
@@ -3124,7 +3122,6 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
         }
      }
   }
-
   messages.clear();
   offd->Finalize(0);
   offd->SetWidth(col_map.size());
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
camierjs	58a2c7ac15	Merge master in lor-gpu	2022-10-23 09:54:23 -07:00
camierjs	2a04311099	LOR H1/ND/RT explicit specializations	2022-09-15 07:28:55 -07:00
camierjs	026c2ae2dd	Add lor_h1 batched kernel for order 1	2022-09-13 17:26:27 -07:00
Will Pazner	b9de6d2b37	Fix issue with memory reuse	2022-09-13 12:56:41 -07:00
Will Pazner	1cca575225	Merge remote-tracking branch 'origin/lor-gpu' into lor-gpu	2022-09-08 16:42:02 -07:00
Will Pazner	a44abee1ba	Fix some issues with CEED	2022-09-08 16:33:48 -07:00
Will Pazner	55497c158f	Add script to run AMR case	2022-09-08 16:33:14 -07:00
camierjs	39185e89bc	Bring simplified lor_h1 kernel	2022-09-06 10:10:59 -07:00
Will Pazner	cc6b3c025d	Merge remote-tracking branch 'origin/quadrature-coeff-lor' into lor-gpu	2022-09-02 16:02:17 -07:00
Will Pazner	e2be4b8d9b	Merge remote-tracking branch 'origin/face-quadrature-space-coefficient' into lor-gpu	2022-09-02 16:01:54 -07:00
Will Pazner	120963609b	Fix comment	2022-09-02 08:12:56 -07:00
Will PaznerandYohann Dudouit	ea310df39d	Replace some MFEM_ASSERT with MFEM_VERIFY Co-authored-by: Yohann Dudouit <dudouit1@llnl.gov>	2022-09-02 08:06:49 -07:00
Will Pazner	bae885021f	make style	2022-09-01 17:01:11 -07:00
Will Pazner	798f529929	NVTX	2022-09-01 16:57:28 -07:00
Will Pazner	72e5222a98	make style	2022-08-27 17:13:45 -07:00
Will Pazner	db9772acd4	Bench LOR AMS and ADS	2022-08-27 17:13:05 -07:00
Will Pazner	7a09a3d772	Reuse memory in discrete gradient and curl	2022-08-27 17:12:52 -07:00
Will Pazner	45402504ea	Allow memory reuse in FormLORVertexCoordinates	2022-08-27 17:12:34 -07:00
Will Pazner	24fd0b1c1e	Reuse memory in LOR benchmarks	2022-08-25 13:32:31 -07:00
Will Pazner	59fa852dc6	ND, RT, AMS, ADS LOR benchmarking	2022-08-25 11:40:52 -07:00
Will Pazner	eee4d9c25c	Specializations	2022-08-24 14:18:43 -07:00
Will Pazner	218c56945d	Add AMG to benchmarks	2022-08-24 13:56:30 -07:00
Will Pazner	7d31215c91	LOR benchmark	2022-08-24 13:37:17 -07:00
Will Pazner	f378aed68d	gitignore	2022-08-24 13:37:08 -07:00
Will Pazner	e4318fe0fb	Increase MD and MQ limits	2022-08-24 13:36:10 -07:00
Will Pazner	c504463321	Add warm up in LOR benchmark	2022-08-24 12:00:13 -07:00
Will Pazner	cba110b1bc	Reuse memory in LOR assembly	2022-08-24 12:00:02 -07:00
Will Pazner	474aa14907	Work on LOR benchmark	2022-08-24 11:40:17 -07:00
Will Pazner	86265d0848	Output total number of elements in plor_solvers	2022-08-24 09:48:21 -07:00
Will Pazner	152cae0c0e	Don't add NVTX for early-return hypre setup	2022-08-24 09:47:49 -07:00
Will Pazner	f98c2b9df5	Ignore more nsys files	2022-08-24 09:47:20 -07:00
Will Pazner	049ba615b3	Don't compute error unless also visualizing	2022-08-23 15:00:17 -07:00
Will Pazner	3f8c7aa5e1	Ignore nsys-rep files	2022-08-23 14:55:30 -07:00
Will Pazner	dd6164a9b4	Separate AMG setup in plor_solvers	2022-08-23 14:54:19 -07:00
Will Pazner	6974ebd250	Turn off visualization by default in plor_solvers	2022-08-23 14:39:36 -07:00
Will Pazner	947c51ff75	NVTX instrumentation	2022-08-23 14:39:24 -07:00
Will Pazner	d754cf4914	Add LOR benchmark	2022-08-23 07:49:44 -07:00
Will Pazner	5aa5cb551e	Add debug.hpp and nvtx.hpp	2022-08-23 07:48:54 -07:00
Will Pazner	7ce22d38c5	Support general coefficient types in batched LOR	2022-08-05 16:30:21 -07:00