make style

Adding Exodus II output option to mesh-explorer
Merge branch 'master' into exodus-writer-dev
2024-05-31 14:43:05 -07:00 · 2024-05-31 14:36:11 -07:00 · 2024-05-31 17:46:04 +01:00 · 2024-05-31 16:44:56 +00:00 · 2024-05-31 16:39:09 +00:00 · 2024-05-31 09:29:36 -07:00
632 changed files with 31723 additions and 17511 deletions
@@ -33,6 +33,7 @@ env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
  HYPRE_TOP_DIR: hypre-2.19.0
  METIS_ARCHIVE: metis-4.0.3.tar.gz
+  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
  METIS_TOP_DIR: metis-4.0.3
  MFEM_TOP_DIR: mfem

@@ -52,6 +53,7 @@ jobs:
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
+        precision: [fp64]
        exclude:
          - os: ubuntu-latest
            build-system: cmake
@@ -75,6 +77,8 @@ jobs:
          - os: ubuntu-latest
            target: dbg
            config-opts: 'CPPFLAGS+=-Og'
+          - os: macos-latest
+            codecov: NO
          - os: windows-latest
            codecov: NO
          - os: windows-latest
@@ -87,6 +91,7 @@ jobs:
            mpi: par
            build-system: cmake
            hypre-target: int32
+            precision: fp64
            # This option can be set to pass additional configuration options to
            # the MFEM configuration command.
            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
@@ -96,7 +101,15 @@ jobs:
            mpi: par
            build-system: make
            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+            precision: fp64
+          - os: ubuntu-latest
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int32
+            precision: fp32
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}

    runs-on: ${{ matrix.os }}

@@ -126,6 +139,17 @@ jobs:
          # Fetch the complete history for codecov to access commits ID
          fetch-depth: 0

+      - name: Xcode version setup (MacOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          XCODE_PATH="/Applications/Xcode_15.3.app"
+          echo "> sudo xcode-select -s ${XCODE_PATH}"
+          sudo xcode-select -s ${XCODE_PATH}
+          echo "> g++ -v"
+          g++ -v
+          echo "> clang++ -v"
+          clang++ -v
+
      # Only get MPI if defined for the job.
      # TODO: It would be nice to have only one step, e.g. with a dedicated
      # action, but I (@adrienbernede) don't see how at the moment.
@@ -169,25 +193,27 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5

      - name: get hypre
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: make
+          precision: ${{ matrix.precision }}

      - name: get hypre (Windows)
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: cmake
+          precision: ${{ matrix.precision }}

      # Get Metis through cache, or build it.
      # Install will only run on cache miss.
@@ -197,13 +223,13 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: install metis
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
-          archive: ${{ env.METIS_ARCHIVE }}
+          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
          dir: ${{ env.METIS_TOP_DIR }}

      - name: cache vcpkg (Windows)
@@ -228,7 +254,7 @@ jobs:

      # MFEM build and test
      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        env:
          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
        with:
@@ -240,6 +266,7 @@ jobs:
          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
          metis-dir: ${{ env.METIS_TOP_DIR }}
          mfem-dir: ${{ env.MFEM_TOP_DIR }}
+          precision: ${{ matrix.precision }}
          config-options: ${{ matrix.config-opts }}
          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}

@@ -282,7 +309,7 @@ jobs:
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
+        uses: mfem/github-actions/upload-coverage@v2.5
        with:
          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
          project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -53,11 +53,11 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.5

      - name: Get Hypre
        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +68,18 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: Install Metis
        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{ env.METIS_ARCHIVE }}
          dir: ${{ env.METIS_TOP_DIR }}

      # MFEM build and test
      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -44,7 +44,7 @@ jobs:
          path: mfem

      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -91,10 +91,6 @@ examples/ex16.mesh
 examples/ex16-mesh.*
 examples/ex16-init.*
 examples/ex16-final.*
-examples/vortex-mesh.*
-examples/vortex.mesh
-examples/vortex-?-init.*
-examples/vortex-?-final.*
 examples/deformation.*
 examples/pressure.*
 examples/ex20.dat
@@ -120,6 +116,8 @@ examples/cond_mesh.*
 examples/port_mesh.*
 examples/port_mode.*

+examples/euler-*
+
 examples/amgx/ex1
 examples/amgx/ex1p
 examples/amgx/.logamgx
@@ -234,7 +232,7 @@ miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
-miniapps/meshing/mesh-explorer.mesh
+miniapps/meshing/mesh-explorer.mesh*
 miniapps/meshing/partitioning.txt
 miniapps/meshing/mesh-explorer-visit*
 miniapps/meshing/mesh-explorer-paraview/
@@ -354,6 +352,8 @@ miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh

 miniapps/multidomain/multidomain
+miniapps/multidomain/multidomain_nd
+miniapps/multidomain/multidomain_rt
 miniapps/hooke/hooke

 miniapps/dpg/diffusion
@@ -369,6 +369,8 @@ miniapps/dpg/ParaView
 miniapps/spde/generate_random_field
 miniapps/spde/ParaView

+miniapps/tribol/contact-patch-test
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -13,6 +13,9 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
@@ -9,6 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
 # We define the following GitLab pipeline variables:
 variables:

@@ -35,9 +35,8 @@ variables:
    - when: on_success

 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
+# pre-allocation the same way slurm does.  We use the pci queue on lassen
+# to speed-up the allocation.
 .build_and_test_on_lassen:
  extends: [.on_lassen]
  stage: build_and_test
@@ -45,5 +44,5 @@ variables:
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -52,4 +52,4 @@ variables:
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -14,14 +14,14 @@ stages:
  - build_and_test
  - report

-opt_mpi_cuda_xl_16_1_1_12:
+opt_mpi_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen

-opt_mpi_cuda_hypre_cuda_xl:
+opt_mpi_cuda_hypre_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
  extends: .build_and_test_on_lassen

 # Jobs report
@@ -32,11 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
@@ -8,56 +8,109 @@
                               https://mfem.org


-Version 4.6.1 (development)
+Version 4.7.1 (development)
 ===========================

+
+Version 4.7, released on May 7, 2024
+====================================
+
+- Added support for single precision (with corresponding hypre build). The MFEM
+  floating point type was generalized from `double` to `real_t`. For details see
+  https://github.com/orgs/mfem/discussions/4207.
+
 Meshing improvements
 --------------------
+- Added the capability to partition (big) serial meshes in serial code, see the
+  new classes MeshPartitioner and MeshPart. This capability is also exposed as a
+  menu option in the mesh-explorer miniapp in miniapps/meshing.
+
+- Added named attribute sets and basic supporting methods to the Mesh class as a
+  convenient means of referring to sets of domain or boundary attribute numbers.
+  See the new Example 39/39p and data/compass.mesh.
+
 - Introduced formulas for refinement of patches in NURBS meshes. Refinement by
  arbitrary integer factors is also enabled, e.g. in the mesh-explorer miniapp.
  NURBS coarsening and knot removal are also introduced.

+- Added support for internal boundary elements in nonconforming meshes.
+
+- Added ExodusII output capability. The writer can handle first-order (Pyramid5, 
+  Wedge6, Hex8, Tet4) and second-order FE types (Pyramid14, Wedge18, Hex27, Tet10).
+
+- The ReadCubit Genesis mesh importer has been rewritten to improve readability.
+
 Discretization improvements
 ---------------------------
- Introduced support for higher order non conformal Nedelec elements on
-  simplices in ParMesh.
- Introduced support for internal boundary elements in nonconformal adapted
-  meshes.
+- Added a new nonlinear integrator, `HyperbolicFormIntegrator` that implements
+  both element-wise weak divergence and face-wise numerical flux for a general
+  system of hyperbolic conservation laws. To use the integrator for a specific
+  flux function, users can define a derived class of `FluxFunction`. Currently,
+  advection, Burgers, shallow-water and Euler equations (see Example 18/18p) are
+  available.

- Added functionality for construction of cut-surface and cut-volume
-  IntegrationRules through a moment-fitting approach. The cut is specified by
-  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
+- Added a capability to construct cut-surface and cut-volume IntegrationRules
+  through a moment-fitting approach. The cut is specified by the zero level set
+  of a Coefficient. See fem/intrules_cut.hpp and the new Example 38.
+
+- Introduced support for high-order nonconforming Nedelec elements on simplices.
+
+GPU computing
+-------------
+- Added partial assembly and GPU support for the DG diffusion integrator.
+
+- Efficient GPU-accelerated LOR assembly is now supported on surface meshes.
+
+- Added functionality to automatically configure hypre's compute policy to match
+  MFEM's compute policy when hypre is built with GPU support. Requires version
+  hypre-2.31.0 or later.

-GPU support
----------------------------
 - Added support for full assembly on simplices.
- Added functionality for BilinearFormIntegrators to use kernels that work for both
-  tensor and unstructured elements.
- Added partial assembly for linear elasticity. Does not use sum factorization for now.
+
+- Added partial assembly for linear elasticity (no sum factorization for now).
+
+- Added functionality for BilinearFormIntegrators to use kernels that work for
+  both tensor and unstructured elements.
+
+- The RAJA backend will use `seq_exec` for serial loop execution when RAJA
+  v2023.06.00 and beyond is detected as `loop_exec` is deprecated.
+
+- API change: The macro MFEM_HYPRE_FORALL (from hypre.hpp) which was intended
+  for internal use, has been removed and replaced by the function template
+  mfem::hypre_forall in general/forall.hpp.

 New and updated examples and miniapps
 -------------------------------------
- Added a new block solver in miniapp/solvers for the Darcy problem.
-  The new solver is based on a Bramble-Pasciak preconditioning. User can
-  use and implement their own preconditioner for the mass matrix.
+- Added a new miniapp illustrating elastic contact based on the Tribol library,
+  (https://github.com/LLNL/Tribol). See miniapps/tribol.

- Added miniapp to demonstrate new elasticity integrator and unstructured element GPU support,
-  and a block diagonal preconditioner using low order refinement. Allows comparison with
-  currently existing legacy mode integrator. See miniapps/solvers/lor_elast.
+- Added a miniapp to demonstrate low order refined (LOR) block preconditioning
+  for linear elasticity on GPUs. See miniapps/solvers/lor_elast.
+
+- Added a new block solver in miniapp/solvers for the Darcy problem. The new
+  solver is based on a Bramble-Pasciak preconditioning. User can use and
+  implement their own preconditioner for the mass matrix.
+
+- Added a small miniapp for printing the shape functions of a KnotVector. See
+  miniapps/nurbs/nurbs_printfunc.cpp.
+
+- Added two new example codes: 38 and 39/39p described above. Substantially
+  updated Example 18/18p.

 Miscellaneous
 -------------
- The ReadCubit Genesis mesh importer has been rewritten to improve readability.
-
 - Updated the Doxygen documentation style, which now requires Doxygen version
  1.9.8 or later. See the doc/ directory.

- Improved thread safety for global variables in the library, for example
-  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
-  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+- Improved thread safety for global variables in the library, e.g. for IntRules,
+  RefinedIntRules, GlobGeometryRefiner, and FiniteElement::dof2quad_array.

- RAJA backend will use seq_exec for serial loop execution when RAJA
-  v2023.06.00 and beyond is detected as loop_exec is deprecated.
+- PETSc integration now generally requires PETSc version 3.21 or later, though
+  depending on the functionality older versions may still work.
+
+- Various other simplifications, extensions, and bugfixes in the code.
+
+- Added GSLIB-based gather-scatter operator.


 Version 4.6, released on September 27, 2023
@@ -79,7 +132,6 @@ Meshing improvements
  * The edge to knot map for NURBS meshes can be determined automatically. It is
    no longer needed to specify this in the NURBS mesh.
  * Added curve interpolation method for NURBS.
-  * Added new small miniapp for printing of shape functions of a KnotVector
  * See miniapps/nurbs for example meshes and miniapps.

 Discretization improvements
@@ -126,8 +178,6 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

- Efficient GPU-accelerated LOR assembly now supports surface meshes.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
@@ -58,7 +58,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.7.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -87,10 +87,11 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
+# SUNDIALS, STRUMPACK, Ginkgo, Tribol, RAJA and Umpire require C++14:
 if ((MFEM_USE_SUNDIALS OR
     MFEM_USE_STRUMPACK OR
     MFEM_USE_GINKGO OR
+     MFEM_USE_TRIBOL OR
     MFEM_USE_RAJA OR
     MFEM_USE_UMPIRE) AND
    ("${CMAKE_CXX_STANDARD}" LESS "14"))
@@ -183,6 +184,19 @@ endif()
 # Process configuration options
 #-------------------------------------------------------------------------------

+# MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
+if (MFEM_PRECISION MATCHES "^(double|Double|DOUBLE)$")
+  set(MFEM_USE_SINGLE OFF)
+  set(MFEM_USE_DOUBLE ON)
+elseif (MFEM_PRECISION MATCHES "^(single|Single|SINGLE)$")
+  set(MFEM_USE_SINGLE ON)
+  set(MFEM_USE_DOUBLE OFF)
+else()
+  message(FATAL_ERROR " *** Invalid floating-point precision: "
+    "MFEM_PRECISION = ${MFEM_PRECISION}")
+endif()
+message(STATUS "Floating-point precision: MFEM_PRECISION = ${MFEM_PRECISION}")
+
 # MFEM_DEBUG
 if (CMAKE_BUILD_TYPE MATCHES "Debug|debug|DEBUG")
  set(MFEM_DEBUG ON)
@@ -490,6 +504,15 @@ if (MFEM_USE_PARELAG)
   find_package(PARELAG REQUIRED)
 endif()

+# Tribol
+if (MFEM_USE_TRIBOL)
+  if (MFEM_USE_MPI)
+    find_package(Tribol REQUIRED tribol redecomp)
+  else()
+    message(FATAL_ERROR " *** Tribol requires that MPI be enabled.")
+  endif()
+endif()
+
 # Enzyme
 if (MFEM_USE_ENZYME)
   find_package(ENZYME REQUIRED)
@@ -535,7 +558,7 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
-    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -151,7 +151,8 @@ The MFEM source code has the following structure:
  │   ├── solvers
  │   ├── spde
  │   ├── tools
-  │   └── toys
+  │   ├── toys
+  │   └── tribol
  └── tests
      ├── benchmarks
      ├── convergence
@@ -362,6 +363,8 @@ Before you can start, you need a GitHub account, here are a few suggestions:
    conflicted files in the commit message.
  - All significant new features and changes should be documented in CHANGELOG.
  - New examples and miniapps should have documentation on the MFEM webpage.
+  - The general floating-point type `real_t` should be used, rather than
+    `float` or `double`, except in special cases where only one is possible.


 ### Pull Requests
@@ -75,6 +75,8 @@ and miniapps. See https://glvis.org and https://mfem.org/building.

 Quick start with GNU make
 =========================
+See also: https://mfem.org/building
+
 Serial build:
   make serial -j 4

@@ -83,6 +85,7 @@ Parallel build:
   (build METIS 4 in ../metis-4.0 relative to mfem/)
   (build hypre in ../hypre relative to mfem/)
   make parallel -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   make cuda -j 4
@@ -116,6 +119,7 @@ Parallel build:
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
   make -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   (this build requires CMake 3.8 or newer)
@@ -284,6 +288,15 @@ MFEM_USE_METIS = YES/NO
   option in the library will be Cartesian partitioning with box meshes, and
   thus most of the parallel examples and miniapps will fail.

+MFEM_PRECISION = double/Double/DOUBLE/single/Single/SINGLE
+   Use single (float type) or double floating-point precision. In the
+   configuration header 'config/_config.hpp' this option is represented by
+   defining exactly one of the macros: MFEM_USE_DOUBLE, or MFEM_USE_SINGLE.
+   In the exported config files 'config.mk' and 'MFEMConfig.cmake', the option
+   is represented by the variables MFEM_USE_DOUBLE and MFEM_USE_SINGLE defined
+   as YES/NO (make) or ON/OFF (cmake). For more details see
+   https://github.com/orgs/mfem/discussions/4207
+
 MFEM_DEBUG = YES/NO
   Choose debug/optimized build. The debug build enables a number of messages
   and consistency checks that may simplify bug-hunting.
@@ -562,6 +575,11 @@ MFEM_USE_PARELAG = YES/NO
   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
   currently only concerns the miniapps.

+MFEM_USE_TRIBOL = YES/NO
+   Enables the miniapps that use the Tribol library. MFEM does not currently
+   use Tribol. In fact, Tribol is dependent on MFEM. Therefore, this option
+   currently only concerns the miniapps.
+
 MFEM_USE_ENZYME = YES/NO
   Enables automatic differentiation support through the LLVM plugin Enzyme.
   This requires the compiler to be set to clang (>=14.0.0). We also advise to
@@ -598,9 +616,13 @@ The specific libraries and their options are:
            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
            HYPRE >= 2.22.1   (HYPRE built with CUDA)
            HYPRE >= 2.23.0   (HYPRE built with HIP)
+            HYPRE >= 2.31.0   (runtime selectable HYPRE execution on CPU/GPU)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
-  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
+  MFEM_USE_METIS_5 = YES (default is to use METIS 4). For building instructions,
+  see the following:
+    - METIS 4.0.3: https://mfem.org/building/#parallel-mpi-version-of-mfem
+    - METIS 5.1.0: https://mfem.org/building/#parallel-build-using-metis-5
  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.
@@ -692,9 +714,10 @@ The specific libraries and their options are:
  Options: NETCDF_OPT, NETCDF_LIB.
  Versions: NetCDF >= 4.4.0.

- PETSc (optional), used when MFEM_USE_PETSC = YES. Version 3.8 or higher of
-  the PETSC dev branch is required. The MFEM and PETSc builds can share common
-  libraries, e.g., hypre and SUNDIALS. Here's an example configuration, assuming
+- PETSc (optional), used when MFEM_USE_PETSC = YES. Version 3.21 or higher of
+  the PETSC dev branch is required, though depending on the functionality older
+  versions may work too. The MFEM and PETSc builds can share common libraries,
+  e.g., hypre and SUNDIALS. Here's an example configuration, assuming
  PETSc has been cloned on the same level as mfem and hypre:
    ./configure --download-fblaslapack=yes --download-scalapack=yes \
                --download-mumps=yes --download-suitesparse=yes \
@@ -704,9 +727,7 @@ The specific libraries and their options are:
  CFLAGS to allow proper parsing of the hipsparse header under C.
  URL: https://www.mcs.anl.gov/petsc
  Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA/HIP)
-            PETSc >= 3.15.0  (PETSc built with CUDA)
-            PETSc >= 3.19.0  (PETSc built with HIP, older versions may work too)
+  Versions: PETSc >= 3.21.0, older versions may work too.

 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
  uses some of the PETSc options when compiled.
@@ -849,6 +870,10 @@ The specific libraries and their options are:
  URL: https://github.com/LLNL/parelag
  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.

+- Tribol, used when MFEM_USE_TRIBOL = YES.
+  URL: https://github.com/LLNL/Tribol
+  Options: TRIBOL_DIR, TRIBOL_OPT, TRIBOL_LIB.
+
 - Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
  URL: https://github.com/EnzymeAD/Enzyme
  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
@@ -959,6 +984,7 @@ The following options are equivalent to the GNU make options with the same name:

 MFEM_USE_MPI
 MFEM_USE_METIS - Set to ${MFEM_USE_MPI}, can be overwritten.
+MFEM_PRECISION
 MFEM_USE_LIBUNWIND
 MFEM_USE_LAPACK
 MFEM_THREAD_SAFE
@@ -992,6 +1018,7 @@ MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
 MFEM_USE_PARELAG
+MFEM_USE_TRIBOL
 MFEM_USE_ENZYME

 The following options are CMake specific:
@@ -287,3 +287,7 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_PARELAG)
  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
 ENDIF()
+
+IF (DEFINED TPL_ENABLE_TRIBOL)
+  SET(MFEM_USE_TRIBOL ${TPL_ENABLE_TRIBOL} CACHE BOOL "Enable Tribol" FORCE)
+ENDIF()
@@ -18,6 +18,8 @@ set(MFEM_GIT_STRING "@MFEM_GIT_STRING@")
 set(MFEM_USE_MPI @MFEM_USE_MPI@)
 set(MFEM_USE_METIS @MFEM_USE_METIS@)
 set(MFEM_USE_METIS_5 @MFEM_USE_METIS_5@)
+set(MFEM_USE_DOUBLE @MFEM_USE_DOUBLE@)
+set(MFEM_USE_SINGLE @MFEM_USE_SINGLE@)
 set(MFEM_DEBUG @MFEM_DEBUG@)
 set(MFEM_USE_EXCEPTIONS @MFEM_USE_EXCEPTIONS@)
 set(MFEM_USE_ZLIB @MFEM_USE_ZLIB@)
@@ -62,6 +64,7 @@ set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
+set(MFEM_USE_TRIBOL @MFEM_USE_TRIBOL@)
 set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
@@ -46,6 +46,12 @@
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 #cmakedefine MFEM_USE_MPI

+// Use double-precision floating point type
+#cmakedefine MFEM_USE_DOUBLE
+
+// Use single-precision floating point type
+#cmakedefine MFEM_USE_SINGLE
+
 // Enable debug checks in MFEM.
 #cmakedefine MFEM_DEBUG

@@ -18,4 +18,13 @@ include(MfemCmakeUtilities)
 # Note: components are enabled based on the find_package() parameters.
 mfem_find_package(Axom AXOM AXOM_DIR "include" "" "lib" ""
  "Paths to headers required by Axom." "Libraries required by Axom."
-  ADD_COMPONENT Axom "include" axom/config.hpp "lib" axom)
+  ADD_COMPONENT core "include" axom/core.hpp "lib" axom_core
+  ADD_COMPONENT inlet "include" axom/inlet.hpp "lib" axom_inlet
+  ADD_COMPONENT klee "include" axom/klee.hpp "lib" axom_klee
+  ADD_COMPONENT lumberjack "include" axom/lumberjack.hpp "lib" axom_lumberjack
+  ADD_COMPONENT mint "include" axom/mint.hpp "lib" axom_mint
+  ADD_COMPONENT multimat "include" axom/multimat.hpp "lib" axom_multimat
+  ADD_COMPONENT quest "include" axom/quest.hpp "lib" axom_quest
+  ADD_COMPONENT sidre "include" axom/sidre.hpp "lib" axom_sidre
+  ADD_COMPONENT slam "include" axom/slam.hpp "lib" axom_slam
+  ADD_COMPONENT slic "include" axom/slic.hpp "lib" axom_slic)
@@ -36,7 +36,11 @@ include(MfemCmakeUtilities)
 mfem_find_package(Conduit CONDUIT CONDUIT_DIR
  "include;include/conduit" conduit.hpp "lib" conduit
  "Paths to headers required by Conduit." "Libraries required by Conduit."
+  ADD_COMPONENT blueprint
+    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint
+  ADD_COMPONENT blueprint_mpi
+    "include;include/conduit" conduit_blueprint_mpi.hpp "lib" conduit_blueprint_mpi
  ADD_COMPONENT relay
    "include;include/conduit" conduit_relay.hpp "lib" conduit_relay
-  ADD_COMPONENT blueprint
-    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
+  ADD_COMPONENT relay_mpi
+    "include;include/conduit" conduit_relay_mpi.hpp "lib" conduit_relay_mpi)
@@ -79,7 +79,9 @@ if (HYPRE_FOUND AND HYPRE_USING_CUDA)
  find_package(CUDAToolkit REQUIRED)
  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
  get_target_property(CURAND_LIBRARIES CUDA::curand LOCATION)
-  list(APPEND HYPRE_LIBRARIES ${CUSPARSE_LIBRARIES} ${CURAND_LIBRARIES})
+  get_target_property(CUBLAS_LIBRARIES CUDA::cublas LOCATION)
+  list(APPEND HYPRE_LIBRARIES ${CUSPARSE_LIBRARIES} ${CURAND_LIBRARIES}
+       ${CUBLAS_LIBRARIES})
  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
      "HYPRE libraries + dependencies." FORCE)
  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
@@ -16,12 +16,22 @@
 #   - MUMPS_VERSION

 include(MfemCmakeUtilities)
+
+# Toggle which precision of MUMPS to use depending on the precision of MFEM.
+if (MFEM_USE_DOUBLE)
+  set(_mumps_header dmumps_c.h)
+  set(_mumps_lib dmumps)
+elseif(MFEM_USE_SINGLE)
+  set(_mumps_header smumps_c.h)
+  set(_mumps_lib smumps)
+endif()
+
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
-  "include" dmumps_c.h "lib" dmumps
+  "include" ${_mumps_header} "lib" ${_mumps_lib}
  "Paths to headers required by MUMPS."
  "Libraries required by MUMPS."
-  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
-  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
+  ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
+  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)

 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - TRIBOL_FOUND
+#   - TRIBOL_LIBRARIES
+#   - TRIBOL_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+# Note: components are enabled based on the find_package() parameters.
+mfem_find_package(Tribol TRIBOL TRIBOL_DIR "include" tribol/config.hpp "lib" tribol
+  "Paths to headers required by Tribol." "Libraries required by Tribol."
+  ADD_COMPONENT redecomp
+    "include" redecomp/redecomp.hpp "lib" redecomp)
@@ -842,16 +842,17 @@ function(mfem_export_mk_files)

  # Convert Boolean vars to YES/NO without writing the values to cache
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
-      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
-      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
-      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
-      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
-      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
+      MFEM_USE_SINGLE MFEM_USE_DOUBLE MFEM_DEBUG MFEM_USE_EXCEPTIONS
+      MFEM_USE_ZLIB MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE
+      MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS
+      MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS
+      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS
+      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB
+      MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED
+      MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2
+      MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_ADFORWARD
+      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL
      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
@@ -23,6 +23,62 @@
 #include "_config.hpp"
 #endif

+namespace mfem
+{
+
+#if (defined(MFEM_USE_CUDA) && defined(__CUDACC__)) || \
+    (defined(MFEM_USE_HIP) && defined(__HIPCC__))
+#define MFEM_HOST_DEVICE __host__ __device__
+#else
+#define MFEM_HOST_DEVICE
+#endif
+
+// MFEM precision configuration
+
+#if defined MFEM_USE_SINGLE && defined MFEM_USE_DOUBLE
+#error "DOUBLE and SINGLE precision cannot both be specified"
+#endif
+
+#ifdef MFEM_USE_SINGLE
+typedef float real_t;
+#elif defined MFEM_USE_DOUBLE
+typedef double real_t;
+#else
+#error "Either DOUBLE or SINGLE precision must be specified"
+#endif
+
+MFEM_HOST_DEVICE
+constexpr real_t operator""_r(long double v)
+{
+   return static_cast<real_t>(v);
+}
+
+MFEM_HOST_DEVICE
+constexpr real_t operator""_r(unsigned long long v)
+{
+   return static_cast<real_t>(v);
+}
+
+} // namespace mfem
+
+// Return value for main function in examples that should be skipped by testing
+// in some case. This return value prevents failures in testing.
+#define MFEM_SKIP_RETURN_VALUE 242
+
+// Request a global object to be instantiated for each thread in its TLS.
+#define MFEM_THREAD_LOCAL thread_local
+
+// MFEM_DEPRECATED macro to mark obsolete functions and methods
+// see https://stackoverflow.com/questions/295120/c-mark-as-deprecated
+#if defined(__GNUC__) || defined(__clang__)
+#define MFEM_DEPRECATED __attribute__((deprecated))
+#elif defined(_MSC_VER)
+#define MFEM_DEPRECATED __declspec(deprecated)
+#else
+#pragma message("WARNING: You need to implement MFEM_DEPRECATED for this compiler")
+#define MFEM_DEPRECATED
+#endif
+
 // Common configuration macros

 #if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7)) || defined(__clang__)
@@ -64,6 +120,15 @@

 // Check dependencies:

+// Define MFEM_MPI_REAL_T to be the appropriate MPI real type
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_SINGLE
+#define MFEM_MPI_REAL_T MPI_FLOAT
+#elif defined MFEM_USE_DOUBLE
+#define MFEM_MPI_REAL_T MPI_DOUBLE
+#endif
+#endif
+
 // Options that require MPI
 #ifndef MFEM_USE_MPI
 #ifdef MFEM_USE_SUPERLU
@@ -46,6 +46,12 @@
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI

+// Use double-precision floating point type
+// #define MFEM_USE_DOUBLE
+
+// Use single-precision floating point type
+// #define MFEM_USE_SINGLE
+
 // Enable debug checks in MFEM.
 // #define MFEM_DEBUG

@@ -18,6 +18,8 @@ MFEM_GIT_STRING        = @MFEM_GIT_STRING@
 MFEM_USE_MPI           = @MFEM_USE_MPI@
 MFEM_USE_METIS         = @MFEM_USE_METIS@
 MFEM_USE_METIS_5       = @MFEM_USE_METIS_5@
+MFEM_USE_DOUBLE        = @MFEM_USE_DOUBLE@
+MFEM_USE_SINGLE        = @MFEM_USE_SINGLE@
 MFEM_DEBUG             = @MFEM_DEBUG@
 MFEM_USE_EXCEPTIONS    = @MFEM_USE_EXCEPTIONS@
 MFEM_USE_ZLIB          = @MFEM_USE_ZLIB@
@@ -63,6 +65,7 @@ MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
+MFEM_USE_TRIBOL        = @MFEM_USE_TRIBOL@
 MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@

 # Compiler, compile options, and link options
@@ -22,6 +22,8 @@ endif()
 option(BUILD_SHARED_LIBS "Enable shared library build of MFEM" OFF)
 option(MFEM_USE_MPI "Enable MPI parallel build" OFF)
 option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
+set(MFEM_PRECISION "double" CACHE STRING
+    "Floating-point precision to use: single, or double")
 option(MFEM_USE_EXCEPTIONS "Enable the use of exceptions" OFF)
 option(MFEM_USE_ZLIB "Enable zlib for compressed data streams." OFF)
 option(MFEM_USE_LIBUNWIND "Enable backtrace for errors." OFF)
@@ -65,6 +67,7 @@ option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
+option(MFEM_USE_TRIBOL "Enable Tribol" OFF)
 option(MFEM_USE_ENZYME "Enable Enzyme" OFF)

 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
@@ -210,8 +213,15 @@ set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH

 set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 # May need to add "Boost" as requirement.
-set(Axom_REQUIRED_PACKAGES "Conduit/relay/blueprint" CACHE STRING
-    "Additional packages required by Axom.")
+if (MFEM_USE_SIDRE)
+    if (MFEM_USE_MPI)
+        set(Axom_REQUIRED_PACKAGES "Conduit/blueprint/blueprint_mpi/relay/relay_mpi" CACHE STRING
+            "Additional packages required by Axom.")
+    elseif()
+        set(Axom_REQUIRED_PACKAGES "Conduit/blueprint/relay" CACHE STRING
+            "Additional packages required by Axom.")
+    endif()
+endif()

 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
    "Directory where PUMI is installed")
@@ -248,6 +258,10 @@ set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
 set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
    "The ParELAG library.")

+set(TRIBOL_DIR "${MFEM_DIR}/../tribol" CACHE PATH "Path to Tribol")
+set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING 
+    "Additional packages required by Tribol")
+
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
@@ -120,6 +120,7 @@ MFEM_MPI_NP = 4
 MFEM_USE_MPI           = NO
 MFEM_USE_METIS         = $(MFEM_USE_MPI)
 MFEM_USE_METIS_5       = NO
+MFEM_PRECISION         = double
 MFEM_DEBUG             = NO
 MFEM_USE_EXCEPTIONS    = NO
 MFEM_USE_ZLIB          = NO
@@ -166,8 +167,21 @@ MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
+MFEM_USE_TRIBOL        = NO
 MFEM_USE_ENZYME        = NO

+# Process MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
+ifneq ($(filter double Double DOUBLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = YES
+   MFEM_USE_SINGLE = NO
+else ifneq ($(filter single Single SINGLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = NO
+   MFEM_USE_SINGLE = YES
+else ifeq ($(MAKECMDGOALS),config)
+   $(error Invalid floating-point precision: \
+     MFEM_PRECISION = $(MFEM_PRECISION))
+endif
+
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
 ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
@@ -317,8 +331,13 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
- -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+ifeq ($(MFEM_USE_SINGLE),YES)
+   MUMPS_LIB += -lsmumps
+else
+   MUMPS_LIB += -ldmumps
+endif
+MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)

 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
@@ -369,7 +388,7 @@ GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -570,6 +589,16 @@ PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
 PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG

+# Tribol library configuration
+ifeq ($(MFEM_USE_TRIBOL),YES)
+   BASE_FLAGS = -std=c++14
+endif
+AXOM_DIR = @MFEM_DIR@/../axom
+TRIBOL_DIR = @MFEM_DIR@/../tribol
+TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
+TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
+   -laxom_slam -laxom_slic -laxom_core
+
 # Enzyme configuration

 # If you want to enable automatic differentiation at compile time, use the
@@ -9,7 +9,11 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

+#ifdef MFEM_USE_SINGLE
+#include "smumps_c.h"
+#else
 #include "dmumps_c.h"
+#endif
 #include <string>
 #include <iostream>
 #include <algorithm>
@@ -110,4 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out *.dSYM
@@ -0,0 +1,118 @@
+SetFactory("OpenCASCADE");
+
+order = 1;
+
+R = 1;
+r = 0.2;
+
+Point(1) = {0,0,0};
+
+Point(2) = {r/Sqrt(2),r/Sqrt(2),0};
+Point(3) = {-r/Sqrt(2),r/Sqrt(2),0};
+Point(4) = {-r/Sqrt(2),-r/Sqrt(2),0};
+Point(5) = {r/Sqrt(2),-r/Sqrt(2),0};
+
+Point(6) = {R,0,0};
+Point(7) = {R/Sqrt(2),R/Sqrt(2),0};
+Point(8) = {0,R,0};
+Point(9) = {-R/Sqrt(2),R/Sqrt(2),0};
+Point(10) = {-R,0,0};
+Point(11) = {-R/Sqrt(2),-R/Sqrt(2),0};
+Point(12) = {0,-R,0};
+Point(13) = {R/Sqrt(2),-R/Sqrt(2),0};
+
+Line(1) = {1,2};
+Line(2) = {1,3};
+Line(3) = {1,4};
+Line(4) = {1,5};
+
+Line(5) = {1,6};
+Line(6) = {1,8};
+Line(7) = {1,10};
+Line(8) = {1,12};
+
+Line(9) = {2,6};
+Line(10) = {2,8};
+Line(11) = {3,8};
+Line(12) = {3,10};
+Line(13) = {4,10};
+Line(14) = {4,12};
+Line(15) = {5,12};
+Line(16) = {5,6};
+
+Line(17) = {6,7};
+Line(18) = {7,8};
+Line(19) = {8,9};
+Line(20) = {9,10};
+Line(21) = {10,11};
+Line(22) = {11,12};
+Line(23) = {12,13};
+Line(24) = {13,6};
+
+Transfinite Curve{1:24} = 2;
+
+Physical Curve("ENE") = {17};
+Physical Curve("NNE") = {18};
+Physical Curve("NNW") = {19};
+Physical Curve("WNW") = {20};
+Physical Curve("WSW") = {21};
+Physical Curve("SSW") = {22};
+Physical Curve("SSE") = {23};
+Physical Curve("ESE") = {24};
+
+Curve Loop(1) = {9,17,18,-10};
+Curve Loop(2) = {11,19,20,-12};
+Curve Loop(3) = {13,21,22,-14};
+Curve Loop(4) = {15,23,24,-16};
+
+Plane Surface(1) = {1};
+Plane Surface(2) = {2};
+Plane Surface(3) = {3};
+Plane Surface(4) = {4};
+
+Transfinite Surface{1} = {2,6,7,8};
+Transfinite Surface{2} = {3,8,9,10};
+Transfinite Surface{3} = {4,10,11,12};
+Transfinite Surface{4} = {5,12,13,6};
+Recombine Surface{1:4};
+
+Physical Surface("Base") = {1,2,3,4};
+
+Curve Loop(5) = {1,10,-6};
+Plane Surface(5) = {5};
+Physical Surface("N Even") = {5};
+
+Curve Loop(6) = {6,-11,-2};
+Plane Surface(6) = {6};
+Physical Surface("N Odd") = {6};
+
+Curve Loop(7) = {2,12,-7};
+Plane Surface(7) = {7};
+Physical Surface("W Even") = {7};
+
+Curve Loop(8) = {7,-13,-3};
+Plane Surface(8) = {8};
+Physical Surface("W Odd") = {8};
+
+Curve Loop(9) = {3,14,-8};
+Plane Surface(9) = {9};
+Physical Surface("S Even") = {9};
+
+Curve Loop(10) = {8,-15,-4};
+Plane Surface(10) = {10};
+Physical Surface("S Odd") = {10};
+
+Curve Loop(11) = {4,16,-5};
+Plane Surface(11) = {11};
+Physical Surface("E Even") = {11};
+
+Curve Loop(12) = {5,-9,-1};
+Plane Surface(12) = {12};
+Physical Surface("E Odd") = {12};
+
+// Generate 2D mesh
+Mesh 2;
+SetOrder order;
+Mesh.MshFileVersion = 2.2;
+
+Save "compass.msh";
@@ -0,0 +1,96 @@
+MFEM mesh v1.3
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+12
+10 2 7 0 1
+11 2 0 7 2
+12 2 9 0 2
+13 2 0 9 3
+14 2 11 0 3
+15 2 0 11 4
+16 2 5 0 4
+17 2 0 5 1
+9 3 1 5 6 7
+9 3 2 7 8 9
+9 3 3 9 10 11
+9 3 4 11 12 5
+
+attribute_sets
+16
+"Base" 1 9
+"E Even" 1 16
+"E Odd" 1 17
+"East" 2 16 17
+"N Even" 1 10
+"N Odd" 1 11
+"North" 2 10 11
+"Rose" 8 10 11 12 13 14 15 16 17
+"Rose Even" 4 10 12 14 16
+"Rose Odd" 4 11 13 15 17
+"S Even" 1 14
+"S Odd" 1 15
+"South" 2 14 15
+"W Even" 1 12
+"W Odd" 1 13
+"West" 2 12 13
+
+boundary
+8
+1 1 5 6
+2 1 6 7
+3 1 7 8
+4 1 8 9
+5 1 9 10
+6 1 10 11
+7 1 11 12
+8 1 12 5
+
+bdr_attribute_sets
+13
+"Boundary" 8 1 2 3 4 5 6 7 8
+"ENE" 1 1
+"ESE" 1 8
+"Eastern Boundary" 2 1 8
+"NNE" 1 2
+"NNW" 1 3
+"Northern Boundary" 2 2 3
+"SSE" 1 7
+"SSW" 1 6
+"Southern Boundary" 2 6 7
+"WNW" 1 4
+"WSW" 1 5
+"Western Boundary" 2 4 5
+
+vertices
+13
+2
+0 0
+0.14142136 0.14142136
+-0.14142136 0.14142136
+-0.14142136 -0.14142136
+0.14142136 -0.14142136
+1 0
+0.70710678 0.70710678
+0 1
+-0.70710678 0.70710678
+-1 0
+-0.70710678 -0.70710678
+0 -1
+0.70710678 -0.70710678
+
+mfem_mesh_end
@@ -0,0 +1,62 @@
+$MeshFormat
+2.2 0 8
+$EndMeshFormat
+$PhysicalNames
+17
+1 1 "ENE"
+1 2 "NNE"
+1 3 "NNW"
+1 4 "WNW"
+1 5 "WSW"
+1 6 "SSW"
+1 7 "SSE"
+1 8 "ESE"
+2 9 "Base"
+2 10 "N Even"
+2 11 "N Odd"
+2 12 "W Even"
+2 13 "W Odd"
+2 14 "S Even"
+2 15 "S Odd"
+2 16 "E Even"
+2 17 "E Odd"
+$EndPhysicalNames
+$Nodes
+13
+1 0 0 0
+2 0.1414213562373095 0.1414213562373095 0
+3 -0.1414213562373095 0.1414213562373095 0
+4 -0.1414213562373095 -0.1414213562373095 0
+5 0.1414213562373095 -0.1414213562373095 0
+6 1 0 0
+7 0.7071067811865475 0.7071067811865475 0
+8 0 1 0
+9 -0.7071067811865475 0.7071067811865475 0
+10 -1 0 0
+11 -0.7071067811865475 -0.7071067811865475 0
+12 0 -1 0
+13 0.7071067811865475 -0.7071067811865475 0
+$EndNodes
+$Elements
+20
+1 1 2 1 17 6 7
+2 1 2 2 18 7 8
+3 1 2 3 19 8 9
+4 1 2 4 20 9 10
+5 1 2 5 21 10 11
+6 1 2 6 22 11 12
+7 1 2 7 23 12 13
+8 1 2 8 24 13 6
+9 2 2 10 5 1 2 8
+10 2 2 11 6 1 8 3
+11 2 2 12 7 1 3 10
+12 2 2 13 8 1 10 4
+13 2 2 14 9 1 4 12
+14 2 2 15 10 1 12 5
+15 2 2 16 11 1 5 6
+16 2 2 17 12 1 6 2
+17 3 2 9 1 2 6 7 8
+18 3 2 9 2 3 8 9 10
+19 3 2 9 3 4 10 11 12
+20 3 2 9 4 5 12 13 6
+$EndElements
@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.6.1
+PROJECT_NUMBER         = v4.7.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -987,6 +987,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/tribol \
                         @MFEM_SOURCE_DIR@/miniapps/spde \
                         @MFEM_SOURCE_DIR@/miniapps/dpg \
                         @MFEM_SOURCE_DIR@/miniapps/dpg/util
@@ -110,9 +110,11 @@ namespace mfem {
 * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
 * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
- * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
+ * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
 * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
+ * - <a class="el" href="ex39_8cpp_source.html">Example 39</a>: named mesh attributes
+ * - <a class="el" href="ex39p_8cpp_source.html">Example 39</a>: parallel named mesh attributes
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -214,6 +216,8 @@ namespace mfem {
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
 * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
 * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
 * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
@@ -46,7 +46,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
        })
        // Update the color scheme when the tab is made visible again.
-        // It is possible that the appearance was changed in another tab 
+        // It is possible that the appearance was changed in another tab
        // while this tab was in the background.
        document.addEventListener("visibilitychange", visibilityState => {
            if (document.visibilityState === 'visible') {
@@ -97,7 +97,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
     * @returns `true` for dark-mode, `false` for light-mode user preference
     */
    static get userPreference() {
-        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) ||
        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
    }

@@ -43,13 +43,9 @@ list(APPEND ALL_EXE_SRCS
  ex34.cpp
  ex36.cpp
  ex37.cpp
-  )
-
-if(MFEM_USE_LAPACK)
-list(APPEND ALL_EXE_SRCS
  ex38.cpp
+  ex39.cpp
  )
-endif()

 if (MFEM_USE_MPI)
  list(APPEND ALL_EXE_SRCS
@@ -90,7 +86,20 @@ if (MFEM_USE_MPI)
    ex35p.cpp
    ex36p.cpp
    ex37p.cpp
-    )
+    ex39p.cpp
+  )
+endif()
+
+# Examples that return MFEM_SKIP_RETURN_VALUE in some cases:
+set(SKIP_TESTS)
+if (HYPRE_USING_CUDA OR HYPRE_USING_HIP)
+  list(APPEND SKIP_TESTS ex19p.cpp ex28p.cpp)
+endif()
+if (MFEM_USE_SINGLE)
+  list(APPEND SKIP_TESTS ex33.cpp ex33p.cpp)
+endif()
+if (NOT MFEM_USE_LAPACK)
+  list(APPEND SKIP_TESTS ex38.cpp)
 endif()

 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
@@ -102,6 +111,9 @@ add_mfem_examples(ALL_EXE_SRCS)
 # Add a test for each example
 if (MFEM_ENABLE_TESTING)
  foreach(SRC_FILE ${ALL_EXE_SRCS})
+    if (SRC_FILE IN_LIST SKIP_TESTS)
+      continue()
+    endif()
    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

@@ -134,10 +146,10 @@ if (MFEM_ENABLE_TESTING)
  # Add CUDA/HIP tests.
  set(DEVICE_EXAMPLES
      # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
+      ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
      # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
-      ex34p ex35p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p ex22p ex24p ex25p
+      ex26p ex34p ex35p)
  set(MFEM_TEST_DEVICE)
  if (MFEM_USE_CUDA)
    set(MFEM_TEST_DEVICE "cuda")
@@ -147,6 +159,11 @@ if (MFEM_ENABLE_TESTING)
  if (MFEM_TEST_DEVICE)
    foreach(TEST_NAME ${DEVICE_EXAMPLES})
      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (${TEST_NAME} MATCHES "ex14p")
+        list(APPEND THIS_TEST_OPTIONS "-rs" "2" "-rp" "0" "-pa")
+      elseif (${TEST_NAME} MATCHES "ex14")
+        list(APPEND THIS_TEST_OPTIONS "-r" "2" "-pa")
+      endif()
      if (NOT (${TEST_NAME} MATCHES ".*p$"))
        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
@@ -62,7 +62,7 @@ protected:

   BilinearForm M, S;
   NonlinearForm H;
-   double viscosity;
+   real_t viscosity;
   HyperelasticModel *model;

   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
@@ -84,16 +84,16 @@ protected:

 public:
   HyperelasticOperator(FiniteElementSpace &f, Array<int> &ess_bdr,
-                        double visc, double mu, double K);
+                        real_t visc, real_t mu, real_t K);

   /// Compute the right-hand side of the ODE system.
   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
   /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);

-   double ElasticEnergy(const Vector &x) const;
-   double KineticEnergy(const Vector &v) const;
+   real_t ElasticEnergy(const Vector &x) const;
+   real_t KineticEnergy(const Vector &v) const;
   void GetElasticEnergyDensity(const GridFunction &x, GridFunction &w) const;

   virtual ~HyperelasticOperator();
@@ -109,7 +109,7 @@ private:
   BilinearForm *M, *S;
   NonlinearForm *H;
   mutable SparseMatrix *Jacobian;
-   double dt;
+   real_t dt;
   const Vector *v, *x;
   mutable Vector w, z;

@@ -117,7 +117,7 @@ public:
   ReducedSystemOperator(BilinearForm *M_, BilinearForm *S_, NonlinearForm *H_);

   /// Set current dt, v, x values - needed to compute action and Jacobian.
-   void SetParameters(double dt_, const Vector *v_, const Vector *x_);
+   void SetParameters(real_t dt_, const Vector *v_, const Vector *x_);

   /// Compute y = H(x + dt (v + dt k)) + M k + S (v + dt k).
   virtual void Mult(const Vector &k, Vector &y) const;
@@ -141,7 +141,7 @@ private:
 public:
   ElasticEnergyCoefficient(HyperelasticModel &m, const GridFunction &x_)
      : model(m), x(x_) { }
-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
   virtual ~ElasticEnergyCoefficient() { }
 };

@@ -161,11 +161,11 @@ int main(int argc, char *argv[])
   int ref_levels = 2;
   int order = 2;
   int ode_solver_type = 3;
-   double t_final = 300.0;
-   double dt = 3.0;
-   double visc = 1e-2;
-   double mu = 0.25;
-   double K = 5.0;
+   real_t t_final = 300.0;
+   real_t dt = 3.0;
+   real_t visc = 1e-2;
+   real_t mu = 0.25;
+   real_t K = 5.0;
   bool visualization = true;
   int vis_steps = 1;

@@ -309,13 +309,13 @@ int main(int argc, char *argv[])
           << " Press space (in the GLVis window) to resume it.\n";
   }

-   double ee0 = oper.ElasticEnergy(x.GetTrueVector());
-   double ke0 = oper.KineticEnergy(v.GetTrueVector());
+   real_t ee0 = oper.ElasticEnergy(x.GetTrueVector());
+   real_t ke0 = oper.KineticEnergy(v.GetTrueVector());
   cout << "initial elastic energy (EE) = " << ee0 << endl;
   cout << "initial kinetic energy (KE) = " << ke0 << endl;
   cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;

-   double t = 0.0;
+   real_t t = 0.0;
   oper.SetTime(t);
   ode_solver->Init(oper);

@@ -324,7 +324,7 @@ int main(int argc, char *argv[])
   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
   {
-      double dt_real = min(dt, t_final - t);
+      real_t dt_real = min(dt, t_final - t);

      ode_solver->Step(vx, t, dt_real);

@@ -332,8 +332,8 @@ int main(int argc, char *argv[])

      if (last_step || (ti % vis_steps) == 0)
      {
-         double ee = oper.ElasticEnergy(x.GetTrueVector());
-         double ke = oper.KineticEnergy(v.GetTrueVector());
+         real_t ee = oper.ElasticEnergy(x.GetTrueVector());
+         real_t ke = oper.KineticEnergy(v.GetTrueVector());

         cout << "step " << ti << ", t = " << t << ", EE = " << ee << ", KE = "
              << ke << ", ΔTE = " << (ee+ke)-(ee0+ke0) << endl;
@@ -419,7 +419,7 @@ ReducedSystemOperator::ReducedSystemOperator(
     dt(0.0), v(NULL), x(NULL), w(height), z(height)
 { }

-void ReducedSystemOperator::SetParameters(double dt_, const Vector *v_,
+void ReducedSystemOperator::SetParameters(real_t dt_, const Vector *v_,
                                          const Vector *x_)
 {
   dt = dt_;  v = v_;  x = x_;
@@ -453,16 +453,26 @@ ReducedSystemOperator::~ReducedSystemOperator()


 HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
-                                           Array<int> &ess_bdr, double visc,
-                                           double mu, double K)
-   : TimeDependentOperator(2*f.GetTrueVSize(), 0.0), fespace(f),
+                                           Array<int> &ess_bdr, real_t visc,
+                                           real_t mu, real_t K)
+   : TimeDependentOperator(2*f.GetTrueVSize(), (real_t) 0.0), fespace(f),
     M(&fespace), S(&fespace), H(&fespace),
     viscosity(visc), z(height/2)
 {
-   const double rel_tol = 1e-8;
+#if defined(MFEM_USE_DOUBLE)
+   const real_t rel_tol = 1e-8;
+   const real_t newton_abs_tol = 0.0;
+#elif defined(MFEM_USE_SINGLE)
+   const real_t rel_tol = 1e-3;
+   const real_t newton_abs_tol = 1e-4;
+#else
+#error "Only single and double precision are supported!"
+   const real_t rel_tol = real_t(1);
+   const real_t newton_abs_tol = real_t(0);
+#endif
   const int skip_zero_entries = 0;

-   const double ref_density = 1.0; // density in the reference configuration
+   const real_t ref_density = 1.0; // density in the reference configuration
   ConstantCoefficient rho0(ref_density);
   M.AddDomainIntegrator(new VectorMassIntegrator(rho0));
   M.Assemble(skip_zero_entries);
@@ -509,7 +519,7 @@ HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
   newton_solver.SetOperator(*reduced_oper);
   newton_solver.SetPrintLevel(1); // print Newton iterations
   newton_solver.SetRelTol(rel_tol);
-   newton_solver.SetAbsTol(0.0);
+   newton_solver.SetAbsTol(newton_abs_tol);
   newton_solver.SetMaxIter(10);
 }

@@ -533,7 +543,7 @@ void HyperelasticOperator::Mult(const Vector &vx, Vector &dvx_dt) const
   dx_dt = v;
 }

-void HyperelasticOperator::ImplicitSolve(const double dt,
+void HyperelasticOperator::ImplicitSolve(const real_t dt,
                                         const Vector &vx, Vector &dvx_dt)
 {
   int sc = height/2;
@@ -555,12 +565,12 @@ void HyperelasticOperator::ImplicitSolve(const double dt,
   add(v, dt, dv_dt, dx_dt);
 }

-double HyperelasticOperator::ElasticEnergy(const Vector &x) const
+real_t HyperelasticOperator::ElasticEnergy(const Vector &x) const
 {
   return H.GetEnergy(x);
 }

-double HyperelasticOperator::KineticEnergy(const Vector &v) const
+real_t HyperelasticOperator::KineticEnergy(const Vector &v) const
 {
   return 0.5*M.InnerProduct(v, v);
 }
@@ -581,7 +591,7 @@ HyperelasticOperator::~HyperelasticOperator()
 }


-double ElasticEnergyCoefficient::Eval(ElementTransformation &T,
+real_t ElasticEnergyCoefficient::Eval(ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
   model.SetTransformation(T);
@@ -601,7 +611,7 @@ void InitialDeformation(const Vector &x, Vector &y)
 void InitialVelocity(const Vector &x, Vector &v)
 {
   const int dim = x.Size();
-   const double s = 0.1/64.;
+   const real_t s = 0.1/64.;

   v = 0.0;
   v(dim-1) = s*x(0)*x(0)*(8.0-x(0));
@@ -63,7 +63,7 @@ protected:

   ParBilinearForm M, S;
   ParNonlinearForm H;
-   double viscosity;
+   real_t viscosity;
   HyperelasticModel *model;

   HypreParMatrix *Mmat; // Mass matrix from ParallelAssemble()
@@ -86,16 +86,16 @@ protected:

 public:
   HyperelasticOperator(ParFiniteElementSpace &f, Array<int> &ess_bdr,
-                        double visc, double mu, double K);
+                        real_t visc, real_t mu, real_t K);

   /// Compute the right-hand side of the ODE system.
   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
   /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);

-   double ElasticEnergy(const ParGridFunction &x) const;
-   double KineticEnergy(const ParGridFunction &v) const;
+   real_t ElasticEnergy(const ParGridFunction &x) const;
+   real_t KineticEnergy(const ParGridFunction &v) const;
   void GetElasticEnergyDensity(const ParGridFunction &x,
                                ParGridFunction &w) const;

@@ -112,7 +112,7 @@ private:
   ParBilinearForm *M, *S;
   ParNonlinearForm *H;
   mutable HypreParMatrix *Jacobian;
-   double dt;
+   real_t dt;
   const Vector *v, *x;
   mutable Vector w, z;
   const Array<int> &ess_tdof_list;
@@ -122,7 +122,7 @@ public:
                         ParNonlinearForm *H_, const Array<int> &ess_tdof_list);

   /// Set current dt, v, x values - needed to compute action and Jacobian.
-   void SetParameters(double dt_, const Vector *v_, const Vector *x_);
+   void SetParameters(real_t dt_, const Vector *v_, const Vector *x_);

   /// Compute y = H(x + dt (v + dt k)) + M k + S (v + dt k).
   virtual void Mult(const Vector &k, Vector &y) const;
@@ -146,7 +146,7 @@ private:
 public:
   ElasticEnergyCoefficient(HyperelasticModel &m, const ParGridFunction &x_)
      : model(m), x(x_) { }
-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
   virtual ~ElasticEnergyCoefficient() { }
 };

@@ -173,11 +173,11 @@ int main(int argc, char *argv[])
   int par_ref_levels = 0;
   int order = 2;
   int ode_solver_type = 3;
-   double t_final = 300.0;
-   double dt = 3.0;
-   double visc = 1e-2;
-   double mu = 0.25;
-   double K = 5.0;
+   real_t t_final = 300.0;
+   real_t dt = 3.0;
+   real_t visc = 1e-2;
+   real_t mu = 0.25;
+   real_t K = 5.0;
   bool adaptive_lin_rtol = true;
   bool visualization = true;
   int vis_steps = 1;
@@ -358,8 +358,8 @@ int main(int argc, char *argv[])
      }
   }

-   double ee0 = oper.ElasticEnergy(x_gf);
-   double ke0 = oper.KineticEnergy(v_gf);
+   real_t ee0 = oper.ElasticEnergy(x_gf);
+   real_t ke0 = oper.KineticEnergy(v_gf);
   if (myid == 0)
   {
      cout << "initial elastic energy (EE) = " << ee0 << endl;
@@ -367,7 +367,7 @@ int main(int argc, char *argv[])
      cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;
   }

-   double t = 0.0;
+   real_t t = 0.0;
   oper.SetTime(t);
   ode_solver->Init(oper);

@@ -376,7 +376,7 @@ int main(int argc, char *argv[])
   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
   {
-      double dt_real = min(dt, t_final - t);
+      real_t dt_real = min(dt, t_final - t);

      ode_solver->Step(vx, t, dt_real);

@@ -386,8 +386,8 @@ int main(int argc, char *argv[])
      {
         v_gf.SetFromTrueVector(); x_gf.SetFromTrueVector();

-         double ee = oper.ElasticEnergy(x_gf);
-         double ke = oper.KineticEnergy(v_gf);
+         real_t ee = oper.ElasticEnergy(x_gf);
+         real_t ke = oper.KineticEnergy(v_gf);

         if (myid == 0)
         {
@@ -485,7 +485,7 @@ ReducedSystemOperator::ReducedSystemOperator(
     ess_tdof_list(ess_tdof_list_)
 { }

-void ReducedSystemOperator::SetParameters(double dt_, const Vector *v_,
+void ReducedSystemOperator::SetParameters(real_t dt_, const Vector *v_,
                                          const Vector *x_)
 {
   dt = dt_;  v = v_;  x = x_;
@@ -523,17 +523,27 @@ ReducedSystemOperator::~ReducedSystemOperator()


 HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
-                                           Array<int> &ess_bdr, double visc,
-                                           double mu, double K)
-   : TimeDependentOperator(2*f.TrueVSize(), 0.0), fespace(f),
+                                           Array<int> &ess_bdr, real_t visc,
+                                           real_t mu, real_t K)
+   : TimeDependentOperator(2*f.TrueVSize(), (real_t) 0.0), fespace(f),
     M(&fespace), S(&fespace), H(&fespace),
     viscosity(visc), M_solver(f.GetComm()), newton_solver(f.GetComm()),
     z(height/2)
 {
-   const double rel_tol = 1e-8;
+#if defined(MFEM_USE_DOUBLE)
+   const real_t rel_tol = 1e-8;
+   const real_t newton_abs_tol = 0.0;
+#elif defined(MFEM_USE_SINGLE)
+   const real_t rel_tol = 1e-3;
+   const real_t newton_abs_tol = 1e-4;
+#else
+#error "Only single and double precision are supported!"
+   const real_t rel_tol = real_t(1);
+   const real_t newton_abs_tol = real_t(0);
+#endif
   const int skip_zero_entries = 0;

-   const double ref_density = 1.0; // density in the reference configuration
+   const real_t ref_density = 1.0; // density in the reference configuration
   ConstantCoefficient rho0(ref_density);
   M.AddDomainIntegrator(new VectorMassIntegrator(rho0));
   M.Assemble(skip_zero_entries);
@@ -581,7 +591,7 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
   newton_solver.SetOperator(*reduced_oper);
   newton_solver.SetPrintLevel(1); // print Newton iterations
   newton_solver.SetRelTol(rel_tol);
-   newton_solver.SetAbsTol(0.0);
+   newton_solver.SetAbsTol(newton_abs_tol);
   newton_solver.SetAdaptiveLinRtol(2, 0.5, 0.9);
   newton_solver.SetMaxIter(10);
 }
@@ -607,7 +617,7 @@ void HyperelasticOperator::Mult(const Vector &vx, Vector &dvx_dt) const
   dx_dt = v;
 }

-void HyperelasticOperator::ImplicitSolve(const double dt,
+void HyperelasticOperator::ImplicitSolve(const real_t dt,
                                         const Vector &vx, Vector &dvx_dt)
 {
   int sc = height/2;
@@ -629,17 +639,14 @@ void HyperelasticOperator::ImplicitSolve(const double dt,
   add(v, dt, dv_dt, dx_dt);
 }

-double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const
+real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const
 {
   return H.GetEnergy(x);
 }

-double HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
+real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   double loc_energy = 0.5*M.InnerProduct(v, v);
-   double energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM,
-                 fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -660,7 +667,7 @@ HyperelasticOperator::~HyperelasticOperator()
 }


-double ElasticEnergyCoefficient::Eval(ElementTransformation &T,
+real_t ElasticEnergyCoefficient::Eval(ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
   model.SetTransformation(T);
@@ -680,7 +687,7 @@ void InitialDeformation(const Vector &x, Vector &y)
 void InitialVelocity(const Vector &x, Vector &v)
 {
   const int dim = x.Size();
-   const double s = 0.1/64.;
+   const real_t s = 0.1/64.;

   v = 0.0;
   v(dim-1) = s*x(0)*x(0)*(8.0-x(0));
@@ -211,7 +211,7 @@ int main(int argc, char *argv[])
   m->AddDomainIntegrator(new MassIntegrator(one));
   m->Assemble();
   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<real_t>::min());
   m->Finalize();

   HypreParMatrix *A = a->ParallelAssemble();
@@ -300,7 +300,7 @@ int main(int argc, char *argv[])
   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
   //    parallel grid function to represent each of the eigenmodes returned by
   //    the solver.
-   Array<double> eigenvalues;
+   Array<real_t> eigenvalues;
   lobpcg->Solve();
   lobpcg->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);
@@ -206,7 +206,7 @@ int main(int argc, char *argv[])
   m->AddDomainIntegrator(new VectorMassIntegrator());
   m->Assemble();
   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<real_t>::min());
   m->Finalize();
   if (myid == 0)
   {
@@ -247,7 +247,7 @@ int main(int argc, char *argv[])
   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
   //     parallel grid function to represent each of the eigenmodes returned by
   //     the solver.
-   Array<double> eigenvalues;
+   Array<real_t> eigenvalues;
   lobpcg->Solve();
   lobpcg->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);
@@ -170,7 +170,7 @@ int main(int argc, char *argv[])
   m->AddDomainIntegrator(new VectorFEMassIntegrator(one));
   m->Assemble();
   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<real_t>::min());
   m->Finalize();

   HypreParMatrix *A = a->ParallelAssemble();
@@ -198,7 +198,7 @@ int main(int argc, char *argv[])
   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
   //     parallel grid function to represent each of the eigenmodes returned by
   //     the solver.
-   Array<double> eigenvalues;
+   Array<real_t> eigenvalues;
   ame->Solve();
   ame->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);
@@ -18,6 +18,12 @@
 //               ex14 -m ../data/amr-quad.mesh -r 3
 //               ex14 -m ../data/amr-hex.mesh
 //               ex14 -m ../data/fichera-amr.mesh
+//               ex14 -pa -r 1 -o 3
+//               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
+//
+// Device sample runs:
+//               ex14 -pa -r 2 -d cuda -o 3
+//               ex14 -pa -r 2 -d cuda -o 3 -m ../data/fichera.mesh
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -43,10 +49,12 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int ref_levels = -1;
   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   double eta = 0.0;
+   real_t sigma = -1.0;
+   real_t kappa = -1.0;
+   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -62,9 +70,13 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -77,117 +89,129 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
   //    NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   const int dim = mesh.Dimension();

-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
   //    we choose it to be the largest number that gives a final mesh with no
   //    more than 50,000 elements.
   {
      if (ref_levels < 0)
      {
-         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

-   // 4. Define a finite element space on the mesh. Here we use discontinuous
+   // 5. Define a finite element space on the mesh. Here we use discontinuous
   //    finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of unknowns: " << fespace->GetVSize() << endl;
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of unknowns: " << fespace.GetVSize() << endl;

-   // 5. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
-   LinearForm *b = new LinearForm(fespace);
+   LinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

-   // 6. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   GridFunction x(fespace);
+   GridFunction x(&fespace);
   x = 0.0;

-   // 7. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator and the interior and boundary DG face integrators.
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After assembly and finalizing we
   //    extract the corresponding sparse matrix A.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   BilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
-   const SparseMatrix &A = a->SpMat();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

-#ifndef MFEM_USE_SUITESPARSE
-   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   // 9. Define a simple symmetric Gauss-Seidel preconditioner and use it to
   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
-   //    non-symmetric one.
-   GSSmoother M(A);
-   if (sigma == -1.0)
+   //    non-symmetric one. (Note that tolerances are squared: 1e-12 corresponds
+   //    to a relative tolerance of 1e-6).
+   //
+   //    If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   if (pa)
   {
-      PCG(A, M, *b, x, 1, 500, 1e-12, 0.0);
+      MFEM_VERIFY(sigma == -1.0,
+                  "The case of PA with sigma != -1 is not yet supported.");
+      CG(a, b, x, 1, 500, 1e-12, 0.0);
   }
   else
   {
-      GMRES(A, M, *b, x, 1, 500, 10, 1e-12, 0.0);
-   }
+      const SparseMatrix &A = a.SpMat();
+#ifndef MFEM_USE_SUITESPARSE
+      GSSmoother M(A);
+      if (sigma == -1.0)
+      {
+         PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
+      }
+      else
+      {
+         GMRES(A, M, b, x, 1, 500, 10, 1e-12, 0.0);
+      }
 #else
-   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(*b, x);
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(b, x);
 #endif
+   }

-   // 9. Save the refined mesh and the solution. This output can be viewed later
-   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
+   // 10. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
+   mesh.Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);

-   // 10. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
+      sol_sock << "solution\n" << mesh << x << flush;
   }

-   // 11. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete mesh;
-
   return 0;
 }
@@ -17,6 +17,12 @@
 //               mpirun -np 4 ex14p -m ../data/inline-segment.mesh -rs 5
 //               mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 3
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
+//
+// Device sample runs:
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -m ../data/fichera.mesh -o 3
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -38,42 +44,39 @@ using namespace mfem;

 class CustomSolverMonitor : public IterativeSolverMonitor
 {
+private:
+   const ParMesh &pmesh;
+   ParGridFunction &pgf;
 public:
-   CustomSolverMonitor(const ParMesh *m,
-                       ParGridFunction *f) :
-      pmesh(m),
-      pgf(f) {}
+   CustomSolverMonitor(const ParMesh &pmesh_,
+                       ParGridFunction &pgf_) :
+      pmesh(pmesh_),
+      pgf(pgf_) {}

-   void MonitorSolution(int i, double norm, const Vector &x, bool final)
+   void MonitorSolution(int i, real_t norm, const Vector &x, bool final)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      int  num_procs, myid;

-      MPI_Comm_size(pmesh->GetComm(),&num_procs);
-      MPI_Comm_rank(pmesh->GetComm(),&myid);
+      MPI_Comm_size(pmesh.GetComm(), &num_procs);
+      MPI_Comm_rank(pmesh.GetComm(), &myid);

-      pgf->SetFromTrueDofs(x);
+      pgf.SetFromTrueDofs(x);

      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << *pgf
+      sol_sock << "solution\n" << pmesh << pgf
               << "window_title 'Iteration no " << i << "'"
               << "keys rRjlc\n" << flush;
   }
-
-private:
-   const ParMesh *pmesh;
-   ParGridFunction *pgf;
 };

 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
   Hypre::Init();

   // 2. Parse command-line options.
@@ -81,10 +84,12 @@ int main(int argc, char *argv[])
   int ser_ref_levels = -1;
   int par_ref_levels = 2;
   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   double eta = 0.0;
+   real_t sigma = -1.0;
+   real_t kappa = -1.0;
+   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -103,13 +108,17 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
-      if (myid == 0)
+      if (Mpi::Root())
      {
         args.PrintUsage(cout);
      }
@@ -119,16 +128,19 @@ int main(int argc, char *argv[])
   {
      kappa = (order+1)*(order+1);
   }
-   if (myid == 0)
+   if (Mpi::Root())
   {
      args.PrintOptions(cout);
   }

+   Device device(device_config);
+   if (Mpi::Root()) { device.Print(); }
+
   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
   //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   int dim = mesh.Dimension();

   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -137,53 +149,54 @@ int main(int argc, char *argv[])
   {
      if (ser_ref_levels < 0)
      {
-         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ser_ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
   {
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh->UniformRefinement();
+         pmesh.UniformRefinement();
      }
   }

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use discontinuous finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
-   if (myid == 0)
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (Mpi::Root())
   {
      cout << "Number of unknowns: " << size << endl;
   }

   // 7. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
-   ParLinearForm *b = new ParLinearForm(fespace);
+   ParLinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

   // 8. Define the solution vector x as a parallel finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   ParGridFunction x(fespace);
+   ParGridFunction x(&fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
@@ -192,42 +205,51 @@ int main(int argc, char *argv[])
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After serial and parallel assembly we
   //    extract the corresponding parallel matrix A.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

   // 10. Define the parallel (hypre) matrix and vectors representing a(.,.),
   //     b(.) and the finite element approximation.
-   HypreParMatrix *A = a->ParallelAssemble();
-   HypreParVector *B = b->ParallelAssemble();
-   HypreParVector *X = x.ParallelProject();
+   OperatorHandle A;

-   delete a;
-   delete b;
-
-   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
-   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
-   HypreSolver *amg = new HypreBoomerAMG(*A);
-   if (sigma == -1.0)
+   std::unique_ptr<HypreBoomerAMG> amg;
+   if (pa)
   {
-      HyprePCG pcg(*A);
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(*amg);
-      pcg.Mult(*B, *X);
+      A.Reset(&a, false);
   }
   else
   {
-      CustomSolverMonitor monitor(pmesh, &x);
+      A.SetType(Operator::Hypre_ParCSR);
+      a.ParallelAssemble(A);
+      amg.reset(new HypreBoomerAMG(*A.As<HypreParMatrix>()));
+   }
+
+   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
+   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
+   if (sigma == -1.0)
+   {
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(500);
+      cg.SetPrintLevel(1);
+      cg.SetOperator(*A);
+      if (amg) { cg.SetPreconditioner(*amg); }
+      cg.Mult(b, x);
+   }
+   else
+   {
+      CustomSolverMonitor monitor(pmesh, x);
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
@@ -235,51 +257,37 @@ int main(int argc, char *argv[])
      gmres.SetKDim(10);
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(*amg);
+      if (amg) { gmres.SetPreconditioner(*amg); }
      gmres.SetMonitor(monitor);
-      gmres.Mult(*B, *X);
+      gmres.Mult(b, x);
   }
-   delete amg;

-   // 12. Extract the parallel grid function corresponding to the finite element
-   //     approximation X. This is the local solution on each processor.
-   x = *X;
-
-   // 13. Save the refined mesh and the solution in parallel. This output can
+   // 12. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      mesh_name << "mesh." << setfill('0') << setw(6) << Mpi::WorldRank();
+      sol_name << "sol." << setfill('0') << setw(6) << Mpi::WorldRank();

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
+      pmesh.Print(mesh_ofs);

      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
      x.Save(sol_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
+      sol_sock << "solution\n" << pmesh << x << flush;
   }

-   // 15. Free the used memory.
-   delete X;
-   delete B;
-   delete A;
-
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
   return 0;
 }
@@ -63,8 +63,8 @@ int problem;
 int nfeatures;

 // Prescribed time-dependent boundary and right-hand side functions.
-double bdr_func(const Vector &pt, double t);
-double rhs_func(const Vector &pt, double t);
+real_t bdr_func(const Vector &pt, real_t t);
+real_t rhs_func(const Vector &pt, real_t t);

 // Update the finite element space, interpolate the solution and perform
 // parallel load balancing.
@@ -79,9 +79,9 @@ int main(int argc, char *argv[])
   nfeatures = 1;
   const char *mesh_file = "../data/star-hilbert.mesh";
   int order = 2;
-   double t_final = 1.0;
-   double max_elem_error = 5.0e-3;
-   double hysteresis = 0.15; // derefinement safety coefficient
+   real_t t_final = 1.0;
+   real_t max_elem_error = 5.0e-3;
+   real_t hysteresis = 0.15; // derefinement safety coefficient
   int ref_levels = 0;
   int nc_limit = 3;         // maximum level of hanging nodes
   bool visualization = true;
@@ -247,7 +247,7 @@ int main(int argc, char *argv[])
   //     refine the mesh as many times as necessary. Then we derefine any
   //     elements which have very small errors.
   x = 0.0;
-   for (double time = 0.0; time < t_final + 1e-10; time += 0.01)
+   for (real_t time = 0.0; time < t_final + 1e-10; time += 0.01)
   {
      cout << "\nTime " << time << "\n\nRefinement:" << endl;

@@ -366,47 +366,47 @@ void UpdateProblem(Mesh &mesh, FiniteElementSpace &fespace,
 }


-const double alpha = 0.02;
+const real_t alpha = 0.02;

 // Spherical front with a Gaussian cross section and radius t
-double front(double x, double y, double z, double t, int)
+real_t front(real_t x, real_t y, real_t z, real_t t, int)
 {
-   double r = sqrt(x*x + y*y + z*z);
+   real_t r = sqrt(x*x + y*y + z*z);
   return exp(-0.5*pow((r - t)/alpha, 2));
 }

-double front_laplace(double x, double y, double z, double t, int dim)
+real_t front_laplace(real_t x, real_t y, real_t z, real_t t, int dim)
 {
-   double x2 = x*x, y2 = y*y, z2 = z*z, t2 = t*t;
-   double r = sqrt(x2 + y2 + z2);
-   double a2 = alpha*alpha, a4 = a2*a2;
+   real_t x2 = x*x, y2 = y*y, z2 = z*z, t2 = t*t;
+   real_t r = sqrt(x2 + y2 + z2);
+   real_t a2 = alpha*alpha, a4 = a2*a2;
   return -exp(-0.5*pow((r - t)/alpha, 2)) / a4 *
          (-2*t*(x2 + y2 + z2 - (dim-1)*a2/2)/r + x2 + y2 + z2 + t2 - dim*a2);
 }

 // Smooth spherical step function with radius t
-double ball(double x, double y, double z, double t, int)
+real_t ball(real_t x, real_t y, real_t z, real_t t, int)
 {
-   double r = sqrt(x*x + y*y + z*z);
+   real_t r = sqrt(x*x + y*y + z*z);
   return -atan(2*(r - t)/alpha);
 }

-double ball_laplace(double x, double y, double z, double t, int dim)
+real_t ball_laplace(real_t x, real_t y, real_t z, real_t t, int dim)
 {
-   double x2 = x*x, y2 = y*y, z2 = z*z, t2 = 4*t*t;
-   double r = sqrt(x2 + y2 + z2);
-   double a2 = alpha*alpha;
-   double den = pow(-a2 - 4*(x2 + y2 + z2 - 2*r*t) - t2, 2.0);
+   real_t x2 = x*x, y2 = y*y, z2 = z*z, t2 = 4*t*t;
+   real_t r = sqrt(x2 + y2 + z2);
+   real_t a2 = alpha*alpha;
+   real_t den = pow(-a2 - 4*(x2 + y2 + z2 - 2*r*t) - t2, 2.0);
   return (dim == 2) ? 2*alpha*(a2 + t2 - 4*x2 - 4*y2)/r/den
          /*      */ : 4*alpha*(a2 + t2 - 4*r*t)/r/den;
 }

 // Composes several features into one function
 template<typename F0, typename F1>
-double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
+real_t composite_func(const Vector &pt, real_t t, F0 f0, F1 f1)
 {
   int dim = pt.Size();
-   double x = pt(0), y = pt(1), z = 0.0;
+   real_t x = pt(0), y = pt(1), z = 0.0;
   if (dim == 3) { z = pt(2); }

   if (problem == 0)
@@ -417,11 +417,11 @@ double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
      }
      else
      {
-         double sum = 0.0;
+         real_t sum = 0.0;
         for (int i = 0; i < nfeatures; i++)
         {
-            double x0 = 0.5*cos(2*M_PI * i / nfeatures);
-            double y0 = 0.5*sin(2*M_PI * i / nfeatures);
+            real_t x0 = 0.5*cos(2*M_PI * i / nfeatures);
+            real_t y0 = 0.5*sin(2*M_PI * i / nfeatures);
            sum += f0(x - x0, y - y0, z, t, dim);
         }
         return sum;
@@ -429,11 +429,11 @@ double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
   }
   else
   {
-      double sum = 0.0;
+      real_t sum = 0.0;
      for (int i = 0; i < nfeatures; i++)
      {
-         double x0 = 0.5*cos(2*M_PI * i / nfeatures + M_PI*t);
-         double y0 = 0.5*sin(2*M_PI * i / nfeatures + M_PI*t);
+         real_t x0 = 0.5*cos(2*M_PI * i / nfeatures + M_PI*t);
+         real_t y0 = 0.5*sin(2*M_PI * i / nfeatures + M_PI*t);
         sum += f1(x - x0, y - y0, z, 0.25, dim);
      }
      return sum;
@@ -441,13 +441,13 @@ double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
 }

 // Exact solution, used for the Dirichlet BC.
-double bdr_func(const Vector &pt, double t)
+real_t bdr_func(const Vector &pt, real_t t)
 {
   return composite_func(pt, t, front, ball);
 }

 // Laplace of the exact solution, used for the right hand side.
-double rhs_func(const Vector &pt, double t)
+real_t rhs_func(const Vector &pt, real_t t)
 {
   return composite_func(pt, t, front_laplace, ball_laplace);
 }
@@ -68,8 +68,8 @@ int problem;
 int nfeatures;

 // Prescribed time-dependent boundary and right-hand side functions.
-double bdr_func(const Vector &pt, double t);
-double rhs_func(const Vector &pt, double t);
+real_t bdr_func(const Vector &pt, real_t t);
+real_t rhs_func(const Vector &pt, real_t t);

 // Update the finite element space, interpolate the solution and perform
 // parallel load balancing.
@@ -91,9 +91,9 @@ int main(int argc, char *argv[])
   nfeatures = 1;
   const char *mesh_file = "../data/star-hilbert.mesh";
   int order = 2;
-   double t_final = 1.0;
-   double max_elem_error = 1.0e-4;
-   double hysteresis = 0.25; // derefinement safety coefficient
+   real_t t_final = 1.0;
+   real_t max_elem_error = 1.0e-4;
+   real_t hysteresis = 0.25; // derefinement safety coefficient
   int ref_levels = 0;
   int nc_limit = 3;         // maximum level of hanging nodes
   bool visualization = true;
@@ -282,7 +282,7 @@ int main(int argc, char *argv[])
   //     solve the problem on the current mesh, visualize the solution and
   //     refine the mesh as many times as necessary. Then we derefine any
   //     elements which have very small errors.
-   for (double time = 0.0; time < t_final + 1e-10; time += 0.01)
+   for (real_t time = 0.0; time < t_final + 1e-10; time += 0.01)
   {
      if (myid == 0)
      {
@@ -427,47 +427,47 @@ void UpdateAndRebalance(ParMesh &pmesh, ParFiniteElementSpace &fespace,
 }


-const double alpha = 0.02;
+const real_t alpha = 0.02;

 // Spherical front with a Gaussian cross section and radius t
-double front(double x, double y, double z, double t, int)
+real_t front(real_t x, real_t y, real_t z, real_t t, int)
 {
-   double r = sqrt(x*x + y*y + z*z);
+   real_t r = sqrt(x*x + y*y + z*z);
   return exp(-0.5*pow((r - t)/alpha, 2));
 }

-double front_laplace(double x, double y, double z, double t, int dim)
+real_t front_laplace(real_t x, real_t y, real_t z, real_t t, int dim)
 {
-   double x2 = x*x, y2 = y*y, z2 = z*z, t2 = t*t;
-   double r = sqrt(x2 + y2 + z2);
-   double a2 = alpha*alpha, a4 = a2*a2;
+   real_t x2 = x*x, y2 = y*y, z2 = z*z, t2 = t*t;
+   real_t r = sqrt(x2 + y2 + z2);
+   real_t a2 = alpha*alpha, a4 = a2*a2;
   return -exp(-0.5*pow((r - t)/alpha, 2)) / a4 *
          (-2*t*(x2 + y2 + z2 - (dim-1)*a2/2)/r + x2 + y2 + z2 + t2 - dim*a2);
 }

 // Smooth spherical step function with radius t
-double ball(double x, double y, double z, double t, int)
+real_t ball(real_t x, real_t y, real_t z, real_t t, int)
 {
-   double r = sqrt(x*x + y*y + z*z);
+   real_t r = sqrt(x*x + y*y + z*z);
   return -atan(2*(r - t)/alpha);
 }

-double ball_laplace(double x, double y, double z, double t, int dim)
+real_t ball_laplace(real_t x, real_t y, real_t z, real_t t, int dim)
 {
-   double x2 = x*x, y2 = y*y, z2 = z*z, t2 = 4*t*t;
-   double r = sqrt(x2 + y2 + z2);
-   double a2 = alpha*alpha;
-   double den = pow(-a2 - 4*(x2 + y2 + z2 - 2*r*t) - t2, 2.0);
+   real_t x2 = x*x, y2 = y*y, z2 = z*z, t2 = 4*t*t;
+   real_t r = sqrt(x2 + y2 + z2);
+   real_t a2 = alpha*alpha;
+   real_t den = pow(-a2 - 4*(x2 + y2 + z2 - 2*r*t) - t2, 2.0);
   return (dim == 2) ? 2*alpha*(a2 + t2 - 4*x2 - 4*y2)/r/den
          /*      */ : 4*alpha*(a2 + t2 - 4*r*t)/r/den;
 }

 // Composes several features into one function
 template<typename F0, typename F1>
-double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
+real_t composite_func(const Vector &pt, real_t t, F0 f0, F1 f1)
 {
   int dim = pt.Size();
-   double x = pt(0), y = pt(1), z = 0.0;
+   real_t x = pt(0), y = pt(1), z = 0.0;
   if (dim == 3) { z = pt(2); }

   if (problem == 0)
@@ -478,11 +478,11 @@ double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
      }
      else
      {
-         double sum = 0.0;
+         real_t sum = 0.0;
         for (int i = 0; i < nfeatures; i++)
         {
-            double x0 = 0.5*cos(2*M_PI * i / nfeatures);
-            double y0 = 0.5*sin(2*M_PI * i / nfeatures);
+            real_t x0 = 0.5*cos(2*M_PI * i / nfeatures);
+            real_t y0 = 0.5*sin(2*M_PI * i / nfeatures);
            sum += f0(x - x0, y - y0, z, t, dim);
         }
         return sum;
@@ -490,11 +490,11 @@ double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
   }
   else
   {
-      double sum = 0.0;
+      real_t sum = 0.0;
      for (int i = 0; i < nfeatures; i++)
      {
-         double x0 = 0.5*cos(2*M_PI * i / nfeatures + M_PI*t);
-         double y0 = 0.5*sin(2*M_PI * i / nfeatures + M_PI*t);
+         real_t x0 = 0.5*cos(2*M_PI * i / nfeatures + M_PI*t);
+         real_t y0 = 0.5*sin(2*M_PI * i / nfeatures + M_PI*t);
         sum += f1(x - x0, y - y0, z, 0.25, dim);
      }
      return sum;
@@ -502,13 +502,13 @@ double composite_func(const Vector &pt, double t, F0 f0, F1 f1)
 }

 // Exact solution, used for the Dirichlet BC.
-double bdr_func(const Vector &pt, double t)
+real_t bdr_func(const Vector &pt, real_t t)
 {
   return composite_func(pt, t, front, ball);
 }

 // Laplace of the exact solution, used for the right hand side.
-double rhs_func(const Vector &pt, double t)
+real_t rhs_func(const Vector &pt, real_t t)
 {
   return composite_func(pt, t, front_laplace, ball_laplace);
 }
@@ -60,7 +60,7 @@ protected:

   SparseMatrix Mmat, Kmat;
   SparseMatrix *T; // T = M + dt K
-   double current_dt;
+   real_t current_dt;

   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
   DSmoother M_prec;  // Preconditioner for the mass matrix M
@@ -68,18 +68,18 @@ protected:
   CGSolver T_solver; // Implicit solver for T = M + dt K
   DSmoother T_prec;  // Preconditioner for the implicit solver

-   double alpha, kappa;
+   real_t alpha, kappa;

   mutable Vector z; // auxiliary vector

 public:
-   ConductionOperator(FiniteElementSpace &f, double alpha, double kappa,
+   ConductionOperator(FiniteElementSpace &f, real_t alpha, real_t kappa,
                      const Vector &u);

   virtual void Mult(const Vector &u, Vector &du_dt) const;
   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
+   virtual void ImplicitSolve(const real_t dt, const Vector &u, Vector &k);

   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
   void SetParameters(const Vector &u);
@@ -87,7 +87,7 @@ public:
   virtual ~ConductionOperator();
 };

-double InitialTemperature(const Vector &x);
+real_t InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
@@ -96,10 +96,10 @@ int main(int argc, char *argv[])
   int ref_levels = 2;
   int order = 2;
   int ode_solver_type = 3;
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double alpha = 1.0e-2;
-   double kappa = 0.5;
+   real_t t_final = 0.5;
+   real_t dt = 1.0e-2;
+   real_t alpha = 1.0e-2;
+   real_t kappa = 0.5;
   bool visualization = true;
   bool visit = false;
   int vis_steps = 5;
@@ -246,7 +246,7 @@ int main(int argc, char *argv[])
   // 8. Perform time-integration (looping over the time iterations, ti, with a
   //    time-step dt).
   ode_solver->Init(oper);
-   double t = 0.0;
+   real_t t = 0.0;

   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -293,12 +293,12 @@ int main(int argc, char *argv[])
   return 0;
 }

-ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
-                                       double kap, const Vector &u)
-   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
-     T(NULL), current_dt(0.0), z(height)
+ConductionOperator::ConductionOperator(FiniteElementSpace &f, real_t al,
+                                       real_t kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), (real_t) 0.0), fespace(f),
+     M(NULL), K(NULL), T(NULL), current_dt(0.0), z(height)
 {
-   const double rel_tol = 1e-8;
+   const real_t rel_tol = 1e-8;

   M = new BilinearForm(&fespace);
   M->AddDomainIntegrator(new MassIntegrator());
@@ -336,7 +336,7 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
   M_solver.Mult(z, du_dt);
 }

-void ConductionOperator::ImplicitSolve(const double dt,
+void ConductionOperator::ImplicitSolve(const real_t dt,
                                       const Vector &u, Vector &du_dt)
 {
   // Solve the equation:
@@ -382,7 +382,7 @@ ConductionOperator::~ConductionOperator()
   delete K;
 }

-double InitialTemperature(const Vector &x)
+real_t InitialTemperature(const Vector &x)
 {
   if (x.Norml2() < 0.5)
   {
@@ -62,7 +62,7 @@ protected:
   HypreParMatrix Mmat;
   HypreParMatrix Kmat;
   HypreParMatrix *T; // T = M + dt K
-   double current_dt;
+   real_t current_dt;

   CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
   HypreSmoother M_prec; // Preconditioner for the mass matrix M
@@ -70,18 +70,18 @@ protected:
   CGSolver T_solver;    // Implicit solver for T = M + dt K
   HypreSmoother T_prec; // Preconditioner for the implicit solver

-   double alpha, kappa;
+   real_t alpha, kappa;

   mutable Vector z; // auxiliary vector

 public:
-   ConductionOperator(ParFiniteElementSpace &f, double alpha, double kappa,
+   ConductionOperator(ParFiniteElementSpace &f, real_t alpha, real_t kappa,
                      const Vector &u);

   virtual void Mult(const Vector &u, Vector &du_dt) const;
   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
+   virtual void ImplicitSolve(const real_t dt, const Vector &u, Vector &k);

   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
   void SetParameters(const Vector &u);
@@ -89,7 +89,7 @@ public:
   virtual ~ConductionOperator();
 };

-double InitialTemperature(const Vector &x);
+real_t InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
@@ -105,10 +105,10 @@ int main(int argc, char *argv[])
   int par_ref_levels = 1;
   int order = 2;
   int ode_solver_type = 3;
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double alpha = 1.0e-2;
-   double kappa = 0.5;
+   real_t t_final = 0.5;
+   real_t dt = 1.0e-2;
+   real_t alpha = 1.0e-2;
+   real_t kappa = 0.5;
   bool visualization = true;
   bool visit = false;
   int vis_steps = 5;
@@ -313,7 +313,7 @@ int main(int argc, char *argv[])
   // 10. Perform time-integration (looping over the time iterations, ti, with a
   //     time-step dt).
   ode_solver->Init(oper);
-   double t = 0.0;
+   real_t t = 0.0;

   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -382,13 +382,13 @@ int main(int argc, char *argv[])
   return 0;
 }

-ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
-                                       double kap, const Vector &u)
-   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
-     T(NULL), current_dt(0.0),
+ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, real_t al,
+                                       real_t kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), (real_t) 0.0), fespace(f),
+     M(NULL), K(NULL), T(NULL), current_dt(0.0),
     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
 {
-   const double rel_tol = 1e-8;
+   const real_t rel_tol = 1e-8;

   M = new ParBilinearForm(&fespace);
   M->AddDomainIntegrator(new MassIntegrator());
@@ -427,7 +427,7 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
   M_solver.Mult(z, du_dt);
 }

-void ConductionOperator::ImplicitSolve(const double dt,
+void ConductionOperator::ImplicitSolve(const real_t dt,
                                       const Vector &u, Vector &du_dt)
 {
   // Solve the equation:
@@ -473,7 +473,7 @@ ConductionOperator::~ConductionOperator()
   delete K;
 }

-double InitialTemperature(const Vector &x)
+real_t InitialTemperature(const Vector &x)
 {
   if (x.Norml2() < 0.5)
   {
@@ -69,7 +69,7 @@ public:
   void SetDisplacement(GridFunction &u_) { u = &u_; }
   void SetComponent(int i, int j) { si = i; sj = j; }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 // Simple GLVis visualization manager.
@@ -104,8 +104,8 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/beam-tri.mesh";
   int ref_levels = -1;
   int order = 1;
-   double alpha = -1.0;
-   double kappa = -1.0;
+   real_t alpha = -1.0;
+   real_t kappa = -1.0;
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -245,7 +245,7 @@ int main(int argc, char *argv[])
   //     solve the system Ax=b with PCG for the symmetric formulation, or GMRES
   //     for the non-symmetric.
   GSSmoother M(A);
-   const double rtol = 1e-6;
+   const real_t rtol = 1e-6;
   if (alpha == -1.0)
   {
      PCG(A, M, B, X, 3, 5000, rtol*rtol, 0.0);
@@ -337,17 +337,17 @@ void InitDisplacement(const Vector &x, Vector &u)
 }


-double StressCoefficient::Eval(ElementTransformation &T,
+real_t StressCoefficient::Eval(ElementTransformation &T,
                               const IntegrationPoint &ip)
 {
   MFEM_ASSERT(u != NULL, "displacement field is not set");

-   double L = lambda.Eval(T, ip);
-   double M = mu.Eval(T, ip);
+   real_t L = lambda.Eval(T, ip);
+   real_t M = mu.Eval(T, ip);
   u->GetVectorGradient(T, grad);
   if (si == sj)
   {
-      double div_u = grad.Trace();
+      real_t div_u = grad.Trace();
      return L*div_u + 2*M*grad(si,si);
   }
   else
@@ -69,7 +69,7 @@ public:
   void SetDisplacement(GridFunction &u_) { u = &u_; }
   void SetComponent(int i, int j) { si = i; sj = j; }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 // Simple GLVis visualization manager.
@@ -108,8 +108,8 @@ int main(int argc, char *argv[])
   int ser_ref_levels = -1;
   int par_ref_levels = 1;
   int order = 1;
-   double alpha = -1.0;
-   double kappa = -1.0;
+   real_t alpha = -1.0;
+   real_t kappa = -1.0;
   bool amg_elast = false;
   bool visualization = 1;

@@ -268,7 +268,7 @@ int main(int argc, char *argv[])
   // 11. Define a simple symmetric Gauss-Seidel preconditioner and use it to
   //     solve the system Ax=b with PCG for the symmetric formulation, or GMRES
   //     for the non-symmetric.
-   const double rtol = 1e-6;
+   const real_t rtol = 1e-6;
   HypreBoomerAMG amg(A);
   if (amg_elast)
   {
@@ -376,17 +376,17 @@ void InitDisplacement(const Vector &x, Vector &u)
 }


-double StressCoefficient::Eval(ElementTransformation &T,
+real_t StressCoefficient::Eval(ElementTransformation &T,
                               const IntegrationPoint &ip)
 {
   MFEM_ASSERT(u != NULL, "displacement field is not set");

-   double L = lambda.Eval(T, ip);
-   double M = mu.Eval(T, ip);
+   real_t L = lambda.Eval(T, ip);
+   real_t M = mu.Eval(T, ip);
   u->GetVectorGradient(T, grad);
   if (si == sj)
   {
-      double div_u = grad.Trace();
+      real_t div_u = grad.Trace();
      return L*div_u + 2*M*grad(si,si);
   }
   else
@@ -7,13 +7,19 @@
 //       ex18 -p 1 -r 2 -o 1 -s 3
 //       ex18 -p 1 -r 1 -o 3 -s 4
 //       ex18 -p 1 -r 0 -o 5 -s 6
-//       ex18 -p 2 -r 1 -o 1 -s 3
-//       ex18 -p 2 -r 0 -o 3 -s 3
+//       ex18 -p 2 -r 1 -o 1 -s 3 -mf
+//       ex18 -p 2 -r 0 -o 3 -s 3 -mf
 //
 // Description:  This example code solves the compressible Euler system of
 //               equations, a model nonlinear hyperbolic PDE, with a
 //               discontinuous Galerkin (DG) formulation.
 //
+//                (u_t, v)_T - (F(u), ∇ v)_T + <F̂(u,n), [[v]]>_F = 0
+//
+//               where (⋅,⋅)_T is volume integration, and <⋅,⋅>_F is face
+//               integration, F is the Euler flux function, and F̂ is the
+//               numerical flux.
+//
 //               Specifically, it solves for an exact solution of the equations
 //               whereby a vortex is transported by a uniform flow. Since all
 //               boundaries are periodic here, the method's accuracy can be
@@ -27,49 +33,47 @@
 //               method. An additional factor can be tuned by passing the --cfl
 //               (or -c shorter) flag.
 //
-//               The example demonstrates user-defined bilinear and nonlinear
-//               form integrators for systems of equations that are defined with
-//               block vectors, and how these are used with an operator for
-//               explicit time integrators. In this case the system also
-//               involves an external approximate Riemann solver for the DG
-//               interface flux. It also demonstrates how to use GLVis for
-//               in-situ visualization of vector grid functions.
+//               The example demonstrates usage of DGHyperbolicConservationLaws
+//               that wraps NonlinearFormIntegrators containing element and face
+//               integration schemes. In this case the system also involves an
+//               external approximate Riemann solver for the DG interface flux.
+//               By default, weak-divergence is pre-assembled in element-wise
+//               manner, which corresponds to (I_h(F(u_h)), ∇ v). This yields
+//               better performance and similar accuracy for the included test
+//               problems. This can be turned off and use nonlinear assembly
+//               similar to matrix-free assembly when -mf flag is provided.
+//               It also demonstrates how to use GLVis for in-situ visualization
+//               of vector grid function and how to set top-view.
 //
 //               We recommend viewing examples 9, 14 and 17 before viewing this
 //               example.

 #include "mfem.hpp"
 #include <fstream>
-#include <sstream>
 #include <iostream>
-
-// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
-// shared between the serial and parallel version of the example.
+#include <sstream>
 #include "ex18.hpp"

-// Choice for the problem setup. See InitialCondition in ex18.hpp.
-int problem;
-
-// Equation constant parameters.
-const int num_equation = 4;
-const double specific_heat_ratio = 1.4;
-const double gas_constant = 1.0;
-
-// Maximum characteristic speed (updated by integrators)
-double max_char_speed;
+using namespace std;
+using namespace mfem;

 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
-   problem = 1;
-   const char *mesh_file = "../data/periodic-square.mesh";
+   int problem = 1;
+   const real_t specific_heat_ratio = 1.4;
+   const real_t gas_constant = 1.0;
+
+   string mesh_file = "";
+   int IntOrderOffset = 1;
   int ref_levels = 1;
   int order = 3;
   int ode_solver_type = 4;
-   double t_final = 2.0;
-   double dt = -0.01;
-   double cfl = 0.3;
+   real_t t_final = 2.0;
+   real_t dt = -0.01;
+   real_t cfl = 0.3;
   bool visualization = true;
+   bool preassembleWeakDiv = true;
   int vis_steps = 50;

   int precision = 8;
@@ -77,9 +81,10 @@ int main(int argc, char *argv[])

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
+                  "Mesh file to use. If not provided, then a periodic square"
+                  " mesh will be used.");
   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
+                  "Problem setup to use. See EulerInitialCondition().");
   args.AddOption(&ref_levels, "-r", "--refine",
                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
@@ -87,8 +92,7 @@ int main(int argc, char *argv[])
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Forward Euler,\n\t"
                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
+   args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step. Positive number skips CFL timestep calculation.");
   args.AddOption(&cfl, "-c", "--cfl-number",
@@ -96,23 +100,28 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&preassembleWeakDiv, "-ea", "--element-assembly-divergence",
+                  "-mf", "--matrix-free-divergence",
+                  "Weak divergence assembly level\n"
+                  "    ea - Element assembly with interpolated F\n"
+                  "    mf - Nonlinear assembly in matrix-free manner");
   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
                  "Visualize every n-th timestep.");
+   args.ParseCheck();

-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. This example requires a 2D
-   //    periodic mesh, such as ../data/periodic-square.mesh.
-   Mesh mesh(mesh_file, 1, 1);
+   // 2. Read the mesh from the given mesh file. When the user does not provide
+   //    mesh file, use the default mesh file for the problem.
+   Mesh mesh = mesh_file.empty() ? EulerMesh(problem) : Mesh(mesh_file);
   const int dim = mesh.Dimension();
+   const int num_equations = dim + 2;

-   MFEM_ASSERT(dim == 2, "Need a two-dimensional mesh for the problem definition");
+   // Refine the mesh to increase the resolution. In this example we do
+   // 'ref_levels' of uniform refinement, where 'ref_levels' is a command-line
+   // parameter.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh.UniformRefinement();
+   }

   // 3. Define the ODE solver used for time integration. Several explicit
   //    Runge-Kutta methods are available.
@@ -129,15 +138,7 @@ int main(int argc, char *argv[])
         return 3;
   }

-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   // 5. Define the discontinuous DG finite element space of the given
+   // 4. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
   DG_FECollection fec(order, dim);
   // Finite element space for a scalar (thermodynamic quantity)
@@ -145,81 +146,74 @@ int main(int argc, char *argv[])
   // Finite element space for a mesh-dim vector quantity (momentum)
   FiniteElementSpace dfes(&mesh, &fec, dim, Ordering::byNODES);
   // Finite element space for all variables together (total thermodynamic state)
-   FiniteElementSpace vfes(&mesh, &fec, num_equation, Ordering::byNODES);
+   FiniteElementSpace vfes(&mesh, &fec, num_equations, Ordering::byNODES);

   // This example depends on this ordering of the space.
   MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES, "");

   cout << "Number of unknowns: " << vfes.GetVSize() << endl;

-   // 6. Define the initial conditions, save the corresponding mesh and grid
-   //    functions to a file. This can be opened with GLVis with the -gc option.
-
-   // The solution u has components {density, x-momentum, y-momentum, energy}.
-   // These are stored contiguously in the BlockVector u_block.
-   Array<int> offsets(num_equation + 1);
-   for (int k = 0; k <= num_equation; k++) { offsets[k] = k * vfes.GetNDofs(); }
-   BlockVector u_block(offsets);
-
-   // Momentum grid function on dfes for visualization.
-   GridFunction mom(&dfes, u_block.GetData() + offsets[1]);
+   // 5. Define the initial conditions, save the corresponding mesh and grid
+   //    functions to files. These can be opened with GLVis using:
+   //    "glvis -m euler-mesh.mesh -g euler-1-init.gf" (for x-momentum).

   // Initialize the state.
-   VectorFunctionCoefficient u0(num_equation, InitialCondition);
-   GridFunction sol(&vfes, u_block.GetData());
+   VectorFunctionCoefficient u0 = EulerInitialCondition(problem,
+                                                        specific_heat_ratio,
+                                                        gas_constant);
+   GridFunction sol(&vfes);
   sol.ProjectCoefficient(u0);
-
+   GridFunction mom(&dfes, sol.GetData() + fes.GetNDofs());
   // Output the initial solution.
   {
-      ofstream mesh_ofs("vortex.mesh");
+      ostringstream mesh_name;
+      mesh_name << "euler-mesh.mesh";
+      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(precision);
      mesh_ofs << mesh;

-      for (int k = 0; k < num_equation; k++)
+      for (int k = 0; k < num_equations; k++)
      {
-         GridFunction uk(&fes, u_block.GetBlock(k));
+         GridFunction uk(&fes, sol.GetData() + k * fes.GetNDofs());
         ostringstream sol_name;
-         sol_name << "vortex-" << k << "-init.gf";
+         sol_name << "euler-" << k << "-init.gf";
         ofstream sol_ofs(sol_name.str().c_str());
         sol_ofs.precision(precision);
         sol_ofs << uk;
      }
   }

-   // 7. Set up the nonlinear form corresponding to the DG discretization of the
-   //    flux divergence, and assemble the corresponding mass matrix.
-   MixedBilinearForm Aflux(&dfes, &fes);
-   Aflux.AddDomainIntegrator(new TransposeIntegrator(new GradientIntegrator()));
-   Aflux.Assemble();
+   // 6. Set up the nonlinear form with euler flux and numerical flux
+   EulerFlux flux(dim, specific_heat_ratio);
+   RusanovFlux numericalFlux(flux);
+   DGHyperbolicConservationLaws euler(
+      vfes, std::unique_ptr<HyperbolicFormIntegrator>(
+         new HyperbolicFormIntegrator(numericalFlux, IntOrderOffset)),
+      preassembleWeakDiv);

-   NonlinearForm A(&vfes);
-   RiemannSolver rsolver;
-   A.AddInteriorFaceIntegrator(new FaceIntegrator(rsolver, dim));
-
-   // 8. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   FE_Evolution euler(vfes, A, Aflux.SpMat());
-
-   // Visualize the density
+   // 7. Visualize momentum with its magnitude
   socketstream sout;
   if (visualization)
   {
      char vishost[] = "localhost";
-      int  visport   = 19916;
+      int visport = 19916;

      sout.open(vishost, visport);
      if (!sout)
      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
         visualization = false;
+         cout << "Unable to connect to GLVis server at " << vishost << ':'
+              << visport << endl;
         cout << "GLVis visualization disabled.\n";
      }
      else
      {
         sout.precision(precision);
+         // Plot magnitude of vector-valued momentum
         sout << "solution\n" << mesh << mom;
+         sout << "window_title 'momentum, t = 0'\n";
+         sout << "view 0 0\n";  // view from top
+         sout << "keys jlm\n";  // turn off perspective and light, show mesh
         sout << "pause\n";
         sout << flush;
         cout << "GLVis visualization paused."
@@ -227,54 +221,57 @@ int main(int argc, char *argv[])
      }
   }

-   // Determine the minimum element size.
-   double hmin = 0.0;
+   // 8. Time integration
+
+   // When dt is not specified, use CFL condition.
+   // Compute h_min and initial maximum characteristic speed
+   real_t hmin = infinity();
   if (cfl > 0)
   {
-      hmin = mesh.GetElementSize(0, 1);
-      for (int i = 1; i < mesh.GetNE(); i++)
+      for (int i = 0; i < mesh.GetNE(); i++)
      {
         hmin = min(mesh.GetElementSize(i, 1), hmin);
      }
+      // Find a safe dt, using a temporary vector. Calling Mult() computes the
+      // maximum char speed at all quadrature points on all faces (and all
+      // elements with -mf).
+      Vector z(sol.Size());
+      euler.Mult(sol, z);
+
+      real_t max_char_speed = euler.GetMaxCharSpeed();
+      dt = cfl * hmin / max_char_speed / (2 * order + 1);
   }

   // Start the timer.
   tic_toc.Clear();
   tic_toc.Start();

-   double t = 0.0;
+   // Init time integration
+   real_t t = 0.0;
   euler.SetTime(t);
   ode_solver->Init(euler);

-   if (cfl > 0)
-   {
-      // Find a safe dt, using a temporary vector. Calling Mult() computes the
-      // maximum char speed at all quadrature points on all faces.
-      Vector z(A.Width());
-      max_char_speed = 0.;
-      A.Mult(sol, z);
-      dt = cfl * hmin / max_char_speed / (2*order+1);
-   }
-
   // Integrate in time.
   bool done = false;
-   for (int ti = 0; !done; )
+   for (int ti = 0; !done;)
   {
-      double dt_real = min(dt, t_final - t);
+      real_t dt_real = min(dt, t_final - t);

      ode_solver->Step(sol, t, dt_real);
-      if (cfl > 0)
+      if (cfl > 0) // update time step size with CFL
      {
-         dt = cfl * hmin / max_char_speed / (2*order+1);
+         real_t max_char_speed = euler.GetMaxCharSpeed();
+         dt = cfl * hmin / max_char_speed / (2 * order + 1);
      }
      ti++;

-      done = (t >= t_final - 1e-8*dt);
+      done = (t >= t_final - 1e-8 * dt);
      if (done || ti % vis_steps == 0)
      {
         cout << "time step: " << ti << ", time: " << t << endl;
         if (visualization)
         {
+            sout << "window_title 'momentum, t = " << t << "'\n";
            sout << "solution\n" << mesh << mom << flush;
         }
      }
@@ -284,23 +281,28 @@ int main(int argc, char *argv[])
   cout << " done, " << tic_toc.RealTime() << "s." << endl;

   // 9. Save the final solution. This output can be viewed later using GLVis:
-   //    "glvis -m vortex.mesh -g vortex-1-final.gf".
-   for (int k = 0; k < num_equation; k++)
+   //    "glvis -m euler-mesh-final.mesh -g euler-1-final.gf" (for x-momentum).
   {
-      GridFunction uk(&fes, u_block.GetBlock(k));
-      ostringstream sol_name;
-      sol_name << "vortex-" << k << "-final.gf";
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(precision);
-      sol_ofs << uk;
+      ostringstream mesh_name;
+      mesh_name << "euler-mesh-final.mesh";
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(precision);
+      mesh_ofs << mesh;
+
+      for (int k = 0; k < num_equations; k++)
+      {
+         GridFunction uk(&fes, sol.GetData() + k * fes.GetNDofs());
+         ostringstream sol_name;
+         sol_name << "euler-" << k << "-final.gf";
+         ofstream sol_ofs(sol_name.str().c_str());
+         sol_ofs.precision(precision);
+         sol_ofs << uk;
+      }
   }

   // 10. Compute the L2 solution error summed for all components.
-   if (t_final == 2.0)
-   {
-      const double error = sol.ComputeLpError(2, u0);
-      cout << "Solution error: " << error << endl;
-   }
+   const real_t error = sol.ComputeLpError(2, u0);
+   cout << "Solution error: " << error << endl;

   // Free the used memory.
   delete ode_solver;
@@ -1,490 +1,367 @@
 //                  MFEM Example 18 - Serial/Parallel Shared Code
+//                      (Implementation of Time-dependent DG Operator)
+//
+// This code provide example problems for the Euler equations and implements
+// the time-dependent DG operator given by the equation:
+//
+//            (u_t, v)_T - (F(u), ∇ v)_T + <F̂(u, n), [[v]]>_F = 0.
+//
+// This operator is designed for explicit time stepping methods. Specifically,
+// the function DGHyperbolicConservationLaws::Mult implements the following
+// transformation:
+//
+//                             u ↦ M⁻¹(-DF(u) + NF(u))
+//
+// where M is the mass matrix, DF is the weak divergence of flux, and NF is the
+// interface flux. The inverse of the mass matrix is computed element-wise by
+// leveraging the block-diagonal structure of the DG mass matrix. Additionally,
+// the flux-related terms are computed using the HyperbolicFormIntegrator.
+//
+// The maximum characteristic speed is determined for each time step. For more
+// details, refer to the documentation of DGHyperbolicConservationLaws::Mult.
+//

+#include <functional>
 #include "mfem.hpp"

-using namespace std;
-using namespace mfem;
+namespace mfem
+{

-// Problem definition
-extern int problem;
-
-// Maximum characteristic speed (updated by integrators)
-extern double max_char_speed;
-
-extern const int num_equation;
-extern const double specific_heat_ratio;
-extern const double gas_constant;
-
-// Time-dependent operator for the right-hand side of the ODE representing the
-// DG weak form.
-class FE_Evolution : public TimeDependentOperator
+/// @brief Time dependent DG operator for hyperbolic conservation laws
+class DGHyperbolicConservationLaws : public TimeDependentOperator
 {
 private:
+   const int num_equations; // the number of equations
   const int dim;
-
-   FiniteElementSpace &vfes;
-   Operator &A;
-   SparseMatrix &Aflux;
-   DenseTensor Me_inv;
-
-   mutable Vector state;
-   mutable DenseMatrix f;
-   mutable DenseTensor flux;
+   FiniteElementSpace &vfes; // vector finite element space
+   // Element integration form. Should contain ComputeFlux
+   std::unique_ptr<HyperbolicFormIntegrator> formIntegrator;
+   // Base Nonlinear Form
+   std::unique_ptr<NonlinearForm> nonlinearForm;
+   // element-wise inverse mass matrix
+   std::vector<DenseMatrix> invmass; // local scalar inverse mass
+   std::vector<DenseMatrix> weakdiv; // local weak divergence (trial space ByDim)
+   // global maximum characteristic speed. Updated by form integrators
+   mutable real_t max_char_speed;
+   // auxiliary variable used in Mult
   mutable Vector z;

-   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;
+   // Compute element-wise inverse mass matrix
+   void ComputeInvMass();
+   // Compute element-wise weak-divergence matrix
+   void ComputeWeakDivergence();

 public:
-   FE_Evolution(FiniteElementSpace &vfes_,
-                Operator &A_, SparseMatrix &Aflux_);
+   /**
+    * @brief Construct a new DGHyperbolicConservationLaws object
+    *
+    * @param vfes_ vector finite element space. Only tested for DG [Pₚ]ⁿ
+    * @param formIntegrator_ integrator (F(u,x), grad v)
+    * @param preassembleWeakDivergence preassemble weak divergence for faster
+    *                                  assembly
+    */
+   DGHyperbolicConservationLaws(
+      FiniteElementSpace &vfes_,
+      std::unique_ptr<HyperbolicFormIntegrator> formIntegrator_,
+      bool preassembleWeakDivergence=true);
+   /**
+    * @brief Apply nonlinear form to obtain M⁻¹(DIVF + JUMP HAT(F))
+    *
+    * @param x current solution vector
+    * @param y resulting dual vector to be used in an EXPLICIT solver
+    */
+   void Mult(const Vector &x, Vector &y) const override;
+   // get global maximum characteristic speed to be used in CFL condition
+   // where max_char_speed is updated during Mult.
+   real_t GetMaxCharSpeed() { return max_char_speed; }
+   void Update();

-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution() { }
 };

-// Implements a simple Rusanov flux
-class RiemannSolver
-{
-private:
-   Vector flux1;
-   Vector flux2;
+//////////////////////////////////////////////////////////////////
+///        HYPERBOLIC CONSERVATION LAWS IMPLEMENTATION         ///
+//////////////////////////////////////////////////////////////////

-public:
-   RiemannSolver();
-   double Eval(const Vector &state1, const Vector &state2,
-               const Vector &nor, Vector &flux);
-};
-
-// Interior face term: <F.n(u),[w]>
-class FaceIntegrator : public NonlinearFormIntegrator
-{
-private:
-   RiemannSolver rsolver;
-   Vector shape1;
-   Vector shape2;
-   Vector funval1;
-   Vector funval2;
-   Vector nor;
-   Vector fluxN;
-
-public:
-   FaceIntegrator(RiemannSolver &rsolver_, const int dim);
-
-   virtual void AssembleFaceVector(const FiniteElement &el1,
-                                   const FiniteElement &el2,
-                                   FaceElementTransformations &Tr,
-                                   const Vector &elfun, Vector &elvect);
-};
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(FiniteElementSpace &vfes_,
-                           Operator &A_, SparseMatrix &Aflux_)
-   : TimeDependentOperator(A_.Height()),
-     dim(vfes_.GetFE(0)->GetDim()),
+// Implementation of class DGHyperbolicConservationLaws
+DGHyperbolicConservationLaws::DGHyperbolicConservationLaws(
+   FiniteElementSpace &vfes_,
+   std::unique_ptr<HyperbolicFormIntegrator> formIntegrator_,
+   bool preassembleWeakDivergence)
+   : TimeDependentOperator(vfes_.GetTrueVSize()),
+     num_equations(formIntegrator_->num_equations),
+     dim(vfes_.GetMesh()->SpaceDimension()),
     vfes(vfes_),
-     A(A_),
-     Aflux(Aflux_),
-     Me_inv(vfes.GetFE(0)->GetDof(), vfes.GetFE(0)->GetDof(), vfes.GetNE()),
-     state(num_equation),
-     f(num_equation, dim),
-     flux(vfes.GetNDofs(), dim, num_equation),
-     z(A.Height())
+     formIntegrator(std::move(formIntegrator_)),
+     z(vfes_.GetTrueVSize())
 {
   // Standard local assembly and inversion for energy mass matrices.
-   const int dof = vfes.GetFE(0)->GetDof();
-   DenseMatrix Me(dof);
-   DenseMatrixInverse inv(&Me);
-   MassIntegrator mi;
-   for (int i = 0; i < vfes.GetNE(); i++)
+   ComputeInvMass();
+#ifndef MFEM_USE_MPI
+   nonlinearForm.reset(new NonlinearForm(&vfes));
+#else
+   ParFiniteElementSpace *pvfes = dynamic_cast<ParFiniteElementSpace *>(&vfes);
+   if (pvfes)
   {
-      mi.AssembleElementMatrix(*vfes.GetFE(i), *vfes.GetElementTransformation(i), Me);
-      inv.Factor();
-      inv.GetInverseMatrix(Me_inv(i));
+      nonlinearForm.reset(new ParNonlinearForm(pvfes));
   }
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // 0. Reset wavespeed computation before operator application.
-   max_char_speed = 0.;
-
-   // 1. Create the vector z with the face terms -<F.n(u), [w]>.
-   A.Mult(x, z);
-
-   // 2. Add the element terms.
-   // i.  computing the flux approximately as a grid function by interpolating
-   //     at the solution nodes.
-   // ii. multiplying this grid function by a (constant) mixed bilinear form for
-   //     each of the num_equation, computing (F(u), grad(w)) for each equation.
-
-   DenseMatrix xmat(x.GetData(), vfes.GetNDofs(), num_equation);
-   GetFlux(xmat, flux);
-
-   for (int k = 0; k < num_equation; k++)
-   {
-      Vector fk(flux(k).GetData(), dim * vfes.GetNDofs());
-      Vector zk(z.GetData() + k * vfes.GetNDofs(), vfes.GetNDofs());
-      Aflux.AddMult(fk, zk);
-   }
-
-   // 3. Multiply element-wise by the inverse mass matrices.
-   Vector zval;
-   Array<int> vdofs;
-   const int dof = vfes.GetFE(0)->GetDof();
-   DenseMatrix zmat, ymat(dof, num_equation);
-
-   for (int i = 0; i < vfes.GetNE(); i++)
-   {
-      // Return the vdofs ordered byNODES
-      vfes.GetElementVDofs(i, vdofs);
-      z.GetSubVector(vdofs, zval);
-      zmat.UseExternalData(zval.GetData(), dof, num_equation);
-      mfem::Mult(Me_inv(i), zmat, ymat);
-      y.SetSubVector(vdofs, ymat.GetData());
-   }
-}
-
-// Physicality check (at end)
-bool StateIsPhysical(const Vector &state, const int dim);
-
-// Pressure (EOS) computation
-inline double ComputePressure(const Vector &state, int dim)
-{
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, dim);
-   const double den_energy = state(1 + dim);
-
-   double den_vel2 = 0;
-   for (int d = 0; d < dim; d++) { den_vel2 += den_vel(d) * den_vel(d); }
-   den_vel2 /= den;
-
-   return (specific_heat_ratio - 1.0) * (den_energy - 0.5 * den_vel2);
-}
-
-// Compute the vector flux F(u)
-void ComputeFlux(const Vector &state, int dim, DenseMatrix &flux)
-{
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, dim);
-   const double den_energy = state(1 + dim);
-
-   MFEM_ASSERT(StateIsPhysical(state, dim), "");
-
-   const double pres = ComputePressure(state, dim);
-
-   for (int d = 0; d < dim; d++)
-   {
-      flux(0, d) = den_vel(d);
-      for (int i = 0; i < dim; i++)
-      {
-         flux(1+i, d) = den_vel(i) * den_vel(d) / den;
-      }
-      flux(1+d, d) += pres;
-   }
-
-   const double H = (den_energy + pres) / den;
-   for (int d = 0; d < dim; d++)
-   {
-      flux(1+dim, d) = den_vel(d) * H;
-   }
-}
-
-// Compute the scalar F(u).n
-void ComputeFluxDotN(const Vector &state, const Vector &nor,
-                     Vector &fluxN)
-{
-   // NOTE: nor in general is not a unit normal
-   const int dim = nor.Size();
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, dim);
-   const double den_energy = state(1 + dim);
-
-   MFEM_ASSERT(StateIsPhysical(state, dim), "");
-
-   const double pres = ComputePressure(state, dim);
-
-   double den_velN = 0;
-   for (int d = 0; d < dim; d++) { den_velN += den_vel(d) * nor(d); }
-
-   fluxN(0) = den_velN;
-   for (int d = 0; d < dim; d++)
-   {
-      fluxN(1+d) = den_velN * den_vel(d) / den + pres * nor(d);
-   }
-
-   const double H = (den_energy + pres) / den;
-   fluxN(1 + dim) = den_velN * H;
-}
-
-// Compute the maximum characteristic speed.
-inline double ComputeMaxCharSpeed(const Vector &state, const int dim)
-{
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, dim);
-
-   double den_vel2 = 0;
-   for (int d = 0; d < dim; d++) { den_vel2 += den_vel(d) * den_vel(d); }
-   den_vel2 /= den;
-
-   const double pres = ComputePressure(state, dim);
-   const double sound = sqrt(specific_heat_ratio * pres / den);
-   const double vel = sqrt(den_vel2 / den);
-
-   return vel + sound;
-}
-
-// Compute the flux at solution nodes.
-void FE_Evolution::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
-{
-   const int flux_dof = flux_.SizeI();
-   const int flux_dim = flux_.SizeJ();
-
-   for (int i = 0; i < flux_dof; i++)
-   {
-      for (int k = 0; k < num_equation; k++) { state(k) = x_(i, k); }
-      ComputeFlux(state, flux_dim, f);
-
-      for (int d = 0; d < flux_dim; d++)
-      {
-         for (int k = 0; k < num_equation; k++)
-         {
-            flux_(i, d, k) = f(k, d);
-         }
-      }
-
-      // Update max char speed
-      const double mcs = ComputeMaxCharSpeed(state, flux_dim);
-      if (mcs > max_char_speed) { max_char_speed = mcs; }
-   }
-}
-
-// Implementation of class RiemannSolver
-RiemannSolver::RiemannSolver() :
-   flux1(num_equation),
-   flux2(num_equation) { }
-
-double RiemannSolver::Eval(const Vector &state1, const Vector &state2,
-                           const Vector &nor, Vector &flux)
-{
-   // NOTE: nor in general is not a unit normal
-   const int dim = nor.Size();
-
-   MFEM_ASSERT(StateIsPhysical(state1, dim), "");
-   MFEM_ASSERT(StateIsPhysical(state2, dim), "");
-
-   const double maxE1 = ComputeMaxCharSpeed(state1, dim);
-   const double maxE2 = ComputeMaxCharSpeed(state2, dim);
-
-   const double maxE = max(maxE1, maxE2);
-
-   ComputeFluxDotN(state1, nor, flux1);
-   ComputeFluxDotN(state2, nor, flux2);
-
-   double normag = 0;
-   for (int i = 0; i < dim; i++)
-   {
-      normag += nor(i) * nor(i);
-   }
-   normag = sqrt(normag);
-
-   for (int i = 0; i < num_equation; i++)
-   {
-      flux(i) = 0.5 * (flux1(i) + flux2(i))
-                - 0.5 * maxE * (state2(i) - state1(i)) * normag;
-   }
-
-   return maxE;
-}
-
-// Implementation of class FaceIntegrator
-FaceIntegrator::FaceIntegrator(RiemannSolver &rsolver_, const int dim) :
-   rsolver(rsolver_),
-   funval1(num_equation),
-   funval2(num_equation),
-   nor(dim),
-   fluxN(num_equation) { }
-
-void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
-                                        const FiniteElement &el2,
-                                        FaceElementTransformations &Tr,
-                                        const Vector &elfun, Vector &elvect)
-{
-   // Compute the term <F.n(u),[w]> on the interior faces.
-   const int dof1 = el1.GetDof();
-   const int dof2 = el2.GetDof();
-
-   shape1.SetSize(dof1);
-   shape2.SetSize(dof2);
-
-   elvect.SetSize((dof1 + dof2) * num_equation);
-   elvect = 0.0;
-
-   DenseMatrix elfun1_mat(elfun.GetData(), dof1, num_equation);
-   DenseMatrix elfun2_mat(elfun.GetData() + dof1 * num_equation, dof2,
-                          num_equation);
-
-   DenseMatrix elvect1_mat(elvect.GetData(), dof1, num_equation);
-   DenseMatrix elvect2_mat(elvect.GetData() + dof1 * num_equation, dof2,
-                           num_equation);
-
-   // Integration order calculation from DGTraceIntegrator
-   int intorder;
-   if (Tr.Elem2No >= 0)
-      intorder = (min(Tr.Elem1->OrderW(), Tr.Elem2->OrderW()) +
-                  2*max(el1.GetOrder(), el2.GetOrder()));
   else
   {
-      intorder = Tr.Elem1->OrderW() + 2*el1.GetOrder();
+      nonlinearForm.reset(new NonlinearForm(&vfes));
   }
-   if (el1.Space() == FunctionSpace::Pk)
+#endif
+   if (preassembleWeakDivergence)
   {
-      intorder++;
+      ComputeWeakDivergence();
   }
-   const IntegrationRule *ir = &IntRules.Get(Tr.GetGeometryType(), intorder);
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
+   else
   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
+      nonlinearForm->AddDomainIntegrator(formIntegrator.get());
+   }
+   nonlinearForm->AddInteriorFaceIntegrator(formIntegrator.get());
+   nonlinearForm->UseExternalIntegrators();

-      Tr.SetAllIntPoints(&ip); // set face and element int. points
+}

-      // Calculate basis functions on both elements at the face
-      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
+void DGHyperbolicConservationLaws::ComputeInvMass()
+{
+   InverseIntegrator inv_mass(new MassIntegrator());

-      // Interpolate elfun at the point
-      elfun1_mat.MultTranspose(shape1, funval1);
-      elfun2_mat.MultTranspose(shape2, funval2);
-
-      // Get the normal vector and the flux on the face
-      CalcOrtho(Tr.Jacobian(), nor);
-      const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);
-
-      // Update max char speed
-      if (mcs > max_char_speed) { max_char_speed = mcs; }
-
-      fluxN *= ip.weight;
-      for (int k = 0; k < num_equation; k++)
-      {
-         for (int s = 0; s < dof1; s++)
-         {
-            elvect1_mat(s, k) -= fluxN(k) * shape1(s);
-         }
-         for (int s = 0; s < dof2; s++)
-         {
-            elvect2_mat(s, k) += fluxN(k) * shape2(s);
-         }
-      }
+   invmass.resize(vfes.GetNE());
+   for (int i=0; i<vfes.GetNE(); i++)
+   {
+      int dof = vfes.GetFE(i)->GetDof();
+      invmass[i].SetSize(dof);
+      inv_mass.AssembleElementMatrix(*vfes.GetFE(i),
+                                     *vfes.GetElementTransformation(i),
+                                     invmass[i]);
   }
 }

-// Check that the state is physical - enabled in debug mode
-bool StateIsPhysical(const Vector &state, const int dim)
+void DGHyperbolicConservationLaws::ComputeWeakDivergence()
 {
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, dim);
-   const double den_energy = state(1 + dim);
+   TransposeIntegrator weak_div(new GradientIntegrator());
+   DenseMatrix weakdiv_bynodes;

-   if (den < 0)
+   weakdiv.resize(vfes.GetNE());
+   for (int i=0; i<vfes.GetNE(); i++)
   {
-      cout << "Negative density: ";
-      for (int i = 0; i < state.Size(); i++)
+      int dof = vfes.GetFE(i)->GetDof();
+      weakdiv_bynodes.SetSize(dof, dof*dim);
+      weak_div.AssembleElementMatrix2(*vfes.GetFE(i), *vfes.GetFE(i),
+                                      *vfes.GetElementTransformation(i),
+                                      weakdiv_bynodes);
+      weakdiv[i].SetSize(dof, dof*dim);
+      // Reorder so that trial space is ByDim.
+      // This makes applying weak divergence to flux value simpler.
+      for (int j=0; j<dof; j++)
      {
-         cout << state(i) << " ";
+         for (int d=0; d<dim; d++)
+         {
+            weakdiv[i].SetCol(j*dim + d, weakdiv_bynodes.GetColumn(d*dof + j));
+         }
      }
-      cout << endl;
-      return false;
+
   }
-   if (den_energy <= 0)
+}
+
+
+void DGHyperbolicConservationLaws::Mult(const Vector &x, Vector &y) const
+{
+   // 0. Reset wavespeed computation before operator application.
+   formIntegrator->ResetMaxCharSpeed();
+   // 1. Apply Nonlinear form to obtain an auxiliary result
+   //         z = - <F̂(u_h,n), [[v]]>_e
+   //    If weak-divergence is not preassembled, we also have weak-divergence
+   //         z = - <F̂(u_h,n), [[v]]>_e + (F(u_h), ∇v)
+   nonlinearForm->Mult(x, z);
+   if (!weakdiv.empty()) // if weak divergence is pre-assembled
   {
-      cout << "Negative energy: ";
-      for (int i = 0; i < state.Size(); i++)
+      // Apply weak divergence to F(u_h), and inverse mass to z_loc + weakdiv_loc
+      Vector current_state; // view of current state at a node
+      DenseMatrix current_flux; // flux of current state
+      DenseMatrix flux; // element flux value. Whose column is ordered by dim.
+      DenseMatrix current_xmat; // view of current states in an element, dof x num_eq
+      DenseMatrix current_zmat; // view of element auxiliary result, dof x num_eq
+      DenseMatrix current_ymat; // view of element result, dof x num_eq
+      const FluxFunction &fluxFunction = formIntegrator->GetFluxFunction();
+      Array<int> vdofs;
+      Vector xval, zval;
+      for (int i=0; i<vfes.GetNE(); i++)
      {
-         cout << state(i) << " ";
+         ElementTransformation* Tr = vfes.GetElementTransformation(i);
+         int dof = vfes.GetFE(i)->GetDof();
+         vfes.GetElementVDofs(i, vdofs);
+         x.GetSubVector(vdofs, xval);
+         current_xmat.UseExternalData(xval.GetData(), dof, num_equations);
+         flux.SetSize(num_equations, dim*dof);
+         for (int j=0; j<dof; j++) // compute flux for all nodes in the element
+         {
+            current_xmat.GetRow(j, current_state);
+            current_flux.UseExternalData(flux.GetData() + num_equations*dim*j,
+                                         num_equations, dof);
+            fluxFunction.ComputeFlux(current_state, *Tr, current_flux);
+         }
+         // Compute weak-divergence and add it to auxiliary result, z
+         // Recalling that weakdiv is reordered by dim, we can apply
+         // weak-divergence to the transpose of flux.
+         z.GetSubVector(vdofs, zval);
+         current_zmat.UseExternalData(zval.GetData(), dof, num_equations);
+         mfem::AddMult_a_ABt(1.0, weakdiv[i], flux, current_zmat);
+         // Apply inverse mass to auxiliary result to obtain the final result
+         current_ymat.SetSize(dof, num_equations);
+         mfem::Mult(invmass[i], current_zmat, current_ymat);
+         y.SetSubVector(vdofs, current_ymat.GetData());
      }
-      cout << endl;
-      return false;
   }
-
-   double den_vel2 = 0;
-   for (int i = 0; i < dim; i++) { den_vel2 += den_vel(i) * den_vel(i); }
-   den_vel2 /= den;
-
-   const double pres = (specific_heat_ratio - 1.0) * (den_energy - 0.5 * den_vel2);
-
-   if (pres <= 0)
+   else
   {
-      cout << "Negative pressure: " << pres << ", state: ";
-      for (int i = 0; i < state.Size(); i++)
+      // Apply block inverse mass
+      Vector zval; // z_loc, dof*num_eq
+
+      DenseMatrix current_zmat; // view of element auxiliary result, dof x num_eq
+      DenseMatrix current_ymat; // view of element result, dof x num_eq
+      Array<int> vdofs;
+      for (int i=0; i<vfes.GetNE(); i++)
      {
-         cout << state(i) << " ";
+         int dof = vfes.GetFE(i)->GetDof();
+         vfes.GetElementVDofs(i, vdofs);
+         z.GetSubVector(vdofs, zval);
+         current_zmat.UseExternalData(zval.GetData(), dof, num_equations);
+         current_ymat.SetSize(dof, num_equations);
+         mfem::Mult(invmass[i], current_zmat, current_ymat);
+         y.SetSubVector(vdofs, current_ymat.GetData());
      }
-      cout << endl;
-      return false;
   }
-   return true;
+   max_char_speed = formIntegrator->GetMaxCharSpeed();
+}
+
+void DGHyperbolicConservationLaws::Update()
+{
+   nonlinearForm->Update();
+   height = nonlinearForm->Height();
+   width = height;
+   z.SetSize(height);
+
+   ComputeInvMass();
+   if (!weakdiv.empty()) {ComputeWeakDivergence();}
+}
+
+std::function<void(const Vector&, Vector&)> GetMovingVortexInit(
+   const real_t radius, const real_t Minf, const real_t beta,
+   const real_t gas_constant, const real_t specific_heat_ratio)
+{
+   return [specific_heat_ratio,
+           gas_constant, Minf, radius, beta](const Vector &x, Vector &y)
+   {
+      MFEM_ASSERT(x.Size() == 2, "");
+
+      const real_t xc = 0.0, yc = 0.0;
+
+      // Nice units
+      const real_t vel_inf = 1.;
+      const real_t den_inf = 1.;
+
+      // Derive remainder of background state from this and Minf
+      const real_t pres_inf = (den_inf / specific_heat_ratio) *
+                              (vel_inf / Minf) * (vel_inf / Minf);
+      const real_t temp_inf = pres_inf / (den_inf * gas_constant);
+
+      real_t r2rad = 0.0;
+      r2rad += (x(0) - xc) * (x(0) - xc);
+      r2rad += (x(1) - yc) * (x(1) - yc);
+      r2rad /= (radius * radius);
+
+      const real_t shrinv1 = 1.0 / (specific_heat_ratio - 1.);
+
+      const real_t velX =
+         vel_inf * (1 - beta * (x(1) - yc) / radius * std::exp(-0.5 * r2rad));
+      const real_t velY =
+         vel_inf * beta * (x(0) - xc) / radius * std::exp(-0.5 * r2rad);
+      const real_t vel2 = velX * velX + velY * velY;
+
+      const real_t specific_heat =
+         gas_constant * specific_heat_ratio * shrinv1;
+      const real_t temp = temp_inf - 0.5 * (vel_inf * beta) *
+                          (vel_inf * beta) / specific_heat *
+                          std::exp(-r2rad);
+
+      const real_t den = den_inf * std::pow(temp / temp_inf, shrinv1);
+      const real_t pres = den * gas_constant * temp;
+      const real_t energy = shrinv1 * pres / den + 0.5 * vel2;
+
+      y(0) = den;
+      y(1) = den * velX;
+      y(2) = den * velY;
+      y(3) = den * energy;
+   };
+}
+
+Mesh EulerMesh(const int problem)
+{
+   switch (problem)
+   {
+      case 1:
+      case 2:
+      case 3:
+         return Mesh("../data/periodic-square.mesh");
+         break;
+      case 4:
+         return Mesh("../data/periodic-segment.mesh");
+         break;
+      default:
+         MFEM_ABORT("Problem Undefined");
+   }
 }

 // Initial condition
-void InitialCondition(const Vector &x, Vector &y)
+VectorFunctionCoefficient EulerInitialCondition(const int problem,
+                                                const real_t specific_heat_ratio,
+                                                const real_t gas_constant)
 {
-   MFEM_ASSERT(x.Size() == 2, "");
-
-   double radius = 0, Minf = 0, beta = 0;
-   if (problem == 1)
+   switch (problem)
   {
-      // "Fast vortex"
-      radius = 0.2;
-      Minf = 0.5;
-      beta = 1. / 5.;
+      case 1: // fast moving vortex
+         return VectorFunctionCoefficient(
+                   4, GetMovingVortexInit(0.2, 0.5, 1. / 5., gas_constant,
+                                          specific_heat_ratio));
+      case 2: // slow moving vortex
+         return VectorFunctionCoefficient(
+                   4, GetMovingVortexInit(0.2, 0.05, 1. / 50., gas_constant,
+                                          specific_heat_ratio));
+      case 3: // moving sine wave
+         return VectorFunctionCoefficient(4, [](const Vector &x, Vector &y)
+         {
+            MFEM_ASSERT(x.Size() == 2, "");
+            const real_t density = 1.0 + 0.2 * std::sin(M_PI*(x(0) + x(1)));
+            const real_t velocity_x = 0.7;
+            const real_t velocity_y = 0.3;
+            const real_t pressure = 1.0;
+            const real_t energy =
+               pressure / (1.4 - 1.0) +
+               density * 0.5 * (velocity_x * velocity_x + velocity_y * velocity_y);
+
+            y(0) = density;
+            y(1) = density * velocity_x;
+            y(2) = density * velocity_y;
+            y(3) = energy;
+         });
+      case 4:
+         return VectorFunctionCoefficient(3, [](const Vector &x, Vector &y)
+         {
+            MFEM_ASSERT(x.Size() == 1, "");
+            const real_t density = 1.0 + 0.2 * std::sin(M_PI * 2 * x(0));
+            const real_t velocity_x = 1.0;
+            const real_t pressure = 1.0;
+            const real_t energy =
+               pressure / (1.4 - 1.0) + density * 0.5 * (velocity_x * velocity_x);
+
+            y(0) = density;
+            y(1) = density * velocity_x;
+            y(2) = energy;
+         });
+      default:
+         MFEM_ABORT("Problem Undefined");
   }
-   else if (problem == 2)
-   {
-      // "Slow vortex"
-      radius = 0.2;
-      Minf = 0.05;
-      beta = 1. / 50.;
-   }
-   else
-   {
-      mfem_error("Cannot recognize problem."
-                 "Options are: 1 - fast vortex, 2 - slow vortex");
-   }
-
-   const double xc = 0.0, yc = 0.0;
-
-   // Nice units
-   const double vel_inf = 1.;
-   const double den_inf = 1.;
-
-   // Derive remainder of background state from this and Minf
-   const double pres_inf = (den_inf / specific_heat_ratio) * (vel_inf / Minf) *
-                           (vel_inf / Minf);
-   const double temp_inf = pres_inf / (den_inf * gas_constant);
-
-   double r2rad = 0.0;
-   r2rad += (x(0) - xc) * (x(0) - xc);
-   r2rad += (x(1) - yc) * (x(1) - yc);
-   r2rad /= (radius * radius);
-
-   const double shrinv1 = 1.0 / (specific_heat_ratio - 1.);
-
-   const double velX = vel_inf * (1 - beta * (x(1) - yc) / radius * exp(
-                                     -0.5 * r2rad));
-   const double velY = vel_inf * beta * (x(0) - xc) / radius * exp(-0.5 * r2rad);
-   const double vel2 = velX * velX + velY * velY;
-
-   const double specific_heat = gas_constant * specific_heat_ratio * shrinv1;
-   const double temp = temp_inf - 0.5 * (vel_inf * beta) *
-                       (vel_inf * beta) / specific_heat * exp(-r2rad);
-
-   const double den = den_inf * pow(temp/temp_inf, shrinv1);
-   const double pres = den * gas_constant * temp;
-   const double energy = shrinv1 * pres / den + 0.5 * vel2;
-
-   y(0) = den;
-   y(1) = den * velX;
-   y(2) = den * velY;
-   y(3) = den * energy;
 }
+
+} // namespace mfem
@@ -1,18 +1,24 @@
-//                       MFEM Example 18 - Parallel Version
+//                                MFEM Example 18 - Parallel Version
 //
-// Compile with: make ex18
+// Compile with: make ex18p
 //
 // Sample runs:
 //
 //       mpirun -np 4 ex18p -p 1 -rs 2 -rp 1 -o 1 -s 3
 //       mpirun -np 4 ex18p -p 1 -rs 1 -rp 1 -o 3 -s 4
 //       mpirun -np 4 ex18p -p 1 -rs 1 -rp 1 -o 5 -s 6
-//       mpirun -np 4 ex18p -p 2 -rs 1 -rp 1 -o 1 -s 3
-//       mpirun -np 4 ex18p -p 2 -rs 1 -rp 1 -o 3 -s 3
+//       mpirun -np 4 ex18p -p 2 -rs 1 -rp 1 -o 1 -s 3 -mf
+//       mpirun -np 4 ex18p -p 2 -rs 1 -rp 1 -o 3 -s 3 -mf
 //
 // Description:  This example code solves the compressible Euler system of
 //               equations, a model nonlinear hyperbolic PDE, with a
-//               discontinuous Galerkin (DG) formulation.
+//               discontinuous Galerkin (DG) formulation in parallel.
+//
+//                (u_t, v)_T - (F(u), ∇ v)_T + <F̂(u,n), [[v]]>_F = 0
+//
+//               where (⋅,⋅)_T is volume integration, and <⋅,⋅>_F is face
+//               integration, F is the Euler flux function, and F̂ is the
+//               numerical flux.
 //
 //               Specifically, it solves for an exact solution of the equations
 //               whereby a vortex is transported by a uniform flow. Since all
@@ -27,54 +33,54 @@
 //               method. An additional factor can be tuned by passing the --cfl
 //               (or -c shorter) flag.
 //
-//               The example demonstrates user-defined bilinear and nonlinear
-//               form integrators for systems of equations that are defined with
-//               block vectors, and how these are used with an operator for
-//               explicit time integrators. In this case the system also
-//               involves an external approximate Riemann solver for the DG
-//               interface flux. It also demonstrates how to use GLVis for
-//               in-situ visualization of vector grid functions.
+//               The example demonstrates usage of DGHyperbolicConservationLaws
+//               that wraps NonlinearFormIntegrators containing element and face
+//               integration schemes. In this case the system also involves an
+//               external approximate Riemann solver for the DG interface flux.
+//               By default, weak-divergence is pre-assembled in element-wise
+//               manner, which corresponds to (I_h(F(u_h)), ∇ v). This yields
+//               better performance and similar accuracy for the included test
+//               problems. This can be turned off and use nonlinear assembly
+//               similar to matrix-free assembly when -mf flag is provided.
+//               It also demonstrates how to use GLVis for in-situ visualization
+//               of vector grid function and how to set top-view.
 //
 //               We recommend viewing examples 9, 14 and 17 before viewing this
 //               example.

 #include "mfem.hpp"
 #include <fstream>
-#include <sstream>
 #include <iostream>
-
-// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
-// shared between the serial and parallel version of the example.
+#include <sstream>
 #include "ex18.hpp"

-// Choice for the problem setup. See InitialCondition in ex18.hpp.
-int problem;
-
-// Equation constant parameters.
-const int num_equation = 4;
-const double specific_heat_ratio = 1.4;
-const double gas_constant = 1.0;
-
-// Maximum characteristic speed (updated by integrators)
-double max_char_speed;
+using namespace std;
+using namespace mfem;

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 0. Parallel setup
   Mpi::Init(argc, argv);
+   const int numProcs = Mpi::WorldSize();
+   const int myRank = Mpi::WorldRank();
   Hypre::Init();

-   // 2. Parse command-line options.
-   problem = 1;
-   const char *mesh_file = "../data/periodic-square.mesh";
+   // 1. Parse command-line options.
+   int problem = 1;
+   const real_t specific_heat_ratio = 1.4;
+   const real_t gas_constant = 1.0;
+
+   string mesh_file = "";
+   int IntOrderOffset = 1;
   int ser_ref_levels = 0;
   int par_ref_levels = 1;
   int order = 3;
   int ode_solver_type = 4;
-   double t_final = 2.0;
-   double dt = -0.01;
-   double cfl = 0.3;
+   real_t t_final = 2.0;
+   real_t dt = -0.01;
+   real_t cfl = 0.3;
   bool visualization = true;
+   bool preassembleWeakDiv = true;
   int vis_steps = 50;

   int precision = 8;
@@ -82,22 +88,20 @@ int main(int argc, char *argv[])

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
+                  "Mesh file to use. If not provided, then a periodic square"
+                  " mesh will be used.");
   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly before parallel"
-                  " partitioning, -1 for auto.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly after parallel"
-                  " partitioning.");
+                  "Problem setup to use. See EulerInitialCondition().");
+   args.AddOption(&ser_ref_levels, "-rs", "--serial-refine",
+                  "Number of times to refine the serial mesh uniformly.");
+   args.AddOption(&par_ref_levels, "-rp", "--parallel-refine",
+                  "Number of times to refine the parallel mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Forward Euler,\n\t"
                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
+   args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step. Positive number skips CFL timestep calculation.");
   args.AddOption(&cfl, "-c", "--cfl-number",
@@ -105,25 +109,44 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&preassembleWeakDiv, "-ea", "--element-assembly-divergence",
+                  "-mf", "--matrix-free-divergence",
+                  "Weak divergence assembly level\n"
+                  "    ea - Element assembly with interpolated F\n"
+                  "    mf - Nonlinear assembly in matrix-free manner");
   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
                  "Visualize every n-th timestep.");
+   args.ParseCheck();

-   args.Parse();
-   if (!args.Good())
-   {
-      if (Mpi::Root()) { args.PrintUsage(cout); }
-      return 1;
-   }
-   if (Mpi::Root()) { args.PrintOptions(cout); }
-
-   // 3. Read the mesh from the given mesh file. This example requires a 2D
-   //    periodic mesh, such as ../data/periodic-square.mesh.
-   Mesh mesh(mesh_file, 1, 1);
+   // 2. Read the mesh from the given mesh file. When the user does not provide
+   //    mesh file, use the default mesh file for the problem.
+   Mesh mesh = mesh_file.empty() ? EulerMesh(problem) : Mesh(mesh_file);
   const int dim = mesh.Dimension();
+   const int num_equations = dim + 2;

-   MFEM_ASSERT(dim == 2, "Need a two-dimensional mesh for the problem definition");
+   // Refine the mesh to increase the resolution. In this example we do
+   // 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is a
+   // command-line parameter.
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh.UniformRefinement();
+   }

-   // 4. Define the ODE solver used for time integration. Several explicit
+   // Define a parallel mesh by a partitioning of the serial mesh. Refine this
+   // mesh further in parallel to increase the resolution. Once the parallel
+   // mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh = ParMesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Refine the mesh to increase the resolution. In this example we do
+   // 'par_ref_levels' of uniform refinement, where 'par_ref_levels' is a
+   // command-line parameter.
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   // 3. Define the ODE solver used for time integration. Several explicit
   //    Runge-Kutta methods are available.
   ODESolver *ode_solver = NULL;
   switch (ode_solver_type)
@@ -134,32 +157,11 @@ int main(int argc, char *argv[])
      case 4: ode_solver = new RK4Solver; break;
      case 6: ode_solver = new RK6Solver; break;
      default:
-         if (Mpi::Root())
-         {
-            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         }
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
         return 3;
   }

-   // 5. Refine the mesh in serial to increase the resolution. In this example
-   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-   //    a command-line parameter.
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh.UniformRefinement();
-   }
-
-   // 7. Define the discontinuous DG finite element space of the given
+   // 4. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
   DG_FECollection fec(order, dim);
   // Finite element space for a scalar (thermodynamic quantity)
@@ -167,7 +169,7 @@ int main(int argc, char *argv[])
   // Finite element space for a mesh-dim vector quantity (momentum)
   ParFiniteElementSpace dfes(&pmesh, &fec, dim, Ordering::byNODES);
   // Finite element space for all variables together (total thermodynamic state)
-   ParFiniteElementSpace vfes(&pmesh, &fec, num_equation, Ordering::byNODES);
+   ParFiniteElementSpace vfes(&pmesh, &fec, num_equations, Ordering::byNODES);

   // This example depends on this ordering of the space.
   MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES, "");
@@ -178,87 +180,72 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << glob_size << endl;
   }

-   // 8. Define the initial conditions, save the corresponding mesh and grid
-   //    functions to a file. This can be opened with GLVis with the -gc option.
-
-   // The solution u has components {density, x-momentum, y-momentum, energy}.
-   // These are stored contiguously in the BlockVector u_block.
-   Array<int> offsets(num_equation + 1);
-   for (int k = 0; k <= num_equation; k++) { offsets[k] = k * vfes.GetNDofs(); }
-   BlockVector u_block(offsets);
-
-   // Momentum grid function on dfes for visualization.
-   ParGridFunction mom(&dfes, u_block.GetData() + offsets[1]);
+   // 5. Define the initial conditions, save the corresponding mesh and grid
+   //    functions to files. These can be opened with GLVis using:
+   //    "glvis -np 4  -m euler-mesh -g euler-1-init" (for x-momentum).

   // Initialize the state.
-   VectorFunctionCoefficient u0(num_equation, InitialCondition);
-   ParGridFunction sol(&vfes, u_block.GetData());
+   VectorFunctionCoefficient u0 = EulerInitialCondition(problem,
+                                                        specific_heat_ratio,
+                                                        gas_constant);
+   ParGridFunction sol(&vfes);
   sol.ProjectCoefficient(u0);
-
+   ParGridFunction mom(&dfes, sol.GetData() + fes.GetNDofs());
   // Output the initial solution.
   {
      ostringstream mesh_name;
-      mesh_name << "vortex-mesh." << setfill('0')
-                << setw(6) << Mpi::WorldRank();
+      mesh_name << "euler-mesh." << setfill('0') << setw(6) << Mpi::WorldRank();
      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(precision);
      mesh_ofs << pmesh;

-      for (int k = 0; k < num_equation; k++)
+      for (int k = 0; k < num_equations; k++)
      {
-         ParGridFunction uk(&fes, u_block.GetBlock(k));
+         ParGridFunction uk(&fes, sol.GetData() + k * fes.GetNDofs());
         ostringstream sol_name;
-         sol_name << "vortex-" << k << "-init."
-                  << setfill('0') << setw(6) << Mpi::WorldRank();
+         sol_name << "euler-" << k << "-init." << setfill('0') << setw(6)
+                  << Mpi::WorldRank();
         ofstream sol_ofs(sol_name.str().c_str());
         sol_ofs.precision(precision);
         sol_ofs << uk;
      }
   }

-   // 9. Set up the nonlinear form corresponding to the DG discretization of the
-   //    flux divergence, and assemble the corresponding mass matrix.
-   MixedBilinearForm Aflux(&dfes, &fes);
-   Aflux.AddDomainIntegrator(new TransposeIntegrator(new GradientIntegrator()));
-   Aflux.Assemble();
+   // 6. Set up the nonlinear form with euler flux and numerical flux
+   EulerFlux flux(dim, specific_heat_ratio);
+   RusanovFlux numericalFlux(flux);
+   DGHyperbolicConservationLaws euler(
+      vfes, std::unique_ptr<HyperbolicFormIntegrator>(
+         new HyperbolicFormIntegrator(numericalFlux, IntOrderOffset)),
+      preassembleWeakDiv);

-   ParNonlinearForm A(&vfes);
-   RiemannSolver rsolver;
-   A.AddInteriorFaceIntegrator(new FaceIntegrator(rsolver, dim));
-
-   // 10. Define the time-dependent evolution operator describing the ODE
-   //     right-hand side, and perform time-integration (looping over the time
-   //     iterations, ti, with a time-step dt).
-   FE_Evolution euler(vfes, A, Aflux.SpMat());
-
-   // Visualize the density
+   // 7. Visualize momentum with its magnitude
   socketstream sout;
   if (visualization)
   {
      char vishost[] = "localhost";
-      int  visport   = 19916;
+      int visport = 19916;

-      MPI_Barrier(pmesh.GetComm());
      sout.open(vishost, visport);
      if (!sout)
      {
-         if (Mpi::Root())
-         {
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-         }
         visualization = false;
         if (Mpi::Root())
         {
+            cout << "Unable to connect to GLVis server at " << vishost << ':'
+                 << visport << endl;
            cout << "GLVis visualization disabled.\n";
         }
      }
      else
      {
-         sout << "parallel " << Mpi::WorldSize()
-              << " " << Mpi::WorldRank() << "\n";
         sout.precision(precision);
+         // Plot magnitude of vector-valued momentum
+         sout << "parallel " << numProcs << " " << myRank << "\n";
         sout << "solution\n" << pmesh << mom;
+         sout << "window_title 'momentum, t = 0'\n";
+         sout << "view 0 0\n";  // view from top
+         sout << "keys jlm\n";  // turn off perspective and light, show mesh
         sout << "pause\n";
         sout << flush;
         if (Mpi::Root())
@@ -266,68 +253,63 @@ int main(int argc, char *argv[])
            cout << "GLVis visualization paused."
                 << " Press space (in the GLVis window) to resume it.\n";
         }
+         MPI_Barrier(pmesh.GetComm());
      }
   }

-   // Determine the minimum element size.
-   double hmin;
+   // 8. Time integration
+
+   // When dt is not specified, use CFL condition.
+   // Compute h_min and initial maximum characteristic speed
+   real_t hmin = infinity();
   if (cfl > 0)
   {
-      double my_hmin = pmesh.GetElementSize(0, 1);
-      for (int i = 1; i < pmesh.GetNE(); i++)
+      for (int i = 0; i < pmesh.GetNE(); i++)
      {
-         my_hmin = min(pmesh.GetElementSize(i, 1), my_hmin);
+         hmin = min(pmesh.GetElementSize(i, 1), hmin);
      }
-      // Reduce to find the global minimum element size
-      MPI_Allreduce(&my_hmin, &hmin, 1, MPI_DOUBLE, MPI_MIN, pmesh.GetComm());
+      MPI_Allreduce(MPI_IN_PLACE, &hmin, 1,  MPITypeMap<real_t>::mpi_type, MPI_MIN,
+                    pmesh.GetComm());
+      // Find a safe dt, using a temporary vector. Calling Mult() computes the
+      // maximum char speed at all quadrature points on all faces (and all
+      // elements with -mf).
+      Vector z(sol.Size());
+      euler.Mult(sol, z);
+
+      real_t max_char_speed = euler.GetMaxCharSpeed();
+      MPI_Allreduce(MPI_IN_PLACE, &max_char_speed, 1,  MPITypeMap<real_t>::mpi_type,
+                    MPI_MAX,
+                    pmesh.GetComm());
+      dt = cfl * hmin / max_char_speed / (2 * order + 1);
   }

   // Start the timer.
   tic_toc.Clear();
   tic_toc.Start();

-   double t = 0.0;
+   // Init time integration
+   real_t t = 0.0;
   euler.SetTime(t);
   ode_solver->Init(euler);

-   if (cfl > 0)
-   {
-      // Find a safe dt, using a temporary vector. Calling Mult() computes the
-      // maximum char speed at all quadrature points on all faces.
-      max_char_speed = 0.;
-      Vector z(sol.Size());
-      A.Mult(sol, z);
-      // Reduce to find the global maximum wave speed
-      {
-         double all_max_char_speed;
-         MPI_Allreduce(&max_char_speed, &all_max_char_speed,
-                       1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-         max_char_speed = all_max_char_speed;
-      }
-      dt = cfl * hmin / max_char_speed / (2*order+1);
-   }
-
   // Integrate in time.
   bool done = false;
-   for (int ti = 0; !done; )
+   for (int ti = 0; !done;)
   {
-      double dt_real = min(dt, t_final - t);
+      real_t dt_real = min(dt, t_final - t);

      ode_solver->Step(sol, t, dt_real);
-      if (cfl > 0)
+      if (cfl > 0) // update time step size with CFL
      {
-         // Reduce to find the global maximum wave speed
-         {
-            double all_max_char_speed;
-            MPI_Allreduce(&max_char_speed, &all_max_char_speed,
-                          1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-            max_char_speed = all_max_char_speed;
-         }
-         dt = cfl * hmin / max_char_speed / (2*order+1);
+         real_t max_char_speed = euler.GetMaxCharSpeed();
+         MPI_Allreduce(MPI_IN_PLACE, &max_char_speed, 1,  MPITypeMap<real_t>::mpi_type,
+                       MPI_MAX,
+                       pmesh.GetComm());
+         dt = cfl * hmin / max_char_speed / (2 * order + 1);
      }
      ti++;

-      done = (t >= t_final - 1e-8*dt);
+      done = (t >= t_final - 1e-8 * dt);
      if (done || ti % vis_steps == 0)
      {
         if (Mpi::Root())
@@ -336,9 +318,8 @@ int main(int argc, char *argv[])
         }
         if (visualization)
         {
-            MPI_Barrier(pmesh.GetComm());
-            sout << "parallel " << Mpi::WorldSize()
-                 << " " << Mpi::WorldRank() << "\n";
+            sout << "window_title 'momentum, t = " << t << "'\n";
+            sout << "parallel " << numProcs << " " << myRank << "\n";
            sout << "solution\n" << pmesh << mom << flush;
         }
      }
@@ -350,27 +331,33 @@ int main(int argc, char *argv[])
      cout << " done, " << tic_toc.RealTime() << "s." << endl;
   }

-   // 11. Save the final solution. This output can be viewed later using GLVis:
-   //     "glvis -np 4 -m vortex-mesh -g vortex-1-final".
-   for (int k = 0; k < num_equation; k++)
+   // 9. Save the final solution. This output can be viewed later using GLVis:
+   //    "glvis -np 4 -m euler-mesh-final -g euler-1-final" (for x-momentum).
   {
-      ParGridFunction uk(&fes, u_block.GetBlock(k));
-      ostringstream sol_name;
-      sol_name << "vortex-" << k << "-final."
-               << setfill('0') << setw(6) << Mpi::WorldRank();
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(precision);
-      sol_ofs << uk;
+      ostringstream mesh_name;
+      mesh_name << "euler-mesh-final." << setfill('0') << setw(6)
+                << Mpi::WorldRank();
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(precision);
+      mesh_ofs << pmesh;
+
+      for (int k = 0; k < num_equations; k++)
+      {
+         ParGridFunction uk(&fes, sol.GetData() + k * fes.GetNDofs());
+         ostringstream sol_name;
+         sol_name << "euler-" << k << "-final." << setfill('0') << setw(6)
+                  << Mpi::WorldRank();
+         ofstream sol_ofs(sol_name.str().c_str());
+         sol_ofs.precision(precision);
+         sol_ofs << uk;
+      }
   }

-   // 12. Compute the L2 solution error summed for all components.
-   if (t_final == 2.0)
+   // 10. Compute the L2 solution error summed for all components.
+   const real_t error = sol.ComputeLpError(2, u0);
+   if (Mpi::Root())
   {
-      const double error = sol.ComputeLpError(2, u0);
-      if (Mpi::Root())
-      {
-         cout << "Solution error: " << error << endl;
-      }
+      cout << "Solution error: " << error << endl;
   }

   // Free the used memory.
@@ -48,15 +48,15 @@ public:
      print_level = print_lvl;
   }

-   virtual void MonitorResidual(int it, double norm, const Vector &r, bool final);
+   virtual void MonitorResidual(int it, real_t norm, const Vector &r, bool final);

 private:
   const std::string prefix;
   int print_level;
-   mutable double norm0;
+   mutable real_t norm0;
 };

-void GeneralResidualMonitor::MonitorResidual(int it, double norm,
+void GeneralResidualMonitor::MonitorResidual(int it, real_t norm,
                                             const Vector &r, bool final)
 {
   if (print_level == 1 || (print_level == 3 && (final || it == 0)))
@@ -103,7 +103,7 @@ protected:
   BlockOperator *jacobian;

   // Scaling factor for the pressure mass matrix in the block preconditioner
-   double gamma;
+   real_t gamma;

   // Objects for the block preconditioner application
   SparseMatrix *pressure_mass;
@@ -157,7 +157,7 @@ protected:

 public:
   RubberOperator(Array<FiniteElementSpace *> &fes, Array<Array<int> *>&ess_bdr,
-                  Array<int> &block_trueOffsets, double rel_tol, double abs_tol,
+                  Array<int> &block_trueOffsets, real_t rel_tol, real_t abs_tol,
                  int iter, Coefficient &mu);

   // Required to use the native newton solver
@@ -187,10 +187,10 @@ int main(int argc, char *argv[])
   int ref_levels = 0;
   int order = 2;
   bool visualization = true;
-   double newton_rel_tol = 1e-4;
-   double newton_abs_tol = 1e-6;
+   real_t newton_rel_tol = 1e-4;
+   real_t newton_abs_tol = 1e-6;
   int newton_iter = 500;
-   double mu = 1.0;
+   real_t mu = 1.0;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -449,8 +449,8 @@ JacobianPreconditioner::~JacobianPreconditioner()
 RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
                               Array<Array<int> *> &ess_bdr,
                               Array<int> &offsets,
-                               double rel_tol,
-                               double abs_tol,
+                               real_t rel_tol,
+                               real_t abs_tol,
                               int iter,
                               Coefficient &c_mu)
   : Operator(fes[0]->GetTrueVSize() + fes[1]->GetTrueVSize()),
@@ -62,15 +62,15 @@ public:
 #endif
   }

-   virtual void MonitorResidual(int it, double norm, const Vector &r, bool final);
+   virtual void MonitorResidual(int it, real_t norm, const Vector &r, bool final);

 private:
   const std::string prefix;
   int print_level;
-   mutable double norm0;
+   mutable real_t norm0;
 };

-void GeneralResidualMonitor::MonitorResidual(int it, double norm,
+void GeneralResidualMonitor::MonitorResidual(int it, real_t norm,
                                             const Vector &r, bool final)
 {
   if (print_level == 1 || (print_level == 3 && (final || it == 0)))
@@ -117,7 +117,7 @@ protected:
   BlockOperator *jacobian;

   // Scaling factor for the pressure mass matrix in the block preconditioner
-   double gamma;
+   real_t gamma;

   // Objects for the block preconditioner application
   Operator *pressure_mass;
@@ -171,7 +171,7 @@ protected:

 public:
   RubberOperator(Array<ParFiniteElementSpace *> &fes, Array<Array<int> *>&ess_bdr,
-                  Array<int> &block_trueOffsets, double rel_tol, double abs_tol,
+                  Array<int> &block_trueOffsets, real_t rel_tol, real_t abs_tol,
                  int iter, Coefficient &mu);

   // Required to use the native newton solver
@@ -200,7 +200,7 @@ int main(int argc, char *argv[])
 #ifdef HYPRE_USING_GPU
   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
        << "is NOT supported with the GPU version of hypre.\n\n";
-   return 242;
+   return MFEM_SKIP_RETURN_VALUE;
 #endif

   // 1. Initialize MPI and HYPRE.
@@ -214,10 +214,10 @@ int main(int argc, char *argv[])
   int par_ref_levels = 0;
   int order = 2;
   bool visualization = true;
-   double newton_rel_tol = 1e-4;
-   double newton_abs_tol = 1e-6;
+   real_t newton_rel_tol = 1e-4;
+   real_t newton_abs_tol = 1e-6;
   int newton_iter = 500;
-   double mu = 1.0;
+   real_t mu = 1.0;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -524,8 +524,8 @@ JacobianPreconditioner::~JacobianPreconditioner()
 RubberOperator::RubberOperator(Array<ParFiniteElementSpace *> &fes,
                               Array<Array<int> *> &ess_bdr,
                               Array<int> &trueOffsets,
-                               double rel_tol,
-                               double abs_tol,
+                               real_t rel_tol,
+                               real_t abs_tol,
                               int iter,
                               Coefficient &c_mu)
   : Operator(fes[0]->TrueVSize() + fes[1]->TrueVSize()),
@@ -69,11 +69,11 @@ using namespace mfem;

 // Constants used in the Hamiltonian
 static int prob_ = 0;
-static double m_ = 1.0;
-static double k_ = 1.0;
+static real_t m_ = 1.0;
+static real_t k_ = 1.0;

 // Hamiltonian functional, see below for implementation
-double hamiltonian(double q, double p, double t);
+real_t hamiltonian(real_t q, real_t p, real_t t);

 class GradT : public Operator
 {
@@ -94,7 +94,7 @@ int main(int argc, char *argv[])
   // 1. Parse command-line options.
   int order  = 1;
   int nsteps = 100;
-   double dt  = 0.1;
+   real_t dt  = 0.1;
   bool visualization = true;
   bool gnuplot = false;

@@ -136,7 +136,7 @@ int main(int argc, char *argv[])
   siaSolver.Init(P,F);

   // 3. Set the initial conditions
-   double t = 0.0;
+   real_t t = 0.0;
   Vector q(1), p(1);
   Vector e(nsteps+1);
   q(0) = 0.0;
@@ -160,7 +160,7 @@ int main(int argc, char *argv[])
   Vector x1(3); x1 = 0.0;

   // 6. Perform time-stepping
-   double e_mean = 0.0;
+   real_t e_mean = 0.0;

   for (int i = 0; i < nsteps; i++)
   {
@@ -210,13 +210,13 @@ int main(int argc, char *argv[])

   // 7. Compute and display mean and standard deviation of the energy
   e_mean /= (nsteps + 1);
-   double e_var = 0.0;
+   real_t e_var = 0.0;
   for (int i=0; i<=nsteps; i++)
   {
      e_var += pow(e[i] - e_mean, 2);
   }
   e_var /= (nsteps + 1);
-   double e_sd = sqrt(e_var);
+   real_t e_sd = sqrt(e_var);
   cout << endl << "Mean and standard deviation of the energy" << endl;
   cout << e_mean << "\t" << e_sd << endl;

@@ -256,9 +256,9 @@ int main(int argc, char *argv[])
   }
 }

-double hamiltonian(double q, double p, double t)
+real_t hamiltonian(real_t q, real_t p, real_t t)
 {
-   double h = 1.0 - 0.5 / m_ + 0.5 * p * p / m_;
+   real_t h = 1.0 - 0.5 / m_ + 0.5 * p * p / m_;
   switch (prob_)
   {
      case 1:
@@ -74,11 +74,11 @@ using namespace mfem;

 // Constants used in the Hamiltonian
 static int prob_ = 0;
-static double m_ = 1.0;
-static double k_ = 1.0;
+static real_t m_ = 1.0;
+static real_t k_ = 1.0;

 // Hamiltonian functional, see below for implementation
-double hamiltonian(double q, double p, double t);
+real_t hamiltonian(real_t q, real_t p, real_t t);

 class GradT : public Operator
 {
@@ -106,7 +106,7 @@ int main(int argc, char *argv[])
   // 2. Parse command-line options.
   int order  = 1;
   int nsteps = 100;
-   double dt  = 0.1;
+   real_t dt  = 0.1;
   bool visualization = true;
   bool gnuplot = false;

@@ -154,11 +154,11 @@ int main(int argc, char *argv[])
   siaSolver.Init(P,F);

   // 4. Set the initial conditions
-   double t = 0.0;
+   real_t t = 0.0;
   Vector q(1), p(1);
   Vector e(nsteps+1);
-   q(0) = sin(2.0*M_PI*(double)myid/num_procs);
-   p(0) = cos(2.0*M_PI*(double)myid/num_procs);
+   q(0) = sin(2.0*M_PI*(real_t)myid/num_procs);
+   p(0) = cos(2.0*M_PI*(real_t)myid/num_procs);

   // 5. Prepare GnuPlot output file if needed
   ostringstream oss;
@@ -181,7 +181,7 @@ int main(int argc, char *argv[])
   Vector x1(3); x1 = 0.0;

   // 7. Perform time-stepping
-   double e_mean = 0.0;
+   real_t e_mean = 0.0;

   for (int i = 0; i < nsteps; i++)
   {
@@ -238,20 +238,21 @@ int main(int argc, char *argv[])

   // 8. Compute and display mean and standard deviation of the energy
   e_mean /= (nsteps + 1);
-   double e_var = 0.0;
+   real_t e_var = 0.0;
   for (int i = 0; i <= nsteps; i++)
   {
      e_var += pow(e[i] - e_mean, 2);
   }
   e_var /= (nsteps + 1);
-   double e_sd = sqrt(e_var);
+   real_t e_sd = sqrt(e_var);

-   double e_loc_stats[2];
-   double *e_stats = (myid == 0) ? new double[2 * num_procs] : (double*)NULL;
+   real_t e_loc_stats[2];
+   real_t *e_stats = (myid == 0) ? new real_t[2 * num_procs] : (real_t*)NULL;

   e_loc_stats[0] = e_mean;
   e_loc_stats[1] = e_sd;
-   MPI_Gather(e_loc_stats, 2, MPI_DOUBLE, e_stats, 2, MPI_DOUBLE, 0, comm);
+   MPI_Gather(e_loc_stats, 2, MPITypeMap<real_t>::mpi_type, e_stats, 2,
+              MPITypeMap<real_t>::mpi_type, 0, comm);

   if (myid == 0)
   {
@@ -324,9 +325,9 @@ int main(int argc, char *argv[])
   }
 }

-double hamiltonian(double q, double p, double t)
+real_t hamiltonian(real_t q, real_t p, real_t t)
 {
-   double h = 1.0 - 0.5 / m_ + 0.5 * p * p / m_;
+   real_t h = 1.0 - 0.5 / m_ + 0.5 * p * p / m_;
   switch (prob_)
   {
      case 1:
@@ -57,13 +57,13 @@
 using namespace std;
 using namespace mfem;

-static double mu_ = 1.0;
-static double epsilon_ = 1.0;
-static double sigma_ = 20.0;
-static double omega_ = 10.0;
+static real_t mu_ = 1.0;
+static real_t epsilon_ = 1.0;
+static real_t sigma_ = 20.0;
+static real_t omega_ = 10.0;

-double u0_real_exact(const Vector &);
-double u0_imag_exact(const Vector &);
+real_t u0_real_exact(const Vector &);
+real_t u0_imag_exact(const Vector &);

 void u1_real_exact(const Vector &, Vector &);
 void u1_imag_exact(const Vector &, Vector &);
@@ -80,8 +80,8 @@ int main(int argc, char *argv[])
   int ref_levels = 0;
   int order = 1;
   int prob = 0;
-   double freq = -1.0;
-   double a_coef = 0.0;
+   real_t freq = -1.0;
+   real_t a_coef = 0.0;
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
@@ -412,7 +412,7 @@ int main(int argc, char *argv[])
               break; // This should be unreachable
         }
      }
-      double s = (prob != 1) ? 1.0 : -1.0;
+      real_t s = (prob != 1) ? 1.0 : -1.0;
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
                                s:-s);
@@ -436,8 +436,8 @@ int main(int argc, char *argv[])

   if (exact_sol)
   {
-      double err_r = -1.0;
-      double err_i = -1.0;
+      real_t err_r = -1.0;
+      real_t err_i = -1.0;

      switch (prob)
      {
@@ -524,7 +524,7 @@ int main(int argc, char *argv[])
      int i = 0;
      while (sol_sock)
      {
-         double t = (double)(i % num_frames) / num_frames;
+         real_t t = (real_t)(i % num_frames) / num_frames;
         ostringstream oss;
         oss << "Harmonic Solution (t = " << t << " T)";

@@ -555,21 +555,21 @@ bool check_for_inline_mesh(const char * mesh_file)
   return s0 == "inline-";
 }

-complex<double> u0_exact(const Vector &x)
+complex<real_t> u0_exact(const Vector &x)
 {
   int dim = x.Size();
-   complex<double> i(0.0, 1.0);
-   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
-   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
+   complex<real_t> i(0.0, 1.0);
+   complex<real_t> alpha = (epsilon_ * omega_ - i * sigma_);
+   complex<real_t> kappa = std::sqrt(mu_ * omega_* alpha);
   return std::exp(-i * kappa * x[dim - 1]);
 }

-double u0_real_exact(const Vector &x)
+real_t u0_real_exact(const Vector &x)
 {
   return u0_exact(x).real();
 }

-double u0_imag_exact(const Vector &x)
+real_t u0_imag_exact(const Vector &x)
 {
   return u0_exact(x).imag();
 }
@@ -57,13 +57,13 @@
 using namespace std;
 using namespace mfem;

-static double mu_ = 1.0;
-static double epsilon_ = 1.0;
-static double sigma_ = 20.0;
-static double omega_ = 10.0;
+static real_t mu_ = 1.0;
+static real_t epsilon_ = 1.0;
+static real_t sigma_ = 20.0;
+static real_t omega_ = 10.0;

-double u0_real_exact(const Vector &);
-double u0_imag_exact(const Vector &);
+real_t u0_real_exact(const Vector &);
+real_t u0_imag_exact(const Vector &);

 void u1_real_exact(const Vector &, Vector &);
 void u1_imag_exact(const Vector &, Vector &);
@@ -87,8 +87,8 @@ int main(int argc, char *argv[])
   int par_ref_levels = 1;
   int order = 1;
   int prob = 0;
-   double freq = -1.0;
-   double a_coef = 0.0;
+   real_t freq = -1.0;
+   real_t a_coef = 0.0;
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
@@ -475,8 +475,8 @@ int main(int argc, char *argv[])

   if (exact_sol)
   {
-      double err_r = -1.0;
-      double err_i = -1.0;
+      real_t err_r = -1.0;
+      real_t err_i = -1.0;

      switch (prob)
      {
@@ -576,7 +576,7 @@ int main(int argc, char *argv[])
      int i = 0;
      while (sol_sock)
      {
-         double t = (double)(i % num_frames) / num_frames;
+         real_t t = (real_t)(i % num_frames) / num_frames;
         ostringstream oss;
         oss << "Harmonic Solution (t = " << t << " T)";

@@ -608,21 +608,21 @@ bool check_for_inline_mesh(const char * mesh_file)
   return s0 == "inline-";
 }

-complex<double> u0_exact(const Vector &x)
+complex<real_t> u0_exact(const Vector &x)
 {
   int dim = x.Size();
-   complex<double> i(0.0, 1.0);
-   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
-   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
+   complex<real_t> i(0.0, 1.0);
+   complex<real_t> alpha = (epsilon_ * omega_ - i * sigma_);
+   complex<real_t> kappa = std::sqrt(mu_ * omega_* alpha);
   return std::exp(-i * kappa * x[dim - 1]);
 }

-double u0_real_exact(const Vector &x)
+real_t u0_real_exact(const Vector &x)
 {
   return u0_exact(x).real();
 }

-double u0_imag_exact(const Vector &x)
+real_t u0_imag_exact(const Vector &x)
 {
   return u0_exact(x).imag();
 }
@@ -46,7 +46,7 @@ protected:

   SparseMatrix Mmat, Kmat, Kmat0;
   SparseMatrix *T; // T = M + dt K
-   double current_dt;
+   real_t current_dt;

   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
   DSmoother M_prec;  // Preconditioner for the mass matrix M
@@ -58,7 +58,7 @@ protected:
   mutable Vector z; // auxiliary vector

 public:
-   WaveOperator(FiniteElementSpace &f, Array<int> &ess_bdr,double speed);
+   WaveOperator(FiniteElementSpace &f, Array<int> &ess_bdr, real_t speed);

   using SecondOrderTimeDependentOperator::Mult;
   virtual void Mult(const Vector &u, const Vector &du_dt,
@@ -68,7 +68,7 @@ public:
       d2udt2 = f(u + fac0*d2udt2,dudt + fac1*d2udt2, t),
       for the unknown d2udt2. */
   using SecondOrderTimeDependentOperator::ImplicitSolve;
-   virtual void ImplicitSolve(const double fac0, const double fac1,
+   virtual void ImplicitSolve(const real_t fac0, const real_t fac1,
                              const Vector &u, const Vector &dudt, Vector &d2udt2);

   ///
@@ -79,12 +79,11 @@ public:


 WaveOperator::WaveOperator(FiniteElementSpace &f,
-                           Array<int> &ess_bdr, double speed)
-   : SecondOrderTimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL),
-     K(NULL),
-     T(NULL), current_dt(0.0), z(height)
+                           Array<int> &ess_bdr, real_t speed)
+   : SecondOrderTimeDependentOperator(f.GetTrueVSize(), (real_t) 0.0),
+     fespace(f), M(NULL), K(NULL), T(NULL), current_dt(0.0), z(height)
 {
-   const double rel_tol = 1e-8;
+   const real_t rel_tol = 1e-8;

   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

@@ -132,7 +131,7 @@ void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
   M_solver.Mult(z, d2udt2);
 }

-void WaveOperator::ImplicitSolve(const double fac0, const double fac1,
+void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
                                 const Vector &u, const Vector &dudt, Vector &d2udt2)
 {
   // Solve the equation:
@@ -167,12 +166,12 @@ WaveOperator::~WaveOperator()
   delete c2;
 }

-double InitialSolution(const Vector &x)
+real_t InitialSolution(const Vector &x)
 {
   return exp(-x.Norml2()*x.Norml2()*30);
 }

-double InitialRate(const Vector &x)
+real_t InitialRate(const Vector &x)
 {
   return 0.0;
 }
@@ -186,9 +185,9 @@ int main(int argc, char *argv[])
   int ref_levels = 2;
   int order = 2;
   int ode_solver_type = 10;
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double speed = 1.0;
+   real_t t_final = 0.5;
+   real_t dt = 1.0e-2;
+   real_t speed = 1.0;
   bool visualization = true;
   bool visit = true;
   bool dirichlet = true;
@@ -366,7 +365,7 @@ int main(int argc, char *argv[])
   // 8. Perform time-integration (looping over the time iterations, ti, with a
   //    time-step dt).
   ode_solver->Init(oper);
-   double t = 0.0;
+   real_t t = 0.0;

   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -44,14 +44,14 @@
 using namespace std;
 using namespace mfem;

-double p_exact(const Vector &x);
+real_t p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
-double div_gradp_exact(const Vector &x);
+real_t div_gradp_exact(const Vector &x);
 void v_exact(const Vector &x, Vector &v);
 void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -304,9 +304,9 @@ int main(int argc, char *argv[])
   // 12. Compute and print the L_2 norm of the error.
   if (prob == 0)
   {
-      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
-      double errProj = exact_proj.ComputeL2Error(gradp_coef);
+      real_t errSol = x.ComputeL2Error(gradp_coef);
+      real_t errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
+      real_t errProj = exact_proj.ComputeL2Error(gradp_coef);

      cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
           "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
@@ -317,9 +317,9 @@ int main(int argc, char *argv[])
   }
   else if (prob == 1)
   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
+      real_t errSol = x.ComputeL2Error(curlv_coef);
+      real_t errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
+      real_t errProj = exact_proj.ComputeL2Error(curlv_coef);

      cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in H(div): "
           "|| E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
@@ -337,9 +337,9 @@ int main(int argc, char *argv[])
         irs[i] = &(IntRules.Get(i, order_quad));
      }

-      double errSol = x.ComputeL2Error(divgradp_coef, irs);
-      double errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
-      double errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);
+      real_t errSol = x.ComputeL2Error(divgradp_coef, irs);
+      real_t errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
+      real_t errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);

      cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
           "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
@@ -376,7 +376,7 @@ int main(int argc, char *argv[])
   return 0;
 }

-double p_exact(const Vector &x)
+real_t p_exact(const Vector &x)
 {
   if (dim == 3)
   {
@@ -406,7 +406,7 @@ void gradp_exact(const Vector &x, Vector &f)
   }
 }

-double div_gradp_exact(const Vector &x)
+real_t div_gradp_exact(const Vector &x)
 {
   if (dim == 3)
   {
@@ -44,14 +44,14 @@
 using namespace std;
 using namespace mfem;

-double p_exact(const Vector &x);
+real_t p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
-double div_gradp_exact(const Vector &x);
+real_t div_gradp_exact(const Vector &x);
 void v_exact(const Vector &x, Vector &v);
 void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -352,9 +352,9 @@ int main(int argc, char *argv[])
   // 14. Compute and print the L_2 norm of the error.
   if (prob == 0)
   {
-      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
-      double errProj = exact_proj.ComputeL2Error(gradp_coef);
+      real_t errSol = x.ComputeL2Error(gradp_coef);
+      real_t errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
+      real_t errProj = exact_proj.ComputeL2Error(gradp_coef);

      if (myid == 0)
      {
@@ -368,9 +368,9 @@ int main(int argc, char *argv[])
   }
   else if (prob == 1)
   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
+      real_t errSol = x.ComputeL2Error(curlv_coef);
+      real_t errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
+      real_t errProj = exact_proj.ComputeL2Error(curlv_coef);

      if (myid == 0)
      {
@@ -391,9 +391,9 @@ int main(int argc, char *argv[])
         irs[i] = &(IntRules.Get(i, order_quad));
      }

-      double errSol = x.ComputeL2Error(divgradp_coef, irs);
-      double errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
-      double errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);
+      real_t errSol = x.ComputeL2Error(divgradp_coef, irs);
+      real_t errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
+      real_t errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);

      if (myid == 0)
      {
@@ -441,7 +441,7 @@ int main(int argc, char *argv[])
   return 0;
 }

-double p_exact(const Vector &x)
+real_t p_exact(const Vector &x)
 {
   if (dim == 3)
   {
@@ -471,7 +471,7 @@ void gradp_exact(const Vector &x, Vector &f)
   }
 }

-double div_gradp_exact(const Vector &x)
+real_t div_gradp_exact(const Vector &x)
 {
   if (dim == 3)
   {
@@ -53,13 +53,13 @@ private:
   int dim;

   // Length of the PML Region in each direction
-   Array2D<double> length;
+   Array2D<real_t> length;

   // Computational Domain Boundary
-   Array2D<double> comp_dom_bdr;
+   Array2D<real_t> comp_dom_bdr;

   // Domain Boundary
-   Array2D<double> dom_bdr;
+   Array2D<real_t> dom_bdr;

   // Integer Array identifying elements in the PML
   // 0: in the PML, 1: not in the PML
@@ -70,13 +70,13 @@ private:

 public:
   // Constructor
-   PML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<real_t> length_);

   // Return Computational Domain Boundary
-   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
+   Array2D<real_t> GetCompDomainBdr() {return comp_dom_bdr;}

   // Return Domain Boundary
-   Array2D<double> GetDomainBdr() {return dom_bdr;}
+   Array2D<real_t> GetDomainBdr() {return dom_bdr;}

   // Return Markers list for elements
   Array<int> * GetMarkedPMLElements() {return &elems;}
@@ -85,7 +85,7 @@ public:
   void SetAttributes(Mesh *mesh_);

   // PML complex stretching function
-   void StretchFunction(const Vector &x, vector<complex<double>> &dxs);
+   void StretchFunction(const Vector &x, vector<complex<real_t>> &dxs);
 };

 // Class for returning the PML coefficients of the bilinear form
@@ -106,7 +106,7 @@ public:
   virtual void Eval(Vector &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
-      double x[3];
+      real_t x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
      K.SetSize(vdim);
@@ -114,7 +114,7 @@ public:
   }
 };

-void maxwell_solution(const Vector &x, vector<complex<double>> &Eval);
+void maxwell_solution(const Vector &x, vector<complex<real_t>> &Eval);

 void E_bdr_data_Re(const Vector &x, Vector &E);
 void E_bdr_data_Im(const Vector &x, Vector &E);
@@ -134,15 +134,17 @@ void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D);
 void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D);
 void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D);

-Array2D<double> comp_domain_bdr;
-Array2D<double> domain_bdr;
+Array2D<real_t> comp_domain_bdr;
+Array2D<real_t> domain_bdr;

-double mu = 1.0;
-double epsilon = 1.0;
-double omega;
+real_t mu = 1.0;
+real_t epsilon = 1.0;
+real_t omega;
 int dim;
 bool exact_known = false;

+template <typename T> T pow2(const T &x) { return x*x; }
+
 enum prob_type
 {
   beam,     // Wave propagating in a beam-like domain
@@ -160,7 +162,7 @@ int main(int argc, char *argv[])
   int order = 1;
   int ref_levels = 3;
   int iprob = 4;
-   double freq = 5.0;
+   real_t freq = 5.0;
   bool herm_conv = true;
   bool umf_solver = false;
   bool visualization = 1;
@@ -241,10 +243,10 @@ int main(int argc, char *argv[])
   dim = mesh->Dimension();

   // Angular frequency
-   omega = 2.0 * M_PI * freq;
+   omega = real_t(2.0 * M_PI) * freq;

   // Setup PML length
-   Array2D<double> length(dim, 2); length = 0.0;
+   Array2D<real_t> length(dim, 2); length = 0.0;

   // 4. Setup the Cartesian PML region.
   switch (prob)
@@ -312,14 +314,15 @@ int main(int argc, char *argv[])
            switch (prob)
            {
               case lshape:
-                  if (center[0] == 1.0 || center[0] == 0.5 || center[1] == 0.5)
+                  if (center[0] == 1_r || center[0] == 0.5_r ||
+                      center[1] == 0.5_r)
                  {
                     ess_bdr[k - 1] = 1;
                  }
                  break;
               case fichera:
-                  if (center[0] == -1.0 || center[0] == 0.0 ||
-                      center[1] ==  0.0 || center[2] == 0.0)
+                  if (center[0] == -1_r || center[0] == 0_r ||
+                      center[1] ==  0_r || center[2] == 0_r)
                  {
                     ess_bdr[k - 1] = 1;
                  }
@@ -378,8 +381,8 @@ int main(int argc, char *argv[])
      }
   }

-   ConstantCoefficient muinv(1.0/mu);
-   ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
+   ConstantCoefficient muinv(1_r / mu);
+   ConstantCoefficient omeg(-pow2(omega) * epsilon);
   RestrictedCoefficient restr_muinv(muinv,attr);
   RestrictedCoefficient restr_omeg(omeg,attr);

@@ -439,7 +442,7 @@ int main(int argc, char *argv[])
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
   if (pa || !umf_solver)
   {
-      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
+      ConstantCoefficient absomeg(pow2(omega) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);

      BilinearForm prec(fespace);
@@ -470,7 +473,7 @@ int main(int argc, char *argv[])

      std::unique_ptr<Operator> pc_r;
      std::unique_ptr<Operator> pc_i;
-      double s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      real_t s = (conv == ComplexOperator::HERMITIAN) ? -1_r : 1_r;
      if (pa)
      {
         // Jacobi Smoother
@@ -519,14 +522,14 @@ int main(int argc, char *argv[])
         irs[i] = &(IntRules.Get(i, order_quad));
      }

-      double L2Error_Re = x.real().ComputeL2Error(E_ex_Re, irs,
+      real_t L2Error_Re = x.real().ComputeL2Error(E_ex_Re, irs,
                                                  pml->GetMarkedPMLElements());
-      double L2Error_Im = x.imag().ComputeL2Error(E_ex_Im, irs,
+      real_t L2Error_Im = x.imag().ComputeL2Error(E_ex_Im, irs,
                                                  pml->GetMarkedPMLElements());

      ComplexGridFunction x_gf0(fespace);
      x_gf0 = 0.0;
-      double norm_E_Re, norm_E_Im;
+      real_t norm_E_Re, norm_E_Im;
      norm_E_Re = x_gf0.real().ComputeL2Error(E_ex_Re, irs,
                                              pml->GetMarkedPMLElements());
      norm_E_Im = x_gf0.imag().ComputeL2Error(E_ex_Im, irs,
@@ -593,12 +596,12 @@ int main(int argc, char *argv[])
      int i = 0;
      while (sol_sock)
      {
-         double t = (double)(i % num_frames) / num_frames;
+         real_t t = (real_t)(i % num_frames) / num_frames;
         ostringstream oss;
         oss << "Harmonic Solution (t = " << t << " T)";

-         add(cos(2.0 * M_PI * t), x.real(),
-             sin(2.0 * M_PI * t), x.imag(), x_t);
+         add(cos(real_t(2.0 * M_PI) * t), x.real(),
+             sin(real_t(2.0 * M_PI) * t), x.imag(), x_t);
         sol_sock << "solution\n"
                  << *mesh << x_t
                  << "window_title '" << oss.str() << "'" << flush;
@@ -617,20 +620,20 @@ int main(int argc, char *argv[])
 void source(const Vector &x, Vector &f)
 {
   Vector center(dim);
-   double r = 0.0;
+   real_t r = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      center(i) = 0.5 * (comp_domain_bdr(i, 0) + comp_domain_bdr(i, 1));
-      r += pow(x[i] - center[i], 2.);
+      center(i) = 0.5_r * (comp_domain_bdr(i, 0) + comp_domain_bdr(i, 1));
+      r += pow2(x[i] - center[i]);
   }
-   double n = 5.0 * omega * sqrt(epsilon * mu) / M_PI;
-   double coeff = pow(n, 2) / M_PI;
-   double alpha = -pow(n, 2) * r;
+   real_t n = 5_r * omega * sqrt(epsilon * mu) / real_t(M_PI);
+   real_t coeff = pow2(n) / real_t(M_PI);
+   real_t alpha = -pow2(n) * r;
   f = 0.0;
   f[0] = coeff * exp(alpha);
 }

-void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+void maxwell_solution(const Vector &x, vector<complex<real_t>> &E)
 {
   // Initialize
   for (int i = 0; i < dim; ++i)
@@ -638,8 +641,8 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)
      E[i] = 0.0;
   }

-   complex<double> zi = complex<double>(0., 1.);
-   double k = omega * sqrt(epsilon * mu);
+   constexpr complex<real_t> zi = complex<real_t>(0., 1.);
+   real_t k = omega * sqrt(epsilon * mu);
   switch (prob)
   {
      case disc:
@@ -654,58 +657,58 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)

         if (dim == 2)
         {
-            double x0 = x(0) + shift(0);
-            double x1 = x(1) + shift(1);
-            double r = sqrt(x0 * x0 + x1 * x1);
-            double beta = k * r;
+            real_t x0 = x(0) + shift(0);
+            real_t x1 = x(1) + shift(1);
+            real_t r = sqrt(x0 * x0 + x1 * x1);
+            real_t beta = k * r;

            // Bessel functions
-            complex<double> Ho, Ho_r, Ho_rr;
-            Ho = jn(0, beta) + zi * yn(0, beta);
-            Ho_r = -k * (jn(1, beta) + zi * yn(1, beta));
-            Ho_rr = -k * k * (1.0 / beta *
-                              (jn(1, beta) + zi * yn(1, beta)) -
-                              (jn(2, beta) + zi * yn(2, beta)));
+            complex<real_t> Ho, Ho_r, Ho_rr;
+            Ho = real_t(jn(0, beta)) + zi * real_t(yn(0, beta));
+            Ho_r = -k * (real_t(jn(1, beta)) + zi * real_t(yn(1, beta)));
+            Ho_rr = -k * k * (1_r / beta *
+                              (real_t(jn(1, beta)) + zi * real_t(yn(1, beta))) -
+                              (real_t(jn(2, beta)) + zi * real_t(yn(2, beta))));

            // First derivatives
-            double r_x = x0 / r;
-            double r_y = x1 / r;
-            double r_xy = -(r_x / r) * r_y;
-            double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
+            real_t r_x = x0 / r;
+            real_t r_y = x1 / r;
+            real_t r_xy = -(r_x / r) * r_y;
+            real_t r_xx = (1_r / r) * (1_r - r_x * r_x);

-            complex<double> val, val_xx, val_xy;
-            val = 0.25 * zi * Ho;
-            val_xx = 0.25 * zi * (r_xx * Ho_r + r_x * r_x * Ho_rr);
-            val_xy = 0.25 * zi * (r_xy * Ho_r + r_x * r_y * Ho_rr);
+            complex<real_t> val, val_xx, val_xy;
+            val = real_t(0.25) * zi * Ho;
+            val_xx = real_t(0.25) * zi * (r_xx * Ho_r + r_x * r_x * Ho_rr);
+            val_xy = real_t(0.25) * zi * (r_xy * Ho_r + r_x * r_y * Ho_rr);
            E[0] = zi / k * (k * k * val + val_xx);
            E[1] = zi / k * val_xy;
         }
         else if (dim == 3)
         {
-            double x0 = x(0) + shift(0);
-            double x1 = x(1) + shift(1);
-            double x2 = x(2) + shift(2);
-            double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
+            real_t x0 = x(0) + shift(0);
+            real_t x1 = x(1) + shift(1);
+            real_t x2 = x(2) + shift(2);
+            real_t r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);

-            double r_x = x0 / r;
-            double r_y = x1 / r;
-            double r_z = x2 / r;
-            double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
-            double r_yx = -(r_y / r) * r_x;
-            double r_zx = -(r_z / r) * r_x;
+            real_t r_x = x0 / r;
+            real_t r_y = x1 / r;
+            real_t r_z = x2 / r;
+            real_t r_xx = (1_r / r) * (1_r - r_x * r_x);
+            real_t r_yx = -(r_y / r) * r_x;
+            real_t r_zx = -(r_z / r) * r_x;

-            complex<double> val, val_r, val_rr;
+            complex<real_t> val, val_r, val_rr;
            val = exp(zi * k * r) / r;
-            val_r = val / r * (zi * k * r - 1.0);
+            val_r = val / r * (zi * k * r - 1_r);
            val_rr = val / (r * r) * (-k * k * r * r
-                                      - 2.0 * zi * k * r + 2.0);
+                                      - real_t(2) * zi * k * r + real_t(2));

-            complex<double> val_xx, val_yx, val_zx;
+            complex<real_t> val_xx, val_yx, val_zx;
            val_xx = val_rr * r_x * r_x + val_r * r_xx;
            val_yx = val_rr * r_x * r_y + val_r * r_yx;
            val_zx = val_rr * r_x * r_z + val_r * r_zx;

-            complex<double> alpha = zi * k / 4.0 / M_PI / k / k;
+            complex<real_t> alpha = zi * k / real_t(4) / (real_t) M_PI / k / k;
            E[0] = alpha * (k * k * val + val_xx);
            E[1] = alpha * val_yx;
            E[2] = alpha * val_zx;
@@ -717,12 +720,13 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)
         // T_10 mode
         if (dim == 3)
         {
-            double k10 = sqrt(k * k - M_PI * M_PI);
-            E[1] = -zi * k / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+            real_t k10 = sqrt(k * k - real_t(M_PI * M_PI));
+            E[1] = -zi * k / (real_t) M_PI *
+                   sin((real_t) M_PI*x(2))*exp(zi * k10 * x(0));
         }
         else if (dim == 2)
         {
-            E[1] = -zi * k / M_PI * exp(zi * k * x(0));
+            E[1] = -zi * k / (real_t) M_PI * exp(zi * k * x(0));
         }
         break;
      }
@@ -733,7 +737,7 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)

 void E_exact_Re(const Vector &x, Vector &E)
 {
-   vector<complex<double>> Eval(E.Size());
+   vector<complex<real_t>> Eval(E.Size());
   maxwell_solution(x, Eval);
   for (int i = 0; i < dim; ++i)
   {
@@ -743,7 +747,7 @@ void E_exact_Re(const Vector &x, Vector &E)

 void E_exact_Im(const Vector &x, Vector &E)
 {
-   vector<complex<double>> Eval(E.Size());
+   vector<complex<real_t>> Eval(E.Size());
   maxwell_solution(x, Eval);
   for (int i = 0; i < dim; ++i)
   {
@@ -759,8 +763,8 @@ void E_bdr_data_Re(const Vector &x, Vector &E)
   for (int i = 0; i < dim; ++i)
   {
      // check if in PML
-      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
-          x(i) - comp_domain_bdr(i, 1) > 0.0)
+      if (x(i) - comp_domain_bdr(i, 0) < 0_r ||
+          x(i) - comp_domain_bdr(i, 1) > 0_r)
      {
         in_pml = true;
         break;
@@ -768,7 +772,7 @@ void E_bdr_data_Re(const Vector &x, Vector &E)
   }
   if (!in_pml)
   {
-      vector<complex<double>> Eval(E.Size());
+      vector<complex<real_t>> Eval(E.Size());
      maxwell_solution(x, Eval);
      for (int i = 0; i < dim; ++i)
      {
@@ -786,8 +790,8 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   for (int i = 0; i < dim; ++i)
   {
      // check if in PML
-      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
-          x(i) - comp_domain_bdr(i, 1) > 0.0)
+      if (x(i) - comp_domain_bdr(i, 0) < 0_r ||
+          x(i) - comp_domain_bdr(i, 1) > 0_r)
      {
         in_pml = true;
         break;
@@ -795,7 +799,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
   if (!in_pml)
   {
-      vector<complex<double>> Eval(E.Size());
+      vector<complex<real_t>> Eval(E.Size());
      maxwell_solution(x, Eval);
      for (int i = 0; i < dim; ++i)
      {
@@ -806,8 +810,8 @@ void E_bdr_data_Im(const Vector &x, Vector &E)

 void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det(1.0, 0.0);
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -817,14 +821,14 @@ void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D)

   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow2(dxs[i])).real();
   }
 }

 void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -834,14 +838,14 @@ void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D)

   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow2(dxs[i])).imag();
   }
 }

 void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -851,14 +855,14 @@ void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D)

   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow2(dxs[i]));
   }
 }

 void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det(1.0, 0.0);
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -869,21 +873,21 @@ void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      D = (1_r / det).real();
   }
   else
   {
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow2(dxs[i]) / det).real();
      }
   }
 }

 void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -893,21 +897,21 @@ void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      D = (1_r / det).imag();
   }
   else
   {
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow2(dxs[i]) / det).imag();
      }
   }
 }

 void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -917,18 +921,18 @@ void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      D = abs(1_r / det);
   }
   else
   {
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow2(dxs[i]) / det);
      }
   }
 }

-PML::PML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<real_t> length_)
   : mesh(mesh_), length(length_)
 {
   dim = mesh->Dimension();
@@ -979,7 +983,7 @@ void PML::SetAttributes(Mesh *mesh_)
      for (int iv = 0; iv < nrvert; ++iv)
      {
         int vert_idx = vertices[iv];
-         double *coords = mesh_->GetVertex(vert_idx);
+         real_t *coords = mesh_->GetVertex(vert_idx);
         for (int comp = 0; comp < dim; ++comp)
         {
            if (coords[comp] > comp_dom_bdr(comp, 1) ||
@@ -1000,14 +1004,14 @@ void PML::SetAttributes(Mesh *mesh_)
 }

 void PML::StretchFunction(const Vector &x,
-                          vector<complex<double>> &dxs)
+                          vector<complex<real_t>> &dxs)
 {
-   complex<double> zi = complex<double>(0., 1.);
+   constexpr complex<real_t> zi = complex<real_t>(0., 1.);

-   double n = 2.0;
-   double c = 5.0;
-   double coeff;
-   double k = omega * sqrt(epsilon * mu);
+   real_t n = 2.0;
+   real_t c = 5.0;
+   real_t coeff;
+   real_t k = omega * sqrt(epsilon * mu);

   // Stretch in each direction independently
   for (int i = 0; i < dim; ++i)
@@ -1016,14 +1020,14 @@ void PML::StretchFunction(const Vector &x,
      if (x(i) >= comp_domain_bdr(i, 1))
      {
         coeff = n * c / k / pow(length(i, 1), n);
-         dxs[i] = 1.0 + zi * coeff *
-                  abs(pow(x(i) - comp_domain_bdr(i, 1), n - 1.0));
+         dxs[i] = 1_r + zi * coeff *
+                  abs(pow(x(i) - comp_domain_bdr(i, 1), n - 1_r));
      }
      if (x(i) <= comp_domain_bdr(i, 0))
      {
         coeff = n * c / k / pow(length(i, 0), n);
-         dxs[i] = 1.0 + zi * coeff *
-                  abs(pow(x(i) - comp_domain_bdr(i, 0), n - 1.0));
+         dxs[i] = 1_r + zi * coeff *
+                  abs(pow(x(i) - comp_domain_bdr(i, 0), n - 1_r));
      }
   }
 }
@@ -52,13 +52,13 @@ private:
   int dim;

   // Length of the PML Region in each direction
-   Array2D<double> length;
+   Array2D<real_t> length;

   // Computational Domain Boundary
-   Array2D<double> comp_dom_bdr;
+   Array2D<real_t> comp_dom_bdr;

   // Domain Boundary
-   Array2D<double> dom_bdr;
+   Array2D<real_t> dom_bdr;

   // Integer Array identifying elements in the PML
   // 0: in the PML, 1: not in the PML
@@ -69,13 +69,13 @@ private:

 public:
   // Constructor
-   PML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<real_t> length_);

   // Return Computational Domain Boundary
-   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
+   Array2D<real_t> GetCompDomainBdr() {return comp_dom_bdr;}

   // Return Domain Boundary
-   Array2D<double> GetDomainBdr() {return dom_bdr;}
+   Array2D<real_t> GetDomainBdr() {return dom_bdr;}

   // Return Markers list for elements
   Array<int> * GetMarkedPMLElements() {return &elems;}
@@ -84,7 +84,7 @@ public:
   void SetAttributes(ParMesh *pmesh);

   // PML complex stretching function
-   void StretchFunction(const Vector &x, vector<complex<double>> &dxs);
+   void StretchFunction(const Vector &x, vector<complex<real_t>> &dxs);
 };

 // Class for returning the PML coefficients of the bilinear form
@@ -105,7 +105,7 @@ public:
   virtual void Eval(Vector &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
-      double x[3];
+      real_t x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
      K.SetSize(vdim);
@@ -113,7 +113,7 @@ public:
   }
 };

-void maxwell_solution(const Vector &x, vector<complex<double>> &Eval);
+void maxwell_solution(const Vector &x, vector<complex<real_t>> &Eval);

 void E_bdr_data_Re(const Vector &x, Vector &E);
 void E_bdr_data_Im(const Vector &x, Vector &E);
@@ -133,15 +133,17 @@ void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D);
 void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D);
 void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D);

-Array2D<double> comp_domain_bdr;
-Array2D<double> domain_bdr;
+Array2D<real_t> comp_domain_bdr;
+Array2D<real_t> domain_bdr;

-double mu = 1.0;
-double epsilon = 1.0;
-double omega;
+real_t mu = 1.0;
+real_t epsilon = 1.0;
+real_t omega;
 int dim;
 bool exact_known = false;

+template <typename T> T pow2(const T &x) { return x*x; }
+
 enum prob_type
 {
   beam,     // Wave propagating in a beam-like domain
@@ -166,7 +168,7 @@ int main(int argc, char *argv[])
   int ref_levels = 1;
   int par_ref_levels = 2;
   int iprob = 4;
-   double freq = 5.0;
+   real_t freq = 5.0;
   bool herm_conv = true;
   bool slu_solver  = false;
   bool mumps_solver = false;
@@ -275,10 +277,10 @@ int main(int argc, char *argv[])
   dim = mesh->Dimension();

   // Angular frequency
-   omega = 2.0 * M_PI * freq;
+   omega = real_t(2.0 * M_PI) * freq;

   // Setup PML length
-   Array2D<double> length(dim, 2); length = 0.0;
+   Array2D<real_t> length(dim, 2); length = 0.0;

   // 5. Setup the Cartesian PML region.
   switch (prob)
@@ -357,14 +359,15 @@ int main(int argc, char *argv[])
            switch (prob)
            {
               case lshape:
-                  if (center[0] == 1.0 || center[0] == 0.5 || center[1] == 0.5)
+                  if (center[0] == 1_r || center[0] == 0.5_r ||
+                      center[1] == 0.5_r)
                  {
                     ess_bdr[k - 1] = 1;
                  }
                  break;
               case fichera:
-                  if (center[0] == -1.0 || center[0] == 0.0 ||
-                      center[1] ==  0.0 || center[2] == 0.0)
+                  if (center[0] == -1_r || center[0] == 0_r ||
+                      center[1] ==  0_r || center[2] == 0_r)
                  {
                     ess_bdr[k - 1] = 1;
                  }
@@ -423,8 +426,8 @@ int main(int argc, char *argv[])
      }
   }

-   ConstantCoefficient muinv(1.0/mu);
-   ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
+   ConstantCoefficient muinv(1_r / mu);
+   ConstantCoefficient omeg(-pow2(omega) * epsilon);
   RestrictedCoefficient restr_muinv(muinv,attr);
   RestrictedCoefficient restr_omeg(omeg,attr);

@@ -520,7 +523,7 @@ int main(int argc, char *argv[])
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
   if (pa || (!slu_solver && !mumps_solver && !strumpack_solver))
   {
-      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
+      ConstantCoefficient absomeg(pow2(omega) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);

      ParBilinearForm prec(fespace);
@@ -551,7 +554,7 @@ int main(int argc, char *argv[])

      std::unique_ptr<Operator> pc_r;
      std::unique_ptr<Operator> pc_i;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      int s = (conv == ComplexOperator::HERMITIAN) ? -1 : 1;
      if (pa)
      {
         // Jacobi Smoother
@@ -599,14 +602,14 @@ int main(int argc, char *argv[])
         irs[i] = &(IntRules.Get(i, order_quad));
      }

-      double L2Error_Re = x.real().ComputeL2Error(E_ex_Re, irs,
+      real_t L2Error_Re = x.real().ComputeL2Error(E_ex_Re, irs,
                                                  pml->GetMarkedPMLElements());
-      double L2Error_Im = x.imag().ComputeL2Error(E_ex_Im, irs,
+      real_t L2Error_Im = x.imag().ComputeL2Error(E_ex_Im, irs,
                                                  pml->GetMarkedPMLElements());

      ParComplexGridFunction x_gf0(fespace);
      x_gf0 = 0.0;
-      double norm_E_Re, norm_E_Im;
+      real_t norm_E_Re, norm_E_Im;
      norm_E_Re = x_gf0.real().ComputeL2Error(E_ex_Re, irs,
                                              pml->GetMarkedPMLElements());
      norm_E_Im = x_gf0.imag().ComputeL2Error(E_ex_Im, irs,
@@ -694,11 +697,12 @@ int main(int argc, char *argv[])
         int i = 0;
         while (sol_sock)
         {
-            double t = (double)(i % num_frames) / num_frames;
+            real_t t = (real_t)(i % num_frames) / num_frames;
            ostringstream oss;
            oss << "Harmonic Solution (t = " << t << " T)";

-            add(cos(2.0*M_PI*t), x.real(), sin(2.0*M_PI*t), x.imag(), x_t);
+            add(cos(real_t(2.0*M_PI)*t), x.real(),
+                sin(real_t(2.0*M_PI)*t), x.imag(), x_t);
            sol_sock << "parallel " << num_procs << " " << myid << "\n";
            sol_sock << "solution\n" << *pmesh << x_t
                     << "window_title '" << oss.str() << "'" << flush;
@@ -718,20 +722,20 @@ int main(int argc, char *argv[])
 void source(const Vector &x, Vector &f)
 {
   Vector center(dim);
-   double r = 0.0;
+   real_t r = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      center(i) = 0.5 * (comp_domain_bdr(i, 0) + comp_domain_bdr(i, 1));
-      r += pow(x[i] - center[i], 2.);
+      center(i) = real_t(0.5) * (comp_domain_bdr(i, 0) + comp_domain_bdr(i, 1));
+      r += pow2(x[i] - center[i]);
   }
-   double n = 5.0 * omega * sqrt(epsilon * mu) / M_PI;
-   double coeff = pow(n, 2) / M_PI;
-   double alpha = -pow(n, 2) * r;
+   real_t n = real_t(5) * omega * sqrt(epsilon * mu) / real_t(M_PI);
+   real_t coeff = pow2(n) / real_t(M_PI);
+   real_t alpha = -pow2(n) * r;
   f = 0.0;
   f[0] = coeff * exp(alpha);
 }

-void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+void maxwell_solution(const Vector &x, vector<complex<real_t>> &E)
 {
   // Initialize
   for (int i = 0; i < dim; ++i)
@@ -739,8 +743,8 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)
      E[i] = 0.0;
   }

-   complex<double> zi = complex<double>(0., 1.);
-   double k = omega * sqrt(epsilon * mu);
+   constexpr complex<real_t> zi = complex<real_t>(0., 1.);
+   real_t k = omega * sqrt(epsilon * mu);
   switch (prob)
   {
      case disc:
@@ -755,58 +759,58 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)

         if (dim == 2)
         {
-            double x0 = x(0) + shift(0);
-            double x1 = x(1) + shift(1);
-            double r = sqrt(x0 * x0 + x1 * x1);
-            double beta = k * r;
+            real_t x0 = x(0) + shift(0);
+            real_t x1 = x(1) + shift(1);
+            real_t r = sqrt(x0 * x0 + x1 * x1);
+            real_t beta = k * r;

            // Bessel functions
-            complex<double> Ho, Ho_r, Ho_rr;
-            Ho = jn(0, beta) + zi * yn(0, beta);
-            Ho_r = -k * (jn(1, beta) + zi * yn(1, beta));
-            Ho_rr = -k * k * (1.0 / beta *
-                              (jn(1, beta) + zi * yn(1, beta)) -
-                              (jn(2, beta) + zi * yn(2, beta)));
+            complex<real_t> Ho, Ho_r, Ho_rr;
+            Ho = real_t(jn(0, beta)) + zi * real_t(yn(0, beta));
+            Ho_r = -k * (real_t(jn(1, beta)) + zi * real_t(yn(1, beta)));
+            Ho_rr = -k * k * (1_r / beta *
+                              (real_t(jn(1, beta)) + zi * real_t(yn(1, beta))) -
+                              (real_t(jn(2, beta)) + zi * real_t(yn(2, beta))));

            // First derivatives
-            double r_x = x0 / r;
-            double r_y = x1 / r;
-            double r_xy = -(r_x / r) * r_y;
-            double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
+            real_t r_x = x0 / r;
+            real_t r_y = x1 / r;
+            real_t r_xy = -(r_x / r) * r_y;
+            real_t r_xx = (1_r / r) * (1_r - r_x * r_x);

-            complex<double> val, val_xx, val_xy;
-            val = 0.25 * zi * Ho;
-            val_xx = 0.25 * zi * (r_xx * Ho_r + r_x * r_x * Ho_rr);
-            val_xy = 0.25 * zi * (r_xy * Ho_r + r_x * r_y * Ho_rr);
+            complex<real_t> val, val_xx, val_xy;
+            val = real_t(0.25) * zi * Ho;
+            val_xx = real_t(0.25) * zi * (r_xx * Ho_r + r_x * r_x * Ho_rr);
+            val_xy = real_t(0.25) * zi * (r_xy * Ho_r + r_x * r_y * Ho_rr);
            E[0] = zi / k * (k * k * val + val_xx);
            E[1] = zi / k * val_xy;
         }
         else if (dim == 3)
         {
-            double x0 = x(0) + shift(0);
-            double x1 = x(1) + shift(1);
-            double x2 = x(2) + shift(2);
-            double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
+            real_t x0 = x(0) + shift(0);
+            real_t x1 = x(1) + shift(1);
+            real_t x2 = x(2) + shift(2);
+            real_t r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);

-            double r_x = x0 / r;
-            double r_y = x1 / r;
-            double r_z = x2 / r;
-            double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
-            double r_yx = -(r_y / r) * r_x;
-            double r_zx = -(r_z / r) * r_x;
+            real_t r_x = x0 / r;
+            real_t r_y = x1 / r;
+            real_t r_z = x2 / r;
+            real_t r_xx = (1_r / r) * (1_r - r_x * r_x);
+            real_t r_yx = -(r_y / r) * r_x;
+            real_t r_zx = -(r_z / r) * r_x;

-            complex<double> val, val_r, val_rr;
+            complex<real_t> val, val_r, val_rr;
            val = exp(zi * k * r) / r;
-            val_r = val / r * (zi * k * r - 1.0);
+            val_r = val / r * (zi * k * r - 1_r);
            val_rr = val / (r * r) * (-k * k * r * r
-                                      - 2.0 * zi * k * r + 2.0);
+                                      - real_t(2) * zi * k * r + real_t(2));

-            complex<double> val_xx, val_yx, val_zx;
+            complex<real_t> val_xx, val_yx, val_zx;
            val_xx = val_rr * r_x * r_x + val_r * r_xx;
            val_yx = val_rr * r_x * r_y + val_r * r_yx;
            val_zx = val_rr * r_x * r_z + val_r * r_zx;

-            complex<double> alpha = zi * k / 4.0 / M_PI / k / k;
+            complex<real_t> alpha = zi * k / real_t(4) / (real_t) M_PI / k / k;
            E[0] = alpha * (k * k * val + val_xx);
            E[1] = alpha * val_yx;
            E[2] = alpha * val_zx;
@@ -818,12 +822,13 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)
         // T_10 mode
         if (dim == 3)
         {
-            double k10 = sqrt(k * k - M_PI * M_PI);
-            E[1] = -zi * k / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+            real_t k10 = sqrt(k * k - real_t(M_PI * M_PI));
+            E[1] = -zi * k / (real_t) M_PI *
+                   sin((real_t) M_PI*x(2))*exp(zi * k10 * x(0));
         }
         else if (dim == 2)
         {
-            E[1] = -zi * k / M_PI * exp(zi * k * x(0));
+            E[1] = -zi * k / (real_t) M_PI * exp(zi * k * x(0));
         }
         break;
      }
@@ -834,7 +839,7 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)

 void E_exact_Re(const Vector &x, Vector &E)
 {
-   vector<complex<double>> Eval(E.Size());
+   vector<complex<real_t>> Eval(E.Size());
   maxwell_solution(x, Eval);
   for (int i = 0; i < dim; ++i)
   {
@@ -844,7 +849,7 @@ void E_exact_Re(const Vector &x, Vector &E)

 void E_exact_Im(const Vector &x, Vector &E)
 {
-   vector<complex<double>> Eval(E.Size());
+   vector<complex<real_t>> Eval(E.Size());
   maxwell_solution(x, Eval);
   for (int i = 0; i < dim; ++i)
   {
@@ -860,8 +865,8 @@ void E_bdr_data_Re(const Vector &x, Vector &E)
   for (int i = 0; i < dim; ++i)
   {
      // check if in PML
-      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
-          x(i) - comp_domain_bdr(i, 1) > 0.0)
+      if (x(i) - comp_domain_bdr(i, 0) < 0_r ||
+          x(i) - comp_domain_bdr(i, 1) > 0_r)
      {
         in_pml = true;
         break;
@@ -869,7 +874,7 @@ void E_bdr_data_Re(const Vector &x, Vector &E)
   }
   if (!in_pml)
   {
-      vector<complex<double>> Eval(E.Size());
+      vector<complex<real_t>> Eval(E.Size());
      maxwell_solution(x, Eval);
      for (int i = 0; i < dim; ++i)
      {
@@ -887,8 +892,8 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   for (int i = 0; i < dim; ++i)
   {
      // check if in PML
-      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
-          x(i) - comp_domain_bdr(i, 1) > 0.0)
+      if (x(i) - comp_domain_bdr(i, 0) < 0_r ||
+          x(i) - comp_domain_bdr(i, 1) > 0_r)
      {
         in_pml = true;
         break;
@@ -896,7 +901,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
   if (!in_pml)
   {
-      vector<complex<double>> Eval(E.Size());
+      vector<complex<real_t>> Eval(E.Size());
      maxwell_solution(x, Eval);
      for (int i = 0; i < dim; ++i)
      {
@@ -907,8 +912,8 @@ void E_bdr_data_Im(const Vector &x, Vector &E)

 void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det(1.0, 0.0);
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -918,14 +923,14 @@ void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D)

   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow2(dxs[i])).real();
   }
 }

 void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -935,14 +940,14 @@ void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D)

   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow2(dxs[i])).imag();
   }
 }

 void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -952,14 +957,14 @@ void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D)

   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow2(dxs[i]));
   }
 }

 void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det(1.0, 0.0);
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -970,21 +975,21 @@ void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      D = (1_r / det).real();
   }
   else
   {
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow2(dxs[i]) / det).real();
      }
   }
 }

 void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -994,21 +999,21 @@ void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      D = (1_r / det).imag();
   }
   else
   {
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow2(dxs[i]) / det).imag();
      }
   }
 }

 void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D)
 {
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
+   vector<complex<real_t>> dxs(dim);
+   complex<real_t> det = 1.0;
   pml->StretchFunction(x, dxs);

   for (int i = 0; i < dim; ++i)
@@ -1018,18 +1023,18 @@ void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      D = abs(1_r / det);
   }
   else
   {
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow2(dxs[i]) / det);
      }
   }
 }

-PML::PML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<real_t> length_)
   : mesh(mesh_), length(length_)
 {
   dim = mesh->Dimension();
@@ -1081,7 +1086,7 @@ void PML::SetAttributes(ParMesh *pmesh)
      for (int iv = 0; iv < nrvert; ++iv)
      {
         int vert_idx = vertices[iv];
-         double *coords = pmesh->GetVertex(vert_idx);
+         real_t *coords = pmesh->GetVertex(vert_idx);
         for (int comp = 0; comp < dim; ++comp)
         {
            if (coords[comp] > comp_dom_bdr(comp, 1) ||
@@ -1102,14 +1107,14 @@ void PML::SetAttributes(ParMesh *pmesh)
 }

 void PML::StretchFunction(const Vector &x,
-                          vector<complex<double>> &dxs)
+                          vector<complex<real_t>> &dxs)
 {
-   complex<double> zi = complex<double>(0., 1.);
+   constexpr complex<real_t> zi = complex<real_t>(0., 1.);

-   double n = 2.0;
-   double c = 5.0;
-   double coeff;
-   double k = omega * sqrt(epsilon * mu);
+   real_t n = 2.0;
+   real_t c = 5.0;
+   real_t coeff;
+   real_t k = omega * sqrt(epsilon * mu);

   // Stretch in each direction independently
   for (int i = 0; i < dim; ++i)
@@ -1118,14 +1123,14 @@ void PML::StretchFunction(const Vector &x,
      if (x(i) >= comp_domain_bdr(i, 1))
      {
         coeff = n * c / k / pow(length(i, 1), n);
-         dxs[i] = 1.0 + zi * coeff *
-                  abs(pow(x(i) - comp_domain_bdr(i, 1), n - 1.0));
+         dxs[i] = 1_r + zi * coeff *
+                  abs(pow(x(i) - comp_domain_bdr(i, 1), n - 1_r));
      }
      if (x(i) <= comp_domain_bdr(i, 0))
      {
         coeff = n * c / k / pow(length(i, 0), n);
-         dxs[i] = 1.0 + zi * coeff *
-                  abs(pow(x(i) - comp_domain_bdr(i, 0), n - 1.0));
+         dxs[i] = 1_r + zi * coeff *
+                  abs(pow(x(i) - comp_domain_bdr(i, 0), n - 1_r));
      }
   }
 }
@@ -63,7 +63,7 @@
 using namespace std;
 using namespace mfem;

-static double a_ = 0.2;
+static real_t a_ = 0.2;

 // Normal to hole with boundary attribute 4
 void n4Vec(const Vector &x, Vector &n) { n = x; n[0] -= 0.5; n /= -n.Norml2(); }
@@ -73,25 +73,25 @@ Mesh * GenerateSerialMesh(int ref);
 // Compute the average value of alpha*n.Grad(sol) + beta*sol over the boundary
 // attributes marked in bdr_marker. Also computes the L2 norm of
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
-double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
-                   double alpha, double beta, double gamma,
-                   double &error);
+real_t IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
+                   real_t alpha, real_t beta, real_t gamma,
+                   real_t &error);

 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
   int ser_ref_levels = 2;
   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
+   real_t sigma = -1.0;
+   real_t kappa = -1.0;
   bool h1 = true;
   bool visualization = true;

-   double mat_val = 1.0;
-   double dbc_val = 0.0;
-   double nbc_val = 1.0;
-   double rbc_a_val = 1.0; // du/dn + a * u = b
-   double rbc_b_val = 1.0;
+   real_t mat_val = 1.0;
+   real_t dbc_val = 0.0;
+   real_t nbc_val = 1.0;
+   real_t rbc_a_val = 1.0; // du/dn + a * u = b
+   real_t rbc_b_val = 1.0;

   OptionsParser args(argc, argv);
   args.AddOption(&h1, "-h1", "--continuous", "-dg", "--discontinuous",
@@ -302,7 +302,7 @@ int main(int argc, char *argv[])
   {
      // Integrate the solution on the Dirichlet boundary and compare to the
      // expected value.
-      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
+      real_t error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);

      bool hom_dbc = (dbc_val == 0.0);
      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
@@ -314,7 +314,7 @@ int main(int argc, char *argv[])
   {
      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
      // to the expected value.
-      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
+      real_t error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);

      bool hom_nbc = (nbc_val == 0.0);
      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
@@ -330,7 +330,7 @@ int main(int argc, char *argv[])
      nbc0_bdr = 0;
      nbc0_bdr[3] = 1;

-      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
+      real_t error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);

      bool hom_nbc = true;
      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
@@ -341,8 +341,8 @@ int main(int argc, char *argv[])
   {
      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
      // expected value.
-      double error;
-      double avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);
+      real_t error;
+      real_t avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);

      bool hom_rbc = (rbc_b_val == 0.0);
      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
@@ -383,22 +383,22 @@ int main(int argc, char *argv[])
   return 0;
 }

-void quad_trans(double u, double v, double &x, double &y, bool log = false)
+void quad_trans(real_t u, real_t v, real_t &x, real_t &y, bool log = false)
 {
-   double a = a_; // Radius of disc
+   real_t a = a_; // Radius of disc

-   double d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);
+   real_t d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);

-   double v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
+   real_t v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
               ((4.0 - 3 * M_SQRT2) * a +
                (8.0 * (M_SQRT2 - 1.0) * a - 2.0) * v) / d;

-   double r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
+   real_t r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
                     2.0 * (1.0 + M_SQRT2 *
                            (1.0 + 2.0 * (2.0 * a - M_SQRT2 - 1.0) * a)) * v * v
                    ) / d;

-   double t = asin(v / r) * u / v;
+   real_t t = asin(v / r) * u / v;
   if (log)
   {
      mfem::out << "u, v, r, v0, t "
@@ -411,7 +411,7 @@ void quad_trans(double u, double v, double &x, double &y, bool log = false)

 void trans(const Vector &u, Vector &x)
 {
-   double tol = 1e-4;
+   real_t tol = 1e-4;

   if (u[1] > 0.5 - tol || u[1] < -0.5 + tol)
   {
@@ -542,8 +542,8 @@ Mesh * GenerateSerialMesh(int ref)
      vi[0] = o +  3; vi[1] = o +  4; mesh->AddBdrSegment(vi, 3 + i);
   }

-   double d[2];
-   double a = a_ / M_SQRT2;
+   real_t d[2];
+   real_t a = a_ / M_SQRT2;

   d[0] = -1.0; d[1] = -0.5; mesh->AddVertex(d);
   d[0] = -1.0; d[1] =  0.0; mesh->AddVertex(d);
@@ -636,12 +636,12 @@ Mesh * GenerateSerialMesh(int ref)
   return mesh;
 }

-double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
-                   double alpha, double beta, double gamma,
-                   double &error)
+real_t IntegrateBC(const GridFunction &x, const Array<int> &bdr,
+                   real_t alpha, real_t beta, real_t gamma,
+                   real_t &error)
 {
-   double nrm = 0.0;
-   double avg = 0.0;
+   real_t nrm = 0.0;
+   real_t avg = 0.0;
   error = 0.0;

   const bool a_is_zero = alpha == 0.0;
@@ -683,8 +683,8 @@ double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
         IntegrationPoint eip;
         FTr->Loc1.Transform(ip, eip);
         FTr->Face->SetIntPoint(&ip);
-         double face_weight = FTr->Face->Weight();
-         double val = 0.0;
+         real_t face_weight = FTr->Face->Weight();
+         real_t val = 0.0;
         if (!a_is_zero)
         {
            FTr->Elem1->SetIntPoint(&eip);
@@ -63,7 +63,7 @@
 using namespace std;
 using namespace mfem;

-static double a_ = 0.2;
+static real_t a_ = 0.2;

 // Normal to hole with boundary attribute 4
 void n4Vec(const Vector &x, Vector &n) { n = x; n[0] -= 0.5; n /= -n.Norml2(); }
@@ -73,9 +73,9 @@ Mesh * GenerateSerialMesh(int ref);
 // Compute the average value of alpha*n.Grad(sol) + beta*sol over the boundary
 // attributes marked in bdr_marker. Also computes the L2 norm of
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
-double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
-                   double alpha, double beta, double gamma,
-                   double &error);
+real_t IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
+                   real_t alpha, real_t beta, real_t gamma,
+                   real_t &error);

 int main(int argc, char *argv[])
 {
@@ -88,16 +88,16 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 1;
   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
+   real_t sigma = -1.0;
+   real_t kappa = -1.0;
   bool h1 = true;
   bool visualization = true;

-   double mat_val = 1.0;
-   double dbc_val = 0.0;
-   double nbc_val = 1.0;
-   double rbc_a_val = 1.0; // du/dn + a * u = b
-   double rbc_b_val = 1.0;
+   real_t mat_val = 1.0;
+   real_t dbc_val = 0.0;
+   real_t nbc_val = 1.0;
+   real_t rbc_a_val = 1.0; // du/dn + a * u = b
+   real_t rbc_b_val = 1.0;

   OptionsParser args(argc, argv);
   args.AddOption(&h1, "-h1", "--continuous", "-dg", "--discontinuous",
@@ -322,7 +322,7 @@ int main(int argc, char *argv[])
   {
      // Integrate the solution on the Dirichlet boundary and compare to the
      // expected value.
-      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
+      real_t error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);

      bool hom_dbc = (dbc_val == 0.0);
      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
@@ -334,7 +334,7 @@ int main(int argc, char *argv[])
   {
      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
      // to the expected value.
-      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
+      real_t error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);

      bool hom_nbc = (nbc_val == 0.0);
      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
@@ -350,7 +350,7 @@ int main(int argc, char *argv[])
      nbc0_bdr = 0;
      nbc0_bdr[3] = 1;

-      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
+      real_t error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);

      bool hom_nbc = true;
      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
@@ -361,7 +361,7 @@ int main(int argc, char *argv[])
   {
      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
      // expected value.
-      double error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
+      real_t error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
                                      error);

      bool hom_rbc = (rbc_b_val == 0.0);
@@ -409,22 +409,22 @@ int main(int argc, char *argv[])
   return 0;
 }

-void quad_trans(double u, double v, double &x, double &y, bool log = false)
+void quad_trans(real_t u, real_t v, real_t &x, real_t &y, bool log = false)
 {
-   double a = a_; // Radius of disc
+   real_t a = a_; // Radius of disc

-   double d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);
+   real_t d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);

-   double v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
+   real_t v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
               ((4.0 - 3 * M_SQRT2) * a +
                (8.0 * (M_SQRT2 - 1.0) * a - 2.0) * v) / d;

-   double r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
+   real_t r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
                     2.0 * (1.0 + M_SQRT2 *
                            (1.0 + 2.0 * (2.0 * a - M_SQRT2 - 1.0) * a)) * v * v
                    ) / d;

-   double t = asin(v / r) * u / v;
+   real_t t = asin(v / r) * u / v;
   if (log)
   {
      mfem::out << "u, v, r, v0, t "
@@ -437,7 +437,7 @@ void quad_trans(double u, double v, double &x, double &y, bool log = false)

 void trans(const Vector &u, Vector &x)
 {
-   double tol = 1e-4;
+   real_t tol = 1e-4;

   if (u[1] > 0.5 - tol || u[1] < -0.5 + tol)
   {
@@ -568,8 +568,8 @@ Mesh * GenerateSerialMesh(int ref)
      vi[0] = o +  3; vi[1] = o +  4; mesh->AddBdrSegment(vi, 3 + i);
   }

-   double d[2];
-   double a = a_ / M_SQRT2;
+   real_t d[2];
+   real_t a = a_ / M_SQRT2;

   d[0] = -1.0; d[1] = -0.5; mesh->AddVertex(d);
   d[0] = -1.0; d[1] =  0.0; mesh->AddVertex(d);
@@ -662,14 +662,14 @@ Mesh * GenerateSerialMesh(int ref)
   return mesh;
 }

-double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
-                   double alpha, double beta, double gamma,
-                   double &glb_err)
+real_t IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
+                   real_t alpha, real_t beta, real_t gamma,
+                   real_t &glb_err)
 {
-   double loc_vals[3];
-   double &nrm = loc_vals[0];
-   double &avg = loc_vals[1];
-   double &error = loc_vals[2];
+   real_t loc_vals[3];
+   real_t &nrm = loc_vals[0];
+   real_t &avg = loc_vals[1];
+   real_t &error = loc_vals[2];

   nrm = 0.0;
   avg = 0.0;
@@ -714,8 +714,8 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
         IntegrationPoint eip;
         FTr->Loc1.Transform(ip, eip);
         FTr->Face->SetIntPoint(&ip);
-         double face_weight = FTr->Face->Weight();
-         double val = 0.0;
+         real_t face_weight = FTr->Face->Weight();
+         real_t val = 0.0;
         if (!a_is_zero)
         {
            FTr->Elem1->SetIntPoint(&eip);
@@ -741,11 +741,12 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
      }
   }

-   double glb_vals[3];
-   MPI_Allreduce(loc_vals, glb_vals, 3, MPI_DOUBLE, MPI_SUM, fes.GetComm());
+   real_t glb_vals[3];
+   MPI_Allreduce(loc_vals, glb_vals, 3, MPITypeMap<real_t>::mpi_type,
+                 MPI_SUM, fes.GetComm());

-   double glb_nrm = glb_vals[0];
-   double glb_avg = glb_vals[1];
+   real_t glb_nrm = glb_vals[0];
+   real_t glb_avg = glb_vals[1];
   glb_err = glb_vals[2];

   // Normalize by the length of the boundary
@@ -35,7 +35,7 @@ using namespace mfem;
 // Return a mesh with a single element with vertices (0, 0), (1, 0), (1, 1),
 // (offset, 1) to demonstrate boundary conditions on a surface that is not
 // axis-aligned.
-Mesh * build_trapezoid_mesh(double offset)
+Mesh * build_trapezoid_mesh(real_t offset)
 {
   MFEM_VERIFY(offset < 0.9, "offset is too large!");

@@ -45,7 +45,7 @@ Mesh * build_trapezoid_mesh(double offset)
   Mesh * mesh = new Mesh(dimension, nvt, 1, nbe);

   // vertices
-   double vc[dimension];
+   real_t vc[dimension];
   vc[0] = 0.0; vc[1] = 0.0;
   mesh->AddVertex(vc);
   vc[0] = 1.0; vc[1] = 0.0;
@@ -81,7 +81,7 @@ int main(int argc, char *argv[])
   // 1. Parse command-line options.
   int order = 1;
   bool visualization = 1;
-   double offset = 0.3;
+   real_t offset = 0.3;
   bool visit = false;

   OptionsParser args(argc, argv);
@@ -38,7 +38,7 @@ using namespace mfem;
 // Return a mesh with a single element with vertices (0, 0), (1, 0), (1, 1),
 // (offset, 1) to demonstrate boundary conditions on a surface that is not
 // axis-aligned.
-Mesh * build_trapezoid_mesh(double offset)
+Mesh * build_trapezoid_mesh(real_t offset)
 {
   MFEM_VERIFY(offset < 0.9, "offset is too large!");

@@ -48,7 +48,7 @@ Mesh * build_trapezoid_mesh(double offset)
   Mesh * mesh = new Mesh(dimension, nvt, 1, nbe);

   // vertices
-   double vc[dimension];
+   real_t vc[dimension];
   vc[0] = 0.0; vc[1] = 0.0;
   mesh->AddVertex(vc);
   vc[0] = 1.0; vc[1] = 0.0;
@@ -84,7 +84,7 @@ int main(int argc, char *argv[])
 #ifdef HYPRE_USING_GPU
   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
        << "is NOT supported with the GPU version of hypre.\n\n";
-   return 242;
+   return MFEM_SKIP_RETURN_VALUE;
 #endif

   // 1. Initialize MPI and HYPRE.
@@ -97,9 +97,9 @@ int main(int argc, char *argv[])
   int order = 1;
   bool visualization = 1;
   bool reorder_space = false;
-   double offset = 0.3;
+   real_t offset = 0.3;
   bool visit = false;
-   double penalty = 0.0;
+   real_t penalty = 0.0;

   OptionsParser args(argc, argv);
   args.AddOption(&order, "-o", "--order",
@@ -34,7 +34,7 @@ void trans(const Vector &x, Vector &r);

 void sigmaFunc(const Vector &x, DenseMatrix &s);

-double uExact(const Vector &x)
+real_t uExact(const Vector &x)
 {
   return (0.25 * (2.0 + x[0]) - x[2]) * (x[2] + 0.25 * (2.0 + x[0]));
 }
@@ -167,7 +167,7 @@ int main(int argc, char *argv[])

   // 13. Compute error in the solution and its flux
   FunctionCoefficient uCoef(uExact);
-   double error = x.ComputeL2Error(uCoef);
+   real_t error = x.ComputeL2Error(uCoef);

   cout << "|u - u_h|_2 = " << error << endl;

@@ -176,7 +176,7 @@ int main(int argc, char *argv[])
   x.ComputeFlux(*integ, flux); flux *= -1.0;

   VectorFunctionCoefficient fluxCoef(3, fluxExact);
-   double flux_err = flux.ComputeL2Error(fluxCoef);
+   real_t flux_err = flux.ComputeL2Error(fluxCoef);

   cout << "|f - f_h|_2 = " << flux_err << endl;

@@ -304,8 +304,8 @@ void trans(const Vector &x, Vector &r)
 {
   r.SetSize(3);

-   double tol = 1e-6;
-   double theta = 0.0;
+   real_t tol = 1e-6;
+   real_t theta = 0.0;
   if (fabs(x[1] + 1.0) < tol)
   {
      theta = 0.25 * M_PI * (x[0] - 2.0);
@@ -337,7 +337,7 @@ void trans(const Vector &x, Vector &r)
 void sigmaFunc(const Vector &x, DenseMatrix &s)
 {
   s.SetSize(3);
-   double a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
+   real_t a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
   s(0,0) = 0.5 + x[0] * x[0] * (8.0 / a - 0.5);
   s(0,1) = x[0] * x[1] * (8.0 / a - 0.5);
   s(0,2) = 0.0;
@@ -34,7 +34,7 @@ void trans(const Vector &x, Vector &r);

 void sigmaFunc(const Vector &x, DenseMatrix &s);

-double uExact(const Vector &x)
+real_t uExact(const Vector &x)
 {
   return (0.25 * (2.0 + x[0]) - x[2]) * (x[2] + 0.25 * (2.0 + x[0]));
 }
@@ -201,7 +201,7 @@ int main(int argc, char *argv[])

   // 15. Compute error in the solution and its flux
   FunctionCoefficient uCoef(uExact);
-   double error = x.ComputeL2Error(uCoef);
+   real_t error = x.ComputeL2Error(uCoef);

   if (myid == 0) { cout << "|u - u_h|_2 = " << error << endl; }

@@ -210,7 +210,7 @@ int main(int argc, char *argv[])
   x.ComputeFlux(*integ, flux); flux *= -1.0;

   VectorFunctionCoefficient fluxCoef(3, fluxExact);
-   double flux_err = flux.ComputeL2Error(fluxCoef);
+   real_t flux_err = flux.ComputeL2Error(fluxCoef);

   if (myid == 0) { cout << "|f - f_h|_2 = " << flux_err << endl; }

@@ -349,8 +349,8 @@ void trans(const Vector &x, Vector &r)
 {
   r.SetSize(3);

-   double tol = 1e-6;
-   double theta = 0.0;
+   real_t tol = 1e-6;
+   real_t theta = 0.0;
   if (fabs(x[1] + 1.0) < tol)
   {
      theta = 0.25 * M_PI * (x[0] - 2.0);
@@ -382,7 +382,7 @@ void trans(const Vector &x, Vector &r)
 void sigmaFunc(const Vector &x, DenseMatrix &s)
 {
   s.SetSize(3);
-   double a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
+   real_t a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
   s(0,0) = 0.5 + x[0] * x[0] * (8.0 / a - 0.5);
   s(0,1) = x[0] * x[1] * (8.0 / a - 0.5);
   s(0,2) = 0.0;
@@ -53,7 +53,7 @@ using namespace mfem;
 // Exact solution, E, and r.h.s., f. See below for implementation.
 void E_exact(const Vector &, Vector &);
 void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;
 int dim;

 int main(int argc, char *argv[])
@@ -42,9 +42,9 @@ using namespace std;
 using namespace mfem;

 // Piecewise-affine function which is sometimes mesh-conforming
-double affine_function(const Vector &p)
+real_t affine_function(const Vector &p)
 {
-   double x = p(0), y = p(1);
+   real_t x = p(0), y = p(1);
   if (x < 0.0)
   {
      return 1.0 + x + y;
@@ -56,7 +56,7 @@ double affine_function(const Vector &p)
 }

 // Piecewise-constant function which is never mesh-conforming
-double jump_function(const Vector &p)
+real_t jump_function(const Vector &p)
 {
   if (p.Normlp(2.0) > 0.4 && p.Normlp(2.0) < 0.6)
   {
@@ -70,17 +70,17 @@ double jump_function(const Vector &p)

 // Singular function derived from the Laplacian of the "steep wavefront" problem
 // in [2].
-double singular_function(const Vector &p)
+real_t singular_function(const Vector &p)
 {
-   double x = p(0), y = p(1);
-   double alpha = 1000.0;
-   double xc = 0.75, yc = 0.5;
-   double r0 = 0.7;
-   double r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
-   double num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
-   double denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
+   real_t x = p(0), y = p(1);
+   real_t alpha = 1000.0;
+   real_t xc = 0.75, yc = 0.5;
+   real_t r0 = 0.7;
+   real_t r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
+   real_t num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
+   real_t denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
                            - 2 * pow(alpha,2) * r0 * r + 1.0 ),2);
-   denom = max(denom,1e-8);
+   denom = std::max(denom, (real_t) 1.0e-8);
   return num / denom;
 }

@@ -91,9 +91,9 @@ int main(int argc, char *argv[])
   int order = 1;
   int nc_limit = 1;
   int max_elems = 100*1000;
-   double double_max_elems = double(max_elems);
+   real_t double_max_elems = real_t(max_elems);
   bool visualization = true;
-   double osc_threshold = 1e-3;
+   real_t osc_threshold = 1e-3;
   int enriched_order = 5;

   OptionsParser args(argc, argv);
@@ -42,9 +42,9 @@ using namespace std;
 using namespace mfem;

 // Piecewise-affine function which is sometimes mesh-conforming
-double affine_function(const Vector &p)
+real_t affine_function(const Vector &p)
 {
-   double x = p(0), y = p(1);
+   real_t x = p(0), y = p(1);
   if (x < 0.0)
   {
      return 1.0 + x + y;
@@ -56,7 +56,7 @@ double affine_function(const Vector &p)
 }

 // Piecewise-constant function which is never mesh-conforming
-double jump_function(const Vector &p)
+real_t jump_function(const Vector &p)
 {
   if (p.Normlp(2.0) > 0.4 && p.Normlp(2.0) < 0.6)
   {
@@ -70,17 +70,17 @@ double jump_function(const Vector &p)

 // Singular function derived from the Laplacian of the "steep wavefront" problem
 // in [2].
-double singular_function(const Vector &p)
+real_t singular_function(const Vector &p)
 {
-   double x = p(0), y = p(1);
-   double alpha = 1000.0;
-   double xc = 0.75, yc = 0.5;
-   double r0 = 0.7;
-   double r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
-   double num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
-   double denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
+   real_t x = p(0), y = p(1);
+   real_t alpha = 1000.0;
+   real_t xc = 0.75, yc = 0.5;
+   real_t r0 = 0.7;
+   real_t r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
+   real_t num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
+   real_t denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
                            - 2 * pow(alpha,2) * r0 * r + 1.0 ),2);
-   denom = max(denom,1e-8);
+   denom = std::max(denom, (real_t) 1.0e-8);
   return num / denom;
 }

@@ -97,10 +97,10 @@ int main(int argc, char *argv[])
   int order = 1;
   int nc_limit = 1;
   int max_elems = 1e5;
-   double double_max_elems = double(max_elems);
+   real_t double_max_elems = real_t(max_elems);
   bool visualization = true;
   bool nc_simplices = true;
-   double osc_threshold = 1e-3;
+   real_t osc_threshold = 1e-3;
   int enriched_order = 5;

   OptionsParser args(argc, argv);
@@ -199,7 +199,7 @@ int main(int argc, char *argv[])
   coeffrefiner.PreprocessMesh(pmesh);

   int globalNE = pmesh.GetGlobalNE();
-   double osc = coeffrefiner.GetOsc();
+   real_t osc = coeffrefiner.GetOsc();
   if (myid == 0)
   {
      mfem::out << "\n";
@@ -39,7 +39,7 @@ using namespace mfem;
 void E_exact(const Vector &, Vector &);
 void CurlE_exact(const Vector &, Vector &);
 void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;
 int dim;

 int main(int argc, char *argv[])
@@ -177,7 +177,7 @@ int main(int argc, char *argv[])

   // 13. Compute and print the H(Curl) norm of the error.
   {
-      double error = sol.ComputeHCurlError(&E, &CurlE);
+      real_t error = sol.ComputeHCurlError(&E, &CurlE);
      cout << "\n|| E_h - E ||_{H(Curl)} = " << error << '\n' << endl;
   }

@@ -376,8 +376,8 @@ void CurlE_exact(const Vector &x, Vector &dE)
 {
   if (dim == 1)
   {
-      double c4 = cos(kappa * x(0) + 0.4 * M_PI);
-      double c9 = cos(kappa * x(0) + 0.9 * M_PI);
+      real_t c4 = cos(kappa * x(0) + 0.4 * M_PI);
+      real_t c9 = cos(kappa * x(0) + 0.9 * M_PI);

      dE(0) =  0.0;
      dE(1) = -1.3 * c9;
@@ -386,9 +386,9 @@ void CurlE_exact(const Vector &x, Vector &dE)
   }
   else if (dim == 2)
   {
-      double c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);

      dE(0) =  1.3 * c9;
      dE(1) = -1.3 * c9;
@@ -397,13 +397,13 @@ void CurlE_exact(const Vector &x, Vector &dE)
   }
   else
   {
-      double s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
-      double sk = sin(kappa * x(2));
-      double ck = cos(kappa * x(2));
+      real_t s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t sk = sin(kappa * x(2));
+      real_t ck = cos(kappa * x(2));

      dE(0) =  1.2 * s4 * sk + 1.3 * M_SQRT1_2 * c9 * ck;
      dE(1) = -1.1 * s0 * sk - 1.3 * M_SQRT1_2 * c9 * ck;
@@ -416,9 +416,9 @@ void f_exact(const Vector &x, Vector &f)
 {
   if (dim == 1)
   {
-      double s0 = sin(kappa * x(0) + 0.0 * M_PI);
-      double s4 = sin(kappa * x(0) + 0.4 * M_PI);
-      double s9 = sin(kappa * x(0) + 0.9 * M_PI);
+      real_t s0 = sin(kappa * x(0) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * x(0) + 0.4 * M_PI);
+      real_t s9 = sin(kappa * x(0) + 0.9 * M_PI);

      f(0) = 2.2 * s0 + 1.2 * M_SQRT1_2 * s4;
      f(1) = 1.2 * (2.0 + kappa * kappa) * s4 +
@@ -427,9 +427,9 @@ void f_exact(const Vector &x, Vector &f)
   }
   else if (dim == 2)
   {
-      double s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);

      f(0) = 0.55 * (4.0 + kappa * kappa) * s0 +
             0.6 * (M_SQRT2 - kappa * kappa) * s4;
@@ -440,14 +440,14 @@ void f_exact(const Vector &x, Vector &f)
   }
   else
   {
-      double s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
-      double c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
-      double sk = sin(kappa * x(2));
-      double ck = cos(kappa * x(2));
+      real_t s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t sk = sin(kappa * x(2));
+      real_t ck = cos(kappa * x(2));

      f(0) = 0.55 * (4.0 + 3.0 * kappa * kappa) * s0 * ck +
             0.6 * (M_SQRT2 - kappa * kappa) * s4 * ck -
@@ -39,7 +39,7 @@ using namespace mfem;
 void E_exact(const Vector &, Vector &);
 void CurlE_exact(const Vector &, Vector &);
 void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;
 int dim;

 int main(int argc, char *argv[])
@@ -224,7 +224,7 @@ int main(int argc, char *argv[])

   // 14. Compute and print the H(Curl) norm of the error.
   {
-      double error = sol.ComputeHCurlError(&E, &CurlE);
+      real_t error = sol.ComputeHCurlError(&E, &CurlE);
      if (Mpi::Root())
      {
         cout << "\n|| E_h - E ||_{H(Curl)} = " << error << '\n' << endl;
@@ -442,8 +442,8 @@ void CurlE_exact(const Vector &x, Vector &dE)
 {
   if (dim == 1)
   {
-      double c4 = cos(kappa * x(0) + 0.4 * M_PI);
-      double c9 = cos(kappa * x(0) + 0.9 * M_PI);
+      real_t c4 = cos(kappa * x(0) + 0.4 * M_PI);
+      real_t c9 = cos(kappa * x(0) + 0.9 * M_PI);

      dE(0) =  0.0;
      dE(1) = -1.3 * c9;
@@ -452,9 +452,9 @@ void CurlE_exact(const Vector &x, Vector &dE)
   }
   else if (dim == 2)
   {
-      double c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);

      dE(0) =  1.3 * c9;
      dE(1) = -1.3 * c9;
@@ -463,13 +463,13 @@ void CurlE_exact(const Vector &x, Vector &dE)
   }
   else
   {
-      double s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
-      double sk = sin(kappa * x(2));
-      double ck = cos(kappa * x(2));
+      real_t s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t sk = sin(kappa * x(2));
+      real_t ck = cos(kappa * x(2));

      dE(0) =  1.2 * s4 * sk + 1.3 * M_SQRT1_2 * c9 * ck;
      dE(1) = -1.1 * s0 * sk - 1.3 * M_SQRT1_2 * c9 * ck;
@@ -482,9 +482,9 @@ void f_exact(const Vector &x, Vector &f)
 {
   if (dim == 1)
   {
-      double s0 = sin(kappa * x(0) + 0.0 * M_PI);
-      double s4 = sin(kappa * x(0) + 0.4 * M_PI);
-      double s9 = sin(kappa * x(0) + 0.9 * M_PI);
+      real_t s0 = sin(kappa * x(0) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * x(0) + 0.4 * M_PI);
+      real_t s9 = sin(kappa * x(0) + 0.9 * M_PI);

      f(0) = 2.2 * s0 + 1.2 * M_SQRT1_2 * s4;
      f(1) = 1.2 * (2.0 + kappa * kappa) * s4 +
@@ -493,9 +493,9 @@ void f_exact(const Vector &x, Vector &f)
   }
   else if (dim == 2)
   {
-      double s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);

      f(0) = 0.55 * (4.0 + kappa * kappa) * s0 +
             0.6 * (M_SQRT2 - kappa * kappa) * s4;
@@ -506,14 +506,14 @@ void f_exact(const Vector &x, Vector &f)
   }
   else
   {
-      double s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
-      double s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
-      double s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
-      double c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
-      double sk = sin(kappa * x(2));
-      double ck = cos(kappa * x(2));
+      real_t s0 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t c0 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.0 * M_PI);
+      real_t s4 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t c4 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.4 * M_PI);
+      real_t s9 = sin(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t c9 = cos(kappa * M_SQRT1_2 * (x(0) + x(1)) + 0.9 * M_PI);
+      real_t sk = sin(kappa * x(2));
+      real_t ck = cos(kappa * x(2));

      f(0) = 0.55 * (4.0 + 3.0 * kappa * kappa) * s0 * ck +
             0.6 * (M_SQRT2 - kappa * kappa) * s4 * ck -
@@ -35,8 +35,8 @@
 using namespace std;
 using namespace mfem;

-double GetVectorMax(int vdim, const ParGridFunction &x);
-double GetScalarMax(const ParGridFunction &x);
+real_t GetVectorMax(int vdim, const ParGridFunction &x);
+real_t GetScalarMax(const ParGridFunction &x);

 int main(int argc, char *argv[])
 {
@@ -140,7 +140,7 @@ int main(int argc, char *argv[])
   //    extract the corresponding parallel matrices A and M.
   HypreParMatrix *A = NULL;
   HypreParMatrix *M = NULL;
-   double shift = 0.0;
+   real_t shift = 0.0;
   {
      DenseMatrix epsilonMat(3);
      epsilonMat(0,0) = 2.0; epsilonMat(1,1) = 2.0; epsilonMat(2,2) = 2.0;
@@ -178,7 +178,7 @@ int main(int argc, char *argv[])
      m.AddDomainIntegrator(new VectorFEMassIntegrator(epsilon));
      m.Assemble();
      // shift the eigenvalue corresponding to eliminated dofs to a large value
-      m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+      m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<real_t>::min());
      m.Finalize();

      A = a.ParallelAssemble();
@@ -204,7 +204,7 @@ int main(int argc, char *argv[])
   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define
   //    parallel grid functions to represent each of the eigenmodes returned by
   //    the solver and their derivatives.
-   Array<double> eigenvalues;
+   Array<real_t> eigenvalues;
   ame->Solve();
   ame->GetEigenvalues(eigenvalues);
   ParGridFunction x(&fespace_nd);
@@ -308,10 +308,10 @@ int main(int argc, char *argv[])
               yComp.ProjectCoefficient(yCoef);
               zComp.ProjectCoefficient(zCoef);

-               double max_x = GetScalarMax(xComp);
-               double max_y = GetScalarMax(yComp);
-               double max_z = GetScalarMax(zComp);
-               double max_r = std::max(max_x, std::max(max_y, max_z));
+               real_t max_x = GetScalarMax(xComp);
+               real_t max_y = GetScalarMax(yComp);
+               real_t max_z = GetScalarMax(zComp);
+               real_t max_r = std::max(max_x, std::max(max_y, max_z));

               ostringstream x_cmd;
               x_cmd << " window_title 'Eigenmode " << i+1 << '/' << nev
@@ -368,7 +368,7 @@ int main(int argc, char *argv[])
               dyComp.ProjectCoefficient(dyCoef);
               dzComp.ProjectCoefficient(dzCoef);

-               double min_d = max_r / (bbMax[0] - bbMin[0]);
+               real_t min_d = max_r / (bbMax[0] - bbMin[0]);

               max_y = GetScalarMax(dyComp);
               max_z = GetScalarMax(dzComp);
@@ -480,9 +480,9 @@ int main(int argc, char *argv[])
               xyComp.ProjectCoefficient(xyCoef);
               zComp.ProjectCoefficient(zCoef);

-               double max_v = GetVectorMax(2, xyComp);
-               double max_s = GetScalarMax(zComp);
-               double max_r = std::max(max_v, max_s);
+               real_t max_v = GetVectorMax(2, xyComp);
+               real_t max_s = GetScalarMax(zComp);
+               real_t max_r = std::max(max_v, max_s);

               ostringstream xy_cmd;
               xy_cmd << " window_title 'Eigenmode " << i+1 << '/' << nev
@@ -523,7 +523,7 @@ int main(int argc, char *argv[])
               dxyComp.ProjectCoefficient(dxyCoef);
               dzComp.ProjectCoefficient(dzCoef);

-               double min_d = max_r / std::min(bbMax[0] - bbMin[0],
+               real_t min_d = max_r / std::min(bbMax[0] - bbMin[0],
                                               bbMax[1] - bbMin[1]);

               max_v = GetVectorMax(2, dxyComp);
@@ -649,17 +649,17 @@ int main(int argc, char *argv[])
   return 0;
 }

-double GetVectorMax(int vdim, const ParGridFunction &x)
+real_t GetVectorMax(int vdim, const ParGridFunction &x)
 {
   Vector zeroVec(vdim); zeroVec = 0.0;
   VectorConstantCoefficient zero(zeroVec);
-   double nrm = x.ComputeMaxError(zero);
+   real_t nrm = x.ComputeMaxError(zero);
   return nrm;
 }

-double GetScalarMax(const ParGridFunction &x)
+real_t GetScalarMax(const ParGridFunction &x)
 {
   ConstantCoefficient zero(0.0);
-   double nrm = x.ComputeMaxError(zero);
+   real_t nrm = x.ComputeMaxError(zero);
   return nrm;
 }
@@ -19,8 +19,11 @@
 //    ex33 -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
 //    ex33 -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
 //
-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
+//  Note: The manufactured solution used in this problem is
+//
+//            u = ∏_{i=0}^{dim-1} sin(π x_i) ,
+//
+//        regardless of the value of alpha.
 //
 // Description:
 //
@@ -86,11 +89,16 @@ using namespace mfem;

 int main(int argc, char *argv[])
 {
+#ifdef MFEM_USE_SINGLE
+   cout << "This example is not supported in single precision.\n\n";
+   return MFEM_SKIP_RETURN_VALUE;
+#endif
+
   // 1. Parse command-line options.
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   int num_refs = 3;
-   double alpha = 0.5;
+   real_t alpha = 0.5;
   bool visualization = true;
   bool verification = false;

@@ -109,7 +117,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
+                  "Use sinusoidal function (f) for manufactured "
+                  "solution test.");
   args.Parse();
   if (!args.Good())
   {
@@ -118,13 +127,13 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   Array<double> coeffs, poles;
+   Array<real_t> coeffs, poles;
   int progress_steps = 1;

   // 2. Compute the rational expansion coefficients that define the
   //    integer-order PDEs.
   const int power_of_laplace = (int)floor(alpha);
-   double exponent_to_approximate = alpha - power_of_laplace;
+   real_t exponent_to_approximate = alpha - power_of_laplace;
   bool integer_order = false;
   // Check if alpha is an integer or not.
   if (abs(exponent_to_approximate) > 1e-12)
@@ -135,7 +144,7 @@ int main(int argc, char *argv[])
      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
                                          poles);

-      // If the example is build without LAPACK, the exponent_to_approximate
+      // If the example is built without LAPACK, the exponent_to_approximate
      // might be modified by the function call above.
      alpha = exponent_to_approximate + power_of_laplace;
   }
@@ -158,7 +167,7 @@ int main(int argc, char *argv[])
   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of finite element unknowns: "
+   cout << "Number of degrees of freedom: "
        << fespace.GetTrueVSize() << endl;

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
@@ -173,7 +182,7 @@ int main(int argc, char *argv[])
   // 7. Define diffusion coefficient, load, and solution GridFunction.
   auto func = [&alpha](const Vector &x)
   {
-      double val = 1.0;
+      real_t val = 1.0;
      for (int i=0; i<x.Size(); i++)
      {
         val *= sin(M_PI*x(i));
@@ -364,7 +373,7 @@ int main(int argc, char *argv[])
   {
      auto solution = [] (const Vector &x)
      {
-         double val = 1.0;
+         real_t val = 1.0;
         for (int i=0; i<x.Size(); i++)
         {
            val *= sin(M_PI*x(i));
@@ -372,31 +381,31 @@ int main(int argc, char *argv[])
         return val;
      };
      FunctionCoefficient sol(solution);
-      double l2_error = u.ComputeL2Error(sol);
+      real_t l2_error = u.ComputeL2Error(sol);

-      string analytic_solution,expected_mesh;
+      string manufactured_solution,expected_mesh;
      switch (dim)
      {
         case 1:
-            analytic_solution = "sin(π x)";
+            manufactured_solution = "sin(π x)";
            expected_mesh = "inline_segment.mesh";
            break;
         case 2:
-            analytic_solution = "sin(π x) sin(π y)";
+            manufactured_solution = "sin(π x) sin(π y)";
            expected_mesh = "inline_quad.mesh";
            break;
         default:
-            analytic_solution = "sin(π x) sin(π y) sin(π z)";
+            manufactured_solution = "sin(π x) sin(π y) sin(π z)";
            expected_mesh = "inline_hex.mesh";
            break;
      }

      mfem::out << "\n" << string(80,'=')
                << "\n\nSolution Verification in "<< dim << "D \n\n"
-                << "Analytic solution : " << analytic_solution << "\n"
-                << "Expected mesh     : " << expected_mesh <<"\n"
-                << "Your mesh         : " << mesh_file << "\n"
-                << "L2 error          : " << l2_error << "\n\n"
+                << "Manufactured solution : " << manufactured_solution << "\n"
+                << "Expected mesh         : " << expected_mesh <<"\n"
+                << "Your mesh             : " << mesh_file << "\n"
+                << "L2 error              : " << l2_error << "\n\n"
                << string(80,'=') << endl;
   }

@@ -50,8 +50,8 @@ using namespace mfem;

    See pg. A1501 of Nakatsukasa et al. [1]. */
 void RationalApproximation_AAA(const Vector &val, const Vector &pt,
-                               Array<double> &z, Array<double> &f, Vector &w,
-                               double tol, int max_order)
+                               Array<real_t> &z, Array<real_t> &f, Vector &w,
+                               real_t tol, int max_order)
 {

   // number of sample points
@@ -67,11 +67,11 @@ void RationalApproximation_AAA(const Vector &val, const Vector &pt,
   DenseMatrix C, Ctemp, A, Am;
   // auxiliary arrays and vectors
   Vector f_vec;
-   Array<double> c_i;
+   Array<real_t> c_i;

   // mean of the value vector
   Vector R(val.Size());
-   double mean_val = val.Sum()/size;
+   real_t mean_val = val.Sum()/size;

   for (int i = 0; i<R.Size(); i++) { R(i) = mean_val; }

@@ -79,10 +79,10 @@ void RationalApproximation_AAA(const Vector &val, const Vector &pt,
   {
      // select next support point
      int idx = 0;
-      double tmp_max = 0;
+      real_t tmp_max = 0;
      for (int j = 0; j < size; j++)
      {
-         double tmp = abs(val(j)-R(j));
+         real_t tmp = abs(val(j)-R(j));
         if (tmp > tmp_max)
         {
            tmp_max = tmp;
@@ -98,7 +98,7 @@ void RationalApproximation_AAA(const Vector &val, const Vector &pt,
      J.DeleteFirst(idx);

      // next column in Cauchy matrix
-      Array<double> C_tmp(size);
+      Array<real_t> C_tmp(size);
      for (int j = 0; j < size; j++)
      {
         C_tmp[j] = 1.0/(pt(j)-pt(idx));
@@ -131,7 +131,7 @@ void RationalApproximation_AAA(const Vector &val, const Vector &pt,
      }

 #ifdef MFEM_USE_LAPACK
-      DenseMatrixSVD svd(Am,false,true);
+      DenseMatrixSVD svd(Am,'N','A');
      svd.Eval(Am);
      DenseMatrix &v = svd.RightSingularvectors();
      v.GetRow(k,w);
@@ -173,7 +173,7 @@ void RationalApproximation_AAA(const Vector &val, const Vector &pt,

    See pg. A1501 of Nakatsukasa et al. [1]. */
 void ComputePolesAndZeros(const Vector &z, const Vector &f, const Vector &w,
-                          Array<double> & poles, Array<double> & zeros, double &scale)
+                          Array<real_t> & poles, Array<real_t> & zeros, real_t &scale)
 {
   // Initialization
   poles.SetSize(0);
@@ -242,8 +242,8 @@ void ComputePolesAndZeros(const Vector &z, const Vector &f, const Vector &w,
    @param[in]  zeros   Array of zeros
    @param[in]  scale   Scaling constant
    @param[out] coeffs  Coefficients c_i */
-void PartialFractionExpansion(double scale, Array<double> & poles,
-                              Array<double> & zeros, Array<double> & coeffs)
+void PartialFractionExpansion(real_t scale, Array<real_t> & poles,
+                              Array<real_t> & zeros, Array<real_t> & coeffs)
 {
   int psize = poles.Size();
   int zsize = zeros.Size();
@@ -259,13 +259,13 @@ void PartialFractionExpansion(double scale, Array<double> & poles,

   for (int i=0; i<psize; i++)
   {
-      double tmp_numer=1.0;
+      real_t tmp_numer=1.0;
      for (int j=0; j<zsize; j++)
      {
         tmp_numer *= poles[i]-zeros[j];
      }

-      double tmp_denom=1.0;
+      real_t tmp_denom=1.0;
      for (int k=0; k<psize; k++)
      {
         if (k != i) { tmp_denom *= poles[i]-poles[k]; }
@@ -292,10 +292,10 @@ void PartialFractionExpansion(double scale, Array<double> & poles,
           @a alpha != 0.99, then @a alpha = 0.5 is used by default.

   See pg. A1501 of Nakatsukasa et al. [1]. */
-void ComputePartialFractionApproximation(double & alpha,
-                                         Array<double> & coeffs, Array<double> & poles,
-                                         double lmax = 1000.,
-                                         double tol=1e-10, int npoints = 1000,
+void ComputePartialFractionApproximation(real_t & alpha,
+                                         Array<real_t> & coeffs, Array<real_t> & poles,
+                                         real_t lmax = 1000.,
+                                         real_t tol=1e-10, int npoints = 1000,
                                         int max_order = 100)
 {
   MFEM_VERIFY(alpha < 1., "alpha must be less than 1");
@@ -320,41 +320,41 @@ void ComputePartialFractionApproximation(double & alpha,
            << "\nThe default is alpha = 0.5.\n" << string(80, '=') << "\n"
            << endl;
   }
-   const double eps = std::numeric_limits<double>::epsilon();
+   const real_t eps = std::numeric_limits<real_t>::epsilon();

   if (abs(alpha - 0.33) < eps)
   {
-      coeffs = Array<double> ({1.821898e+03, 9.101221e+01, 2.650611e+01,
+      coeffs = Array<real_t> ({1.821898e+03, 9.101221e+01, 2.650611e+01,
                               1.174937e+01, 6.140444e+00, 3.441713e+00,
                               1.985735e+00, 1.162634e+00, 6.891560e-01,
                               4.111574e-01, 2.298736e-01});
-      poles = Array<double> ({-4.155583e+04, -2.956285e+03, -8.331715e+02,
+      poles = Array<real_t> ({-4.155583e+04, -2.956285e+03, -8.331715e+02,
                              -3.139332e+02, -1.303448e+02, -5.563385e+01,
                              -2.356255e+01, -9.595516e+00, -3.552160e+00,
                              -1.032136e+00, -1.241480e-01});
   }
   else if (abs(alpha - 0.99) < eps)
   {
-      coeffs = Array<double>({2.919591e-02, 1.419750e-02, 1.065798e-02,
+      coeffs = Array<real_t>({2.919591e-02, 1.419750e-02, 1.065798e-02,
                              9.395094e-03, 8.915329e-03, 8.822991e-03,
                              9.058247e-03, 9.814521e-03, 1.180396e-02,
                              1.834554e-02, 9.840482e-01});
-      poles = Array<double> ({-1.069683e+04, -1.769370e+03, -5.718374e+02,
+      poles = Array<real_t> ({-1.069683e+04, -1.769370e+03, -5.718374e+02,
                              -2.242095e+02, -9.419132e+01, -4.031012e+01,
                              -1.701525e+01, -6.810088e+00, -2.382810e+00,
                              -5.700059e-01, -1.384324e-03});
   }
   else
   {
-      if (abs(alpha - 0.5) > eps && print_warning)
+      if (abs(alpha - 0.5) > eps)
      {
         alpha = 0.5;
      }
-      coeffs = Array<double>({2.290262e+02, 2.641819e+01, 1.005566e+01,
+      coeffs = Array<real_t>({2.290262e+02, 2.641819e+01, 1.005566e+01,
                              5.390411e+00, 3.340725e+00, 2.211205e+00,
                              1.508883e+00, 1.049474e+00, 7.462709e-01,
                              5.482686e-01, 4.232510e-01, 3.578967e-01});
-      poles = Array<double>({-3.168211e+04, -3.236077e+03, -9.868287e+02,
+      poles = Array<real_t>({-3.168211e+04, -3.236077e+03, -9.868287e+02,
                             -3.945597e+02, -1.738889e+02, -7.925178e+01,
                             -3.624992e+01, -1.629196e+01, -6.982956e+00,
                             -2.679984e+00, -7.782607e-01, -7.649166e-02});
@@ -368,19 +368,21 @@ void ComputePartialFractionApproximation(double & alpha,


   return;
+#else
+   MFEM_CONTRACT_VAR(print_warning);
 #endif

   Vector x(npoints);
   Vector val(npoints);
-   double dx = lmax / (double)(npoints-1);
+   real_t dx = lmax / (real_t)(npoints-1);
   for (int i = 0; i<npoints; i++)
   {
-      x(i) = dx * (double)i;
+      x(i) = dx * (real_t)i;
      val(i) = pow(x(i),1.-alpha);
   }

   // Apply triple-A algorithm to f(x) = x^{1-a}
-   Array<double> z, f;
+   Array<real_t> z, f;
   Vector w;
   RationalApproximation_AAA(val,x,z,f,w,tol,max_order);

@@ -389,8 +391,8 @@ void ComputePartialFractionApproximation(double & alpha,
   vecf.SetDataAndSize(f.GetData(), f.Size());

   // Compute poles and zeros for RA of f(x) = x^{1-a}
-   double scale;
-   Array<double> zeros;
+   real_t scale;
+   Array<real_t> zeros;
   ComputePolesAndZeros(vecz, vecf, w, poles, zeros, scale);

   // Remove the zero at x=0, thus, delivering a RA for f(x) = x^{-a}
@@ -19,8 +19,11 @@
 //    mpirun -np 4 ex33p -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
 //    mpirun -np 4 ex33p -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1

-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
+//  Note: The manufactured solution used in this problem is
+//
+//            u = ∏_{i=0}^{dim-1} sin(π x_i) ,
+//
+//        regardless of the value of alpha.
 //
 // Description:
 //
@@ -86,6 +89,11 @@ using namespace mfem;

 int main(int argc, char *argv[])
 {
+#ifdef MFEM_USE_SINGLE
+   cout << "This example is not supported in single precision.\n\n";
+   return MFEM_SKIP_RETURN_VALUE;
+#endif
+
   // 0. Initialize MPI.
   Mpi::Init(argc, argv);
   int num_procs = Mpi::WorldSize();
@@ -96,7 +104,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   int num_refs = 3;
-   double alpha = 0.5;
+   real_t alpha = 0.5;
   bool visualization = true;
   bool verification = false;

@@ -115,7 +123,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
+                  "Use sinusoidal function (f) for manufactured "
+                  "solution test.");
   args.Parse();
   if (!args.Good())
   {
@@ -127,13 +136,13 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   Array<double> coeffs, poles;
+   Array<real_t> coeffs, poles;
   int progress_steps = 1;

   // 2. Compute the rational expansion coefficients that define the
   //    integer-order PDEs.
   const int power_of_laplace = floor(alpha);
-   double exponent_to_approximate = alpha - power_of_laplace;
+   real_t exponent_to_approximate = alpha - power_of_laplace;
   bool integer_order = false;
   // Check if alpha is an integer or not.
   if (abs(exponent_to_approximate) > 1e-12)
@@ -175,10 +184,11 @@ int main(int argc, char *argv[])
   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
   if (Mpi::Root())
   {
-      cout << "Number of finite element unknowns: "
-           << fespace.GetTrueVSize() << endl;
+      cout << "Number of degrees of freedom: "
+           << size << endl;
   }

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
@@ -193,7 +203,7 @@ int main(int argc, char *argv[])
   // 7. Define diffusion coefficient, load, and solution GridFunction.
   auto func = [&alpha](const Vector &x)
   {
-      double val = 1.0;
+      real_t val = 1.0;
      for (int i=0; i<x.Size(); i++)
      {
         val *= sin(M_PI*x(i));
@@ -218,7 +228,7 @@ int main(int argc, char *argv[])
   if (verification)
   {
      // This statement is only relevant for the verification of the code. It
-      // uses a different f such that an analytic solution is known and easy
+      // uses a different f such that an manufactured solution is known and easy
      // to compare with the numerical one. The FPDE becomes:
      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
      // -> u(x,y) = sin(\pi x) sin(\pi y)
@@ -398,7 +408,7 @@ int main(int argc, char *argv[])
   {
      auto solution = [] (const Vector &x)
      {
-         double val = 1.0;
+         real_t val = 1.0;
         for (int i=0; i<x.Size(); i++)
         {
            val *= sin(M_PI*x(i));
@@ -406,33 +416,33 @@ int main(int argc, char *argv[])
         return val;
      };
      FunctionCoefficient sol(solution);
-      double l2_error = u.ComputeL2Error(sol);
+      real_t l2_error = u.ComputeL2Error(sol);

      if (Mpi::Root())
      {
-         string analytic_solution,expected_mesh;
+         string manufactured_solution,expected_mesh;
         switch (dim)
         {
            case 1:
-               analytic_solution = "sin(π x)";
+               manufactured_solution = "sin(π x)";
               expected_mesh = "inline_segment.mesh";
               break;
            case 2:
-               analytic_solution = "sin(π x) sin(π y)";
+               manufactured_solution = "sin(π x) sin(π y)";
               expected_mesh = "inline_quad.mesh";
               break;
            default:
-               analytic_solution = "sin(π x) sin(π y) sin(π z)";
+               manufactured_solution = "sin(π x) sin(π y) sin(π z)";
               expected_mesh = "inline_hex.mesh";
               break;
         }

         mfem::out << "\n" << string(80,'=')
                   << "\n\nSolution Verification in "<< dim << "D \n\n"
-                   << "Analytic solution : " << analytic_solution << "\n"
-                   << "Expected mesh     : " << expected_mesh <<"\n"
-                   << "Your mesh         : " << mesh_file << "\n"
-                   << "L2 error          : " << l2_error << "\n\n"
+                   << "Manufactured solution : " << manufactured_solution << "\n"
+                   << "Expected mesh         : " << expected_mesh <<"\n"
+                   << "Your mesh             : " << mesh_file << "\n"
+                   << "L2 error              : " << l2_error << "\n\n"
                   << string(80,'=') << endl;
      }
   }
@@ -52,6 +52,7 @@ static bool pa_ = false;
 static bool algebraic_ceed_ = false;

 void ComputeCurrentDensityOnSubMesh(int order,
+                                    bool visualization,
                                    const Array<int> &phi0_attr,
                                    const Array<int> &phi1_attr,
                                    const Array<int> &jn_zero_attr,
@@ -69,7 +70,7 @@ int main(int argc, char *argv[])
   Array<int> jn_zero_attr;
   int ref_levels = 1;
   int order = 1;
-   double delta_const = 1e-6;
+   real_t delta_const = 1e-6;
   bool mixed = true;
   bool static_cond = false;
   const char *device_config = "cpu";
@@ -236,8 +237,8 @@ int main(int argc, char *argv[])
   FiniteElementSpace fes_cond_rt(&mesh_cond, &fec_cond_rt);
   GridFunction j_cond(&fes_cond_rt);

-   ComputeCurrentDensityOnSubMesh(order, phi0_attr, phi1_attr, jn_zero_attr,
-                                  j_cond);
+   ComputeCurrentDensityOnSubMesh(order, visualization,
+                                  phi0_attr, phi1_attr, jn_zero_attr, j_cond);

   // 6a. Save the SubMesh and associated current density in parallel. This
   //     output can be viewed later using GLVis:
@@ -255,6 +256,7 @@ int main(int argc, char *argv[])
      cond_ofs.precision(8);
      j_cond.Save(cond_ofs);
   }
+
   // 6b. Send the current density, computed on the SubMesh, to a GLVis server.
   if (visualization)
   {
@@ -450,6 +452,7 @@ int main(int argc, char *argv[])
 }

 void ComputeCurrentDensityOnSubMesh(int order,
+                                    bool visualization,
                                    const Array<int> &phi0_attr,
                                    const Array<int> &phi1_attr,
                                    const Array<int> &jn_zero_attr,
@@ -567,6 +570,7 @@ void ComputeCurrentDensityOnSubMesh(int order,
      a_h1.RecoverFEMSolution(X, b_h1, phi_h1);
   }

+   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
@@ -49,6 +49,7 @@ using namespace std;
 using namespace mfem;

 void ComputeCurrentDensityOnSubMesh(int order,
+                                    bool visualization,
                                    const Array<int> &phi0_attr,
                                    const Array<int> &phi1_attr,
                                    const Array<int> &jn_zero_attr,
@@ -73,7 +74,7 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 1;
   int par_ref_levels = 1;
   int order = 1;
-   double delta_const = 1e-6;
+   real_t delta_const = 1e-6;
   bool mixed = true;
   bool static_cond = false;
   bool pa = false;
@@ -270,8 +271,8 @@ int main(int argc, char *argv[])
   ParFiniteElementSpace fes_cond_rt(&pmesh_cond, &fec_cond_rt);
   ParGridFunction j_cond(&fes_cond_rt);

-   ComputeCurrentDensityOnSubMesh(order, phi0_attr, phi1_attr, jn_zero_attr,
-                                  j_cond);
+   ComputeCurrentDensityOnSubMesh(order, visualization,
+                                  phi0_attr, phi1_attr, jn_zero_attr, j_cond);

   // 7a. Save the SubMesh and associated current density in parallel. This
   //     output can be viewed later using GLVis:
@@ -289,6 +290,7 @@ int main(int argc, char *argv[])
      cond_ofs.precision(8);
      j_cond.Save(cond_ofs);
   }
+
   // 7b. Send the current density, computed on the SubMesh, to a GLVis server.
   if (visualization)
   {
@@ -498,6 +500,7 @@ int main(int argc, char *argv[])
 }

 void ComputeCurrentDensityOnSubMesh(int order,
+                                    bool visualization,
                                    const Array<int> &phi0_attr,
                                    const Array<int> &phi1_attr,
                                    const Array<int> &jn_zero_attr,
@@ -586,6 +589,8 @@ void ComputeCurrentDensityOnSubMesh(int order,
      cg.Mult(B, X);
      a_h1.RecoverFEMSolution(X, b_h1, phi_h1);
   }
+
+   if (visualization)
   {
      int num_procs = fes_cond_h1.GetNRanks();
      char vishost[] = "localhost";
@@ -55,9 +55,9 @@
 using namespace std;
 using namespace mfem;

-static double mu_ = 1.0;
-static double epsilon_ = 1.0;
-static double sigma_ = 2.0;
+static real_t mu_ = 1.0;
+static real_t epsilon_ = 1.0;
+static real_t sigma_ = 2.0;

 void SetPortBC(int prob, int dim, int mode, ParGridFunction &port_bc);

@@ -77,9 +77,9 @@ int main(int argc, char *argv[])
   Array<int> port_bc_attr;
   int prob = 0;
   int mode = 1;
-   double freq = -1.0;
-   double omega = 2.0 * M_PI;
-   double a_coef = 0.0;
+   real_t freq = -1.0;
+   real_t omega = 2.0 * M_PI;
+   real_t a_coef = 0.0;
   bool herm_conv = true;
   bool slu_solver  = false;
   bool visualization = 1;
@@ -587,7 +587,7 @@ int main(int argc, char *argv[])
      int i = 0;
      while (sol_sock)
      {
-         double t = (double)(i % num_frames) / num_frames;
+         real_t t = (real_t)(i % num_frames) / num_frames;
         ostringstream oss;
         oss << "Harmonic Solution (t = " << t << " T)";

@@ -637,7 +637,7 @@ void ScalarWaveGuide(int mode, ParGridFunction &x)
   m.AddDomainIntegrator(new MassIntegrator);
   m.Assemble();
   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<real_t>::min());
   m.Finalize();

   HypreParMatrix *A = a.ParallelAssemble();
@@ -694,7 +694,7 @@ void VectorWaveGuide(int mode, ParGridFunction &x)
   m.AddDomainIntegrator(new VectorFEMassIntegrator);
   m.Assemble();
   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<real_t>::min());
   m.Finalize();

   HypreParMatrix *A = a.ParallelAssemble();
@@ -37,8 +37,8 @@
 using namespace std;
 using namespace mfem;

-double spherical_obstacle(const Vector &pt);
-double exact_solution_obstacle(const Vector &pt);
+real_t spherical_obstacle(const Vector &pt);
+real_t exact_solution_obstacle(const Vector &pt);
 void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad);

 class LogarithmGridFunctionCoefficient : public Coefficient
@@ -46,14 +46,14 @@ class LogarithmGridFunctionCoefficient : public Coefficient
 protected:
   GridFunction *u; // grid function
   Coefficient *obstacle;
-   double min_val;
+   real_t min_val;

 public:
   LogarithmGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
-                                    double min_val_=-36)
+                                    real_t min_val_=-36)
      : u(&u_), obstacle(&obst_), min_val(min_val_) { }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 class ExponentialGridFunctionCoefficient : public Coefficient
@@ -61,15 +61,15 @@ class ExponentialGridFunctionCoefficient : public Coefficient
 protected:
   GridFunction *u;
   Coefficient *obstacle;
-   double min_val;
-   double max_val;
+   real_t min_val;
+   real_t max_val;

 public:
   ExponentialGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
-                                      double min_val_=0.0, double max_val_=1e6)
+                                      real_t min_val_=0.0, real_t max_val_=1e6)
      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 int main(int argc, char *argv[])
@@ -78,8 +78,8 @@ int main(int argc, char *argv[])
   int order = 1;
   int max_it = 10;
   int ref_levels = 3;
-   double alpha = 1.0;
-   double tol = 1e-5;
+   real_t alpha = 1.0;
+   real_t tol = 1e-5;
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -124,7 +124,7 @@ int main(int argc, char *argv[])

   // 3C. Rescale the domain to a unit circle (radius = 1).
   GridFunction *nodes = mesh.GetNodes();
-   double scale = 2*sqrt(2);
+   real_t scale = 2*sqrt(2);
   *nodes /= scale;

   // 4. Define the necessary finite element spaces on the mesh.
@@ -159,8 +159,8 @@ int main(int argc, char *argv[])
   // 6. Define an initial guess for the solution.
   auto IC_func = [](const Vector &x)
   {
-      double r0 = 1.0;
-      double rr = 0.0;
+      real_t r0 = 1.0;
+      real_t rr = 0.0;
      for (int i=0; i<x.Size(); i++)
      {
         rr += x(i)*x(i);
@@ -211,7 +211,7 @@ int main(int argc, char *argv[])
   // 10. Iterate
   int k;
   int total_iterations = 0;
-   double increment_u = 0.1;
+   real_t increment_u = 0.1;
   for (k = 0; k < max_it; k++)
   {
      GridFunction u_tmp(&H1fes);
@@ -300,10 +300,10 @@ int main(int argc, char *argv[])
         delta_psi_gf.MakeRef(&L2fes, x.GetBlock(1), 0);

         u_tmp -= u_gf;
-         double Newton_update_size = u_tmp.ComputeL2Error(zero);
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
         u_tmp = u_gf;

-         double gamma = 1.0;
+         real_t gamma = 1.0;
         delta_psi_gf *= gamma;
         psi_gf += delta_psi_gf;

@@ -337,7 +337,7 @@ int main(int argc, char *argv[])
         break;
      }

-      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      real_t H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
      mfem::out << "H1-error  (|| u - uₕᵏ||)       = " << H1_error << endl;

   }
@@ -362,13 +362,13 @@ int main(int argc, char *argv[])
   }

   {
-      double L2_error = u_gf.ComputeL2Error(exact_coef);
-      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      real_t L2_error = u_gf.ComputeL2Error(exact_coef);
+      real_t H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);

      ExponentialGridFunctionCoefficient u_alt_cf(psi_gf,obstacle);
      GridFunction u_alt_gf(&L2fes);
      u_alt_gf.ProjectCoefficient(u_alt_cf);
-      double L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);
+      real_t L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);

      mfem::out << "\n Final L2-error (|| u - uₕ||)          = " << L2_error <<
                endl;
@@ -380,35 +380,35 @@ int main(int argc, char *argv[])
   return 0;
 }

-double LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
+real_t LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
                                              const IntegrationPoint &ip)
 {
   MFEM_ASSERT(u != NULL, "grid function is not set");

-   double val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
+   real_t val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
   return max(min_val, log(val));
 }

-double ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
+real_t ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
                                                const IntegrationPoint &ip)
 {
   MFEM_ASSERT(u != NULL, "grid function is not set");

-   double val = u->GetValue(T, ip);
+   real_t val = u->GetValue(T, ip);
   return min(max_val, max(min_val, exp(val) + obstacle->Eval(T, ip)));
 }

-double spherical_obstacle(const Vector &pt)
+real_t spherical_obstacle(const Vector &pt)
 {
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double beta = 0.9;
+   real_t x = pt(0), y = pt(1);
+   real_t r = sqrt(x*x + y*y);
+   real_t r0 = 0.5;
+   real_t beta = 0.9;

-   double b = r0*beta;
-   double tmp = sqrt(r0*r0 - b*b);
-   double B = tmp + b*b/tmp;
-   double C = -b/tmp;
+   real_t b = r0*beta;
+   real_t tmp = sqrt(r0*r0 - b*b);
+   real_t B = tmp + b*b/tmp;
+   real_t C = -b/tmp;

   if (r > b)
   {
@@ -420,13 +420,13 @@ double spherical_obstacle(const Vector &pt)
   }
 }

-double exact_solution_obstacle(const Vector &pt)
+real_t exact_solution_obstacle(const Vector &pt)
 {
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double a =  0.348982574111686;
-   double A = -0.340129705945858;
+   real_t x = pt(0), y = pt(1);
+   real_t r = sqrt(x*x + y*y);
+   real_t r0 = 0.5;
+   real_t a =  0.348982574111686;
+   real_t A = -0.340129705945858;

   if (r > a)
   {
@@ -440,11 +440,11 @@ double exact_solution_obstacle(const Vector &pt)

 void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad)
 {
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double a =  0.348982574111686;
-   double A = -0.340129705945858;
+   real_t x = pt(0), y = pt(1);
+   real_t r = sqrt(x*x + y*y);
+   real_t r0 = 0.5;
+   real_t a =  0.348982574111686;
+   real_t A = -0.340129705945858;

   if (r > a)
   {
@@ -37,8 +37,8 @@
 using namespace std;
 using namespace mfem;

-double spherical_obstacle(const Vector &pt);
-double exact_solution_obstacle(const Vector &pt);
+real_t spherical_obstacle(const Vector &pt);
+real_t exact_solution_obstacle(const Vector &pt);
 void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad);

 class LogarithmGridFunctionCoefficient : public Coefficient
@@ -46,14 +46,14 @@ class LogarithmGridFunctionCoefficient : public Coefficient
 protected:
   GridFunction *u; // grid function
   Coefficient *obstacle;
-   double min_val;
+   real_t min_val;

 public:
   LogarithmGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
-                                    double min_val_=-36)
+                                    real_t min_val_=-36)
      : u(&u_), obstacle(&obst_), min_val(min_val_) { }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 class ExponentialGridFunctionCoefficient : public Coefficient
@@ -61,15 +61,15 @@ class ExponentialGridFunctionCoefficient : public Coefficient
 protected:
   GridFunction *u;
   Coefficient *obstacle;
-   double min_val;
-   double max_val;
+   real_t min_val;
+   real_t max_val;

 public:
   ExponentialGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
-                                      double min_val_=0.0, double max_val_=1e6)
+                                      real_t min_val_=0.0, real_t max_val_=1e6)
      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 int main(int argc, char *argv[])
@@ -84,8 +84,8 @@ int main(int argc, char *argv[])
   int order = 1;
   int max_it = 10;
   int ref_levels = 3;
-   double alpha = 1.0;
-   double tol = 1e-5;
+   real_t alpha = 1.0;
+   real_t tol = 1e-5;
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -136,7 +136,7 @@ int main(int argc, char *argv[])

   // 3C. Rescale the domain to a unit circle (radius = 1).
   GridFunction *nodes = mesh.GetNodes();
-   double scale = 2*sqrt(2);
+   real_t scale = 2*sqrt(2);
   *nodes /= scale;

   ParMesh pmesh(MPI_COMM_WORLD, mesh);
@@ -192,8 +192,8 @@ int main(int argc, char *argv[])
   // 6. Define an initial guess for the solution.
   auto IC_func = [](const Vector &x)
   {
-      double r0 = 1.0;
-      double rr = 0.0;
+      real_t r0 = 1.0;
+      real_t rr = 0.0;
      for (int i=0; i<x.Size(); i++)
      {
         rr += x(i)*x(i);
@@ -243,7 +243,7 @@ int main(int argc, char *argv[])
   // 10. Iterate
   int k;
   int total_iterations = 0;
-   double increment_u = 0.1;
+   real_t increment_u = 0.1;
   for (k = 0; k < max_it; k++)
   {
      ParGridFunction u_tmp(&H1fes);
@@ -346,10 +346,10 @@ int main(int argc, char *argv[])
         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(1));

         u_tmp -= u_gf;
-         double Newton_update_size = u_tmp.ComputeL2Error(zero);
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
         u_tmp = u_gf;

-         double gamma = 1.0;
+         real_t gamma = 1.0;
         delta_psi_gf *= gamma;
         psi_gf += delta_psi_gf;

@@ -391,7 +391,7 @@ int main(int argc, char *argv[])
         break;
      }

-      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      real_t H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
      if (myid == 0)
      {
         mfem::out << "H1-error  (|| u - uₕᵏ||)       = " << H1_error << endl;
@@ -423,13 +423,13 @@ int main(int argc, char *argv[])
   }

   {
-      double L2_error = u_gf.ComputeL2Error(exact_coef);
-      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      real_t L2_error = u_gf.ComputeL2Error(exact_coef);
+      real_t H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);

      ExponentialGridFunctionCoefficient u_alt_cf(psi_gf,obstacle);
      ParGridFunction u_alt_gf(&L2fes);
      u_alt_gf.ProjectCoefficient(u_alt_cf);
-      double L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);
+      real_t L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);

      if (myid == 0)
      {
@@ -444,35 +444,35 @@ int main(int argc, char *argv[])
   return 0;
 }

-double LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
+real_t LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
                                              const IntegrationPoint &ip)
 {
   MFEM_ASSERT(u != NULL, "grid function is not set");

-   double val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
+   real_t val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
   return max(min_val, log(val));
 }

-double ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
+real_t ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
                                                const IntegrationPoint &ip)
 {
   MFEM_ASSERT(u != NULL, "grid function is not set");

-   double val = u->GetValue(T, ip);
+   real_t val = u->GetValue(T, ip);
   return min(max_val, max(min_val, exp(val) + obstacle->Eval(T, ip)));
 }

-double spherical_obstacle(const Vector &pt)
+real_t spherical_obstacle(const Vector &pt)
 {
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double beta = 0.9;
+   real_t x = pt(0), y = pt(1);
+   real_t r = sqrt(x*x + y*y);
+   real_t r0 = 0.5;
+   real_t beta = 0.9;

-   double b = r0*beta;
-   double tmp = sqrt(r0*r0 - b*b);
-   double B = tmp + b*b/tmp;
-   double C = -b/tmp;
+   real_t b = r0*beta;
+   real_t tmp = sqrt(r0*r0 - b*b);
+   real_t B = tmp + b*b/tmp;
+   real_t C = -b/tmp;

   if (r > b)
   {
@@ -484,13 +484,13 @@ double spherical_obstacle(const Vector &pt)
   }
 }

-double exact_solution_obstacle(const Vector &pt)
+real_t exact_solution_obstacle(const Vector &pt)
 {
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double a =  0.348982574111686;
-   double A = -0.340129705945858;
+   real_t x = pt(0), y = pt(1);
+   real_t r = sqrt(x*x + y*y);
+   real_t r0 = 0.5;
+   real_t a =  0.348982574111686;
+   real_t A = -0.340129705945858;

   if (r > a)
   {
@@ -504,11 +504,11 @@ double exact_solution_obstacle(const Vector &pt)

 void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad)
 {
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double a =  0.348982574111686;
-   double A = -0.340129705945858;
+   real_t x = pt(0), y = pt(1);
+   real_t r = sqrt(x*x + y*y);
+   real_t r0 = 0.5;
+   real_t a =  0.348982574111686;
+   real_t A = -0.340129705945858;

   if (r > a)
   {
@@ -67,9 +67,9 @@ using namespace mfem;
 * @param target_volume θ vol(Ω)
 * @param tol Newton iteration tolerance
 * @param max_its Newton maximum iteration number
- * @return double Final volume, ∫_Ω sigmoid(ψ)
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
 */
-double proj(GridFunction &psi, double target_volume, double tol=1e-12,
+real_t proj(GridFunction &psi, real_t target_volume, real_t tol=1e-12,
            int max_its=10)
 {
   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
@@ -84,12 +84,12 @@ double proj(GridFunction &psi, double target_volume, double tol=1e-12,
   for (int k=0; k<max_its; k++) // Newton iteration
   {
      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
-      const double f = int_sigmoid_psi.Sum() - target_volume;
+      const real_t f = int_sigmoid_psi.Sum() - target_volume;

      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
-      const double df = int_der_sigmoid_psi.Sum();
+      const real_t df = int_der_sigmoid_psi.Sum();

-      const double dc = -f/df;
+      const real_t dc = -f/df;
      psi += dc;
      if (abs(dc) < tol) { done = true; break; }
   }
@@ -179,15 +179,15 @@ int main(int argc, char *argv[])
   // 1. Parse command-line options.
   int ref_levels = 5;
   int order = 2;
-   double alpha = 1.0;
-   double epsilon = 0.01;
-   double vol_fraction = 0.5;
+   real_t alpha = 1.0;
+   real_t epsilon = 0.01;
+   real_t vol_fraction = 0.5;
   int max_it = 1e3;
-   double itol = 1e-1;
-   double ntol = 1e-4;
-   double rho_min = 1e-6;
-   double lambda = 1.0;
-   double mu = 1.0;
+   real_t itol = 1e-1;
+   real_t ntol = 1e-4;
+   real_t rho_min = 1e-6;
+   real_t lambda = 1.0;
+   real_t mu = 1.0;
   bool glvis_visualization = true;
   bool paraview_output = false;

@@ -239,8 +239,8 @@ int main(int argc, char *argv[])
      Array<int> vertices;
      be->GetVertices(vertices);

-      double * coords1 = mesh.GetVertex(vertices[0]);
-      double * coords2 = mesh.GetVertex(vertices[1]);
+      real_t * coords1 = mesh.GetVertex(vertices[0]);
+      real_t * coords2 = mesh.GetVertex(vertices[1]);

      Vector center(2);
      center(0) = 0.5*(coords1[0] + coords2[0]);
@@ -312,7 +312,7 @@ int main(int argc, char *argv[])
   ElasticitySolver->SetupFEM();
   Vector center(2); center(0) = 2.9; center(1) = 0.5;
   Vector force(2); force(0) = 0.0; force(1) = -1.0;
-   double r = 0.05;
+   real_t r = 0.05;
   VolumeForceCoefficient vforce_cf(r,center,force);
   ElasticitySolver->SetRHSCoefficient(&vforce_cf);
   ElasticitySolver->SetEssentialBoundary(ess_bdr);
@@ -353,8 +353,8 @@ int main(int argc, char *argv[])
   LinearForm vol_form(&control_fes);
   vol_form.AddDomainIntegrator(new DomainLFIntegrator(one));
   vol_form.Assemble();
-   double domain_volume = vol_form(onegf);
-   const double target_volume = domain_volume * vol_fraction;
+   real_t domain_volume = vol_form(onegf);
+   const real_t target_volume = domain_volume * vol_fraction;

   // 10. Connect to GLVis. Prepare for VisIt output.
   char vishost[] = "localhost";
@@ -385,7 +385,7 @@ int main(int argc, char *argv[])
   // 11. Iterate:
   for (int k = 1; k <= max_it; k++)
   {
-      if (k > 1) { alpha *= ((double) k) / ((double) k-1); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }

      mfem::out << "\nStep = " << k << std::endl;

@@ -422,14 +422,14 @@ int main(int argc, char *argv[])

      // Step 5 - Update design variable ψ ← proj(ψ - αG)
      psi.Add(-alpha, grad);
-      const double material_volume = proj(psi, target_volume);
+      const real_t material_volume = proj(psi, target_volume);

      // Compute ||ρ - ρ_old|| in control fes.
-      double norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
-      double norm_reduced_gradient = norm_increment/alpha;
+      real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
+      real_t norm_reduced_gradient = norm_increment/alpha;
      psi_old = psi;

-      double compliance = (*(ElasticitySolver->GetLinearForm()))(u);
+      real_t compliance = (*(ElasticitySolver->GetLinearForm()))(u);
      mfem::out << "norm of the reduced gradient = " << norm_reduced_gradient <<
                std::endl;
      mfem::out << "norm of the increment = " << norm_increment << endl;
@@ -449,7 +449,7 @@ int main(int argc, char *argv[])
      {
         rho_gf.ProjectCoefficient(rho);
         paraview_dc.SetCycle(k);
-         paraview_dc.SetTime((double)k);
+         paraview_dc.SetTime((real_t)k);
         paraview_dc.Save();
      }

@@ -9,15 +9,15 @@ namespace mfem
 {

 /// @brief Inverse sigmoid function
-double inv_sigmoid(double x)
+real_t inv_sigmoid(real_t x)
 {
-   double tol = 1e-12;
-   x = std::min(std::max(tol,x),1.0-tol);
+   real_t tol = 1e-12;
+   x = std::min(std::max(tol,x), real_t(1.0)-tol);
   return std::log(x/(1.0-x));
 }

 /// @brief Sigmoid function
-double sigmoid(double x)
+real_t sigmoid(real_t x)
 {
   if (x >= 0)
   {
@@ -30,9 +30,9 @@ double sigmoid(double x)
 }

 /// @brief Derivative of sigmoid function
-double der_sigmoid(double x)
+real_t der_sigmoid(real_t x)
 {
-   double tmp = sigmoid(-x);
+   real_t tmp = sigmoid(-x);
   return tmp - std::pow(tmp,2);
 }

@@ -40,24 +40,24 @@ double der_sigmoid(double x)
 class MappedGridFunctionCoefficient : public GridFunctionCoefficient
 {
 protected:
-   std::function<double(const double)> fun; // f:R → R
+   std::function<real_t(const real_t)> fun; // f:R → R
 public:
   MappedGridFunctionCoefficient()
      :GridFunctionCoefficient(),
-       fun([](double x) {return x;}) {}
+       fun([](real_t x) {return x;}) {}
   MappedGridFunctionCoefficient(const GridFunction *gf,
-                                 std::function<double(const double)> fun_,
+                                 std::function<real_t(const real_t)> fun_,
                                 int comp=1)
      :GridFunctionCoefficient(gf, comp),
       fun(fun_) {}


-   virtual double Eval(ElementTransformation &T,
+   virtual real_t Eval(ElementTransformation &T,
                       const IntegrationPoint &ip)
   {
      return fun(GridFunctionCoefficient::Eval(T, ip));
   }
-   void SetFunction(std::function<double(const double)> fun_) { fun = fun_; }
+   void SetFunction(std::function<real_t(const real_t)> fun_) { fun = fun_; }
 };


@@ -67,30 +67,30 @@ class DiffMappedGridFunctionCoefficient : public GridFunctionCoefficient
 protected:
   const GridFunction *OtherGridF;
   GridFunctionCoefficient OtherGridF_cf;
-   std::function<double(const double)> fun; // f:R → R
+   std::function<real_t(const real_t)> fun; // f:R → R
 public:
   DiffMappedGridFunctionCoefficient()
      :GridFunctionCoefficient(),
       OtherGridF(nullptr),
       OtherGridF_cf(),
-       fun([](double x) {return x;}) {}
+       fun([](real_t x) {return x;}) {}
   DiffMappedGridFunctionCoefficient(const GridFunction *gf,
                                     const GridFunction *other_gf,
-                                     std::function<double(const double)> fun_,
+                                     std::function<real_t(const real_t)> fun_,
                                     int comp=1)
      :GridFunctionCoefficient(gf, comp),
       OtherGridF(other_gf),
       OtherGridF_cf(OtherGridF),
       fun(fun_) {}

-   virtual double Eval(ElementTransformation &T,
+   virtual real_t Eval(ElementTransformation &T,
                       const IntegrationPoint &ip)
   {
-      const double value1 = fun(GridFunctionCoefficient::Eval(T, ip));
-      const double value2 = fun(OtherGridF_cf.Eval(T, ip));
+      const real_t value1 = fun(GridFunctionCoefficient::Eval(T, ip));
+      const real_t value2 = fun(OtherGridF_cf.Eval(T, ip));
      return value1 - value2;
   }
-   void SetFunction(std::function<double(const double)> fun_) { fun = fun_; }
+   void SetFunction(std::function<real_t(const real_t)> fun_) { fun = fun_; }
 };

 /// @brief Solid isotropic material penalization (SIMP) coefficient
@@ -98,20 +98,20 @@ class SIMPInterpolationCoefficient : public Coefficient
 {
 protected:
   GridFunction *rho_filter;
-   double min_val;
-   double max_val;
-   double exponent;
+   real_t min_val;
+   real_t max_val;
+   real_t exponent;

 public:
-   SIMPInterpolationCoefficient(GridFunction *rho_filter_, double min_val_= 1e-6,
-                                double max_val_ = 1.0, double exponent_ = 3)
+   SIMPInterpolationCoefficient(GridFunction *rho_filter_, real_t min_val_= 1e-6,
+                                real_t max_val_ = 1.0, real_t exponent_ = 3)
      : rho_filter(rho_filter_), min_val(min_val_), max_val(max_val_),
        exponent(exponent_) { }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip)
   {
-      double val = rho_filter->GetValue(T, ip);
-      double coeff = min_val + pow(val,exponent)*(max_val-min_val);
+      real_t val = rho_filter->GetValue(T, ip);
+      real_t coeff = min_val + pow(val,exponent)*(max_val-min_val);
      return coeff;
   }
 };
@@ -126,13 +126,13 @@ protected:
   GridFunction *u = nullptr; // displacement
   GridFunction *rho_filter = nullptr; // filter density
   DenseMatrix grad; // auxiliary matrix, used in Eval
-   double exponent;
-   double rho_min;
+   real_t exponent;
+   real_t rho_min;

 public:
   StrainEnergyDensityCoefficient(Coefficient *lambda_, Coefficient *mu_,
-                                  GridFunction * u_, GridFunction * rho_filter_, double rho_min_=1e-6,
-                                  double exponent_ = 3.0)
+                                  GridFunction * u_, GridFunction * rho_filter_, real_t rho_min_=1e-6,
+                                  real_t exponent_ = 3.0)
      : lambda(lambda_), mu(mu_),  u(u_), rho_filter(rho_filter_),
        exponent(exponent_), rho_min(rho_min_)
   {
@@ -142,13 +142,13 @@ public:
      MFEM_ASSERT(rho_filter, "density field is not set");
   }

-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip)
   {
-      double L = lambda->Eval(T, ip);
-      double M = mu->Eval(T, ip);
+      real_t L = lambda->Eval(T, ip);
+      real_t M = mu->Eval(T, ip);
      u->GetVectorGradient(T, grad);
-      double div_u = grad.Trace();
-      double density = L*div_u*div_u;
+      real_t div_u = grad.Trace();
+      real_t density = L*div_u*div_u;
      int dim = T.GetSpaceDim();
      for (int i=0; i<dim; i++)
      {
@@ -157,7 +157,7 @@ public:
            density += M*grad(i,j)*(grad(i,j)+grad(j,i));
         }
      }
-      double val = rho_filter->GetValue(T,ip);
+      real_t val = rho_filter->GetValue(T,ip);

      return -exponent * pow(val, exponent-1.0) * (1-rho_min) * density;
   }
@@ -167,11 +167,11 @@ public:
 class VolumeForceCoefficient : public VectorCoefficient
 {
 private:
-   double r;
+   real_t r;
   Vector center;
   Vector force;
 public:
-   VolumeForceCoefficient(double r_,Vector &  center_, Vector & force_) :
+   VolumeForceCoefficient(real_t r_,Vector &  center_, Vector & force_) :
      VectorCoefficient(center_.Size()), r(r_), center(center_), force(force_) { }

   using VectorCoefficient::Eval;
@@ -186,7 +186,7 @@ public:
         xx[i]=xx[i]-center[i];
      }

-      double cr=xx.Norml2();
+      real_t cr=xx.Norml2();
      V.SetSize(T.GetDimension());
      if (cr <= r)
      {
@@ -198,7 +198,7 @@ public:
      }
   }

-   void Set(double r_,Vector & center_, Vector & force_)
+   void Set(real_t r_,Vector & center_, Vector & force_)
   {
      r=r_;
      center = center_;
@@ -66,9 +66,9 @@ using namespace mfem;
 * @param target_volume θ vol(Ω)
 * @param tol Newton iteration tolerance
 * @param max_its Newton maximum iteration number
- * @return double Final volume, ∫_Ω sigmoid(ψ)
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
 */
-double proj(ParGridFunction &psi, double target_volume, double tol=1e-12,
+real_t proj(ParGridFunction &psi, real_t target_volume, real_t tol=1e-12,
            int max_its=10)
 {
   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
@@ -83,15 +83,17 @@ double proj(ParGridFunction &psi, double target_volume, double tol=1e-12,
   for (int k=0; k<max_its; k++) // Newton iteration
   {
      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
-      double f = int_sigmoid_psi.Sum();
-      MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+      real_t f = int_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
      f -= target_volume;

      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
-      double df = int_der_sigmoid_psi.Sum();
-      MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+      real_t df = int_der_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);

-      const double dc = -f/df;
+      const real_t dc = -f/df;
      psi += dc;
      if (abs(dc) < tol) { done = true; break; }
   }
@@ -101,9 +103,9 @@ double proj(ParGridFunction &psi, double target_volume, double tol=1e-12,
                   "Result may not be accurate.");
   }
   int_sigmoid_psi.Assemble();
-   double material_volume = int_sigmoid_psi.Sum();
-   MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1, MPI_DOUBLE, MPI_SUM,
-                 MPI_COMM_WORLD);
+   real_t material_volume = int_sigmoid_psi.Sum();
+   MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
+                 MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
   return material_volume;
 }

@@ -190,15 +192,15 @@ int main(int argc, char *argv[])
   // 1. Parse command-line options.
   int ref_levels = 5;
   int order = 2;
-   double alpha = 1.0;
-   double epsilon = 0.01;
-   double vol_fraction = 0.5;
+   real_t alpha = 1.0;
+   real_t epsilon = 0.01;
+   real_t vol_fraction = 0.5;
   int max_it = 1e3;
-   double itol = 1e-1;
-   double ntol = 1e-4;
-   double rho_min = 1e-6;
-   double lambda = 1.0;
-   double mu = 1.0;
+   real_t itol = 1e-1;
+   real_t ntol = 1e-4;
+   real_t rho_min = 1e-6;
+   real_t lambda = 1.0;
+   real_t mu = 1.0;
   bool glvis_visualization = true;
   bool paraview_output = false;

@@ -258,8 +260,8 @@ int main(int argc, char *argv[])
      Array<int> vertices;
      be->GetVertices(vertices);

-      double * coords1 = mesh.GetVertex(vertices[0]);
-      double * coords2 = mesh.GetVertex(vertices[1]);
+      real_t * coords1 = mesh.GetVertex(vertices[0]);
+      real_t * coords2 = mesh.GetVertex(vertices[1]);

      Vector center(2);
      center(0) = 0.5*(coords1[0] + coords2[0]);
@@ -337,7 +339,7 @@ int main(int argc, char *argv[])
   ElasticitySolver->SetupFEM();
   Vector center(2); center(0) = 2.9; center(1) = 0.5;
   Vector force(2); force(0) = 0.0; force(1) = -1.0;
-   double r = 0.05;
+   real_t r = 0.05;
   VolumeForceCoefficient vforce_cf(r,center,force);
   ElasticitySolver->SetRHSCoefficient(&vforce_cf);
   ElasticitySolver->SetEssentialBoundary(ess_bdr);
@@ -378,8 +380,8 @@ int main(int argc, char *argv[])
   ParLinearForm vol_form(&control_fes);
   vol_form.AddDomainIntegrator(new DomainLFIntegrator(one));
   vol_form.Assemble();
-   double domain_volume = vol_form(onegf);
-   const double target_volume = domain_volume * vol_fraction;
+   real_t domain_volume = vol_form(onegf);
+   const real_t target_volume = domain_volume * vol_fraction;

   // 10. Connect to GLVis. Prepare for VisIt output.
   char vishost[] = "localhost";
@@ -410,7 +412,7 @@ int main(int argc, char *argv[])
   // 11. Iterate:
   for (int k = 1; k <= max_it; k++)
   {
-      if (k > 1) { alpha *= ((double) k) / ((double) k-1); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }

      if (myid == 0)
      {
@@ -450,15 +452,16 @@ int main(int argc, char *argv[])

      // Step 5 - Update design variable ψ ← proj(ψ - αG)
      psi.Add(-alpha, grad);
-      const double material_volume = proj(psi, target_volume);
+      const real_t material_volume = proj(psi, target_volume);

      // Compute ||ρ - ρ_old|| in control fes.
-      double norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
-      double norm_reduced_gradient = norm_increment/alpha;
+      real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
+      real_t norm_reduced_gradient = norm_increment/alpha;
      psi_old = psi;

-      double compliance = (*(ElasticitySolver->GetLinearForm()))(u);
-      MPI_Allreduce(MPI_IN_PLACE,&compliance,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+      real_t compliance = (*(ElasticitySolver->GetLinearForm()))(u);
+      MPI_Allreduce(MPI_IN_PLACE, &compliance, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
      if (myid == 0)
      {
         mfem::out << "norm of the reduced gradient = " << norm_reduced_gradient << endl;
@@ -480,7 +483,7 @@ int main(int argc, char *argv[])
      {
         rho_gf.ProjectCoefficient(rho);
         paraview_dc.SetCycle(k);
-         paraview_dc.SetTime((double)k);
+         paraview_dc.SetTime((real_t)k);
         paraview_dc.Save();
      }

@@ -46,7 +46,7 @@ enum class IntegrationType { Volumetric1D, Surface2D, Volumetric2D,
 IntegrationType itype;

 /// @brief Level-set function defining the implicit interface
-double lvlset(const Vector& X)
+real_t lvlset(const Vector& X)
 {
   switch (itype)
   {
@@ -66,7 +66,7 @@ double lvlset(const Vector& X)
 }

 /// @brief Function that should be integrated
-double integrand(const Vector& X)
+real_t integrand(const Vector& X)
 {
   switch (itype)
   {
@@ -86,7 +86,7 @@ double integrand(const Vector& X)
 }

 /// @brief Analytic surface integral
-double Surface()
+real_t Surface()
 {
   switch (itype)
   {
@@ -106,7 +106,7 @@ double Surface()
 }

 /// @brief Analytic volume integral over subdomain with positive level-set
-double Volume()
+real_t Volume()
 {
   switch (itype)
   {
@@ -199,7 +199,6 @@ public:
      {
         mesh->GetElementTransformation(elem, &Tr);
         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
-         Vector w;
         MFIRs.GetSurfaceWeights(Tr, ir, w);
         SurfaceWeights.SetCol(elem, w);

@@ -424,7 +423,7 @@ public:
      for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
      {
         Tr.SetIntPoint((&(SIntRule->IntPoint(ip))));
-         double val = Tr.Weight() * Q.Eval(Tr, SIntRule->IntPoint(ip));
+         real_t val = Tr.Weight() * Q.Eval(Tr, SIntRule->IntPoint(ip));
         el.CalcShape(SIntRule->IntPoint(ip), shape);
         add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
      }
@@ -492,7 +491,7 @@ public:
      for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
      {
         Tr.SetIntPoint((&(CIntRule->IntPoint(ip))));
-         double val = Tr.Weight()
+         real_t val = Tr.Weight()
                      * Q.Eval(Tr, CIntRule->IntPoint(ip));
         el.CalcPhysShape(Tr, shape);
         add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
@@ -504,8 +503,8 @@ public:
 int main(int argc, char *argv[])
 {
 #ifndef MFEM_USE_LAPACK
-   cout << "MFEM must be build with LAPACK for this example." << endl;
-   return EXIT_FAILURE;
+   cout << "MFEM must be built with LAPACK for this example." << endl;
+   return MFEM_SKIP_RETURN_VALUE;
 #else
   // 1. Parse he command-line options.
   int ref_levels = 3;
@@ -636,11 +635,11 @@ int main(int argc, char *argv[])
   cout << "Mesh size dx:                       ";
   if (itype != IntegrationType::Volumetric1D)
   {
-      cout << 3.2 / pow(2., (double)ref_levels) << endl;
+      cout << 3.2 / pow(2., (real_t)ref_levels) << endl;
   }
   else
   {
-      cout << .25 / pow(2., (double)ref_levels) << endl;
+      cout << .25 / pow(2., (real_t)ref_levels) << endl;
   }
   if (itype == IntegrationType::Surface2D
       || itype == IntegrationType::Volumetric2D)
@@ -652,7 +651,7 @@ int main(int argc, char *argv[])
   cout << "============================================" << endl;
   cout << "Computed value of surface integral: " << surface.Sum() << endl;
   cout << "True value of surface integral:     " << Surface() << endl;
-   cout << "Absolut Error (Surface):            ";
+   cout << "Absolute Error (Surface):            ";
   cout << abs(surface.Sum() - Surface()) << endl;
   cout << "Relative Error (Surface):           ";
   cout << abs(surface.Sum() - Surface()) / Surface() << endl;
@@ -663,7 +662,7 @@ int main(int argc, char *argv[])
      cout << "--------------------------------------------" << endl;
      cout << "Computed value of volume integral:  " << volume.Sum() << endl;
      cout << "True value of volume integral:      " << Volume() << endl;
-      cout << "Absolut Error (Volume):             ";
+      cout << "Absolute Error (Volume):             ";
      cout << abs(volume.Sum() - Volume()) << endl;
      cout << "Relative Error (Volume):            ";
      cout << abs(volume.Sum() - Volume()) / Volume() << endl;
@@ -0,0 +1,285 @@
+//                                MFEM Example 39
+//
+// Compile with: make ex39
+//
+// Sample runs:  ex39
+//               ex39 -ess "Southern Boundary"
+//               ex39 -src Base
+//
+// Description:  This example code demonstrates the use of named attribute
+//               sets in MFEM to specify material regions, boundary regions,
+//               or source regions by name rather than attribute numbers. It
+//               also demonstrates how new named attribute sets may be created
+//               from arbitrary groupings of attribute numbers and used as a
+//               convenient shorthand to refer to those groupings in other
+//               portions of the application or through the command line.
+//
+//               The particular problem being solved here is nearly the same
+//               as that in example 1 i.e. a simple finite element
+//               discretization of the Laplace problem -Delta u = 1 with
+//               homogeneous Dirichlet boundary conditions and, in this case,
+//               an inhomogeneous diffusion coefficient. The diffusion
+//               coefficient is given a small default value throughout the
+//               domain which is increased by two separate amounts in two named
+//               regions.
+//
+//               This example makes use of a specific input mesh, "compass.msh",
+//               containing named domain and boundary regions generated by Gmsh
+//               and stored in their "msh" format (version 2.2). This file
+//               defines eight boundary regions corresponding to eight compass
+//               headings; "ENE", "NNE", "NNW", "WSW", "SSW", "SSE", and "ESE".
+//               It also defines nine domain regions; "Base", "N Even", "N Odd",
+//               "W Even", "W Odd", "S Even", "S Odd", "E Even", and "E Odd".
+//               These regions split the four compass pointers into two halves
+//               each and also label the remaining elements as "Base". Starting
+//               with these named regions we test the construction of named
+//               sets as well as reading and writing these named groupings from
+//               and to mesh files.
+//
+//               The example highlights the use of named attribute sets for
+//               both subdomains and boundaries in different contexts as well
+//               as basic methods to create named sets from existing attributes.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/compass.msh";
+   int order = 1;
+   string source_name = "Rose Even";
+   string ess_name = "Boundary";
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&source_name,"-src","--source-attr-name",
+                  "Name of attribute set containing source.");
+   args.AddOption(&ess_name,"-ess","--ess-attr-name",
+                  "Name of attribute set containing essential BC.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.ParseCheck();
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 50,000
+   //    elements.
+   {
+      int ref_levels =
+         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   // 4a. Display attribute set names contained in the initial mesh
+   AttributeSets &attr_sets = mesh.attribute_sets;
+   AttributeSets &bdr_attr_sets = mesh.bdr_attribute_sets;
+   {
+      std::set<string> names = attr_sets.GetAttributeSetNames();
+      cout << "Element Attribute Set Names: ";
+      for (auto const &set_name : names)
+      {
+         cout << " \"" << set_name << "\"";
+      }
+      cout << endl;
+
+      std::set<string> bdr_names = bdr_attr_sets.GetAttributeSetNames();
+      cout << "Boundary Attribute Set Names: ";
+      for (auto const &bdr_set_name : bdr_names)
+      {
+         cout << " \"" << bdr_set_name << "\"";
+      }
+      cout << endl;
+   }
+
+   // 4b. Define new regions based on existing attribute sets
+   {
+      Array<int> & Na = attr_sets.GetAttributeSet("N Even");
+      Array<int> & Nb = attr_sets.GetAttributeSet("N Odd");
+      Array<int> & Sa = attr_sets.GetAttributeSet("S Even");
+      Array<int> & Sb = attr_sets.GetAttributeSet("S Odd");
+      Array<int> & Ea = attr_sets.GetAttributeSet("E Even");
+      Array<int> & Eb = attr_sets.GetAttributeSet("E Odd");
+      Array<int> & Wa = attr_sets.GetAttributeSet("W Even");
+      Array<int> & Wb = attr_sets.GetAttributeSet("W Odd");
+
+      // Create a new set spanning the North point
+      attr_sets.SetAttributeSet("North", Na);
+      attr_sets.AddToAttributeSet("North", Nb);
+
+      // Create a new set spanning the South point
+      attr_sets.SetAttributeSet("South", Sa);
+      attr_sets.AddToAttributeSet("South", Sb);
+
+      // Create a new set spanning the East point
+      attr_sets.SetAttributeSet("East", Ea);
+      attr_sets.AddToAttributeSet("East", Eb);
+
+      // Create a new set spanning the West point
+      attr_sets.SetAttributeSet("West", Wa);
+      attr_sets.AddToAttributeSet("West", Wb);
+
+      // Create a new set consisting of the "a" sides of the compass rose
+      attr_sets.SetAttributeSet("Rose Even", Na);
+      attr_sets.AddToAttributeSet("Rose Even", Sa);
+      attr_sets.AddToAttributeSet("Rose Even", Ea);
+      attr_sets.AddToAttributeSet("Rose Even", Wa);
+
+      // Create a new set consisting of the "b" sides of the compass rose
+      attr_sets.SetAttributeSet("Rose Odd", Nb);
+      attr_sets.AddToAttributeSet("Rose Odd", Sb);
+      attr_sets.AddToAttributeSet("Rose Odd", Eb);
+      attr_sets.AddToAttributeSet("Rose Odd", Wb);
+
+      // Create a new set consisting of the full compass rose
+      Array<int> & Ra = attr_sets.GetAttributeSet("Rose Even");
+      Array<int> & Rb = attr_sets.GetAttributeSet("Rose Odd");
+      attr_sets.SetAttributeSet("Rose", Ra);
+      attr_sets.AddToAttributeSet("Rose", Rb);
+   }
+   // 4c. Define new boundary regions based on existing boundary attribute sets
+   {
+      Array<int> & NNE = bdr_attr_sets.GetAttributeSet("NNE");
+      Array<int> & NNW = bdr_attr_sets.GetAttributeSet("NNW");
+      Array<int> & ENE = bdr_attr_sets.GetAttributeSet("ENE");
+      Array<int> & ESE = bdr_attr_sets.GetAttributeSet("ESE");
+      Array<int> & SSE = bdr_attr_sets.GetAttributeSet("SSE");
+      Array<int> & SSW = bdr_attr_sets.GetAttributeSet("SSW");
+      Array<int> & WNW = bdr_attr_sets.GetAttributeSet("WNW");
+      Array<int> & WSW = bdr_attr_sets.GetAttributeSet("WSW");
+
+      bdr_attr_sets.SetAttributeSet("Northern Boundary", NNE);
+      bdr_attr_sets.AddToAttributeSet("Northern Boundary", NNW);
+
+      bdr_attr_sets.SetAttributeSet("Southern Boundary", SSE);
+      bdr_attr_sets.AddToAttributeSet("Southern Boundary", SSW);
+
+      bdr_attr_sets.SetAttributeSet("Eastern Boundary", ENE);
+      bdr_attr_sets.AddToAttributeSet("Eastern Boundary", ESE);
+
+      bdr_attr_sets.SetAttributeSet("Western Boundary", WNW);
+      bdr_attr_sets.AddToAttributeSet("Western Boundary", WSW);
+
+      bdr_attr_sets.SetAttributeSet("Boundary",
+                                    bdr_attr_sets.GetAttributeSet
+                                    ("Northern Boundary"));
+      bdr_attr_sets.AddToAttributeSet("Boundary",
+                                      bdr_attr_sets.GetAttributeSet
+                                      ("Southern Boundary"));
+      bdr_attr_sets.AddToAttributeSet("Boundary",
+                                      bdr_attr_sets.GetAttributeSet
+                                      ("Eastern Boundary"));
+      bdr_attr_sets.AddToAttributeSet("Boundary",
+                                      bdr_attr_sets.GetAttributeSet
+                                      ("Western Boundary"));
+   }
+
+   // 5. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order.
+   H1_FECollection fec(order, mesh.Dimension());
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of finite element unknowns: "
+        << fespace.GetTrueVSize() << endl;
+
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary regions corresponding to the boundary attributes
+   //    contained in the set named "ess_name" as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (bdr_attr_sets.AttributeSetExists(ess_name))
+   {
+      Array<int> ess_bdr_marker = bdr_attr_sets.GetAttributeSetMarker(ess_name);
+      fespace.GetEssentialTrueDofs(ess_bdr_marker, ess_tdof_list);
+   }
+
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1_s,phi_i) where phi_i
+   //    are the basis functions in fespace and 1_s is an indicator function
+   //    equal to 1 on the region defined by the named set "source_name" and
+   //    zero elsewhere.
+   Array<int> source_marker = attr_sets.GetAttributeSetMarker(source_name);
+
+   LinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one), source_marker);
+   b.Assemble();
+
+   // 8. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(&fespace);
+   x = 0.0;
+
+   // 9. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the
+   //    Diffusion domain integrator.
+   BilinearForm a(&fespace);
+
+   ConstantCoefficient defaultCoef(1.0e-6);
+   ConstantCoefficient baseCoef(1.0);
+   ConstantCoefficient roseCoef(2.0);
+
+   Array<int> base_marker = attr_sets.GetAttributeSetMarker("Base");
+   Array<int> rose_marker = attr_sets.GetAttributeSetMarker("Rose Even");
+
+   // Impose a very small diffusion coefficient across the entire mesh
+   a.AddDomainIntegrator(new DiffusionIntegrator(defaultCoef));
+
+   // Impose an additional, stronger diffusion coefficient in select regions
+   a.AddDomainIntegrator(new DiffusionIntegrator(baseCoef), base_marker);
+   a.AddDomainIntegrator(new DiffusionIntegrator(roseCoef), rose_marker);
+
+   // 10. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations.
+   a.Assemble();
+
+   SparseMatrix A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   cout << "Size of linear system: " << A.Height() << endl;
+
+   // 11. Solve the system using PCG with symmetric Gauss-Seidel preconditioner.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 800, 1e-12, 0.0);
+
+   // 12. Recover the solution as a finite element grid function.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 13. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
+   mesh.Save("refined.mesh");
+   x.Save("sol.gf");
+
+   // 14. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << x << "keys Rjmm" << flush;
+   }
+
+   return 0;
+}
@@ -0,0 +1,314 @@
+//                       MFEM Example 39 - Parallel Version
+//
+// Compile with: make ex39p
+//
+// Sample runs:  mpirun -np 4 ex39p
+//               mpirun -np 4 ex39p -ess "Southern Boundary"
+//               mpirun -np 4 ex39p -src Base
+//
+// Description:  This example code demonstrates the use of named attribute
+//               sets in MFEM to specify material regions, boundary regions,
+//               or source regions by name rather than attribute numbers. It
+//               also demonstrates how new named attribute sets may be created
+//               from arbitrary groupings of attribute numbers and used as a
+//               convenient shorthand to refer to those groupings in other
+//               portions of the application or through the command line.
+//
+//               The particular problem being solved here is nearly the same
+//               as that in example 1 i.e. a simple finite element
+//               discretization of the Laplace problem -Delta u = 1 with
+//               homogeneous Dirichlet boundary conditions and, in this case,
+//               an inhomogeneous diffusion coefficient. The diffusion
+//               coefficient is given a small default value throughout the
+//               domain which is increased by two separate amounts in two named
+//               regions.
+//
+//               This example makes use of a specific input mesh, "compass.msh",
+//               containing named domain and boundary regions generated by Gmsh
+//               and stored in their "msh" format (version 2.2). This file
+//               defines eight boundary regions corresponding to eight compass
+//               headings; "ENE", "NNE", "NNW", "WSW", "SSW", "SSE", and "ESE".
+//               It also defines nine domain regions; "Base", "N Even", "N Odd",
+//               "W Even", "W Odd", "S Even", "S Odd", "E Even", and "E Odd".
+//               These regions split the four compass pointers into two halves
+//               each and also label the remaining elements as "Base". Starting
+//               with these named regions we test the construction of named
+//               sets as well as reading and writing these named groupings from
+//               and to mesh files.
+//
+//               The example highlights the use of named attribute sets for
+//               both subdomains and boundaries in different contexts as well
+//               as basic methods to create named sets from existing attributes.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/compass.msh";
+   int order = 1;
+   string source_name = "Rose Even";
+   string ess_name = "Boundary";
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&source_name,"-src","--source-attr-name",
+                  "Name of attribute set containing source.");
+   args.AddOption(&ess_name,"-ess","--ess-attr-name",
+                  "Name of attribute set containing essential BC.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.ParseCheck();
+
+   // 3. Read the serial mesh from the given mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh.UniformRefinement();
+      }
+   }
+
+   // 6a. Display attribute set names contained in the initial mesh
+   AttributeSets &attr_sets = pmesh.attribute_sets;
+   AttributeSets &bdr_attr_sets = pmesh.bdr_attribute_sets;
+   if (Mpi::Root())
+   {
+      std::set<string> names = attr_sets.GetAttributeSetNames();
+      cout << "Element Attribute Set Names: ";
+      for (auto const &set_name : names)
+      {
+         cout << " \"" << set_name << "\"";
+      }
+      cout << endl;
+
+      std::set<string> bdr_names = bdr_attr_sets.GetAttributeSetNames();
+      cout << "Boundary Attribute Set Names: ";
+      for (auto const &bdr_set_name : bdr_names)
+      {
+         cout << " \"" << bdr_set_name << "\"";
+      }
+      cout << endl;
+   }
+
+   // 6b. Define new regions based on existing attribute sets
+   {
+      Array<int> & Na = attr_sets.GetAttributeSet("N Even");
+      Array<int> & Nb = attr_sets.GetAttributeSet("N Odd");
+      Array<int> & Sa = attr_sets.GetAttributeSet("S Even");
+      Array<int> & Sb = attr_sets.GetAttributeSet("S Odd");
+      Array<int> & Ea = attr_sets.GetAttributeSet("E Even");
+      Array<int> & Eb = attr_sets.GetAttributeSet("E Odd");
+      Array<int> & Wa = attr_sets.GetAttributeSet("W Even");
+      Array<int> & Wb = attr_sets.GetAttributeSet("W Odd");
+
+      // Create a new set spanning the North point
+      attr_sets.SetAttributeSet("North", Na);
+      attr_sets.AddToAttributeSet("North", Nb);
+
+      // Create a new set spanning the South point
+      attr_sets.SetAttributeSet("South", Sa);
+      attr_sets.AddToAttributeSet("South", Sb);
+
+      // Create a new set spanning the East point
+      attr_sets.SetAttributeSet("East", Ea);
+      attr_sets.AddToAttributeSet("East", Eb);
+
+      // Create a new set spanning the West point
+      attr_sets.SetAttributeSet("West", Wa);
+      attr_sets.AddToAttributeSet("West", Wb);
+
+      // Create a new set consisting of the "a" sides of the compass rose
+      attr_sets.SetAttributeSet("Rose Even", Na);
+      attr_sets.AddToAttributeSet("Rose Even", Sa);
+      attr_sets.AddToAttributeSet("Rose Even", Ea);
+      attr_sets.AddToAttributeSet("Rose Even", Wa);
+
+      // Create a new set consisting of the "b" sides of the compass rose
+      attr_sets.SetAttributeSet("Rose Odd", Nb);
+      attr_sets.AddToAttributeSet("Rose Odd", Sb);
+      attr_sets.AddToAttributeSet("Rose Odd", Eb);
+      attr_sets.AddToAttributeSet("Rose Odd", Wb);
+
+
+      // Create a new set consisting of the full compass rose
+      Array<int> & Ra = attr_sets.GetAttributeSet("Rose Even");
+      Array<int> & Rb = attr_sets.GetAttributeSet("Rose Odd");
+      attr_sets.SetAttributeSet("Rose", Ra);
+      attr_sets.AddToAttributeSet("Rose", Rb);
+   }
+   // 6c. Define new boundary regions based on existing boundary attribute sets
+   {
+      Array<int> & NNE = bdr_attr_sets.GetAttributeSet("NNE");
+      Array<int> & NNW = bdr_attr_sets.GetAttributeSet("NNW");
+      Array<int> & ENE = bdr_attr_sets.GetAttributeSet("ENE");
+      Array<int> & ESE = bdr_attr_sets.GetAttributeSet("ESE");
+      Array<int> & SSE = bdr_attr_sets.GetAttributeSet("SSE");
+      Array<int> & SSW = bdr_attr_sets.GetAttributeSet("SSW");
+      Array<int> & WNW = bdr_attr_sets.GetAttributeSet("WNW");
+      Array<int> & WSW = bdr_attr_sets.GetAttributeSet("WSW");
+
+      bdr_attr_sets.SetAttributeSet("Northern Boundary", NNE);
+      bdr_attr_sets.AddToAttributeSet("Northern Boundary", NNW);
+
+      bdr_attr_sets.SetAttributeSet("Southern Boundary", SSE);
+      bdr_attr_sets.AddToAttributeSet("Southern Boundary", SSW);
+
+      bdr_attr_sets.SetAttributeSet("Eastern Boundary", ENE);
+      bdr_attr_sets.AddToAttributeSet("Eastern Boundary", ESE);
+
+      bdr_attr_sets.SetAttributeSet("Western Boundary", WNW);
+      bdr_attr_sets.AddToAttributeSet("Western Boundary", WSW);
+
+      bdr_attr_sets.SetAttributeSet("Boundary",
+                                    bdr_attr_sets.GetAttributeSet
+                                    ("Northern Boundary"));
+      bdr_attr_sets.AddToAttributeSet("Boundary",
+                                      bdr_attr_sets.GetAttributeSet
+                                      ("Southern Boundary"));
+      bdr_attr_sets.AddToAttributeSet("Boundary",
+                                      bdr_attr_sets.GetAttributeSet
+                                      ("Eastern Boundary"));
+      bdr_attr_sets.AddToAttributeSet("Boundary",
+                                      bdr_attr_sets.GetAttributeSet
+                                      ("Western Boundary"));
+   }
+
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (Mpi::Root())
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary regions corresponding to the boundary
+   //    attributes contained in the set named "ess_name" as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (bdr_attr_sets.AttributeSetExists(ess_name))
+   {
+      Array<int> ess_bdr_marker = bdr_attr_sets.GetAttributeSetMarker(ess_name);
+      fespace.GetEssentialTrueDofs(ess_bdr_marker, ess_tdof_list);
+   }
+
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1_s,phi_i) where phi_i are the basis functions in fespace and 1_s
+   //    is an indicator function equal to 1 on the region defined by the
+   //    named set "source_name" and zero elsewhere.
+   Array<int> source_marker = attr_sets.GetAttributeSetMarker(source_name);
+
+   ParLinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one), source_marker);
+   b.Assemble();
+
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   ParBilinearForm a(&fespace);
+
+   ConstantCoefficient defaultCoef(1.0e-6);
+   ConstantCoefficient baseCoef(1.0);
+   ConstantCoefficient roseCoef(2.0);
+
+   Array<int> base_marker = attr_sets.GetAttributeSetMarker("Base");
+   Array<int> rose_marker = attr_sets.GetAttributeSetMarker("Rose Even");
+
+   // Impose a very small diffusion coefficient across the entire mesh
+   a.AddDomainIntegrator(new DiffusionIntegrator(defaultCoef));
+
+   // Impose an additional, stronger diffusion coefficient in select regions
+   a.AddDomainIntegrator(new DiffusionIntegrator(baseCoef), base_marker);
+   a.AddDomainIntegrator(new DiffusionIntegrator(roseCoef), rose_marker);
+
+   // 12. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations.
+   a.Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // 13. Solve the system using PCG with hypre's BoomerAMG preconditioner.
+   HypreBoomerAMG M(A);
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(A);
+   cg.Mult(B, X);
+
+   // 14. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   pmesh.Save("mesh");
+   x.Save("sol");
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      int  num_procs = Mpi::WorldSize();
+      int  myid      = Mpi::WorldRank();
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x << "keys Rjmm" << flush;
+   }
+
+   return 0;
+}
@@ -55,7 +55,7 @@ using namespace mfem;
 // Exact solution, E, and r.h.s., f. See below for implementation.
 void E_exact(const Vector &, Vector &);
 void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;
 int dim;

 int main(int argc, char *argv[])
@@ -263,7 +263,7 @@ int main(int argc, char *argv[])

   // 15. Compute and print the L^2 norm of the error.
   {
-      double error = x.ComputeL2Error(E);
+      real_t error = x.ComputeL2Error(E);
      if (myid == 0)
      {
         cout << "\n|| E_h - E ||_{L^2} = " << error << '\n' << endl;
@@ -54,7 +54,7 @@ using namespace mfem;
 // Exact solution, F, and r.h.s., f. See below for implementation.
 void F_exact(const Vector &, Vector &);
 void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -269,9 +269,9 @@ void F_exact(const Vector &p, Vector &F)
 {
   int dim = p.Size();

-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+   real_t x = p(0);
+   real_t y = p(1);
+   // real_t z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z

   F(0) = cos(kappa*x)*sin(kappa*y);
   F(1) = cos(kappa*y)*sin(kappa*x);
@@ -286,11 +286,11 @@ void f_exact(const Vector &p, Vector &f)
 {
   int dim = p.Size();

-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+   real_t x = p(0);
+   real_t y = p(1);
+   // real_t z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z

-   double temp = 1 + 2*kappa*kappa;
+   real_t temp = 1 + 2*kappa*kappa;

   f(0) = temp*cos(kappa*x)*sin(kappa*y);
   f(1) = temp*cos(kappa*y)*sin(kappa*x);
@@ -54,7 +54,7 @@ using namespace mfem;
 // Exact solution, F, and r.h.s., f. See below for implementation.
 void F_exact(const Vector &, Vector &);
 void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
+real_t freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -255,7 +255,7 @@ int main(int argc, char *argv[])

   // 15. Compute and print the L^2 norm of the error.
   {
-      double error = x.ComputeL2Error(F);
+      real_t error = x.ComputeL2Error(F);
      if (myid == 0)
      {
         cout << "\n|| F_h - F ||_{L^2} = " << error << '\n' << endl;
@@ -311,9 +311,9 @@ void F_exact(const Vector &p, Vector &F)
 {
   int dim = p.Size();

-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+   real_t x = p(0);
+   real_t y = p(1);
+   // real_t z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z

   F(0) = cos(kappa*x)*sin(kappa*y);
   F(1) = cos(kappa*y)*sin(kappa*x);
@@ -328,11 +328,11 @@ void f_exact(const Vector &p, Vector &f)
 {
   int dim = p.Size();

-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+   real_t x = p(0);
+   real_t y = p(1);
+   // real_t z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z

-   double temp = 1 + 2*kappa*kappa;
+   real_t temp = 1 + 2*kappa*kappa;

   f(0) = temp*cos(kappa*x)*sin(kappa*y);
   f(1) = temp*cos(kappa*y)*sin(kappa*x);
@@ -45,10 +45,10 @@ using namespace mfem;

 // Define the analytical solution and forcing terms / boundary conditions
 void uFun_ex(const Vector & x, Vector & u);
-double pFun_ex(const Vector & x);
+real_t pFun_ex(const Vector & x);
 void fFun(const Vector & x, Vector & f);
-double gFun(const Vector & x);
-double f_natural(const Vector & x);
+real_t gFun(const Vector & x);
+real_t f_natural(const Vector & x);

 int main(int argc, char *argv[])
 {
@@ -270,8 +270,8 @@ int main(int argc, char *argv[])
   // 11. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(1000);
-   double rtol(1.e-6);
-   double atol(1.e-10);
+   real_t rtol(1.e-6);
+   real_t atol(1.e-10);

   chrono.Clear();
   chrono.Start();
@@ -313,10 +313,10 @@ int main(int argc, char *argv[])
      irs[i] = &(IntRules.Get(i, order_quad));
   }

-   double err_u  = u.ComputeL2Error(ucoeff, irs);
-   double norm_u = ComputeLpNorm(2., ucoeff, *mesh, irs);
-   double err_p  = p.ComputeL2Error(pcoeff, irs);
-   double norm_p = ComputeLpNorm(2., pcoeff, *mesh, irs);
+   real_t err_u  = u.ComputeL2Error(ucoeff, irs);
+   real_t norm_u = ComputeLpNorm(2., ucoeff, *mesh, irs);
+   real_t err_p  = p.ComputeL2Error(pcoeff, irs);
+   real_t norm_p = ComputeLpNorm(2., pcoeff, *mesh, irs);

   std::cout << "|| u_h - u_ex || / || u_ex || = " << err_u / norm_u << "\n";
   std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";
@@ -391,9 +391,9 @@ int main(int argc, char *argv[])

 void uFun_ex(const Vector & x, Vector & u)
 {
-   double xi(x(0));
-   double yi(x(1));
-   double zi(0.0);
+   real_t xi(x(0));
+   real_t yi(x(1));
+   real_t zi(0.0);
   if (x.Size() == 3)
   {
      zi = x(2);
@@ -409,11 +409,11 @@ void uFun_ex(const Vector & x, Vector & u)
 }

 // Change if needed
-double pFun_ex(const Vector & x)
+real_t pFun_ex(const Vector & x)
 {
-   double xi(x(0));
-   double yi(x(1));
-   double zi(0.0);
+   real_t xi(x(0));
+   real_t yi(x(1));
+   real_t zi(0.0);

   if (x.Size() == 3)
   {
@@ -428,7 +428,7 @@ void fFun(const Vector & x, Vector & f)
   f = 0.0;
 }

-double gFun(const Vector & x)
+real_t gFun(const Vector & x)
 {
   if (x.Size() == 3)
   {
@@ -440,7 +440,7 @@ double gFun(const Vector & x)
   }
 }

-double f_natural(const Vector & x)
+real_t f_natural(const Vector & x)
 {
   return (-pFun_ex(x));
 }
@@ -46,10 +46,10 @@ using namespace mfem;

 // Define the analytical solution and forcing terms / boundary conditions
 void uFun_ex(const Vector & x, Vector & u);
-double pFun_ex(const Vector & x);
+real_t pFun_ex(const Vector & x);
 void fFun(const Vector & x, Vector & f);
-double gFun(const Vector & x);
-double f_natural(const Vector & x);
+real_t gFun(const Vector & x);
+real_t f_natural(const Vector & x);

 int main(int argc, char *argv[])
 {
@@ -326,8 +326,8 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(pa ? 1000 : 500);
-   double rtol(1.e-6);
-   double atol(1.e-10);
+   real_t rtol(1.e-6);
+   real_t atol(1.e-10);

   chrono.Clear();
   chrono.Start();
@@ -371,10 +371,10 @@ int main(int argc, char *argv[])
      irs[i] = &(IntRules.Get(i, order_quad));
   }

-   double err_u  = u->ComputeL2Error(ucoeff, irs);
-   double norm_u = ComputeGlobalLpNorm(2, ucoeff, *pmesh, irs);
-   double err_p  = p->ComputeL2Error(pcoeff, irs);
-   double norm_p = ComputeGlobalLpNorm(2, pcoeff, *pmesh, irs);
+   real_t err_u  = u->ComputeL2Error(ucoeff, irs);
+   real_t norm_u = ComputeGlobalLpNorm(2, ucoeff, *pmesh, irs);
+   real_t err_p  = p->ComputeL2Error(pcoeff, irs);
+   real_t norm_p = ComputeGlobalLpNorm(2, pcoeff, *pmesh, irs);

   if (verbose)
   {
@@ -493,9 +493,9 @@ int main(int argc, char *argv[])

 void uFun_ex(const Vector & x, Vector & u)
 {
-   double xi(x(0));
-   double yi(x(1));
-   double zi(0.0);
+   real_t xi(x(0));
+   real_t yi(x(1));
+   real_t zi(0.0);
   if (x.Size() == 3)
   {
      zi = x(2);
@@ -511,11 +511,11 @@ void uFun_ex(const Vector & x, Vector & u)
 }

 // Change if needed
-double pFun_ex(const Vector & x)
+real_t pFun_ex(const Vector & x)
 {
-   double xi(x(0));
-   double yi(x(1));
-   double zi(0.0);
+   real_t xi(x(0));
+   real_t yi(x(1));
+   real_t zi(0.0);

   if (x.Size() == 3)
   {
@@ -530,7 +530,7 @@ void fFun(const Vector & x, Vector & f)
   f = 0.0;
 }

-double gFun(const Vector & x)
+real_t gFun(const Vector & x)
 {
   if (x.Size() == 3)
   {
@@ -542,7 +542,7 @@ double gFun(const Vector & x)
   }
 }

-double f_natural(const Vector & x)
+real_t f_natural(const Vector & x)
 {
   return (-pFun_ex(x));
 }
@@ -28,8 +28,8 @@ using namespace std;
 using namespace mfem;

 // Exact solution and r.h.s., see below for implementation.
-double analytic_solution(const Vector &x);
-double analytic_rhs(const Vector &x);
+real_t analytic_solution(const Vector &x);
+real_t analytic_rhs(const Vector &x);
 void SnapNodes(Mesh &mesh);

 int main(int argc, char *argv[])
@@ -81,7 +81,7 @@ int main(int argc, char *argv[])

   if (elem_type == 0) // inscribed octahedron
   {
-      const double tri_v[6][3] =
+      const real_t tri_v[6][3] =
      {
         { 1,  0,  0}, { 0,  1,  0}, {-1,  0,  0},
         { 0, -1,  0}, { 0,  0,  1}, { 0,  0, -1}
@@ -105,7 +105,7 @@ int main(int argc, char *argv[])
   }
   else // inscribed cube
   {
-      const double quad_v[8][3] =
+      const real_t quad_v[8][3] =
      {
         {-1, -1, -1}, {+1, -1, -1}, {+1, +1, -1}, {-1, +1, -1},
         {-1, -1, +1}, {+1, -1, +1}, {+1, +1, +1}, {-1, +1, +1}
@@ -249,15 +249,15 @@ int main(int argc, char *argv[])
   return 0;
 }

-double analytic_solution(const Vector &x)
+real_t analytic_solution(const Vector &x)
 {
-   double l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
+   real_t l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
   return x(0)*x(1)/l2;
 }

-double analytic_rhs(const Vector &x)
+real_t analytic_rhs(const Vector &x)
 {
-   double l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
+   real_t l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
   return 7*x(0)*x(1)/l2;
 }

@@ -28,8 +28,8 @@ using namespace std;
 using namespace mfem;

 // Exact solution and r.h.s., see below for implementation.
-double analytic_solution(const Vector &x);
-double analytic_rhs(const Vector &x);
+real_t analytic_solution(const Vector &x);
+real_t analytic_rhs(const Vector &x);
 void SnapNodes(Mesh &mesh);

 int main(int argc, char *argv[])
@@ -101,7 +101,7 @@ int main(int argc, char *argv[])

   if (elem_type == 0) // inscribed octahedron
   {
-      const double tri_v[6][3] =
+      const real_t tri_v[6][3] =
      {
         { 1,  0,  0}, { 0,  1,  0}, {-1,  0,  0},
         { 0, -1,  0}, { 0,  0,  1}, { 0,  0, -1}
@@ -125,7 +125,7 @@ int main(int argc, char *argv[])
   }
   else // inscribed cube
   {
-      const double quad_v[8][3] =
+      const real_t quad_v[8][3] =
      {
         {-1, -1, -1}, {+1, -1, -1}, {+1, +1, -1}, {-1, +1, -1},
         {-1, -1, +1}, {+1, -1, +1}, {+1, +1, +1}, {-1, +1, +1}
@@ -281,7 +281,7 @@ int main(int argc, char *argv[])
   delete b;

   // 12. Compute and print the L^2 norm of the error.
-   double error = x.ComputeL2Error(sol_coef);
+   real_t error = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
      cout << "\nL2 norm of error: " << error << endl;
@@ -323,15 +323,15 @@ int main(int argc, char *argv[])
   return 0;
 }

-double analytic_solution(const Vector &x)
+real_t analytic_solution(const Vector &x)
 {
-   double l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
+   real_t l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
   return x(0)*x(1)/l2;
 }

-double analytic_rhs(const Vector &x)
+real_t analytic_rhs(const Vector &x)
 {
-   double l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
+   real_t l2 = x(0)*x(0) + x(1)*x(1) + x(2)*x(2);
   return 7*x(0)*x(1)/l2;
 }

@@ -206,7 +206,7 @@ int main(int argc, char *argv[])
   SparseMatrix * Shat = RAP(matBhat, matSinv, matBhat);

 #ifndef MFEM_USE_SUITESPARSE
-   const double prec_rtol = 1e-3;
+   const real_t prec_rtol = 1e-3;
   const int prec_maxit = 200;
   CGSolver *S0inv = new CGSolver;
   S0inv->SetOperator(matS0);
@@ -240,7 +240,7 @@ int main(int argc, char *argv[])
      Vector LSres(s_test);
      B.Mult(x, LSres);
      LSres -= F;
-      double res = sqrt(matSinv.InnerProduct(LSres, LSres));
+      real_t res = sqrt(matSinv.InnerProduct(LSres, LSres));
      cout << "\n|| B0*x0 + Bhat*xhat - F ||_{S^-1} = " << res << endl;
   }

@@ -283,7 +283,7 @@ int main(int argc, char *argv[])
      B.Mult(x, LSres);
      LSres -= *trueF;
      matSinv->Mult(LSres, tmp);
-      double res = sqrt(InnerProduct(LSres, tmp));
+      real_t res = sqrt(InnerProduct(LSres, tmp));
      if (myid == 0)
      {
         cout << "\n|| B0*x0 + Bhat*xhat - F ||_{S^-1} = " << res << endl;
@@ -58,10 +58,10 @@ int problem;
 void velocity_function(const Vector &x, Vector &v);

 // Initial condition
-double u0_function(const Vector &x);
+real_t u0_function(const Vector &x);

 // Inflow boundary condition
-double inflow_function(const Vector &x);
+real_t inflow_function(const Vector &x);

 // Mesh bounding box
 Vector bb_min, bb_max;
@@ -72,7 +72,7 @@ private:
   SparseMatrix &M, &K, A;
   GMRESSolver linear_solver;
   BlockILU prec;
-   double dt;
+   real_t dt;
 public:
   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
      : M(M_),
@@ -89,7 +89,7 @@ public:
      linear_solver.SetPreconditioner(prec);
   }

-   void SetTimeStep(double dt_)
+   void SetTimeStep(real_t dt_)
   {
      if (dt_ != dt)
      {
@@ -135,7 +135,7 @@ public:
   FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);

   virtual ~FE_Evolution();
 };
@@ -153,8 +153,8 @@ int main(int argc, char *argv[])
   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
-   double t_final = 10.0;
-   double dt = 0.01;
+   real_t t_final = 10.0;
+   real_t dt = 0.01;
   bool visualization = true;
   bool visit = false;
   bool paraview = false;
@@ -293,7 +293,7 @@ int main(int argc, char *argv[])
      k.SetAssemblyLevel(AssemblyLevel::FULL);
   }
   m.AddDomainIntegrator(new MassIntegrator);
-   constexpr double alpha = -1.0;
+   constexpr real_t alpha = -1.0;
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
   k.AddInteriorFaceIntegrator(
      new NonconservativeDGTraceIntegrator(velocity, alpha));
@@ -393,14 +393,14 @@ int main(int argc, char *argv[])
   //    iterations, ti, with a time-step dt).
   FE_Evolution adv(m, k, b);

-   double t = 0.0;
+   real_t t = 0.0;
   adv.SetTime(t);
   ode_solver->Init(adv);

   bool done = false;
   for (int ti = 0; !done; )
   {
-      double dt_real = min(dt, t_final - t);
+      real_t dt_real = min(dt, t_final - t);
      ode_solver->Step(u, t, dt_real);
      ti++;

@@ -482,7 +482,7 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
   M_solver.Mult(z, y);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
   MFEM_VERIFY(dg_solver != NULL,
               "Implicit time integration is not supported with partial assembly");
@@ -507,7 +507,7 @@ void velocity_function(const Vector &x, Vector &v)
   Vector X(dim);
   for (int i = 0; i < dim; i++)
   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
+      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
   }

@@ -529,7 +529,7 @@ void velocity_function(const Vector &x, Vector &v)
      case 2:
      {
         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
+         const real_t w = M_PI/2;
         switch (dim)
         {
            case 1: v(0) = 1.0; break;
@@ -541,8 +541,8 @@ void velocity_function(const Vector &x, Vector &v)
      case 3:
      {
         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
+         const real_t w = M_PI/2;
+         real_t d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
         d = d*d;
         switch (dim)
         {
@@ -556,7 +556,7 @@ void velocity_function(const Vector &x, Vector &v)
 }

 // Initial condition
-double u0_function(const Vector &x)
+real_t u0_function(const Vector &x)
 {
   int dim = x.Size();

@@ -564,7 +564,7 @@ double u0_function(const Vector &x)
   Vector X(dim);
   for (int i = 0; i < dim; i++)
   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
+      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
   }

@@ -580,28 +580,28 @@ double u0_function(const Vector &x)
            case 2:
            case 3:
            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
+               real_t rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
               if (dim == 3)
               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
+                  const real_t s = (1. + 0.25*cos(2*M_PI*X(2)));
                  rx *= s;
                  ry *= s;
               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
+               return ( std::erfc(w*(X(0)-cx-rx))*std::erfc(-w*(X(0)-cx+rx)) *
+                        std::erfc(w*(X(1)-cy-ry))*std::erfc(-w*(X(1)-cy+ry)) )/16;
            }
         }
      }
      case 2:
      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
+         real_t x_ = X(0), y_ = X(1), rho, phi;
+         rho = std::hypot(x_, y_);
         phi = atan2(y_, x_);
         return pow(sin(M_PI*rho),2)*sin(3*phi);
      }
      case 3:
      {
-         const double f = M_PI;
+         const real_t f = M_PI;
         return sin(f*X(0))*sin(f*X(1));
      }
   }
@@ -609,7 +609,7 @@ double u0_function(const Vector &x)
 }

 // Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
+real_t inflow_function(const Vector &x)
 {
   switch (problem)
   {
--- a/Show More
+++ b/Show More