unit tests from complex dense matrix: Mult, MultAtB, Inverse, SystemMatrix

Fixing shadow variables in fem classes [shadow-fem-dev]

.gitignore

@@ -259,6 +259,8 @@ miniapps/performance/sol.*
 
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
+miniapps/shifted/extrapolate
+miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf

.gitlab/lassen-build-and-test.yml

@@ -19,6 +19,11 @@ opt_mpi_cuda_xl_16_1_1_8:
     SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
+opt_mpi_cuda_hypre_cuda_xl:
+  variables:
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+  extends: .build_and_test_on_lassen
+
 # Jobs report
 report_job_success:
   stage: report

.gitlab/quartz-build-and-test.yml

@@ -59,7 +59,7 @@ opt_par_gcc_6_1_0_sundials:
 
 opt_par_gcc_6_1_0_petsc:
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
   extends: .build_and_test_on_quartz
 
 opt_par_gcc_6_1_0_pumi:

.gitlab/scripts/baseline

@@ -29,9 +29,9 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
   lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else

CHANGELOG

@@ -10,9 +10,20 @@
 
 Version 4.3.1 (development)
 ===========================
+
+- Add hipSPARSE support for sparse mat-vec multiplications.
+
+- Added support for using the HYPRE library built with HIP support. Similar to
+  the HYPRE + CUDA support added earlier, most of the MFEM examples and miniapps
+  work transparently with HYPRE + HIP builds. This includes the BoomerAMG, AMS,
+  and ADS solvers.
+
 - More explicit and consistent formating of the output of iterative solvers
   with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
 
+- Added a miniapp for PDE-based extrapolation of finite element functions. See
+  miniapps/shifted/extrapolate.cpp.
+
 - Added support for automatic differentiation. Users can select between native
   implementation and external library implementation during configuration. One
   parallel and two serial examples are implemented in the miniapps/autodiff/
@@ -87,6 +98,10 @@ Version 4.3.1 (development)
 - The HPC versions of ex1 and ex1p (in miniapps/performance) now support
   runtime selection of either 2D or 3D meshes.
 
+- Added ParaView visualization of `QuadratureFunction` fields, through both
+  `QuadratureFunction::SaveVTU` and `ParaViewDataCollection::RegisterQField`.
+
+
 Version 4.3, released on July 29, 2021
 ======================================
 
@@ -126,6 +141,12 @@ Discretization improvements
 
 - Added support for nonscalar coefficient with VectorDiffusionIntegrator.
 
+- Added support for Partial Assembly with Discontinuous Galerkin methods on
+  nonconforming meshes.
+
+- Added a simpler interface to request face information: see
+  `Mesh::FaceInformation` and `Mesh::GetFaceInformation`.
+
 Linear and nonlinear solvers
 ----------------------------
 - Added support for AMG preconditioners on GPUs based on the hypre library

CMakeLists.txt

@@ -16,9 +16,6 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO)
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -84,6 +81,9 @@ if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
+if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14)
+endif()
 
 # Include xSDK default CMake file.
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
@@ -180,6 +180,19 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
+# AMD HIP
+if (MFEM_USE_HIP)
+  if (HIP_ARCH)
+    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
+    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
+  endif()
+  if (ROCM_PATH)
+    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
+  endif()
+  find_package(HIP REQUIRED)
+  find_package(HIPSPARSE REQUIRED)
+endif()
+
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -354,7 +367,7 @@ endif()
 if (MFEM_USE_PUMI)
   # If PUMI_DIR was specified, only link to that directory,
   # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+  find_package(SCOREC 2.2.6 REQUIRED OPTIONAL_COMPONENTS gmi_sim
     CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
   if (SCOREC_FOUND)
     # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
@@ -403,18 +416,6 @@ if (MFEM_USE_CALIPER)
   find_package(Caliper REQUIRED)
 endif()
 
-# AMD HIP
-if (MFEM_USE_HIP)
-  find_package(HIP REQUIRED)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-    if (MFEM_USE_GINKGO)
-      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
-    endif()
-  endif()
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
@@ -454,10 +455,11 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG MPI_CXX)
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
+    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
+    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
+    MPI_CXX HIP HIPSPARSE)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -500,8 +502,6 @@ endforeach()
 
 if (MFEM_USE_CUDA)
   set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
-elseif(MFEM_USE_HIP)
-  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 
 add_subdirectory(config)

INSTALL

@@ -37,7 +37,7 @@ as CUDA, HIP, OCCA, OpenMP and RAJA.
   https://developer.nvidia.com/cuda-toolkit
 
 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
+  https://rocmdocs.amd.com
 
 - OCCA support requires the OCCA library
   https://libocca.org
@@ -58,7 +58,8 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
+- Conda-forge, https://conda-forge.org (pre-built binaries linked with
+  OpenMPI/MPICH, hypre, and METIS)
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
 
 We also recommend downloading and building the MFEM-based GLVis visualization
@@ -78,7 +79,7 @@ Parallel build:
 
 CUDA build:
    make cuda -j 4
-   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')
+   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_70')
 
 HIP build:
    make hip -j 4
@@ -566,7 +567,7 @@ The specific libraries and their options are:
   Options: HYPRE_OPT, HYPRE_LIB.
   Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
             HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
+            HYPRE >= 2.22.1   (HYPRE built with CUDA or HIP)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -639,7 +640,8 @@ The specific libraries and their options are:
   C++ compiler that supports the C++-14 standard. For additional requirements
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
+  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
+           Debug).
   Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
@@ -704,7 +706,7 @@ The specific libraries and their options are:
   URL: https://scorec.rpi.edu/pumi
        https://github.com/SCOREC/core
   Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI == 2.2.3.
+  Versions: PUMI >= 2.2.6.
 
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
@@ -737,7 +739,7 @@ The specific libraries and their options are:
   Versions: CUDA >= 10.1.168.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
-  URL: https://rocm.github.io/ROCmInstall.html
+  URL: https://rocmdocs.amd.com
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
 
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
@@ -924,6 +926,7 @@ MFEM_USE_HIOP
 MFEM_USE_CODIPACK
 MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
+MFEM_USE_HIP
 MFEM_USE_OCCA
 MFEM_USE_CEED
 MFEM_USE_RAJA

config/cmake/modules/FindHIP.cmake

@@ -1,692 +0,0 @@
-###############################################################################
-# FindHIP.cmake
-###############################################################################
-include(CheckCXXCompilerFlag)
-###############################################################################
-# SET: Variable defaults
-###############################################################################
-# User defined flags
-set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
-set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
-set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
-set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
-mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
-
-set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
-list(REMOVE_DUPLICATES _hip_configuration_types)
-foreach(config ${_hip_configuration_types})
-    string(TOUPPER ${config} config_upper)
-    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
-    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
-    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
-    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
-    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
-endforeach()
-option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
-option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
-mark_as_advanced(HIP_HOST_COMPILATION_CPP)
-
-###############################################################################
-# FIND: HIP and associated helper binaries
-###############################################################################
-
-get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
-
-# HIP is supported on Linux only
-if(UNIX AND NOT APPLE AND NOT CYGWIN)
-    # Search for HIP installation
-    if(NOT HIP_ROOT_DIR)
-        # Search in user specified path first
-        find_path(
-            HIP_ROOT_DIR
-            NAMES bin/hipconfig
-            PATHS
-            "$ENV{ROCM_PATH}/hip"
-            ENV HIP_PATH
-            ${_IMPORT_PREFIX}
-            /opt/rocm/hip
-            DOC "HIP installed location"
-            NO_DEFAULT_PATH
-            )
-        if(NOT EXISTS ${HIP_ROOT_DIR})
-            if(HIP_FIND_REQUIRED)
-                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
-            elseif(NOT HIP_FIND_QUIETLY)
-                message("HIP_ROOT_DIR not found or specified")
-            endif()
-        endif()
-        # And push it back to the cache
-        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
-    endif()
-
-    # Find HIPCC executable
-    find_program(
-        HIP_HIPCC_EXECUTABLE
-        NAMES hipcc
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCC_EXECUTABLE hipcc)
-    endif()
-    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
-
-    # Find HIPCONFIG executable
-    find_program(
-        HIP_HIPCONFIG_EXECUTABLE
-        NAMES hipconfig
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCONFIG_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
-    endif()
-    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
-
-    # Find HIPCC_CMAKE_LINKER_HELPER executable
-    find_program(
-        HIP_HIPCC_CMAKE_LINKER_HELPER
-        NAMES hipcc_cmake_linker_helper
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
-        # Now search in default paths
-        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
-    endif()
-    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
-        # Compute the version
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
-            OUTPUT_VARIABLE _hip_version
-            ERROR_VARIABLE _hip_error
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE
-            )
-        if(NOT _hip_error)
-            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
-        else()
-            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
-        endif()
-        mark_as_advanced(HIP_VERSION)
-    endif()
-    if(HIP_VERSION)
-        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
-        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
-        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
-        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
-        set(HIP_VERSION_STRING "${HIP_VERSION}")
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
-        # Compute the platform
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
-            OUTPUT_VARIABLE _hip_platform
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
-        mark_as_advanced(HIP_PLATFORM)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
-        # Compute the compiler
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
-            OUTPUT_VARIABLE _hip_compiler
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
-        mark_as_advanced(HIP_COMPILER)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
-        # Compute the runtime
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
-            OUTPUT_VARIABLE _hip_runtime
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
-        mark_as_advanced(HIP_RUNTIME)
-    endif()
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(
-    HIP
-    REQUIRED_VARS
-    HIP_ROOT_DIR
-    HIP_HIPCC_EXECUTABLE
-    HIP_HIPCONFIG_EXECUTABLE
-    HIP_PLATFORM
-    HIP_COMPILER
-    HIP_RUNTIME
-    VERSION_VAR HIP_VERSION
-    )
-
-###############################################################################
-# Set HIP CMAKE Flags
-###############################################################################
-# Copy the invocation styles from CXX to HIP
-set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
-set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
-set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
-set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
-#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
-set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
-
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
-set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
-
-if("${HIP_COMPILER}" STREQUAL "nvcc")
-    # Set the CMake Flags to use the nvcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "hcc")
-    # Set the CMake Flags to use the hcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "clang")
-    #Number of parallel jobs by default is 1
-    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
-      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
-    endif()
-    #Add support for parallel build and link
-    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
-      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
-    endif()
-    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
-      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
-        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-      else()
-        message("clang compiler doesn't support parallel jobs")
-      endif()
-    endif()
-
-    # Set the CMake Flags to use the HIP-Clang Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-
-    if("${HIP_RUNTIME}" STREQUAL "rocclr")
-      if(TARGET host)
-        message(STATUS "host interface - found")
-        set(HIP_HOST_INTERFACE host)
-      endif()
-    endif()
-endif()
-
-###############################################################################
-# MACRO: Locate helper files
-###############################################################################
-macro(HIP_FIND_HELPER_FILE _name _extension)
-    set(_hip_full_name "${_name}.${_extension}")
-    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
-    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
-    if(NOT EXISTS "${HIP_${_name}}")
-        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
-        if(HIP_FIND_REQUIRED)
-            message(FATAL_ERROR "${error_message}")
-        else()
-            if(NOT HIP_FIND_QUIETLY)
-                message(STATUS "${error_message}")
-            endif()
-        endif()
-    endif()
-    # Set this variable as internal, so the user isn't bugged with it.
-    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
-endmacro()
-
-###############################################################################
-hip_find_helper_file(run_make2cmake cmake)
-hip_find_helper_file(run_hipcc cmake)
-###############################################################################
-
-###############################################################################
-# MACRO: Reset compiler flags
-###############################################################################
-macro(HIP_RESET_FLAGS)
-    unset(HIP_HIPCC_FLAGS)
-    unset(HIP_HCC_FLAGS)
-    unset(HIP_CLANG_FLAGS)
-    unset(HIP_NVCC_FLAGS)
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        unset(HIP_HIPCC_FLAGS_${config_upper})
-        unset(HIP_HCC_FLAGS_${config_upper})
-        unset(HIP_CLANG_FLAGS_${config_upper})
-        unset(HIP_NVCC_FLAGS_${config_upper})
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Separate the options from the sources
-###############################################################################
-macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
-    set(${_sources})
-    set(${_cmake_options})
-    set(${_hipcc_options})
-    set(${_hcc_options})
-    set(${_clang_options})
-    set(${_nvcc_options})
-    set(_hipcc_found_options FALSE)
-    set(_hcc_found_options FALSE)
-    set(_clang_found_options FALSE)
-    set(_nvcc_found_options FALSE)
-    foreach(arg ${ARGN})
-        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
-            set(_hipcc_found_options TRUE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options TRUE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options TRUE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options TRUE)
-        elseif(
-                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
-                "x${arg}" STREQUAL "xSTATIC" OR
-                "x${arg}" STREQUAL "xSHARED" OR
-                "x${arg}" STREQUAL "xMODULE"
-                )
-            list(APPEND ${_cmake_options} ${arg})
-        else()
-            if(_hipcc_found_options)
-                list(APPEND ${_hipcc_options} ${arg})
-            elseif(_hcc_found_options)
-                list(APPEND ${_hcc_options} ${arg})
-            elseif(_clang_found_options)
-                list(APPEND ${_clang_options} ${arg})
-            elseif(_nvcc_found_options)
-                list(APPEND ${_nvcc_options} ${arg})
-            else()
-                # Assume this is a file
-                list(APPEND ${_sources} ${arg})
-            endif()
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Add include directories to pass to the hipcc command
-###############################################################################
-set(HIP_HIPCC_INCLUDE_ARGS_USER "")
-macro(HIP_INCLUDE_DIRECTORIES)
-    foreach(dir ${ARGN})
-        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
-    endforeach()
-endmacro()
-
-###############################################################################
-# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
-###############################################################################
-function(HIP_COMPUTE_BUILD_PATH path build_path)
-    # Convert to cmake style paths
-    file(TO_CMAKE_PATH "${path}" bpath)
-    if(IS_ABSOLUTE "${bpath}")
-        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
-        if(_binary_dir_pos EQUAL 0)
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
-        else()
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
-        endif()
-    endif()
-
-    # Remove leading /
-    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
-    # Avoid absolute paths by removing ':'
-    string(REPLACE ":" "_" bpath "${bpath}")
-    # Avoid relative paths that go up the tree
-    string(REPLACE "../" "__/" bpath "${bpath}")
-    # Avoid spaces
-    string(REPLACE " " "_" bpath "${bpath}")
-    # Strip off the filename
-    get_filename_component(bpath "${bpath}" PATH)
-
-    set(${build_path} "${bpath}" PARENT_SCOPE)
-endfunction()
-
-###############################################################################
-# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
-###############################################################################
-macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
-    set(_hip_found_config)
-    foreach(arg ${ARGN})
-        # Determine if we are dealing with a per-configuration flag
-        foreach(config ${_hip_configuration_types})
-            string(TOUPPER ${config} config_upper)
-            if(arg STREQUAL "${config_upper}")
-                set(_hip_found_config _${arg})
-                # Clear arg to prevent it from being processed anymore
-                set(arg)
-            endif()
-        endforeach()
-        if(arg)
-            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Try and include dependency file if it exists
-###############################################################################
-macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
-    set(HIP_HIPCC_DEPEND)
-    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
-
-    # Create the dependency file if it doesn't exist
-    if(NOT EXISTS ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-    # Include the dependency file
-    include(${dependency_file})
-
-    # Verify the existence of all the included files
-    if(HIP_HIPCC_DEPEND)
-        foreach(f ${HIP_HIPCC_DEPEND})
-            if(NOT EXISTS ${f})
-                # If they aren't there, regenerate the file again
-                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-            endif()
-        endforeach()
-    else()
-        # No dependencies, so regenerate the file
-        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-    endif()
-
-    # Regenerate the dependency file if needed
-    if(HIP_HIPCC_DEPEND_REGENERATE)
-        set(HIP_HIPCC_DEPEND ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-endmacro()
-
-###############################################################################
-# MACRO: Prepare cmake commands for the target
-###############################################################################
-macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
-    set(_hip_flags "")
-    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
-    if(HIP_HOST_COMPILATION_CPP)
-        set(HIP_C_OR_CXX CXX)
-    else()
-        set(HIP_C_OR_CXX C)
-    endif()
-    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
-
-    # Initialize list of includes with those specified by the user. Append with
-    # ones specified to cmake directly.
-    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
-
-    # Add the include directories
-    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
-    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
-
-    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
-    list(REMOVE_DUPLICATES _hip_include_directories)
-    if(_hip_include_directories)
-        foreach(dir ${_hip_include_directories})
-            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
-        endforeach()
-    endif()
-
-    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
-
-    # Add the compile definitions
-    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
-    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
-
-    # Check if we are building shared library.
-    set(_hip_build_shared_libs FALSE)
-    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
-    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
-    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
-        set(_hip_build_shared_libs TRUE)
-    endif()
-    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
-    if(_hip_found_STATIC GREATER -1)
-        set(_hip_build_shared_libs FALSE)
-    endif()
-
-    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
-    if(_hip_build_shared_libs)
-        list(APPEND HIP_HCC_FLAGS "-fPIC")
-        list(APPEND HIP_CLANG_FLAGS "-fPIC")
-        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
-    endif()
-
-    # Set host compiler
-    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
-
-    # Set compiler flags
-    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
-    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
-    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
-    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
-    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
-        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
-        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
-        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
-        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
-    endforeach()
-
-    # Reset the output variable
-    set(_hip_generated_files "")
-    set(_hip_source_files "")
-
-    # Iterate over all arguments and create custom commands for all source files
-    foreach(file ${ARGN})
-        # Ignore any file marked as a HEADER_FILE_ONLY
-        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
-        # Allow per source file overrides of the format. Also allows compiling non .cu files.
-        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
-        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
-            set(host_flag FALSE)
-        else()
-            set(host_flag TRUE)
-        endif()
-
-        if(NOT host_flag)
-            # Determine output directory
-            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
-            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
-
-            get_filename_component(basename ${file} NAME)
-            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
-            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
-
-            # Set file names
-            set(generated_file "${generated_file_path}/${generated_file_basename}")
-            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
-            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
-            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
-
-            # Set properties for object files
-            set_source_files_properties("${generated_file}"
-                PROPERTIES
-                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
-                )
-
-            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
-            get_filename_component(file_path "${file}" PATH)
-            if(IS_ABSOLUTE "${file_path}")
-                set(source_file "${file}")
-            else()
-                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
-            endif()
-
-            # Bring in the dependencies
-            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
-
-            # Configure the build script
-            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
-            file(GENERATE
-                OUTPUT "${custom_target_script}"
-                INPUT "${custom_target_script_pregen}"
-                )
-            set(main_dep DEPENDS ${source_file})
-            if(CMAKE_GENERATOR MATCHES "Makefiles")
-                set(verbose_output "$(VERBOSE)")
-            elseif(HIP_VERBOSE_BUILD)
-                set(verbose_output ON)
-            else()
-                set(verbose_output OFF)
-            endif()
-
-            # Create up the comment string
-            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
-            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
-
-            # Build the generated file and dependency file
-            add_custom_command(
-                OUTPUT ${generated_file}
-                # These output files depend on the source_file and the contents of cmake_dependency_file
-                ${main_dep}
-                DEPENDS ${HIP_HIPCC_DEPEND}
-                DEPENDS ${custom_target_script}
-                # Make sure the output directory exists before trying to write to it.
-                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
-                COMMAND ${CMAKE_COMMAND} ARGS
-                -D verbose:BOOL=${verbose_output}
-                -D build_configuration:STRING=${_hip_build_configuration}
-                -D "generated_file:STRING=${generated_file}"
-                -P "${custom_target_script}"
-                WORKING_DIRECTORY "${hip_compile_output_dir}"
-                COMMENT "${hip_build_comment_string}"
-                )
-
-            # Make sure the build system knows the file is generated
-            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
-            list(APPEND _hip_generated_files ${generated_file})
-            list(APPEND _hip_source_files ${file})
-        endif()
-    endforeach()
-
-    # Set the return parameter
-    set(${_generated_files} ${_hip_generated_files})
-    set(${_source_files} ${_hip_source_files})
-endmacro()
-
-###############################################################################
-# HIP_ADD_EXECUTABLE
-###############################################################################
-macro(HIP_ADD_EXECUTABLE hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if("${HIP_COMPILER}" STREQUAL "hcc")
-        if("x${HCC_HOME}" STREQUAL "x")
-            if (DEFINED ENV{ROCM_PATH})
-                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
-            else()
-                set(HCC_HOME "/opt/rocm/hcc")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    elseif("${HIP_COMPILER}" STREQUAL "clang")
-        if("x${HIP_CLANG_PATH}" STREQUAL "x")
-            if(DEFINED ENV{HIP_CLANG_PATH})
-                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
-            elseif(DEFINED ENV{ROCM_PATH})
-                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
-            else()
-                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    else()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-###############################################################################
-# HIP_ADD_LIBRARY
-###############################################################################
-macro(HIP_ADD_LIBRARY hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_hipcc.cmake

@@ -1,182 +0,0 @@
-###############################################################################
-# Runs commands using HIPCC
-###############################################################################
-
-###############################################################################
-# This file runs the hipcc commands to produce the desired output file
-# along with the dependency file needed by CMake to compute dependencies.
-#
-# Input variables:
-#
-# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
-#                               ON : Describe each step
-# build_configuration:STRING=<> Build configuration. Defaults to Debug.
-# generated_file:STRING=<>      File to generate. Mandatory argument.
-
-if(NOT build_configuration)
-    set(build_configuration Debug)
-endif()
-if(NOT generated_file)
-    message(FATAL_ERROR "You must specify generated_file on the command line")
-endif()
-
-# Set these up as variables to make reading the generated file easier
-set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
-set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
-set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
-set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
-set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
-set(HCC_HOME "@HCC_HOME@") #path
-set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
-
-@HIP_HOST_FLAGS@
-@_HIP_HIPCC_FLAGS@
-@_HIP_HCC_FLAGS@
-@_HIP_CLANG_FLAGS@
-@_HIP_NVCC_FLAGS@
-#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
-set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
-
-set(cmake_dependency_file "@cmake_dependency_file@") # path
-set(source_file "@source_file@") # path
-set(host_flag "@host_flag@") # bool
-
-# Determine compiler and compiler flags
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
-if(NOT host_flag)
-    set(__CC ${HIP_HIPCC_EXECUTABLE})
-    if("${HIP_PLATFORM}" STREQUAL "amd")
-        if("${HIP_COMPILER}" STREQUAL "hcc")
-            if(NOT "x${HCC_HOME}" STREQUAL "x")
-                set(ENV{HCC_HOME} ${HCC_HOME})
-            endif()
-            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
-        elseif("${HIP_COMPILER}" STREQUAL "clang")
-            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
-                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
-            endif()
-            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
-            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
-        endif()
-    else()
-        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
-    endif()
-else()
-    set(__CC ${HIP_HOST_COMPILER})
-    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
-endif()
-set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
-
-# hip_execute_process - Executes a command with optional command echo and status message.
-#   status     - Status message to print if verbose is true
-#   command    - COMMAND argument from the usual execute_process argument structure
-#   ARGN       - Remaining arguments are the command with arguments
-#   HIP_result - Return value from running the command
-macro(hip_execute_process status command)
-    set(_command ${command})
-    if(NOT "x${_command}" STREQUAL "xCOMMAND")
-        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
-    endif()
-    if(verbose)
-        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
-        # Build command string to print
-        set(hip_execute_process_string)
-        foreach(arg ${ARGN})
-            # Escape quotes if any
-            string(REPLACE "\"" "\\\"" arg ${arg})
-            # Surround args with spaces with quotes
-            if(arg MATCHES " ")
-                list(APPEND hip_execute_process_string "\"${arg}\"")
-            else()
-                list(APPEND hip_execute_process_string ${arg})
-            endif()
-        endforeach()
-        # Echo the command
-        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
-    endif()
-    # Run the command
-    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
-endmacro()
-
-# Delete the target file
-hip_execute_process(
-    "Removing ${generated_file}"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-    )
-
-# Generate the dependency file
-hip_execute_process(
-    "Generating dependency file: ${cmake_dependency_file}.pre"
-    COMMAND "${__CC}"
-    -M
-    "${source_file}"
-    -o "${cmake_dependency_file}.pre"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the cmake readable dependency file to a temp file
-hip_execute_process(
-    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
-    COMMAND "${CMAKE_COMMAND}"
-    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
-    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
-    -D "verbose=${verbose}"
-    -P "${HIP_run_make2cmake}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Copy the file if it is different
-hip_execute_process(
-    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
-    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Delete the temporary file
-hip_execute_process(
-    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the output file
-hip_execute_process(
-    "Generating ${generated_file}"
-    COMMAND "${__CC}"
-    -c
-    "${source_file}"
-    -o "${generated_file}"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    # Make sure that we delete the output file
-    hip_execute_process(
-        "Removing ${generated_file}"
-        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-        )
-    message(FATAL_ERROR "Error generating file ${generated_file}")
-else()
-    if(verbose)
-        message("Generated ${generated_file} successfully.")
-    endif()
-endif()
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_make2cmake.cmake

@@ -1,50 +0,0 @@
-###############################################################################
-# Computes dependencies using HIPCC
-###############################################################################
-
-###############################################################################
-# This file converts dependency files generated using hipcc to a format that
-# cmake can understand.
-
-# Input variables:
-#
-# input_file:STRING=<> Dependency file to parse. Required argument
-# output_file:STRING=<> Output file to generate. Required argument
-
-if(NOT input_file OR NOT output_file)
-    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
-endif()
-
-file(READ ${input_file} depend_text)
-
-if (NOT "${depend_text}" STREQUAL "")
-    string(REPLACE " /" "\n/" depend_text ${depend_text})
-    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
-    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
-
-    set(dependency_list "")
-
-    foreach(file ${depend_text})
-        string(REGEX REPLACE "^ +" "" file ${file})
-        if(NOT EXISTS "${file}")
-            message(WARNING " Removing non-existent dependency file: ${file}")
-            set(file "")
-        endif()
-
-        if(NOT IS_DIRECTORY "${file}")
-            get_filename_component(file_absolute "${file}" ABSOLUTE)
-            list(APPEND dependency_list "${file_absolute}")
-        endif()
-    endforeach()
-endif()
-
-# Remove the duplicate entries and sort them.
-list(REMOVE_DUPLICATES dependency_list)
-list(SORT dependency_list)
-
-foreach(file ${dependency_list})
-    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
-endforeach()
-
-file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHYPRE.cmake

@@ -14,10 +14,33 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
+#   - HYPRE_USING_HIP (internal)
+
+if (HYPRE_FOUND)
+  if (HYPRE_USING_HIP)
+    find_package(rocsparse REQUIRED)
+    find_package(rocrand REQUIRED)
+  endif()
+  return()
+endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
-  "Paths to headers required by HYPRE." "Libraries required by HYPRE.")
+  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
+  CHECK_BUILD HYPRE_USING_HIP FALSE
+  "
+#undef HYPRE_USING_HIP
+#include <HYPRE_config.h>
+
+#ifndef HYPRE_USING_HIP
+#error HYPRE is built without HIP.
+#endif
+
+int main()
+{
+   return 0;
+}
+")
 
 if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
   try_run(HYPRE_VERSION_RUN_RESULT HYPRE_VERSION_COMPILE_RESULT
@@ -33,3 +56,12 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
     message(FATAL_ERROR "Unable to determine HYPRE version.")
   endif()
 endif()
+
+if (HYPRE_FOUND AND HYPRE_USING_HIP)
+  find_package(rocsparse REQUIRED)
+  find_package(rocrand REQUIRED)
+  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
+  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
+      "HYPRE libraries + dependencies." FORCE)
+  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
+endif()

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -46,8 +46,7 @@ endfunction()
 # Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
   if (MFEM_USE_HIP)
-    hip_add_executable(${NAME} ${ARGN})
-    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
+    add_executable(${NAME} ${ARGN})
   else()
     add_executable(${NAME} ${ARGN})
   endif()
@@ -56,7 +55,7 @@ endmacro()
 # Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
   if (MFEM_USE_HIP)
-    hip_add_library(${NAME} ${ARGN})
+    add_library(${NAME} ${ARGN})
   else()
     add_library(${NAME} ${ARGN})
   endif()
@@ -92,8 +91,6 @@ macro(add_mfem_examples EXE_SRCS)
     # If CUDA is enabled, tag source files to be compiled with nvcc.
     if (MFEM_USE_CUDA)
       set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
-    elseif(MFEM_USE_HIP)
-      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
     endif()
 
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
@@ -159,8 +156,6 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
       endforeach()
       set(EXTRA_OPTIONS_LIST ${LIST_})
     endif()
-  elseif(MFEM_USE_HIP)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
   endif()
 
   # Actually add the executable
@@ -534,12 +529,15 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             if (NOT ImportConfig)
               set(ImportConfig RELEASE)
             endif()
+            set(ImportConfigSuffix "_${ImportConfig}")
             get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
             list(FIND ImpConfigs ${ImportConfig} _Index)
-            if (_Index EQUAL -1)
-              message(FATAL_ERROR " *** ${ReqPack}: configuration "
-                "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
-                "from the list: ${ImpConfigs}.")
+            if ((_Index EQUAL -1) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
+              set(ImportConfig "NO_CONFIG")
+              set(ImportConfigSuffix "")
+              # message(FATAL_ERROR " *** ${ReqPack}: configuration "
+              #   "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
+              #   "from the list: ${ImpConfigs}.")
             endif()
           endif()
           # Set _Pack_LIBS
@@ -551,8 +549,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           else()
             # Set _Pack_LIBS from the target properties for ImportConfig
-            foreach (_prop IMPORTED_LOCATION_${ImportConfig}
-                IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig})
+            foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
+                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix})
               get_target_property(_value ${TargetName} ${_prop})
               if (_value)
                 list(APPEND _Pack_LIBS ${_value})
@@ -564,7 +562,7 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           endif()
           # Set _Pack_INCS
-          foreach (_prop INCLUDE_DIRECTORIES)
+          foreach (_prop INCLUDE_DIRECTORIES INTERFACE_INCLUDE_DIRECTORIES)
             get_target_property(_value ${TargetName} ${_prop})
             if (_value)
               list(APPEND _Pack_INCS ${_value})
@@ -742,6 +740,133 @@ function(mfem_find_library Name Prefix Lib LibDoc CheckVar CheckSrc)
 endfunction(mfem_find_library)
 
 
+#
+# Extract compile and link options needed by the given target.
+#
+function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
+
+  if (NOT TARGET ${Target})
+    return()
+  endif()
+
+  # CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG -> '-Wl,-rpath,'
+  set(shared_link_flag ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
+  if (NOT shared_link_flag)
+    set(shared_link_flag "-Wl,-rpath,")
+  endif()
+
+  set(tgt "${Target}")
+  unset(CompileOpts)
+  unset(LinkOpts)
+  get_target_property(IsImported ${tgt} IMPORTED)
+  # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
+  # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
+  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
+  get_target_property(type ${tgt} TYPE)
+  # message(STATUS "${tgt}[TYPE]: ${type}")
+  unset(ImportConfig)
+  get_target_property(ImportConfigs ${tgt} IMPORTED_CONFIGURATIONS)
+  if (ImportConfigs)
+    list(GET ImportConfigs 0 ImportConfig)
+  endif()
+  if (NOT ImportConfig)
+    set(ImportConfig RELEASE)
+  endif()
+  # message(STATUS "${tgt}[ImportConfig]: ${ImportConfig}")
+  # List all properties with: cmake --help-property-list
+  get_target_property(Defs ${tgt} INTERFACE_COMPILE_DEFINITIONS)
+  if (Defs)
+    list(REMOVE_DUPLICATES Defs)
+    foreach(Def ${Defs})
+      list(APPEND CompileOpts "-D${Def}")
+    endforeach()
+  endif()
+  get_target_property(Opts ${tgt} INTERFACE_COMPILE_OPTIONS)
+  if (Opts)
+    foreach(Opt ${Opts})
+      list(APPEND CompileOpts "${Opt}")
+    endforeach()
+  endif()
+  get_target_property(Dirs ${tgt} INTERFACE_INCLUDE_DIRECTORIES)
+  if (Dirs)
+    list(REMOVE_DUPLICATES Dirs)
+    foreach(Dir ${Dirs})
+      list(APPEND CompileOpts "-I\"${Dir}\"")
+    endforeach()
+  endif()
+  get_target_property(SysDirs ${tgt} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+  if (SysDirs)
+    list(REMOVE_DUPLICATES SysDirs)
+    foreach(SysDir ${SysDirs})
+      list(APPEND CompileOpts "-isystem \"${SysDir}\"")
+    endforeach()
+  endif()
+  if ("${type}" STREQUAL "STATIC_LIBRARY")
+    get_target_property(Location ${tgt} LOCATION)
+    if (Location)
+      list(APPEND LinkOpts "\"${Location}\"")
+    else()
+      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
+    endif()
+  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
+    get_target_property(Location ${tgt} LOCATION)
+    if (Location)
+      get_filename_component(Dir ${Location} DIRECTORY)
+      get_filename_component(NameWE ${Location} NAME_WE)
+      string(REGEX REPLACE "^lib" "" LibName ${NameWE})
+      list(APPEND LinkOpts
+        "-L\"${Dir}\""
+        "${shared_link_flag}\"${Dir}\""
+        "-l${LibName}")
+    else()
+      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
+    endif()
+  elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
+    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
+    if (Libs)
+      foreach(Lib ${Libs})
+        if (NOT (TARGET ${Lib}))
+          list(APPEND LinkOpts "${Lib}")
+        else()
+          mfem_get_target_options(${Lib} COpts LOpts)
+          list(APPEND CompileOpts ${COpts})
+          list(APPEND LinkOpts ${LOpts})
+        endif()
+      endforeach()
+    endif()
+    # Other properties we may need to handle:
+    # INTERFACE_LINK_DEPENDS
+    # INTERFACE_LINK_DIRECTORIES
+    # INTERFACE_LINK_OPTIONS
+  else()
+    message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
+      " which is not supported!")
+  endif()
+
+  # Other potentially relevant properties:
+  # - For all target types:
+  # IMPORTED_LIBNAME
+  # IMPORTED_LIBNAME_${ImportConfig}
+  # INTERFACE_AUTOUIC_OPTIONS
+  # INTERFACE_COMPILE_FEATURES
+  # INTERFACE_POSITION_INDEPENDENT_CODE
+  # INTERFACE_SOURCES
+  # INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+  # - For non-"INTERFACE_LIBRARY" target types only:
+  # IMPORTED_LOCATION
+  # IMPORTED_LOCATION_${ImportConfig}
+  # IMPORTED_LINK_INTERFACE_LIBRARIES
+  # IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig}
+  # LINK_FLAGS
+  # LINK_FLAGS_${ImportConfig}
+  # LOCATION_${ImportConfig})
+
+  set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
+  set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)
+
+endfunction(mfem_get_target_options)
+
+
 #
 #   Function that creates 'config.mk' from 'config.mk.in' for the both the
 #   build- and the install-locations and define install rules for 'config.mk'
@@ -760,13 +885,15 @@ function(mfem_export_mk_files)
   # Convert Boolean vars to YES/NO without writing the values to cache
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
+      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
-      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
-      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
+      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
+      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
+      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
+      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
+      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
+      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -868,8 +995,18 @@ function(mfem_export_mk_files)
     get_filename_component(suffix ${lib} EXT)
     # handle interfaces (e.g., SCOREC::apf)
     if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
-    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
-      message(FATAL_ERROR "***** interface lib found ... exiting *****")
+    elseif (TARGET "${lib}")
+      mfem_get_target_options(${lib} CompileOpts LinkOpts)
+      # Removing duplicates may lead to issues:
+      # list(REMOVE_DUPLICATES CompileOpts)
+      # list(REMOVE_DUPLICATES LinkOpts)
+      string(REPLACE ";" " " COpts "${CompileOpts}")
+      string(REPLACE ";" " " LOpts "${LinkOpts}")
+      # message(STATUS "${lib}[COpts]: '${COpts}'")
+      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
+      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
+      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
+      # message(FATAL_ERROR "***** interface lib found ... exiting *****")
       # handle static and shared libs
     elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
       get_filename_component(dir ${lib} DIRECTORY)

config/config.mk.in

@@ -91,7 +91,7 @@ MFEM_MPIEXEC_NP = @MFEM_MPIEXEC_NP@
 MFEM_MPI_NP     = @MFEM_MPI_NP@
 
 # The NVCC compiler cannot link with -x=cu
-MFEM_LINK_FLAGS := $(filter-out -x=cu, $(MFEM_FLAGS))
+MFEM_LINK_FLAGS := $(filter-out -x=cu -xhip, $(MFEM_FLAGS))
 
 # Optional extra configuration
 @MFEM_CONFIG_EXTRA@

config/defaults.cmake

@@ -50,6 +50,7 @@ option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
+option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
@@ -107,6 +108,7 @@ if (MFEM_USE_CUDA)
    set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
        "Libraries that HYPRE depends on.")
 endif()
+# HIP dependency for HYPRE is handled in FindHYPRE.cmake.
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -226,6 +228,11 @@ set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
+# If RAJA is built with external CAMP:
+# set(RAJA_REQUIRED_PACKAGES "camp"
+#     CACHE STRING "Packages that RAJA depends on.")
+# set(camp_DIR "${MFEM_DIR}/../camp/lib/cmake/camp"
+#     CACHE PATH "Path to CAMP CMake files.")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
@@ -245,7 +252,8 @@ set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")
 
-set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/inlude" CACHE STRING "Path to CoDiPack headers.")
+set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/include" CACHE STRING
+    "Path to CoDiPack headers.")
 set(CODIPACK_LIBRARIES "")
 
 # Some useful variables:

config/defaults.mk

@@ -172,6 +172,20 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
    MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
 endif
 
+# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
+# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
+ifeq ($(MFEM_USE_HIP),YES)
+   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
+   HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR)))
+   ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
+      HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null)
+      ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
+         $(error Unable to determine HIP_DIR. Please set it manually.)
+      endif
+   endif
+endif
+
 # Compile and link options for zlib.
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
@@ -189,6 +203,11 @@ ifeq (YES,$(MFEM_USE_CUDA))
    # This is only necessary when hypre is built with cuda:
    HYPRE_LIB += -lcusparse -lcurand
 endif
+ifeq (YES,$(MFEM_USE_HIP))
+   # This is only necessary when hypre is built with hip:
+   HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\
+ -lrocsparse -lrocrand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -430,9 +449,9 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 CUDA_OPT =
 CUDA_LIB = -lcusparse
 
-# HIP library configuration (currently not needed)
+# HIP library configuration
 HIP_OPT =
-HIP_LIB =
+HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa

config/test.mk

@@ -58,14 +58,14 @@ ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
 # Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 255 means the test was skipped, anything else means error
+# 0 means success, 242 means the test was skipped, anything else means error
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 242 ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
    rm -f $(1).stderr; exit $$err
 
@@ -76,7 +76,7 @@ mfem-test-file = \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 242 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
    rm -f $(1).stderr; exit $$err
 

examples/ex19p.cpp

@@ -197,10 +197,10 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
+        << "is NOT supported with the GPU version of hypre.\n\n";
+   return 242;
 #endif
 
    // 1. Initialize MPI
@@ -487,8 +487,8 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
 
       if (!spaces[0]->GetParMesh()->Nonconforming())
       {
-#ifndef HYPRE_USING_CUDA
-         // Not available yet when hypre is built with CUDA
+#if !defined(HYPRE_USING_GPU)
+         // Not available yet when hypre is built with GPU support
          stiff_prec_amg->SetElasticityOptions(spaces[0]);
 #endif
       }

examples/ex28p.cpp

@@ -81,10 +81,10 @@ Mesh * build_trapezoid_mesh(double offset)
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
+        << "is NOT supported with the GPU version of hypre.\n\n";
+   return 242;
 #endif
 
    // 1. Initialize MPI.

examples/ex5.cpp

@@ -197,7 +197,7 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
-      if (Device::IsEnabled()) { B.BuildTranspose(); }
+      B.EnsureMultTranspose();
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);

examples/ex9.cpp

@@ -27,6 +27,7 @@
 //    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
 //    ex9 -ea -m ../data/periodic-cube.mesh -d cuda
 //    ex9 -fa -m ../data/periodic-cube.mesh -d cuda
+//    ex9 -pa -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9 -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and

examples/ex9p.cpp

@@ -28,6 +28,7 @@
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
 //    mpirun -np 4 ex9p -ea -m ../data/periodic-cube.mesh -d cuda
 //    mpirun -np 4 ex9p -fa -m ../data/periodic-cube.mesh -d cuda
+//    mpirun -np 4 ex9p -pa -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9 -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and

examples/petsc/ex2p.cpp

@@ -31,6 +31,10 @@
 //               also illustrated. The example also shows how to form a linear
 //               system using a PETSc matrix and solve with a PETSc solver.
 //
+//               The example also show how to use the non-overlapping feature of
+//               the ParBilinearForm class to obtain the linear operator in
+//               a format suitable for the BDDC preconditioner in PETSc.
+//
 //               We recommend viewing Example 1 before viewing this example.
 
 #include "mfem.hpp"
@@ -61,10 +65,15 @@ int main(int argc, char *argv[])
    bool use_petsc = true;
    const char *petscrc_file = "";
    bool use_nonoverlapping = false;
+   int ser_ref_levels = -1, par_ref_levels = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
    args.AddOption(&amg_elast, "-elast", "--amg-for-elasticity", "-sys",
@@ -131,8 +140,8 @@ int main(int argc, char *argv[])
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
    {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels = ser_ref_levels >= 0 ? ser_ref_levels :
+                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -145,7 +154,6 @@ int main(int argc, char *argv[])
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
-      int par_ref_levels = 1;
       for (int l = 0; l < par_ref_levels; l++)
       {
          pmesh->UniformRefinement();
@@ -296,12 +304,20 @@ int main(int argc, char *argv[])
       PetscPreconditioner *prec = NULL;
       if (use_nonoverlapping)
       {
+         // Compute dofs belonging to the natural boundary
+         Array<int> nat_tdof_list, nat_bdr(pmesh->bdr_attributes.Max());
+         nat_bdr = 1;
+         nat_bdr[0] = 0;
+         fespace->GetEssentialTrueDofs(nat_bdr, nat_tdof_list);
+
          // Auxiliary class for BDDC customization
          PetscBDDCSolverParams opts;
          // Inform the solver about the finite element space
          opts.SetSpace(fespace);
          // Inform the solver about essential dofs
          opts.SetEssBdrDofs(&ess_tdof_list);
+         // Inform the solver about natural dofs
+         opts.SetNatBdrDofs(&nat_tdof_list);
          // Create a BDDC solver with parameters
          prec = new PetscBDDCSolver(A,opts);
          pcg->SetPreconditioner(*prec);

examples/petsc/makefile

@@ -77,6 +77,7 @@ EX1_ARGS_P            := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_e
 EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_cuda
 EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
+EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
 EX3_ARGS              := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
 EX4_ARGS              := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
 EX4_HYB_ARGS          := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
@@ -107,6 +108,7 @@ ifeq ($(MFEM_USE_CUDA),YES)
 endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_BDDC))
 ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p

examples/petsc/rc_ex2p_bddc

@@ -0,0 +1,25 @@
+# Sample options for BDDC
+
+-ksp_converged_reason
+-ksp_view
+-pc_type bddc
+
+# Turn on diagnostic for errors
+#-pc_bddc_check_level 1
+
+# This is an H1 problem, local problems may be singular
+# Turn on automatic corner selection
+-pc_bddc_corner_selection
+
+# Advanced customization
+
+# Deluxe scaling
+-pc_bddc_use_deluxe_scaling
+
+# Adaptive primal space (requires PETSc configured with MUMPS or PARDISO support)
+#-pc_bddc_adaptive_threshold 1.2 # tolerance for eigenvalue selection
+#-pc_bddc_adaptive_userdefined # preserve RBMs
+#-pc_bddc_monolithic # treat all displacements components at once -> smaller primal spaces, larger eigenvalue problems
+
+# Select solver for coarse problem
+# -pc_bddc_coarse_pc_type cholesky

examples/pumi/ex1p.cpp

@@ -145,7 +145,7 @@ int main(int argc, char *argv[])
    // Perform Uniform refinement
    if (ref_levels > 1)
    {
-      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+      auto uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
 
       if (geom_order > 1)
       {

examples/pumi/ex6p.cpp

@@ -150,7 +150,7 @@ int main(int argc, char *argv[])
 
    if (ref_levels > 1)
    {
-      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+      auto uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
 
       if ( geom_order > 1)
       {
@@ -345,9 +345,7 @@ int main(int argc, char *argv[])
       apf::destroyField(ipfield);
 
       // 18. Perform MesAdapt.
-      ma::Input* erinput = ma::configure(pumi_mesh, sizefield);
-      erinput->shouldFixShape = true;
-      erinput->maximumIterations = 2;
+      auto erinput = ma::configure(pumi_mesh, sizefield);
       if ( geom_order > 1)
       {
          crv::adapt(erinput);

fem/bilinearform.cpp

@@ -333,9 +333,9 @@ void BilinearForm::AssembleElementMatrix(
 }
 
 void BilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
+   int i, const DenseMatrix &elmat, Array<int> &vdofs_, int skip_zeros)
 {
-   fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs_);
    if (static_cond)
    {
       static_cond->AssembleMatrix(i, elmat);
@@ -346,7 +346,7 @@ void BilinearForm::AssembleElementMatrix(
       {
          AllocMat();
       }
-      mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
+      mat->AddSubMatrix(vdofs_, vdofs_, elmat, skip_zeros);
       if (hybridization)
       {
          hybridization->AssembleMatrix(i, elmat);
@@ -361,9 +361,9 @@ void BilinearForm::AssembleBdrElementMatrix(
 }
 
 void BilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
+   int i, const DenseMatrix &elmat, Array<int> &vdofs_, int skip_zeros)
 {
-   fes->GetBdrElementVDofs(i, vdofs);
+   fes->GetBdrElementVDofs(i, vdofs_);
    if (static_cond)
    {
       static_cond->AssembleBdrMatrix(i, elmat);
@@ -374,7 +374,7 @@ void BilinearForm::AssembleBdrElementMatrix(
       {
          AllocMat();
       }
-      mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
+      mat->AddSubMatrix(vdofs_, vdofs_, elmat, skip_zeros);
       if (hybridization)
       {
          hybridization->AssembleBdrMatrix(i, elmat);
@@ -965,14 +965,14 @@ void BilinearForm::EliminateEssentialBCDiag (const Array<int> &bdr_attr_is_ess,
    }
 }
 
-void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
+void BilinearForm::EliminateVDofs(const Array<int> &vdofs_,
                                   const Vector &sol, Vector &rhs,
                                   DiagonalPolicy dpolicy)
 {
-   vdofs.HostRead();
-   for (int i = 0; i < vdofs.Size(); i++)
+   vdofs_.HostRead();
+   for (int i = 0; i < vdofs_.Size(); i++)
    {
-      int vdof = vdofs[i];
+      int vdof = vdofs_[i];
       if ( vdof >= 0 )
       {
          mat -> EliminateRowCol (vdof, sol(vdof), rhs, dpolicy);
@@ -984,7 +984,7 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
    }
 }
 
-void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
+void BilinearForm::EliminateVDofs(const Array<int> &vdofs_,
                                   DiagonalPolicy dpolicy)
 {
    if (mat_e == NULL)
@@ -992,9 +992,9 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
       mat_e = new SparseMatrix(height);
    }
 
-   for (int i = 0; i < vdofs.Size(); i++)
+   for (int i = 0; i < vdofs_.Size(); i++)
    {
-      int vdof = vdofs[i];
+      int vdof = vdofs_[i];
       if ( vdof >= 0 )
       {
          mat -> EliminateRowCol (vdof, *mat_e, dpolicy);
@@ -1046,10 +1046,10 @@ void BilinearForm::EliminateEssentialBCFromDofsDiag (const Array<int> &ess_dofs,
 }
 
 void BilinearForm::EliminateVDofsInRHS(
-   const Array<int> &vdofs, const Vector &x, Vector &b)
+   const Array<int> &vdofs_, const Vector &x, Vector &b)
 {
    mat_e->AddMult(x, b, -1.);
-   mat->PartMult(vdofs, x, b);
+   mat->PartMult(vdofs_, x, b);
 }
 
 void BilinearForm::Mult(const Vector &x, Vector &y) const
@@ -1653,16 +1653,16 @@ void MixedBilinearForm::AssembleElementMatrix(
 }
 
 void MixedBilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
+   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs_,
+   Array<int> &test_vdofs_, int skip_zeros)
 {
-   trial_fes->GetElementVDofs(i, trial_vdofs);
-   test_fes->GetElementVDofs(i, test_vdofs);
+   trial_fes->GetElementVDofs(i, trial_vdofs_);
+   test_fes->GetElementVDofs(i, test_vdofs_);
    if (mat == NULL)
    {
       mat = new SparseMatrix(height, width);
    }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
+   mat->AddSubMatrix(test_vdofs_, trial_vdofs_, elmat, skip_zeros);
 }
 
 void MixedBilinearForm::AssembleBdrElementMatrix(
@@ -1672,16 +1672,16 @@ void MixedBilinearForm::AssembleBdrElementMatrix(
 }
 
 void MixedBilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
+   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs_,
+   Array<int> &test_vdofs_, int skip_zeros)
 {
-   trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-   test_fes->GetBdrElementVDofs(i, test_vdofs);
+   trial_fes->GetBdrElementVDofs(i, trial_vdofs_);
+   test_fes->GetBdrElementVDofs(i, test_vdofs_);
    if (mat == NULL)
    {
       mat = new SparseMatrix(height, width);
    }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
+   mat->AddSubMatrix(test_vdofs_, trial_vdofs_, elmat, skip_zeros);
 }
 
 void MixedBilinearForm::EliminateTrialDofs (

fem/bilinearform_ext.cpp

@@ -514,6 +514,10 @@ void EABilinearFormExtension::Assemble()
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
    const int integratorCount = integrators.Size();
+   if ( integratorCount == 0 )
+   {
+      ea_data = 0.0;
+   }
    for (int i = 0; i < integratorCount; ++i)
    {
       integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);

fem/bilininteg.cpp

@@ -659,7 +659,7 @@ void GradientIntegrator::AssembleElementMatrix2(
    const FiniteElement &trial_fe, const FiniteElement &test_fe,
    ElementTransformation &Trans,  DenseMatrix &elmat)
 {
-   int dim = test_fe.GetDim();
+   dim = test_fe.GetDim();
    int trial_dof = trial_fe.GetDof();
    int test_dof = test_fe.GetDof();
    double c;
@@ -726,7 +726,7 @@ void DiffusionIntegrator::AssembleElementMatrix
   DenseMatrix &elmat )
 {
    int nd = el.GetDof();
-   int dim = el.GetDim();
+   dim = el.GetDim();
    int spaceDim = Trans.GetSpaceDim();
    bool square = (dim == spaceDim);
    double w;
@@ -802,7 +802,7 @@ void DiffusionIntegrator::AssembleElementMatrix2(
 {
    int tr_nd = trial_fe.GetDof();
    int te_nd = test_fe.GetDof();
-   int dim = trial_fe.GetDim();
+   dim = trial_fe.GetDim();
    int spaceDim = Trans.GetSpaceDim();
    bool square = (dim == spaceDim);
    double w;
@@ -885,7 +885,7 @@ void DiffusionIntegrator::AssembleElementVector(
    Vector &elvect)
 {
    int nd = el.GetDof();
-   int dim = el.GetDim();
+   dim = el.GetDim();
    int spaceDim = Tr.GetSpaceDim();
    double w;
 
@@ -966,7 +966,7 @@ void DiffusionIntegrator::ComputeElementFlux
 ( const FiniteElement &el, ElementTransformation &Trans,
   Vector &u, const FiniteElement &fluxelem, Vector &flux, bool with_coef )
 {
-   int i, j, nd, dim, spaceDim, fnd;
+   int nd, spaceDim, fnd;
 
    nd = el.GetDof();
    dim = el.GetDim();
@@ -1005,7 +1005,7 @@ void DiffusionIntegrator::ComputeElementFlux
    fnd = ir.GetNPoints();
    flux.SetSize( fnd * spaceDim );
 
-   for (i = 0; i < fnd; i++)
+   for (int i = 0; i < fnd; i++)
    {
       const IntegrationPoint &ip = ir.IntPoint(i);
       el.CalcDShape(ip, dshape);
@@ -1023,7 +1023,7 @@ void DiffusionIntegrator::ComputeElementFlux
             {
                vecdxt *= Q->Eval(Trans,ip);
             }
-            for (j = 0; j < spaceDim; j++)
+            for (int j = 0; j < spaceDim; j++)
             {
                flux(fnd*j+i) = vecdxt(j);
             }
@@ -1043,7 +1043,7 @@ void DiffusionIntegrator::ComputeElementFlux
                   pointflux[j] = D[j] * vecdxt[j];
                }
             }
-            for (j = 0; j < spaceDim; j++)
+            for (int j = 0; j < spaceDim; j++)
             {
                flux(fnd*j+i) = pointflux(j);
             }
@@ -1051,7 +1051,7 @@ void DiffusionIntegrator::ComputeElementFlux
       }
       else
       {
-         for (j = 0; j < spaceDim; j++)
+         for (int j = 0; j < spaceDim; j++)
          {
             flux(fnd*j+i) = vecdxt(j);
          }
@@ -1064,7 +1064,7 @@ double DiffusionIntegrator::ComputeFluxEnergy
   Vector &flux, Vector* d_energy)
 {
    int nd = fluxelem.GetDof();
-   int dim = fluxelem.GetDim();
+   dim = fluxelem.GetDim();
    int spaceDim = Trans.GetSpaceDim();
 
 #ifdef MFEM_THREAD_SAFE
@@ -1295,7 +1295,7 @@ void ConvectionIntegrator::AssembleElementMatrix(
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
    int nd = el.GetDof();
-   int dim = el.GetDim();
+   dim = el.GetDim();
 
 #ifdef MFEM_THREAD_SAFE
    DenseMatrix dshape, adjJ, Q_ir;
@@ -1864,7 +1864,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
   DenseMatrix &elmat )
 {
    int nd = el.GetDof();
-   int dim = el.GetDim();
+   dim = el.GetDim();
    int dimc = (dim == 3) ? 3 : 1;
    double w;
 
@@ -1961,7 +1961,7 @@ double CurlCurlIntegrator::ComputeFluxEnergy(const FiniteElement &fluxelem,
                                              Vector &flux, Vector *d_energy)
 {
    int nd = fluxelem.GetDof();
-   int dim = fluxelem.GetDim();
+   dim = fluxelem.GetDim();
 
 #ifdef MFEM_THREAD_SAFE
    DenseMatrix vshape;
@@ -2420,7 +2420,7 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(
    ElementTransformation &Trans,
    DenseMatrix &elmat)
 {
-   int dim  = trial_fe.GetDim();
+   dim  = trial_fe.GetDim();
    int trial_dof = trial_fe.GetDof();
    int test_dof = test_fe.GetDof();
    double c;
@@ -2523,9 +2523,9 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
    ElementTransformation &Trans,
    DenseMatrix &elmat)
 {
-   const int dim = el.GetDim();
    const int dof = el.GetDof();
-   const int sdim = Trans.GetSpaceDim();
+   dim = el.GetDim();
+   sdim = Trans.GetSpaceDim();
 
    // If vdim is not set, set it to the space dimension;
    vdim = (vdim <= 0) ? sdim : vdim;
@@ -2579,12 +2579,12 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
       else if (MQ)
       {
          MQ->Eval(mcoeff, Trans, ip);
-         for (int i = 0; i < vdim; ++i)
+         for (int ii = 0; ii < vdim; ++ii)
          {
-            for (int j = 0; j < vdim; ++j)
+            for (int jj = 0; jj < vdim; ++jj)
             {
-               Mult_a_AAt(w*mcoeff(i,j), dshapedxt, pelmat);
-               elmat.AddMatrix(pelmat, dof*i, dof*j);
+               Mult_a_AAt(w*mcoeff(ii,jj), dshapedxt, pelmat);
+               elmat.AddMatrix(pelmat, dof*ii, dof*jj);
             }
          }
       }
@@ -2604,9 +2604,9 @@ void VectorDiffusionIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr,
    const Vector &elfun, Vector &elvect)
 {
-   const int dim = el.GetDim();
    const int dof = el.GetDof();
-   const int sdim = Tr.GetSpaceDim();
+   dim = el.GetDim();
+   sdim = Tr.GetSpaceDim();
 
    // If vdim is not set, set it to the space dimension;
    vdim = (vdim <= 0) ? sdim : vdim;
@@ -2665,13 +2665,13 @@ void VectorDiffusionIntegrator::AssembleElementVector(
       else if (MQ)
       {
          MQ->Eval(mcoeff, Tr, ip);
-         for (int i = 0; i < vdim; ++i)
+         for (int ii = 0; ii < vdim; ++ii)
          {
-            Vector vec_out(mat_out.GetColumn(i), dof);
-            for (int j = 0; j < vdim; ++j)
+            Vector vec_out(mat_out.GetColumn(ii), dof);
+            for (int jj = 0; jj < vdim; ++jj)
             {
-               pelmat *= w*mcoeff(i,j);
-               const Vector vec_in(mat_in.GetColumn(j), dof);
+               pelmat *= w*mcoeff(ii,jj);
+               const Vector vec_in(mat_in.GetColumn(jj), dof);
                pelmat.Mult(vec_in, vec_out);
             }
          }
@@ -2759,14 +2759,14 @@ void ElasticityIntegrator::AssembleElementMatrix(
                   elmat (dof*d+k, dof*d+l) += (M * w) * pelmat(k, l);
                }
          }
-         for (int i = 0; i < dim; i++)
-            for (int j = 0; j < dim; j++)
+         for (int ii = 0; ii < dim; ii++)
+            for (int jj = 0; jj < dim; jj++)
             {
-               for (int k = 0; k < dof; k++)
-                  for (int l = 0; l < dof; l++)
+               for (int kk = 0; kk < dof; kk++)
+                  for (int ll = 0; ll < dof; ll++)
                   {
-                     elmat(dof*i+k, dof*j+l) +=
-                        (M * w) * gshape(k, j) * gshape(l, i);
+                     elmat(dof*ii+kk, dof*jj+ll) +=
+                        (M * w) * gshape(kk, jj) * gshape(ll, ii);
                   }
             }
       }
@@ -2947,7 +2947,7 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
                                            FaceElementTransformations &Trans,
                                            DenseMatrix &elmat)
 {
-   int dim, ndof1, ndof2;
+   int ndof1, ndof2;
 
    double un, a, b, w;
 

fem/bilininteg.hpp

@@ -644,13 +644,13 @@ protected:
 
    inline virtual void CalcVShape(const FiniteElement & vector_fe,
                                   ElementTransformation &Trans,
-                                  DenseMatrix & shape)
-   { vector_fe.CalcVShape(Trans, shape); }
+                                  DenseMatrix & shape_)
+   { vector_fe.CalcVShape(Trans, shape_); }
 
    inline virtual void CalcShape(const FiniteElement & scalar_fe,
                                  ElementTransformation &Trans,
-                                 Vector & shape)
-   { scalar_fe.CalcPhysShape(Trans, shape); }
+                                 Vector & shape_)
+   { scalar_fe.CalcPhysShape(Trans, shape_); }
 
    VectorCoefficient *VQ;
    bool transpose;
@@ -2258,7 +2258,7 @@ public:
       : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(NULL), MQ(&q) { }
 
    int GetVDim() const { return vdim; }
-   void SetVDim(int vdim) { this->vdim = vdim; }
+   void SetVDim(int vdim_) { vdim = vdim_; }
 
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,

fem/bilininteg_dgtrace_pa.cpp

@@ -182,22 +182,26 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       auto C = Reshape(vel.HostWrite(), dim, nq, nf);
       Vector Vq(dim);
       int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
+      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
       {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
+         if (face.IsNonconformingCoarse())
          {
+            // We skip nonconforming coarse faces as they are treated
+            // by the corresponding nonconforming fine faces.
+            continue;
+         }
+         else if ( face.IsOfFaceType(type) )
+         {
+            const int mask = FaceElementTransformations::HAVE_ELEM1 |
+                             FaceElementTransformations::HAVE_LOC1;
             FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
+               *fes.GetMesh()->GetFaceElementTransformations(f, mask);
             for (int q = 0; q < nq; ++q)
             {
                // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
                T.SetAllIntPoints(&ir->IntPoint(q));
                const IntegrationPoint &eip1 = T.GetElement1IntPoint();
                u->Eval(Vq, *T.Elem1, eip1);
@@ -242,29 +246,31 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       auto n = Reshape(geom->normal.HostRead(), nq, dim, nf);
       auto C = Reshape(r.HostWrite(), nq, nf);
       int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
+      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
       {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
+         if (face.IsNonconformingCoarse())
+         {
+            // We skip nonconforming coarse faces as they are treated
+            // by the corresponding nonconforming fine faces.
+            continue;
+         }
+         else if ( face.IsOfFaceType(type) )
          {
             FaceElementTransformations &T =
                *fes.GetMesh()->GetFaceElementTransformations(f);
             for (int q = 0; q < nq; ++q)
             {
                // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
 
                T.SetAllIntPoints(&ir->IntPoint(q));
                const IntegrationPoint &eip1 = T.GetElement1IntPoint();
                const IntegrationPoint &eip2 = T.GetElement2IntPoint();
                double r;
 
-               if (inf2 < 0)
+               if ( face.IsBoundary() )
                {
                   r = rho->Eval(*T.Elem1, eip1);
                }

fem/bilininteg_mass_pa.cpp

@@ -41,6 +41,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
       return;
    }
+   int map_type = el.GetMapType();
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
@@ -93,6 +94,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       const int NE = ne;
       const int Q1D = quad1D;
       const bool const_c = coeff.Size() == 1;
+      const bool by_val = map_type == FiniteElement::VALUE;
       const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
       const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
       const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
@@ -110,7 +112,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
                const double J22 = J(qx,qy,1,1,e);
                const double detJ = (J11*J22)-(J21*J12);
                const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-               v(qx,qy,e) =  W(qx,qy) * coeff * detJ;
+               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
             }
          }
       });
@@ -120,6 +122,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       const int NE = ne;
       const int Q1D = quad1D;
       const bool const_c = coeff.Size() == 1;
+      const bool by_val = map_type == FiniteElement::VALUE;
       const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
       const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
       const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
@@ -146,7 +149,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
                   /* */               J21 * (J12 * J33 - J32 * J13) +
                   /* */               J31 * (J12 * J23 - J22 * J13);
                   const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * detJ;
+                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
                }
             }
          }

fem/ceed/algebraic.cpp

@@ -667,7 +667,7 @@ AlgebraicSpaceHierarchy::AlgebraicSpaceHierarchy(FiniteElementSpace &fes)
       const SparseMatrix *R = fespaces[ilevel+1]->GetRestrictionMatrix();
       if (R)
       {
-         R->BuildTranspose();
+         R->EnsureMultTranspose();
          R_tr[ilevel] = new TransposeOperator(*R);
       }
       else

fem/coefficient.hpp

@@ -1605,22 +1605,22 @@ public:
    void SetTime(double t);
 
    /// Reset the first vector coefficient
-   void SetACoef(VectorCoefficient &A) { ACoef = &A; }
+   void SetACoef(VectorCoefficient &A_) { ACoef = &A_; }
    /// Return the first vector coefficient
    VectorCoefficient * GetACoef() const { return ACoef; }
 
    /// Reset the second vector coefficient
-   void SetBCoef(VectorCoefficient &B) { BCoef = &B; }
+   void SetBCoef(VectorCoefficient &B_) { BCoef = &B_; }
    /// Return the second vector coefficient
    VectorCoefficient * GetBCoef() const { return BCoef; }
 
    /// Reset the factor in front of the first vector coefficient
-   void SetAlphaCoef(Coefficient &A) { alphaCoef = &A; }
+   void SetAlphaCoef(Coefficient &A_) { alphaCoef = &A_; }
    /// Return the factor in front of the first vector coefficient
    Coefficient * GetAlphaCoef() const { return alphaCoef; }
 
    /// Reset the factor in front of the second vector coefficient
-   void SetBetaCoef(Coefficient &B) { betaCoef = &B; }
+   void SetBetaCoef(Coefficient &B_) { betaCoef = &B_; }
    /// Return the factor in front of the second vector coefficient
    Coefficient * GetBetaCoef() const { return betaCoef; }
 

fem/complex_fem.cpp

@@ -1243,7 +1243,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
          ess_tdof_list.HostRead();
          for (int k = 0; k < n; k++)
          {
@@ -1256,7 +1256,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
             ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
          const int *d_diag_i = Aih->diag->i;
          double *d_diag_data = Aih->diag->data;
-         CuWrap1D(n, [=] MFEM_DEVICE (int k)
+         MFEM_GPU_FORALL(k, n,
          {
             const int j = d_ess_tdof_list[k];
             d_diag_data[d_diag_i[j]] = 0.0;

fem/datacollection.cpp

@@ -11,6 +11,7 @@
 
 #include "fem.hpp"
 #include "../mesh/nurbs.hpp"
+#include "../mesh/vtk.hpp"
 #include "../general/binaryio.hpp"
 #include "../general/text.hpp"
 #include "picojson.h"
@@ -187,7 +188,7 @@ void DataCollection::SetPrefixPath(const std::string& prefix)
    }
 }
 
-void DataCollection::Load(int cycle)
+void DataCollection::Load(int cycle_)
 {
    MFEM_ABORT("this method is not implemented");
 }
@@ -787,53 +788,44 @@ void ParaViewDataCollection::Load(int )
 
 std::string ParaViewDataCollection::GenerateCollectionPath()
 {
-   std::string out = "";
-   out = prefix_path + DataCollection::GetCollectionName();
-   return out;
+   return prefix_path + DataCollection::GetCollectionName();
 }
 
 std::string ParaViewDataCollection::GeneratePVTUPath()
 {
-   std::string out = "Cycle" + to_padded_string(cycle,pad_digits_cycle);
-   return out;
+   return "Cycle" + to_padded_string(cycle,pad_digits_cycle);
 }
 
 std::string ParaViewDataCollection::GenerateVTUPath()
 {
-   std::string out = GeneratePVTUPath();
-   return out;
+   return GeneratePVTUPath();
 }
 
 std::string ParaViewDataCollection::GeneratePVDFileName()
 {
-   std::string out = GetCollectionName()+".pvd";
-   return out;
+   return GetCollectionName() + ".pvd";
 }
 
-std::string ParaViewDataCollection::GeneratePVTUFileName()
+std::string ParaViewDataCollection::GeneratePVTUFileName(
+   const std::string &prefix)
 {
-   std::string out = "data.pvtu";
-   return out;
+   return prefix + ".pvtu";
 }
 
-std::string ParaViewDataCollection::GenerateVTUFileName()
+std::string ParaViewDataCollection::GenerateVTUFileName(
+   const std::string &prefix, int rank)
 {
-   std::string out = "proc" + to_padded_string(myid,pad_digits_rank)+".vtu";
-   return out;
-}
-std::string ParaViewDataCollection::GenerateVTUFileName(int crank)
-{
-   std::string out = "proc" + to_padded_string(crank,pad_digits_rank)+".vtu";
-   return out;
+   return prefix + to_padded_string(rank, pad_digits_rank) + ".vtu";
 }
 
 void ParaViewDataCollection::Save()
 {
    // add a new collection to the PDV file
 
+   std::string col_path = GenerateCollectionPath();
    // check if the directories are created
    {
-      std::string path = GenerateCollectionPath()+"/"+GenerateVTUPath();
+      std::string path = col_path + "/" + GenerateVTUPath();
       int err = create_directory(path, mesh, myid);
       if (err)
       {
@@ -850,8 +842,7 @@ void ParaViewDataCollection::Save()
 
    if (myid == 0 && !pvd_stream.is_open())
    {
-      std::string dpath=GenerateCollectionPath();
-      std::string pvdname=dpath+"/"+GeneratePVDFileName();
+      std::string pvdname = col_path + "/" + GeneratePVDFileName();
 
       bool write_header = true;
       std::ifstream pvd_in;
@@ -915,80 +906,87 @@ void ParaViewDataCollection::Save()
       }
    }
 
-   // define the vtu file
+   std::string vtu_prefix = col_path + "/" + GenerateVTUPath() + "/";
+
+   // Save the local part of the mesh and grid functions fields to the local
+   // VTU file
    {
-      std::string fname = GenerateCollectionPath()+"/"+GenerateVTUPath()+"/"
-                          +GenerateVTUFileName();
-      std::fstream out(fname, std::ios::out);
+      std::ofstream out(vtu_prefix + GenerateVTUFileName("proc", myid));
       out.precision(precision);
-      SaveDataVTU(out,levels_of_detail);
-      out.close();
+      SaveDataVTU(out, levels_of_detail);
    }
 
-   // define the pvtu file only on process 0
-   if (myid==0)
+   // Save the local part of the quadrature function fields
+   for (const auto &qfield : q_field_map)
    {
-      std::string fname = GenerateCollectionPath()+"/"+GeneratePVTUPath()+"/"
-                          +GeneratePVTUFileName();
-      std::fstream out(fname, std::ios::out);
+      const std::string &field_name = qfield.first;
+      std::ofstream out(vtu_prefix + GenerateVTUFileName(field_name, myid));
+      qfield.second->SaveVTU(out, pv_data_format, compression);
+   }
 
-      out << "<?xml version=\"1.0\"?>\n";
-      out << "<VTKFile type=\"PUnstructuredGrid\"";
-      out << " version =\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
-      out << "<PUnstructuredGrid GhostLevel=\"0\">\n";
-
-      out << "<PPoints>\n";
-      out << "\t<PDataArray type=\"" << GetDataTypeString() << "\" ";
-      out << " Name=\"Points\" NumberOfComponents=\"3\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PPoints>\n";
-
-      out << "<PCells>\n";
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"connectivity\" NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"offsets\"      NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "\t<PDataArray type=\"UInt8\" ";
-      out << " Name=\"types\"        NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PCells>\n";
-
-      out << "<PPointData>\n";
-      for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it)
+   // MPI rank 0 also creates a "PVTU" file that points to all of the separately
+   // written VTU files.
+   // This file path is then appended to the PVD file.
+   if (myid == 0)
+   {
+      // Create the main PVTU file
       {
-         int vec_dim=it->second->VectorDim();
-         out << "<PDataArray type=\"" << GetDataTypeString()
-             << "\" Name=\"" << it->first
-             << "\" NumberOfComponents=\"" << vec_dim << "\" "
-             << "format=\"" << GetDataFormatString() << "\" />\n";
+         std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data"));
+         WritePVTUHeader(pvtu_out);
+
+         // Grid function fields
+         pvtu_out << "<PPointData>\n";
+         for (auto &field_it : field_map)
+         {
+            int vec_dim = field_it.second->VectorDim();
+            pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                     << "\" Name=\"" << field_it.first
+                     << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                     << "format=\"" << GetDataFormatString() << "\" />\n";
+         }
+         pvtu_out << "</PPointData>\n";
+         // Element attributes
+         pvtu_out << "<PCellData>\n";
+         pvtu_out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
+                  << "\" NumberOfComponents=\"1\""
+                  << " format=\"" << GetDataFormatString() << "\"/>\n";
+         pvtu_out << "</PCellData>\n";
+
+         WritePVTUFooter(pvtu_out, "proc");
       }
-      out << "</PPointData>\n";
 
-      // CELL DATA
-      out << "<PCellData>\n";
-      out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
-          << "\" NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PCellData>\n";
+      // Add the latest PVTU to the PVD
+      pvd_stream << "<DataSet timestep=\"" << GetTime()
+                 << "\" group=\"\" part=\"" << 0 << "\" file=\""
+                 << GeneratePVTUPath() + "/" + GeneratePVTUFileName("data")
+                 << "\" name=\"mesh\"/>\n";
 
-      for (int ii=0; ii<num_procs; ii++)
+      // Create PVTU files for each quadrature field and add them to the PVD
+      // file
+      for (auto &q_field : q_field_map)
       {
-         // this one is generated without the path
-         std::string nfname=GenerateVTUFileName(ii);
-         out << "<Piece Source=\"" << nfname << "\"/>\n";
-      }
-      out << "</PUnstructuredGrid>\n";
-      out << "</VTKFile>\n";
-      out.close();
+         const std::string &q_field_name = q_field.first;
+         std::string q_fname = GeneratePVTUPath() + "/"
+                               + GeneratePVTUFileName(q_field_name);
 
-      fname = GeneratePVTUPath()+"/"+GeneratePVTUFileName();
-      // add the pvtu file to the pvd_stream
-      pvd_stream << "<DataSet timestep=\"" << GetTime();  // GetCycle();
-      pvd_stream << "\" group=\"\" part=\"" << 0 << "\" file=\"";
-      pvd_stream << fname << "\"/>\n";
+         std::ofstream pvtu_out(col_path + "/" + q_fname);
+         WritePVTUHeader(pvtu_out);
+         int vec_dim = q_field.second->GetVDim();
+         pvtu_out << "<PPointData>\n";
+         pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                  << "\" Name=\"" << q_field_name
+                  << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                  << "format=\"" << GetDataFormatString() << "\" />\n";
+         pvtu_out << "</PPointData>\n";
+         WritePVTUFooter(pvtu_out, q_field_name);
+
+         pvd_stream << "<DataSet timestep=\"" << GetTime()
+                    << "\" group=\"\" part=\"" << 0 << "\" file=\""
+                    << q_fname << "\" name=\"" << q_field_name << "\"/>\n";
+      }
       pvd_stream.flush();
+      // Move the insertion point before the closing collection tag, so that
+      // the PVD file is valid even when writing incrementally.
       std::fstream::pos_type pos = pvd_stream.tellp();
       pvd_stream << "</Collection>\n";
       pvd_stream << "</VTKFile>" << std::endl;
@@ -996,6 +994,44 @@ void ParaViewDataCollection::Save()
    }
 }
 
+void ParaViewDataCollection::WritePVTUHeader(std::ostream &out)
+{
+   out << "<?xml version=\"1.0\"?>\n";
+   out << "<VTKFile type=\"PUnstructuredGrid\"";
+   out << " version =\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
+   out << "<PUnstructuredGrid GhostLevel=\"0\">\n";
+
+   out << "<PPoints>\n";
+   out << "\t<PDataArray type=\"" << GetDataTypeString() << "\" ";
+   out << " Name=\"Points\" NumberOfComponents=\"3\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "</PPoints>\n";
+
+   out << "<PCells>\n";
+   out << "\t<PDataArray type=\"Int32\" ";
+   out << " Name=\"connectivity\" NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "\t<PDataArray type=\"Int32\" ";
+   out << " Name=\"offsets\"      NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "\t<PDataArray type=\"UInt8\" ";
+   out << " Name=\"types\"        NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "</PCells>\n";
+}
+
+void ParaViewDataCollection::WritePVTUFooter(std::ostream &out,
+                                             const std::string &vtu_prefix)
+{
+   for (int ii=0; ii<num_procs; ii++)
+   {
+      std::string vtu_filename = GenerateVTUFileName(vtu_prefix, ii);
+      out << "<Piece Source=\"" << vtu_filename << "\"/>\n";
+   }
+   out << "</PUnstructuredGrid>\n";
+   out << "</VTKFile>\n";
+}
+
 void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
 {
    out << "<VTKFile type=\"UnstructuredGrid\"";
@@ -1015,16 +1051,6 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
    {
       SaveGFieldVTU(out,ref,it);
    }
-   // iterate over all quadrature functions
-   // if the Quadrature functions are dumped as cell data
-   // the cycle should be moved before the grid functions
-   // and the PrintVTU CellData section should be open in the mesh dump
-   for (QFieldMapIterator it=q_field_map.begin(); it!=q_field_map.end(); ++it)
-   {
-      // save the quadrature functions
-      // this one is not implemented yet
-      SaveQFieldVTU(out,ref,it);
-   }
    out << "</PointData>\n";
    // close the mesh
    out << "</Piece>\n"; // close the piece open in the PrintVTU method
@@ -1032,27 +1058,21 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
    out << "</VTKFile>" << std::endl;
 }
 
-void ParaViewDataCollection::SaveQFieldVTU(std::ostream &out, int ref,
-                                           const QFieldMapIterator& it )
-{
-   MFEM_WARNING("SaveQFieldVTU is not currently implemented - field name:"<<it->second);
-}
-
 void ParaViewDataCollection::SaveGFieldVTU(std::ostream &out, int ref_,
-                                           const FieldMapIterator& it)
+                                           const FieldMapIterator &it)
 {
    RefinedGeometry *RefG;
    Vector val;
    DenseMatrix vval, pmat;
    std::vector<char> buf;
    int vec_dim = it->second->VectorDim();
+   out << "<DataArray type=\"" << GetDataTypeString()
+       << "\" Name=\"" << it->first;
+   out << "\" NumberOfComponents=\"" << vec_dim << "\""
+       << " format=\"" << GetDataFormatString() << "\" >" << '\n';
    if (vec_dim == 1)
    {
       // scalar data
-      out << "<DataArray type=\"" << GetDataTypeString()
-          << "\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"1\" format=\""
-          << GetDataFormatString() << "\" >\n";
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
@@ -1060,51 +1080,23 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &out, int ref_,
          it->second->GetValues(i, RefG->RefPts, val, pmat);
          for (int j = 0; j < val.Size(); j++)
          {
-            if (pv_data_format == VTKFormat::ASCII)
-            {
-               out << ZeroSubnormal(val(j)) << '\n';
-            }
-            else if (pv_data_format == VTKFormat::BINARY)
-            {
-               bin_io::AppendBytes(buf, val(j));
-            }
-            else
-            {
-               bin_io::AppendBytes<float>(buf, float(val(j)));
-            }
+            WriteBinaryOrASCII(out, buf, val(j), "\n", pv_data_format);
          }
       }
    }
    else
    {
       // vector data
-      out << "<DataArray type=\"" << GetDataTypeString()
-          << "\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"" << vec_dim << "\""
-          << " format=\"" << GetDataFormatString() << "\" >" << '\n';
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
                    mesh->GetElementBaseGeometry(i), ref_, 1);
-
          it->second->GetVectorValues(i, RefG->RefPts, vval, pmat);
-
          for (int jj = 0; jj < vval.Width(); jj++)
          {
             for (int ii = 0; ii < vval.Height(); ii++)
             {
-               if (pv_data_format == VTKFormat::ASCII)
-               {
-                  out << ZeroSubnormal(vval(ii,jj)) << ' ';
-               }
-               else if (pv_data_format == VTKFormat::BINARY)
-               {
-                  bin_io::AppendBytes(buf, vval(ii,jj));
-               }
-               else
-               {
-                  bin_io::AppendBytes<float>(buf, float(vval(ii,jj)));
-               }
+               WriteBinaryOrASCII(out, buf, vval(ii,jj), " ", pv_data_format);
             }
             if (pv_data_format == VTKFormat::ASCII) { out << '\n'; }
          }

fem/datacollection.hpp

@@ -258,8 +258,8 @@ public:
    { q_field_map.Deregister(field_name, own_data); }
 
    /// Check if a grid function is part of the collection
-   bool HasField(const std::string& name) const
-   { return field_map.Has(name); }
+   bool HasField(const std::string& field_name) const
+   { return field_map.Has(field_name); }
 
    /// Get a pointer to a grid function in the collection.
    /** Returns NULL if @a field_name is not in the collection. */
@@ -491,19 +491,20 @@ private:
    bool restart_mode;
 
 protected:
+   void WritePVTUHeader(std::ostream &out);
+   void WritePVTUFooter(std::ostream &out, const std::string &vtu_prefix);
    void SaveDataVTU(std::ostream &out, int ref);
    void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
-   void SaveQFieldVTU(std::ostream &out, int ref, const QFieldMapIterator& it);
    const char *GetDataFormatString() const;
    const char *GetDataTypeString() const;
 
-   std::string  GenerateCollectionPath();
-   std::string  GenerateVTUFileName();
-   std::string  GenerateVTUFileName(int rank);
-   std::string  GenerateVTUPath();
-   std::string  GeneratePVDFileName();
-   std::string  GeneratePVTUFileName();
-   std::string  GeneratePVTUPath();
+   std::string GenerateCollectionPath();
+   std::string GenerateVTUFileName(const std::string &prefix, int rank);
+   std::string GenerateVTUPath();
+   std::string GeneratePVDFileName();
+   std::string GeneratePVTUFileName(const std::string &prefix);
+   std::string GeneratePVTUPath();
+
 
 public:
    /// Constructor. The collection name is used when saving the data.

fem/estimators.hpp

@@ -234,7 +234,7 @@ protected:
    bool own_flux_fes; ///< Ownership flag for flux_space and smooth_flux_space.
 
    /// Initialize with the integrator, solution, and flux finite element spaces.
-   void Init(BilinearFormIntegrator &integ,
+   void Init(BilinearFormIntegrator &integ_,
              ParGridFunction &sol,
              ParFiniteElementSpace *flux_fes,
              ParFiniteElementSpace *smooth_flux_fes)
@@ -242,7 +242,7 @@ protected:
       current_sequence = -1;
       local_norm_p = 1;
       total_error = 0.0;
-      this->integ = &integ;
+      integ = &integ_;
       solution = &sol;
       flux_space = flux_fes;
       smooth_flux_space = smooth_flux_fes;

fem/fe/fe_base.cpp

@@ -1538,7 +1538,7 @@ void VectorFiniteElement::LocalRestriction_ND(
 
 
 Poly_1D::Basis::Basis(const int p, const double *nodes, EvalType etype)
-   : etype(etype), auxiliary_basis(NULL)
+   : etype(etype), auxiliary_basis(NULL), scale_integrated(false)
 {
    switch (etype)
    {
@@ -1838,11 +1838,29 @@ void Poly_1D::Basis::EvalIntegrated(const Vector &d_aux, Vector &u) const
    MFEM_VERIFY(etype == Integrated,
                "EvalIntegrated is only valid for Integrated basis type");
    int p = d_aux.Size() - 1;
+   // See Gerritsma, M. (2010).  "Edge functions for spectral element methods",
+   // in Lecture Notes in Computational Science and Engineering, 199--207.
    u[0] = -d_aux[0];
    for (int j=1; j<p; ++j)
    {
       u[j] = u[j-1] - d_aux[j];
    }
+   // If scale_integrated is true, the degrees of freedom represent mean values,
+   // otherwise they represent subcell integrals. Generally, scale_integrated
+   // should be true for MapType::VALUE, and false for other map types.
+   if (scale_integrated)
+   {
+      Vector &aux_nodes = auxiliary_basis->x;
+      for (int j=0; j<aux_nodes.Size()-1; ++j)
+      {
+         u[j] *= aux_nodes[j+1] - aux_nodes[j];
+      }
+   }
+}
+
+void Poly_1D::Basis::ScaleIntegrated(bool scale_integrated_)
+{
+   scale_integrated = scale_integrated_;
 }
 
 Poly_1D::Basis::~Basis()
@@ -2379,6 +2397,18 @@ NodalTensorFiniteElement::NodalTensorFiniteElement(const int dims,
    lex_ordering = dof_map;
 }
 
+void NodalTensorFiniteElement::SetMapType(const int map_type)
+{
+   ScalarFiniteElement::SetMapType(map_type);
+   // If we are using the "integrated" basis, the basis functions should be
+   // scaled for MapType::VALUE, and not scaled for MapType::INTEGRAL. This
+   // ensures spectral equivalence of the mass matrix with its low-order-refined
+   // counterpart (cf. LORDiscretization)
+   if (basis1d.IsIntegratedType())
+   {
+      basis1d.ScaleIntegrated(map_type == VALUE);
+   }
+}
 
 VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                                                      const int d,

fem/fe/fe_base.hpp

@@ -657,7 +657,7 @@ public:
    /** @brief Set the FiniteElement::MapType of the element to either VALUE or
        INTEGRAL. Also sets the FiniteElement::DerivType to GRAD if the
        FiniteElement::MapType is VALUE. */
-   void SetMapType(int M)
+   virtual void SetMapType(int M)
    {
       MFEM_VERIFY(M == VALUE || M == INTEGRAL, "unknown MapType");
       map_type = M;
@@ -955,41 +955,68 @@ public:
 };
 
 
-/// Class for computing 1D special polynomials and their associated basis
+/// @brief Class for computing 1D special polynomials and their associated basis
 /// functions
 class Poly_1D
 {
 public:
+   /// One-dimensional basis evaluation type
    enum EvalType
    {
-      ChangeOfBasis = 0, // Use change of basis, O(p^2) Evals
-      Barycentric   = 1, // Use barycentric Lagrangian interpolation, O(p) Evals
-      Positive      = 2, // Fast evaluation of Bernstein polynomials
-      Integrated    = 3, // Integrated indicator functions (cf. Gerritsma)
-      NumEvalTypes  = 4  // Keep count of the number of eval types
+      ChangeOfBasis = 0, ///< Use change of basis, O(p^2) Evals
+      Barycentric   = 1, ///< Use barycentric Lagrangian interpolation, O(p) Evals
+      Positive      = 2, ///< Fast evaluation of Bernstein polynomials
+      Integrated    = 3, ///< Integrated indicator functions (cf. Gerritsma)
+      NumEvalTypes  = 4  ///< Keep count of the number of eval types
    };
 
+   /// @brief Class for evaluating 1D nodal, positive (Bernstein), or integrated
+   /// (Gerritsma) bases.
    class Basis
    {
    private:
-      int etype;
+      EvalType etype; ///< Determines how the basis functions should be evaluated.
       DenseMatrixInverse Ai;
       mutable Vector x, w;
-      // The following data members are used for "integrated basis type", which
-      // is defined in terms of nodal basis of one degree higher.
+      /// The following data members are used for "integrated basis type", which
+      /// is defined in terms of nodal basis of one degree higher.
+      ///@{
       mutable Vector u_aux, d_aux, d2_aux;
-      Basis *auxiliary_basis; // Non-NULL only for etype == Integrated
+      ///@}
+      /// @brief An auxiliary nodal basis used to evaluate the integrated basis.
+      /// This member variable is NULL whenever etype != Integrated.
+      Basis *auxiliary_basis;
+      /// Should the integrated basis functions be scaled? See ScaleIntegrated.
+      bool scale_integrated;
 
    public:
-      /// Create a nodal or positive (Bernstein) basis
+      /// Create a nodal or positive (Bernstein) basis of degree @a p
       Basis(const int p, const double *nodes, EvalType etype = Barycentric);
+      /// Evaluate the basis functions at point @a x in [0,1]
       void Eval(const double x, Vector &u) const;
+      /// @brief Evaluate the basis functions and their derivatives at point @a
+      /// x in [0,1]
       void Eval(const double x, Vector &u, Vector &d) const;
+      /// @brief Evaluate the basis functions and their first two derivatives at
+      /// point @a x in [0,1]
       void Eval(const double x, Vector &u, Vector &d, Vector &d2) const;
-      /// Evaluate the "integrated" basis, which is given by the negative
-      /// partial sum of the corresponding closed basis derivatives. The closed
-      /// basis derivatives are given by @a d, and the result is stored in @a i.
+      /// @brief Evaluate the "integrated" basis type using pre-computed closed
+      /// basis derivatives.
+      ///
+      /// This basis is given by the negative partial sum of the corresponding
+      /// closed basis derivatives. The closed basis derivatives are given by @a
+      /// d, and the result is stored in @a i.
       void EvalIntegrated(const Vector &d, Vector &i) const;
+      /// @brief Set whether the "integrated" basis should be scaled by the
+      /// subcell sizes. Has no effect for non-integrated bases.
+      ///
+      /// Generally, this should be true for mfem::FiniteElement::MapType VALUE
+      /// and false for all other map types. If this option is enabled, the
+      /// basis functions will be scaled by the widths of the subintervals, so
+      /// that the basis functions represent mean values. Otherwise, the basis
+      /// functions represent integrated values.
+      void ScaleIntegrated(bool scale_integrated_);
+      /// Returns true if the basis is "integrated", false otherwise.
       bool IsIntegratedType() const { return etype == Integrated; }
       ~Basis();
    };
@@ -1193,6 +1220,8 @@ public:
              ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
    }
 
+   virtual void SetMapType(const int map_type_);
+
    virtual void GetTransferMatrix(const FiniteElement &fe,
                                   ElementTransformation &Trans,
                                   DenseMatrix &I) const

fem/fe_coll.cpp

@@ -1897,21 +1897,21 @@ const int *H1_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 FiniteElementCollection *H1_FECollection::GetTraceCollection() const
 {
-   int p = H1_dof[Geometry::SEGMENT] + 1;
-   int dim = -1;
+   int tr_p = H1_dof[Geometry::SEGMENT] + 1;
+   int tr_dim = -1;
    if (!strncmp(h1_name, "H1_", 3))
    {
-      dim = atoi(h1_name + 3);
+      tr_dim = atoi(h1_name + 3);
    }
    else if (!strncmp(h1_name, "H1Pos_", 6))
    {
-      dim = atoi(h1_name + 6);
+      tr_dim = atoi(h1_name + 6);
    }
    else if (!strncmp(h1_name, "H1@", 3))
    {
-      dim = atoi(h1_name + 5);
+      tr_dim = atoi(h1_name + 5);
    }
-   return (dim < 0) ? NULL : new H1_Trace_FECollection(p, dim, b_type);
+   return (dim < 0) ? NULL : new H1_Trace_FECollection(tr_p, tr_dim, b_type);
 }
 
 const int *H1_FECollection::GetDofMap(Geometry::Type GeomType) const
@@ -2374,7 +2374,7 @@ RT_FECollection::RT_FECollection(const int p, const int dim,
    InitFaces(p, dim, map_type, signs);
 }
 
-void RT_FECollection::InitFaces(const int p, const int dim,
+void RT_FECollection::InitFaces(const int p, const int dim_,
                                 const int map_type,
                                 const bool signs)
 {
@@ -2404,7 +2404,7 @@ void RT_FECollection::InitFaces(const int p, const int dim,
       QuadDofOrd[i] = NULL;
    }
 
-   if (dim == 2)
+   if (dim_ == 2)
    {
       L2_SegmentElement *l2_seg = new L2_SegmentElement(p, ob_type);
       l2_seg->SetMapType(map_type);
@@ -2419,7 +2419,7 @@ void RT_FECollection::InitFaces(const int p, const int dim,
          SegDofOrd[1][i] = signs ? (-1 - (p - i)) : (p - i);
       }
    }
-   else if (dim == 3)
+   else if (dim_ == 3)
    {
       L2_TriangleElement *l2_tri = new L2_TriangleElement(p, ob_type);
       l2_tri->SetMapType(map_type);
@@ -2453,9 +2453,9 @@ void RT_FECollection::InitFaces(const int p, const int dim,
             TriDofOrd[5][o] = -1-(TriDof-((pp2-i)*(pp1-i))/2+j);  // (0,2,1)
             if (!signs)
             {
-               for (int k = 1; k < 6; k += 2)
+               for (int kk = 1; kk < 6; kk += 2)
                {
-                  TriDofOrd[k][o] = -1 - TriDofOrd[k][o];
+                  TriDofOrd[kk][o] = -1 - TriDofOrd[kk][o];
                }
             }
          }
@@ -2529,18 +2529,19 @@ const int *RT_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 FiniteElementCollection *RT_FECollection::GetTraceCollection() const
 {
-   int dim, p;
+   int tr_dim, tr_p;
    if (!strncmp(rt_name, "RT_", 3))
    {
-      dim = atoi(rt_name + 3);
-      p = atoi(rt_name + 7);
+      tr_dim = atoi(rt_name + 3);
+      tr_p = atoi(rt_name + 7);
    }
    else // rt_name = RT@.._.D_P*
    {
-      dim = atoi(rt_name + 6);
-      p = atoi(rt_name + 10);
+      tr_dim = atoi(rt_name + 6);
+      tr_p = atoi(rt_name + 10);
    }
-   return new RT_Trace_FECollection(p, dim, FiniteElement::INTEGRAL, ob_type);
+   return new RT_Trace_FECollection(tr_p, tr_dim, FiniteElement::INTEGRAL,
+                                    ob_type);
 }
 
 RT_FECollection::~RT_FECollection()
@@ -2814,22 +2815,22 @@ const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 FiniteElementCollection *ND_FECollection::GetTraceCollection() const
 {
-   int p, dim, cb_type, ob_type;
+   int tr_p, tr_dim, tr_cb_type, tr_ob_type;
 
-   p = ND_dof[Geometry::SEGMENT];
+   tr_p = ND_dof[Geometry::SEGMENT];
    if (nd_name[2] == '_') // ND_
    {
-      dim = atoi(nd_name + 3);
-      cb_type = BasisType::GaussLobatto;
-      ob_type = BasisType::GaussLegendre;
+      tr_dim = atoi(nd_name + 3);
+      tr_cb_type = BasisType::GaussLobatto;
+      tr_ob_type = BasisType::GaussLegendre;
    }
    else // ND@
    {
-      dim = atoi(nd_name + 6);
-      cb_type = BasisType::GetType(nd_name[3]);
-      ob_type = BasisType::GetType(nd_name[4]);
+      tr_dim = atoi(nd_name + 6);
+      tr_cb_type = BasisType::GetType(nd_name[3]);
+      tr_ob_type = BasisType::GetType(nd_name[4]);
    }
-   return new ND_Trace_FECollection(p, dim, cb_type, ob_type);
+   return new ND_Trace_FECollection(tr_p, tr_dim, tr_cb_type, tr_ob_type);
 }
 
 ND_FECollection::~ND_FECollection()

fem/fe_coll.hpp

@@ -1117,7 +1117,7 @@ public:
    { return (GeomType == GeomType_) ? Local_Element : NULL; }
    virtual int DofForGeometry(Geometry::Type GeomType_) const
    { return (GeomType == GeomType_) ? Local_Element->GetDof() : 0; }
-   virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
+   virtual const int *DofOrderForOrientation(Geometry::Type GeomType_,
                                              int Or) const
    { return NULL; }
    virtual const char *Name() const { return d_name; }

fem/fespace.cpp

@@ -70,14 +70,14 @@ FiniteElementSpace::FiniteElementSpace()
 { }
 
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
-                                       Mesh *mesh,
-                                       const FiniteElementCollection *fec)
+                                       Mesh *mesh_,
+                                       const FiniteElementCollection *fec_)
    : VDoFTrans(orig.vdim, orig.ordering)
 {
-   mesh = mesh ? mesh : orig.mesh;
-   fec = fec ? fec : orig.fec;
+   mesh_ = mesh_ ? mesh_ : orig.mesh;
+   fec_ = fec_ ? fec_ : orig.fec;
 
-   NURBSExtension *NURBSext = NULL;
+   NURBSExtension *nurbs_ext = NULL;
    if (orig.NURBSext && orig.NURBSext != orig.mesh->NURBSext)
    {
 #ifdef MFEM_USE_MPI
@@ -85,16 +85,16 @@ FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
          dynamic_cast<ParNURBSExtension *>(orig.NURBSext);
       if (pNURBSext)
       {
-         NURBSext = new ParNURBSExtension(*pNURBSext);
+         nurbs_ext = new ParNURBSExtension(*pNURBSext);
       }
       else
 #endif
       {
-         NURBSext = new NURBSExtension(*orig.NURBSext);
+         nurbs_ext = new NURBSExtension(*orig.NURBSext);
       }
    }
 
-   Constructor(mesh, NURBSext, fec, orig.vdim, orig.ordering);
+   Constructor(mesh_, nurbs_ext, fec_, orig.vdim, orig.ordering);
 }
 
 void FiniteElementSpace::CopyProlongationAndRestriction(
@@ -192,74 +192,74 @@ int FiniteElementSpace::GetElementOrderImpl(int i) const
    return elem_order.Size() ? elem_order[i] : fec->GetOrder();
 }
 
-void FiniteElementSpace::GetVDofs(int vd, Array<int>& dofs, int ndofs) const
+void FiniteElementSpace::GetVDofs(int vd, Array<int>& dofs, int ndofs_) const
 {
-   if (ndofs < 0) { ndofs = this->ndofs; }
+   if (ndofs_ < 0) { ndofs_ = this->ndofs; }
 
    if (ordering == Ordering::byNODES)
    {
       for (int i = 0; i < dofs.Size(); i++)
       {
-         dofs[i] = Ordering::Map<Ordering::byNODES>(ndofs, vdim, i, vd);
+         dofs[i] = Ordering::Map<Ordering::byNODES>(ndofs_, vdim, i, vd);
       }
    }
    else
    {
       for (int i = 0; i < dofs.Size(); i++)
       {
-         dofs[i] = Ordering::Map<Ordering::byVDIM>(ndofs, vdim, i, vd);
+         dofs[i] = Ordering::Map<Ordering::byVDIM>(ndofs_, vdim, i, vd);
       }
    }
 }
 
-void FiniteElementSpace::DofsToVDofs (Array<int> &dofs, int ndofs) const
+void FiniteElementSpace::DofsToVDofs (Array<int> &dofs, int ndofs_) const
 {
    if (vdim == 1) { return; }
-   if (ndofs < 0) { ndofs = this->ndofs; }
+   if (ndofs_ < 0) { ndofs_ = this->ndofs; }
 
    if (ordering == Ordering::byNODES)
    {
-      Ordering::DofsToVDofs<Ordering::byNODES>(ndofs, vdim, dofs);
+      Ordering::DofsToVDofs<Ordering::byNODES>(ndofs_, vdim, dofs);
    }
    else
    {
-      Ordering::DofsToVDofs<Ordering::byVDIM>(ndofs, vdim, dofs);
+      Ordering::DofsToVDofs<Ordering::byVDIM>(ndofs_, vdim, dofs);
    }
 }
 
-void FiniteElementSpace::DofsToVDofs(int vd, Array<int> &dofs, int ndofs) const
+void FiniteElementSpace::DofsToVDofs(int vd, Array<int> &dofs, int ndofs_) const
 {
    if (vdim == 1) { return; }
-   if (ndofs < 0) { ndofs = this->ndofs; }
+   if (ndofs_ < 0) { ndofs_ = this->ndofs; }
 
    if (ordering == Ordering::byNODES)
    {
       for (int i = 0; i < dofs.Size(); i++)
       {
-         dofs[i] = Ordering::Map<Ordering::byNODES>(ndofs, vdim, dofs[i], vd);
+         dofs[i] = Ordering::Map<Ordering::byNODES>(ndofs_, vdim, dofs[i], vd);
       }
    }
    else
    {
       for (int i = 0; i < dofs.Size(); i++)
       {
-         dofs[i] = Ordering::Map<Ordering::byVDIM>(ndofs, vdim, dofs[i], vd);
+         dofs[i] = Ordering::Map<Ordering::byVDIM>(ndofs_, vdim, dofs[i], vd);
       }
    }
 }
 
-int FiniteElementSpace::DofToVDof(int dof, int vd, int ndofs) const
+int FiniteElementSpace::DofToVDof(int dof, int vd, int ndofs_) const
 {
    if (vdim == 1) { return dof; }
-   if (ndofs < 0) { ndofs = this->ndofs; }
+   if (ndofs_ < 0) { ndofs_ = this->ndofs; }
 
    if (ordering == Ordering::byNODES)
    {
-      return Ordering::Map<Ordering::byNODES>(ndofs, vdim, dof, vd);
+      return Ordering::Map<Ordering::byNODES>(ndofs_, vdim, dof, vd);
    }
    else
    {
-      return Ordering::Map<Ordering::byVDIM>(ndofs, vdim, dof, vd);
+      return Ordering::Map<Ordering::byVDIM>(ndofs_, vdim, dof, vd);
    }
 }
 
@@ -713,8 +713,8 @@ FiniteElementSpace::H2L_GlobalRestrictionMatrix (FiniteElementSpace *lfes)
    DenseMatrix loc_restr;
    Array<int> l_dofs, h_dofs, l_vdofs, h_vdofs;
 
-   int vdim = lfes->GetVDim();
-   R = new SparseMatrix (vdim * lfes -> GetNDofs(), vdim * ndofs);
+   int lvdim = lfes->GetVDim();
+   R = new SparseMatrix (lvdim * lfes -> GetNDofs(), lvdim * ndofs);
 
    Geometry::Type cached_geom = Geometry::INVALID;
    const FiniteElement *h_fe = NULL;
@@ -737,7 +737,7 @@ FiniteElementSpace::H2L_GlobalRestrictionMatrix (FiniteElementSpace *lfes)
          cached_geom = geom;
       }
 
-      for (int vd = 0; vd < vdim; vd++)
+      for (int vd = 0; vd < lvdim; vd++)
       {
          l_dofs.Copy(l_vdofs);
          lfes->DofsToVDofs(vd, l_vdofs);
@@ -894,9 +894,10 @@ int FiniteElementSpace::GetDegenerateFaceDofs(int index, Array<int> &dofs,
 int FiniteElementSpace::GetNumBorderDofs(Geometry::Type geom, int order) const
 {
    // return the number of vertex and edge DOFs that precede inner DOFs
-   int nv = fec->GetNumDof(Geometry::POINT, order);
-   int ne = fec->GetNumDof(Geometry::SEGMENT, order);
-   return Geometry::NumVerts[geom] * (nv + ne);
+   const int nv = fec->GetNumDof(Geometry::POINT, order);
+   const int ne = fec->GetNumDof(Geometry::SEGMENT, order);
+
+   return Geometry::NumVerts[geom] * (geom == Geometry::SEGMENT ? nv : (nv + ne));
 }
 
 int FiniteElementSpace::GetEntityDofs(int entity, int index, Array<int> &dofs,
@@ -934,6 +935,12 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    if (cP_is_set) { return; }
    cP_is_set = true;
 
+   if (FEColl()->GetContType() == FiniteElementCollection::DISCONTINUOUS)
+   {
+      cP = cR = cR_hp = NULL; // will be treated as identities
+      return;
+   }
+
    Array<int> master_dofs, slave_dofs, highest_dofs;
 
    IsoparametricTransformation T;
@@ -1054,6 +1061,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
             // get lowest order variant DOFs and FE
             int p = GetEntityDofs(entity, i, master_dofs, geom, 0);
             const auto *master_fe = fec->GetFE(geom, p);
+            if (!master_fe) { break; }
 
             // constrain all higher order DOFs: interpolate lowest order function
             for (int variant = 1; ; variant++)
@@ -1192,7 +1200,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
       if (cR_hp) { MakeVDimMatrix(*cR_hp); }
    }
 
-   if (Device::IsEnabled()) { cP->BuildTranspose(); }
+   cP->EnsureMultTranspose();
 }
 
 void FiniteElementSpace::MakeVDimMatrix(SparseMatrix &mat) const
@@ -1300,7 +1308,14 @@ const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
       FaceRestriction *res;
       if (is_dg_space)
       {
-         res = new L2FaceRestriction(*this, e_ordering, type, m);
+         if (Conforming())
+         {
+            res = new L2FaceRestriction(*this, e_ordering, type, m);
+         }
+         else
+         {
+            res = new NCL2FaceRestriction(*this, e_ordering, type, m);
+         }
       }
       else
       {
@@ -1540,11 +1555,11 @@ void FiniteElementSpace::RefinementOperator
       old_DoFTrans[i] = NULL;
    }
 
-   const FiniteElementCollection *fec = fespace->FEColl();
-   if (dynamic_cast<const ND_FECollection*>(fec))
+   const FiniteElementCollection *fec_ref = fespace->FEColl();
+   if (dynamic_cast<const ND_FECollection*>(fec_ref))
    {
       const FiniteElement * nd_tri =
-         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
+         fec_ref->FiniteElementForGeometry(Geometry::TRIANGLE);
       if (nd_tri)
       {
          old_DoFTrans[Geometry::TRIANGLE] =
@@ -1552,7 +1567,7 @@ void FiniteElementSpace::RefinementOperator
       }
 
       const FiniteElement * nd_tet =
-         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
+         fec_ref->FiniteElementForGeometry(Geometry::TETRAHEDRON);
       if (nd_tet)
       {
          old_DoFTrans[Geometry::TETRAHEDRON] =
@@ -1564,20 +1579,21 @@ void FiniteElementSpace::RefinementOperator
 void FiniteElementSpace::RefinementOperator
 ::Mult(const Vector &x, Vector &y) const
 {
-   Mesh* mesh = fespace->GetMesh();
-   const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();
+   Mesh* mesh_ref = fespace->GetMesh();
+   const CoarseFineTransformations &trans_ref =
+      mesh_ref->GetRefinementTransforms();
 
    Array<int> dofs, vdofs, old_dofs, old_vdofs, old_Fo;
 
-   int vdim = fespace->GetVDim();
-   int old_ndofs = width / vdim;
+   int rvdim = fespace->GetVDim();
+   int old_ndofs = width / rvdim;
 
    Vector subY, subX;
 
-   for (int k = 0; k < mesh->GetNE(); k++)
+   for (int k = 0; k < mesh_ref->GetNE(); k++)
    {
-      const Embedding &emb = rtrans.embeddings[k];
-      const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
+      const Embedding &emb = trans_ref.embeddings[k];
+      const Geometry::Type geom = mesh_ref->GetElementBaseGeometry(k);
       const DenseMatrix &lP = localP[geom](emb.matrix);
 
       subY.SetSize(lP.Height());
@@ -1587,7 +1603,7 @@ void FiniteElementSpace::RefinementOperator
 
       if (!doftrans)
       {
-         for (int vd = 0; vd < vdim; vd++)
+         for (int vd = 0; vd < rvdim; vd++)
          {
             dofs.Copy(vdofs);
             fespace->DofsToVDofs(vd, vdofs);
@@ -1612,7 +1628,7 @@ void FiniteElementSpace::RefinementOperator
             doftrans = vdoftrans->GetDofTransformation();
          }
 
-         for (int vd = 0; vd < vdim; vd++)
+         for (int vd = 0; vd < rvdim; vd++)
          {
             dofs.Copy(vdofs);
             fespace->DofsToVDofs(vd, vdofs);
@@ -1638,23 +1654,24 @@ void FiniteElementSpace::RefinementOperator
 {
    y = 0.0;
 
-   Mesh* mesh = fespace->GetMesh();
-   const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();
+   Mesh* mesh_ref = fespace->GetMesh();
+   const CoarseFineTransformations &trans_ref =
+      mesh_ref->GetRefinementTransforms();
 
    Array<char> processed(fespace->GetVSize());
    processed = 0;
 
    Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs, old_Fo;
 
-   int vdim = fespace->GetVDim();
-   int old_ndofs = width / vdim;
+   int rvdim = fespace->GetVDim();
+   int old_ndofs = width / rvdim;
 
-   Vector subY, subX, subYt, subXt;
+   Vector subY, subX, subYt;
 
-   for (int k = 0; k < mesh->GetNE(); k++)
+   for (int k = 0; k < mesh_ref->GetNE(); k++)
    {
-      const Embedding &emb = rtrans.embeddings[k];
-      const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
+      const Embedding &emb = trans_ref.embeddings[k];
+      const Geometry::Type geom = mesh_ref->GetElementBaseGeometry(k);
       const DenseMatrix &lP = localP[geom](emb.matrix);
 
       DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
@@ -1664,7 +1681,7 @@ void FiniteElementSpace::RefinementOperator
       {
          subY.SetSize(lP.Width());
 
-         for (int vd = 0; vd < vdim; vd++)
+         for (int vd = 0; vd < rvdim; vd++)
          {
             f_dofs.Copy(f_vdofs);
             fespace->DofsToVDofs(vd, f_vdofs);
@@ -1701,7 +1718,7 @@ void FiniteElementSpace::RefinementOperator
             doftrans = vdoftrans->GetDofTransformation();
          }
 
-         for (int vd = 0; vd < vdim; vd++)
+         for (int vd = 0; vd < rvdim; vd++)
          {
             f_dofs.Copy(f_vdofs);
             fespace->DofsToVDofs(vd, f_vdofs);
@@ -1960,15 +1977,16 @@ void FiniteElementSpace::DerefinementOperator
    Array<int> c_vdofs, f_vdofs;
    Vector loc_x, loc_y;
    DenseMatrix loc_x_mat, loc_y_mat;
-   const int vdim = fine_fes->GetVDim();
-   const int coarse_ndofs = height/vdim;
+   const int fine_vdim = fine_fes->GetVDim();
+   const int coarse_ndofs = height/fine_vdim;
    for (int coarse_el = 0; coarse_el < coarse_to_fine.Size(); coarse_el++)
    {
       coarse_elem_dof->GetRow(coarse_el, c_vdofs);
       fine_fes->DofsToVDofs(c_vdofs, coarse_ndofs);
       loc_y.SetSize(c_vdofs.Size());
       loc_y = 0.0;
-      loc_y_mat.UseExternalData(loc_y.GetData(), c_vdofs.Size()/vdim, vdim);
+      loc_y_mat.UseExternalData(loc_y.GetData(), c_vdofs.Size()/fine_vdim,
+                                fine_vdim);
       const int ref_type = coarse_to_ref_type[coarse_el];
       const Geometry::Type geom = ref_type_to_geom[ref_type];
       const int *fine_elems = coarse_to_fine.GetRow(coarse_el);
@@ -1979,7 +1997,8 @@ void FiniteElementSpace::DerefinementOperator
          const DenseMatrix &lR = localR[geom](lR_offset+s);
          fine_fes->GetElementVDofs(fine_elems[s], f_vdofs);
          x.GetSubVector(f_vdofs, loc_x);
-         loc_x_mat.UseExternalData(loc_x.GetData(), f_vdofs.Size()/vdim, vdim);
+         loc_x_mat.UseExternalData(loc_x.GetData(), f_vdofs.Size()/fine_vdim,
+                                   fine_vdim);
          AddMult(lR, loc_x_mat, loc_y_mat);
       }
       y.SetSubVector(c_vdofs, loc_y);
@@ -2111,14 +2130,14 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
    }
 }
 
-void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
-                                     const FiniteElementCollection *fec,
-                                     int vdim, int ordering)
+void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
+                                     const FiniteElementCollection *fec_,
+                                     int vdim_, int ordering_)
 {
-   this->mesh = mesh;
-   this->fec = fec;
-   this->vdim = vdim;
-   this->ordering = (Ordering::Type) ordering;
+   mesh = mesh_;
+   fec = fec_;
+   vdim = vdim_;
+   ordering = (Ordering::Type) ordering_;
 
    elem_dof = NULL;
    elem_fos = NULL;
@@ -2131,19 +2150,19 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    Th.SetType(Operator::ANY_TYPE);
 
    const NURBSFECollection *nurbs_fec =
-      dynamic_cast<const NURBSFECollection *>(fec);
+      dynamic_cast<const NURBSFECollection *>(fec_);
    if (nurbs_fec)
    {
-      MFEM_VERIFY(mesh->NURBSext, "NURBS FE space requires a NURBS mesh.");
+      MFEM_VERIFY(mesh_->NURBSext, "NURBS FE space requires a NURBS mesh.");
 
-      if (NURBSext == NULL)
+      if (NURBSext_ == NULL)
       {
-         this->NURBSext = mesh->NURBSext;
+         NURBSext = mesh_->NURBSext;
          own_ext = 0;
       }
       else
       {
-         this->NURBSext = NURBSext;
+         NURBSext = NURBSext_;
          own_ext = 1;
       }
       UpdateNURBS();
@@ -2154,7 +2173,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    }
    else
    {
-      this->NURBSext = NULL;
+      NURBSext = NULL;
       own_ext = 0;
       Construct();
    }
@@ -3502,7 +3521,7 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
    input >> ord;
 
    NURBSFECollection *nurbs_fec = dynamic_cast<NURBSFECollection*>(r_fec);
-   NURBSExtension *NURBSext = NULL;
+   NURBSExtension *nurbs_ext = NULL;
    if (fes_format == 90) // original format, v0.9
    {
       if (nurbs_fec)
@@ -3512,7 +3531,7 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
          if (order != m->NURBSext->GetOrder() &&
              order != NURBSFECollection::VariableOrder)
          {
-            NURBSext = new NURBSExtension(m->NURBSext, order);
+            nurbs_ext = new NURBSExtension(m->NURBSext, order);
          }
       }
    }
@@ -3529,18 +3548,18 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
             MFEM_VERIFY(nurbs_fec,
                         buff << ": NURBS FE collection is required!");
             MFEM_VERIFY(m->NURBSext, buff << ": NURBS mesh is required!");
-            MFEM_VERIFY(!NURBSext, buff << ": order redefinition!");
+            MFEM_VERIFY(!nurbs_ext, buff << ": order redefinition!");
             if (buff == "NURBS_order")
             {
                int order;
                input >> order;
-               NURBSext = new NURBSExtension(m->NURBSext, order);
+               nurbs_ext = new NURBSExtension(m->NURBSext, order);
             }
             else
             {
                Array<int> orders;
                orders.Load(m->NURBSext->GetNKV(), input);
-               NURBSext = new NURBSExtension(m->NURBSext, orders);
+               nurbs_ext = new NURBSExtension(m->NURBSext, orders);
             }
          }
          else if (buff == "NURBS_periodic")
@@ -3548,13 +3567,13 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
             Array<int> master, slave;
             master.Load(input);
             slave.Load(input);
-            NURBSext->ConnectBoundaries(master,slave);
+            nurbs_ext->ConnectBoundaries(master,slave);
          }
          else if (buff == "NURBS_weights")
          {
-            MFEM_VERIFY(NURBSext, "NURBS_weights: NURBS_orders have to be "
+            MFEM_VERIFY(nurbs_ext, "NURBS_weights: NURBS_orders have to be "
                         "specified before NURBS_weights!");
-            NURBSext->GetWeights().Load(input, NURBSext->GetNDof());
+            nurbs_ext->GetWeights().Load(input, nurbs_ext->GetNDof());
          }
          else if (buff == "element_orders")
          {
@@ -3573,7 +3592,7 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
       }
    }
 
-   Constructor(m, NURBSext, r_fec, vdim, ord);
+   Constructor(m, nurbs_ext, r_fec, vdim, ord);
 
    return r_fec;
 }

fem/geom.cpp

@@ -1337,9 +1337,9 @@ RefinedGeometry * GeometryRefiner::Refine(Geometry::Type Geom,
                   // (ii,jj,kk) are coordinates in the reference tetrahedron,
                   // transform to coordinates (i,j,k) in the auxiliary
                   // tetrahedron defined by (0,0,0), (0,0,1), (1,1,1), (0,1,1)
-                  int i = jj;
-                  int j = jj+kk;
-                  int k = ii+jj+kk;
+                  i = jj;
+                  j = jj+kk;
+                  k = ii+jj+kk;
                   l = i + (j + k * (n+1)) * (n+1);
                   // map from linear Cartesian hex index in the auxiliary tet
                   // to lexicographic in the reference tet

fem/gridfunc.cpp

@@ -323,10 +323,11 @@ int GridFunction::VectorDim() const
    const FiniteElement *fe;
    if (!fes->GetNE())
    {
-      const FiniteElementCollection *fec = fes->FEColl();
+      const FiniteElementCollection *fe_coll = fes->FEColl();
       static const Geometry::Type geoms[3] =
       { Geometry::SEGMENT, Geometry::TRIANGLE, Geometry::TETRAHEDRON };
-      fe = fec->FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
+      fe = fe_coll->
+           FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
    }
    else
    {
@@ -605,11 +606,11 @@ const
       ET->SetIntPoint(&ip);
       FElem->CalcPhysHessian(*ET, DofHes);
 
-      for (int i = 0; i < size; i++)
+      for (int j = 0; j < size; j++)
       {
          for (int d = 0; d < dof; d++)
          {
-            hess(k,i) += DofHes(d,i) * loc_data[d];
+            hess(k,j) += DofHes(d,j) * loc_data[d];
          }
       }
    }
@@ -3948,6 +3949,129 @@ std::ostream &operator<<(std::ostream &out, const QuadratureFunction &qf)
    return out;
 }
 
+void QuadratureFunction::SaveVTU(std::ostream &out, VTKFormat format,
+                                 int compression_level) const
+{
+   out << R"(<VTKFile type="UnstructuredGrid" version="0.1")";
+   if (compression_level != 0)
+   {
+      out << R"( compressor="vtkZLibDataCompressor")";
+   }
+   out << " byte_order=\"" << VTKByteOrder() << "\">\n";
+   out << "<UnstructuredGrid>\n";
+
+   const char *fmt_str = (format == VTKFormat::ASCII) ? "ascii" : "binary";
+   const char *type_str = (format != VTKFormat::BINARY32) ? "Float64" : "Float32";
+   std::vector<char> buf;
+
+   int np = qspace->GetSize();
+   int ne = qspace->GetNE();
+   int sdim = qspace->GetMesh()->SpaceDimension();
+
+   // For quadrature functions, each point is a vertex cell, so number of cells
+   // is equal to number of points
+   out << "<Piece NumberOfPoints=\"" << np
+       << "\" NumberOfCells=\"" << np << "\">\n";
+
+   // print out the points
+   out << "<Points>\n";
+   out << "<DataArray type=\"" << type_str
+       << "\" NumberOfComponents=\"3\" format=\"" << fmt_str << "\">\n";
+
+   Vector pt(sdim);
+   for (int i = 0; i < ne; i++)
+   {
+      ElementTransformation &T = *qspace->GetMesh()->GetElementTransformation(i);
+      const IntegrationRule &ir = GetElementIntRule(i);
+      for (int j = 0; j < ir.Size(); j++)
+      {
+         T.Transform(ir[j], pt);
+         WriteBinaryOrASCII(out, buf, pt[0], " ", format);
+         if (sdim > 1) { WriteBinaryOrASCII(out, buf, pt[1], " ", format); }
+         else { WriteBinaryOrASCII(out, buf, 0.0, " ", format); }
+         if (sdim > 2) { WriteBinaryOrASCII(out, buf, pt[2], "", format); }
+         else { WriteBinaryOrASCII(out, buf, 0.0, "", format); }
+         if (format == VTKFormat::ASCII) { out << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</Points>\n";
+
+   // Write cells (each cell is just a vertex)
+   out << "<Cells>\n";
+   // Connectivity
+   out << R"(<DataArray type="Int32" Name="connectivity" format=")"
+       << fmt_str << "\">\n";
+
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(out, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   // Offsets
+   out << R"(<DataArray type="Int32" Name="offsets" format=")"
+       << fmt_str << "\">\n";
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(out, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   // Types
+   out << R"(<DataArray type="UInt8" Name="types" format=")"
+       << fmt_str << "\">\n";
+   for (int i = 0; i < np; i++)
+   {
+      uint8_t vtk_cell_type = VTKGeometry::POINT;
+      WriteBinaryOrASCII(out, buf, vtk_cell_type, "\n", format);
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</Cells>\n";
+
+   out << "<PointData>\n";
+   out << "<DataArray type=\"" << type_str << "\" Name=\"u\" format=\""
+       << fmt_str << "\" NumberOfComponents=\"" << vdim << "\">\n";
+   for (int i = 0; i < ne; i++)
+   {
+      DenseMatrix vals;
+      GetElementValues(i, vals);
+      for (int j = 0; j < vals.Size(); ++j)
+      {
+         for (int vd = 0; vd < vdim; ++vd)
+         {
+            WriteBinaryOrASCII(out, buf, vals(vd, j), " ", format);
+         }
+         if (format == VTKFormat::ASCII) { out << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</PointData>\n";
+
+   out << "</Piece>\n";
+   out << "</UnstructuredGrid>\n";
+   out << "</VTKFile>" << std::endl;
+}
+
+void QuadratureFunction::SaveVTU(const std::string &filename, VTKFormat format,
+                                 int compression_level) const
+{
+   std::ofstream f(filename + ".vtu");
+   SaveVTU(f, format, compression_level);
+}
+
 
 double ZZErrorEstimator(BilinearFormIntegrator &blfi,
                         GridFunction &u,

fem/gridfunc.hpp

@@ -902,6 +902,22 @@ public:
 
    /// Write the QuadratureFunction to the stream @a out.
    void Save(std::ostream &out) const;
+
+   /// @brief Write the QuadratureFunction to @a out in VTU (ParaView) format.
+   ///
+   /// The data will be uncompressed if @a compression_level is zero, or if the
+   /// format is VTKFormat::ASCII. Otherwise, zlib compression will be used for
+   /// binary data.
+   void SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
+
+   /// @brief Save the QuadratureFunction to a VTU (ParaView) file.
+   ///
+   /// The extension ".vtu" will be appended to @a filename.
+   /// @sa SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+   ///             int compression_level=0)
+   void SaveVTU(const std::string &filename, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
 };
 
 /// Overload operator<< for std::ostream and QuadratureFunction.

fem/gslib.cpp

@@ -78,6 +78,8 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
    MFEM_VERIFY(m.GetNumGeometries(m.Dimension()) == 1,
                "Mixed meshes are not currently supported in FindPointsGSLIB.");
+   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
+               "Variable order mesh is not currently supported.");
 
    // call FreeData if FindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -590,7 +592,8 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
    const L2_FECollection *fec_l2 = dynamic_cast<const L2_FECollection *>(fec_in);
 
    if (fec_h1 && gf_order == mesh_order &&
-       fec_h1->GetBasisType() == BasisType::GaussLobatto)
+       fec_h1->GetBasisType() == BasisType::GaussLobatto &&
+       !field_in.FESpace()->IsVariableOrder())
    {
       InterpolateH1(field_in, field_out);
       return;
@@ -857,6 +860,8 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
    MFEM_VERIFY(m.GetNumGeometries(m.Dimension()) == 1,
                "Mixed meshes are not currently supported in FindPointsGSLIB.");
+   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
+               "Variable order mesh is not currently supported.");
 
    // FreeData if OversetFindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }

fem/hybridization.cpp

@@ -776,7 +776,7 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
          if (vdof_marker[vdof]) { el_vals(j) = 0.0; }
          else { vdof_marker[vdof] = true; }
       }
-      bf_i.SetDataAndSize(&bf[hat_offsets[i]], vdofs.Size());
+      bf_i.MakeRef(bf, hat_offsets[i], vdofs.Size());
       if (mode == 1)
       {
          el_vals -= bf_i;
@@ -821,7 +821,15 @@ void Hybridization::ReduceRHS(const Vector &b, Vector &b_r) const
    else
    {
       Vector bl(pC ? pC->Height() : Ct->Width());
-      pC ? pC->Mult(bf, bl) : Ct->MultTranspose(bf, bl);
+      if (pC)
+      {
+         pC->Mult(bf, bl);
+      }
+      else
+      {
+         Ct->EnsureMultTranspose();
+         Ct->MultTranspose(bf, bl);
+      }
       b_r.SetSize(pH.Ptr()->Height());
       (P_pc ? P_pc : c_pfes->GetProlongationMatrix())->MultTranspose(bl, b_r);
    }

fem/linearform.cpp

@@ -106,8 +106,6 @@ void LinearForm::Assemble()
    DofTransformation *doftrans;
    Vector elemvect;
 
-   int i;
-
    Vector::operator=(0.0);
 
    // The above operation is executed on device because of UseDevice().
@@ -127,7 +125,7 @@ void LinearForm::Assemble()
          }
       }
 
-      for (i = 0; i < fes -> GetNE(); i++)
+      for (int i = 0; i < fes -> GetNE(); i++)
       {
          int elem_attr = fes->GetMesh()->GetAttribute(i);
          for (int k = 0; k < domain_integs.Size(); k++)
@@ -175,7 +173,7 @@ void LinearForm::Assemble()
          }
       }
 
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -223,7 +221,7 @@ void LinearForm::Assemble()
          }
       }
 
-      for (i = 0; i < mesh->GetNBE(); i++)
+      for (int i = 0; i < mesh->GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -253,7 +251,7 @@ void LinearForm::Assemble()
 
       for (int k = 0; k < interior_face_integs.Size(); k++)
       {
-         for (i = 0; i < mesh->GetNumFaces(); i++)
+         for (int i = 0; i < mesh->GetNumFaces(); i++)
          {
             FaceElementTransformations *tr = NULL;
             tr = mesh->GetInteriorFaceTransformations (i);

fem/lor.cpp

@@ -67,11 +67,11 @@ void LORBase::ResetIntegrationRules(GetIntegratorsFn get_integrators)
 
 LORBase::FESpaceType LORBase::GetFESpaceType() const
 {
-   const FiniteElementCollection *fec = fes_ho.FEColl();
-   if (dynamic_cast<const H1_FECollection*>(fec)) { return H1; }
-   else if (dynamic_cast<const ND_FECollection*>(fec)) { return ND; }
-   else if (dynamic_cast<const RT_FECollection*>(fec)) { return RT; }
-   else if (dynamic_cast<const L2_FECollection*>(fec)) { return L2; }
+   const FiniteElementCollection *fec_ho = fes_ho.FEColl();
+   if (dynamic_cast<const H1_FECollection*>(fec_ho)) { return H1; }
+   else if (dynamic_cast<const ND_FECollection*>(fec_ho)) { return ND; }
+   else if (dynamic_cast<const RT_FECollection*>(fec_ho)) { return RT; }
+   else if (dynamic_cast<const L2_FECollection*>(fec_ho)) { return L2; }
    else { MFEM_ABORT("Bad LOR space type."); }
    return INVALID;
 }
@@ -87,9 +87,9 @@ void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
    FESpaceType type = GetFESpaceType();
    MFEM_VERIFY(type != H1 && type != L2, "");
 
-   auto get_dof_map = [](FiniteElementSpace &fes, int i)
+   auto get_dof_map = [](FiniteElementSpace &fes_, int i)
    {
-      const FiniteElement *fe = fes.GetFE(i);
+      const FiniteElement *fe = fes_.GetFE(i);
       auto tfe = dynamic_cast<const TensorBasisElement*>(fe);
       MFEM_ASSERT(tfe != NULL, "");
       return tfe->GetDofMap();

fem/nonlinearform.hpp

@@ -151,8 +151,8 @@ public:
    void SetEssentialVDofs(const Array<int> &ess_vdofs_list);
 
    /// Specify essential boundary conditions.
-   void SetEssentialTrueDofs(const Array<int> &ess_tdof_list)
-   { ess_tdof_list.Copy(this->ess_tdof_list); }
+   void SetEssentialTrueDofs(const Array<int> &ess_tdof_list_)
+   { ess_tdof_list_.Copy(this->ess_tdof_list); }
 
    /// Return a (read-only) list of all essential true dofs.
    const Array<int> &GetEssentialTrueDofs() const { return ess_tdof_list; }

fem/nonlininteg.cpp

@@ -746,7 +746,7 @@ void VectorConvectionNLFIntegrator::AssembleElementVector(
    Vector &elvect)
 {
    const int nd = el.GetDof();
-   const int dim = el.GetDim();
+   dim = el.GetDim();
 
    shape.SetSize(nd);
    dshape.SetSize(nd, dim);
@@ -783,7 +783,7 @@ void VectorConvectionNLFIntegrator::AssembleElementGrad(
    DenseMatrix &elmat)
 {
    const int nd = el.GetDof();
-   const int dim = el.GetDim();
+   dim = el.GetDim();
 
    shape.SetSize(nd);
    dshape.SetSize(nd, dim);
@@ -826,9 +826,9 @@ void VectorConvectionNLFIntegrator::AssembleElementGrad(
       dshape.Mult(vec2, vec3);
       MultVWt(shape, vec3, elmat_comp);
 
-      for (int i = 0; i < dim; i++)
+      for (int ii = 0; ii < dim; ii++)
       {
-         elmat.AddMatrix(elmat_comp, i * nd, i * nd);
+         elmat.AddMatrix(elmat_comp, ii * nd, ii * nd);
       }
 
       MultVVt(shape, elmat_comp);
@@ -837,11 +837,11 @@ void VectorConvectionNLFIntegrator::AssembleElementGrad(
       {
          w *= Q->Eval(trans, ip);
       }
-      for (int i = 0; i < dim; i++)
+      for (int ii = 0; ii < dim; ii++)
       {
-         for (int j = 0; j < dim; j++)
+         for (int jj = 0; jj < dim; jj++)
          {
-            elmat.AddMatrix(w * gradEF(i, j), elmat_comp, i * nd, j * nd);
+            elmat.AddMatrix(w * gradEF(ii, jj), elmat_comp, ii * nd, jj * nd);
          }
       }
    }
@@ -889,9 +889,9 @@ void ConvectiveVectorConvectionNLFIntegrator::AssembleElementGrad(
       dshape.Mult(vec2, vec3); // (u^n \cdot grad u^{n+1})
       MultVWt(shape, vec3, elmat_comp); // (u^n \cdot grad u^{n+1},v)
 
-      for (int i = 0; i < dim; i++)
+      for (int ii = 0; ii < dim; ii++)
       {
-         elmat.AddMatrix(elmat_comp, i * nd, i * nd);
+         elmat.AddMatrix(elmat_comp, ii * nd, ii * nd);
       }
    }
 }
@@ -944,10 +944,10 @@ void SkewSymmetricVectorConvectionNLFIntegrator::AssembleElementGrad(
       MultVWt(shape, vec3, elmat_comp); // (u^n \cdot grad u^{n+1},v)
       elmat_comp_T.Transpose(elmat_comp);
 
-      for (int i = 0; i < dim; i++)
+      for (int ii = 0; ii < dim; ii++)
       {
-         elmat.AddMatrix(.5, elmat_comp, i * nd, i * nd);
-         elmat.AddMatrix(-.5, elmat_comp_T, i * nd, i * nd);
+         elmat.AddMatrix(.5, elmat_comp, ii * nd, ii * nd);
+         elmat.AddMatrix(-.5, elmat_comp_T, ii * nd, ii * nd);
       }
    }
 }

fem/pbilinearform.cpp

@@ -317,26 +317,26 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const double a)
 const
 {
-   if (X.ParFESpace() != pfes)
+   if (Xaux.ParFESpace() != pfes)
    {
-      X.SetSpace(pfes);
-      Y.SetSpace(pfes);
+      Xaux.SetSpace(pfes);
+      Yaux.SetSpace(pfes);
       Ytmp.SetSize(pfes->GetTrueVSize());
    }
 
-   X.Distribute(&x);
+   Xaux.Distribute(&x);
    if (ext)
    {
-      ext->Mult(X, Y);
+      ext->Mult(Xaux, Yaux);
    }
    else
    {
       MFEM_VERIFY(interior_face_integs.Size() == 0,
                   "the case of interior face integrators is not"
                   " implemented");
-      mat->Mult(X, Y);
+      mat->Mult(Xaux, Yaux);
    }
-   pfes->GetProlongationMatrix()->MultTranspose(Y, Ytmp);
+   pfes->GetProlongationMatrix()->MultTranspose(Yaux, Ytmp);
    y.Add(a,Ytmp);
 }
 
@@ -373,6 +373,7 @@ void ParBilinearForm::FormLinearSystem(
       P.MultTranspose(b, true_B);
       R.Mult(x, true_X);
       p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B);
+      R.EnsureMultTranspose();
       R.MultTranspose(true_B, b);
       hybridization->ReduceRHS(true_B, B);
       X.SetSize(B.Size());
@@ -537,15 +538,15 @@ void ParMixedBilinearForm::ParallelAssemble(OperatorHandle &A)
 void ParMixedBilinearForm::TrueAddMult(const Vector &x, Vector &y,
                                        const double a) const
 {
-   if (X.ParFESpace() != trial_pfes)
+   if (Xaux.ParFESpace() != trial_pfes)
    {
-      X.SetSpace(trial_pfes);
-      Y.SetSpace(test_pfes);
+      Xaux.SetSpace(trial_pfes);
+      Yaux.SetSpace(test_pfes);
    }
 
-   X.Distribute(&x);
-   mat->Mult(X, Y);
-   test_pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Y, 1.0, y);
+   Xaux.Distribute(&x);
+   mat->Mult(Xaux, Yaux);
+   test_pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Yaux, 1.0, y);
 }
 
 void ParMixedBilinearForm::FormRectangularSystemMatrix(

fem/pbilinearform.hpp

@@ -32,7 +32,7 @@ protected:
    ParFiniteElementSpace *pfes; ///< Points to the same object as #fes
 
    /// Auxiliary objects used in TrueAddMult().
-   mutable ParGridFunction X, Y;
+   mutable ParGridFunction Xaux, Yaux;
    mutable Vector Ytmp;
 
    OperatorHandle p_mat, p_mat_e;
@@ -211,7 +211,7 @@ protected:
    /// Points to the same object as #test_fes
    ParFiniteElementSpace *test_pfes;
    /// Auxiliary objects used in TrueAddMult().
-   mutable ParGridFunction X, Y;
+   mutable ParGridFunction Xaux, Yaux;
 
    /// Matrix and eliminated matrix
    OperatorHandle p_mat, p_mat_e;

fem/pfespace.cpp

@@ -232,7 +232,7 @@ void ParFiniteElementSpace::PrintPartitionStats()
 }
 
 void ParFiniteElementSpace::GetGroupComm(
-   GroupCommunicator &gc, int ldof_type, Array<int> *ldof_sign)
+   GroupCommunicator &gc, int ldof_type, Array<int> *g_ldof_sign)
 {
    int gr;
    int ng = pmesh->GetNGroups();
@@ -257,10 +257,10 @@ void ParFiniteElementSpace::GetGroupComm(
       }
    }
 
-   if (ldof_sign)
+   if (g_ldof_sign)
    {
-      ldof_sign->SetSize(GetNDofs());
-      *ldof_sign = 1;
+      g_ldof_sign->SetSize(GetNDofs());
+      *g_ldof_sign = 1;
    }
 
    // count the number of ldofs in all groups (excluding the local group 0)
@@ -333,9 +333,9 @@ void ParFiniteElementSpace::GetGroupComm(
                if (ind[l] < 0)
                {
                   dofs[l] = m + (-1-ind[l]);
-                  if (ldof_sign)
+                  if (g_ldof_sign)
                   {
-                     (*ldof_sign)[dofs[l]] = -1;
+                     (*g_ldof_sign)[dofs[l]] = -1;
                   }
                }
                else
@@ -371,9 +371,9 @@ void ParFiniteElementSpace::GetGroupComm(
                if (ind[l] < 0)
                {
                   dofs[l] = m + (-1-ind[l]);
-                  if (ldof_sign)
+                  if (g_ldof_sign)
                   {
-                     (*ldof_sign)[dofs[l]] = -1;
+                     (*g_ldof_sign)[dofs[l]] = -1;
                   }
                }
                else
@@ -409,9 +409,9 @@ void ParFiniteElementSpace::GetGroupComm(
                if (ind[l] < 0)
                {
                   dofs[l] = m + (-1-ind[l]);
-                  if (ldof_sign)
+                  if (g_ldof_sign)
                   {
-                     (*ldof_sign)[dofs[l]] = -1;
+                     (*g_ldof_sign)[dofs[l]] = -1;
                   }
                }
                else
@@ -556,11 +556,25 @@ const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
       FaceRestriction *res;
       if (is_dg_space)
       {
-         res = new ParL2FaceRestriction(*this, e_ordering, type, m);
+         if (Conforming())
+         {
+            res = new ParL2FaceRestriction(*this, e_ordering, type, m);
+         }
+         else
+         {
+            res = new ParNCL2FaceRestriction(*this, e_ordering, type, m);
+         }
       }
       else
       {
-         res = new H1FaceRestriction(*this, e_ordering, type);
+         if (Conforming())
+         {
+            res = new H1FaceRestriction(*this, e_ordering, type);
+         }
+         else
+         {
+            res = new ParNCH1FaceRestriction(*this, e_ordering, type);
+         }
       }
       L2F[key] = res;
       return res;
@@ -750,10 +764,10 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
       diag_counter = offd_counter = 0;
       for (int i = 0; i < ldof; i++)
       {
-         int ltdof = GetLocalTDofNumber(i);
-         if (ltdof >= 0)
+         int ltdof_i = GetLocalTDofNumber(i);
+         if (ltdof_i >= 0)
          {
-            j_diag[diag_counter++] = ltdof;
+            j_diag[diag_counter++] = ltdof_i;
          }
          else
          {
@@ -870,10 +884,10 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
       int offd_col_counter = 0;
       for (int i = 0; i < ldof; i++)
       {
-         int ltdof = GetLocalTDofNumber(i);
-         if (ltdof >= 0)
+         int ltdofi = GetLocalTDofNumber(i);
+         if (ltdofi >= 0)
          {
-            j_diag[diag_counter]   = ltdof;
+            j_diag[diag_counter]   = ltdofi;
             d_diag[diag_counter++] = 1.0;
          }
          else
@@ -1327,12 +1341,12 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
    for (int fn = 0, j = 0; fn < num_face_nbrs; fn++)
    {
       int  num_ldofs = send_face_nbr_ldof.RowSize(fn);
-      int *ldofs     = send_face_nbr_ldof.GetRow(fn);
+      int *ldofs_fn  = send_face_nbr_ldof.GetRow(fn);
       int  j_end     = send_I[send_el_off[fn+1]];
 
       for (int i = 0; i < num_ldofs; i++)
       {
-         int ldof = (ldofs[i] >= 0 ? ldofs[i] : -1-ldofs[i]);
+         int ldof = (ldofs_fn[i] >= 0 ? ldofs_fn[i] : -1-ldofs_fn[i]);
          ldof_marker[ldof] = i;
       }
 
@@ -2023,7 +2037,7 @@ public:
    const RowInfo::List& GetRows() const { return rows; }
 
    void SetNCMesh(ParNCMesh* pnc) { pncmesh = pnc; }
-   void SetFEC(const FiniteElementCollection* fec) { this->fec = fec; }
+   void SetFEC(const FiniteElementCollection* fec_) { this->fec = fec_; }
 
    typedef std::map<int, NeighborRowMessage> Map;
 
@@ -2286,7 +2300,7 @@ void ParFiniteElementSpace
 #endif
 
 int ParFiniteElementSpace
-::BuildParallelConformingInterpolation(HypreParMatrix **P, SparseMatrix **R,
+::BuildParallelConformingInterpolation(HypreParMatrix **P_, SparseMatrix **R_,
                                        Array<HYPRE_BigInt> &dof_offs,
                                        Array<HYPRE_BigInt> &tdof_offs,
                                        Array<int> *dof_tdof,
@@ -2443,10 +2457,10 @@ int ParFiniteElementSpace
    HYPRE_BigInt my_tdof_offset =
       tdof_offs[HYPRE_AssumedPartitionCheck() ? 0 : MyRank];
 
-   if (R)
+   if (R_)
    {
       // initialize the restriction matrix (also parallel but block-diagonal)
-      *R = new SparseMatrix(num_true_dofs*vdim, ndofs*vdim);
+      *R_ = new SparseMatrix(num_true_dofs*vdim, ndofs*vdim);
    }
    if (dof_tdof)
    {
@@ -2484,7 +2498,7 @@ int ParFiniteElementSpace
             int vdof = dof*vdim_factor + vd*dof_stride;
             int vtdof = tdof*vdim_factor + vd*tdof_stride;
 
-            if (R) { (*R)->Add(vtdof, vdof, 1.0); }
+            if (R_) { (*R_)->Add(vtdof, vdof, 1.0); }
             if (dof_tdof) { (*dof_tdof)[vdof] = vtdof; }
          }
          tdof++;
@@ -2497,7 +2511,7 @@ int ParFiniteElementSpace
    n_msgs_sent += send_msg.back().size();
 #endif
 
-   if (R) { (*R)->Finalize(); }
+   if (R_) { (*R_)->Finalize(); }
 
    // *** STEP 4: main loop ***
 
@@ -2606,10 +2620,10 @@ int ParFiniteElementSpace
 #endif
    }
 
-   if (P)
+   if (P_)
    {
-      *P = MakeVDimHypreMatrix(pmatrix, ndofs, num_true_dofs,
-                               dof_offs, tdof_offs);
+      *P_ = MakeVDimHypreMatrix(pmatrix, ndofs, num_true_dofs,
+                                dof_offs, tdof_offs);
    }
 
    // clean up possible remaining messages in the queue to avoid receiving
@@ -2798,13 +2812,13 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
                              ? old_dof_offsets[0] : old_dof_offsets[MyRank];
 
    // send old DOFs of elements we used to own
-   ParNCMesh* pncmesh = pmesh->pncmesh;
-   pncmesh->SendRebalanceDofs(old_ndofs, *old_elem_dof, old_offset, this);
+   ParNCMesh* old_pncmesh = pmesh->pncmesh;
+   old_pncmesh->SendRebalanceDofs(old_ndofs, *old_elem_dof, old_offset, this);
 
    Array<int> dofs;
    int vsize = GetVSize();
 
-   const Array<int> &old_index = pncmesh->GetRebalanceOldIndex();
+   const Array<int> &old_index = old_pncmesh->GetRebalanceOldIndex();
    MFEM_VERIFY(old_index.Size() == pmesh->GetNE(),
                "Mesh::Rebalance was not called before "
                "ParFiniteElementSpace::RebalanceMatrix");
@@ -2838,7 +2852,7 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
    // receive old DOFs for elements we obtained from others in Rebalance
    Array<int> new_elements;
    Array<long> old_remote_dofs;
-   pncmesh->RecvRebalanceDofs(new_elements, old_remote_dofs);
+   old_pncmesh->RecvRebalanceDofs(new_elements, old_remote_dofs);
 
    // create the offdiagonal part of the matrix
    HYPRE_BigInt* i_offd = make_i_array<HYPRE_BigInt>(vsize);
@@ -2936,7 +2950,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    Array<int> dofs, old_dofs, old_vdofs;
    Vector row;
 
-   ParNCMesh* pncmesh = pmesh->pncmesh;
+   ParNCMesh* old_pncmesh = pmesh->pncmesh;
 
    int ldof[Geometry::NumGeom];
    for (int i = 0; i < Geometry::NumGeom; i++)
@@ -2949,8 +2963,9 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
       ldof[geom] = fec->FiniteElementForGeometry(geom)->GetDof();
    }
 
-   const CoarseFineTransformations &dtrans = pncmesh->GetDerefinementTransforms();
-   const Array<int> &old_ranks = pncmesh->GetDerefineOldRanks();
+   const CoarseFineTransformations &dtrans =
+      old_pncmesh->GetDerefinementTransforms();
+   const Array<int> &old_ranks = old_pncmesh->GetDerefineOldRanks();
 
    std::map<int, DerefDofMessage> messages;
 
@@ -2965,7 +2980,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
 
       int fine_rank = old_ranks[k];
       int coarse_rank = (emb.parent < 0) ? (-1 - emb.parent)
-                        : pncmesh->ElementRank(emb.parent);
+                        : old_pncmesh->ElementRank(emb.parent);
 
       if (coarse_rank != MyRank && fine_rank == MyRank)
       {
@@ -3015,7 +3030,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
       const Embedding &emb = dtrans.embeddings[k];
       if (emb.parent < 0) { continue; }
 
-      int coarse_rank = pncmesh->ElementRank(emb.parent);
+      int coarse_rank = old_pncmesh->ElementRank(emb.parent);
       int fine_rank = old_ranks[k];
 
       if (coarse_rank == MyRank && fine_rank == MyRank)
@@ -3065,7 +3080,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
       const Embedding &emb = dtrans.embeddings[k];
       if (emb.parent < 0) { continue; }
 
-      int coarse_rank = pncmesh->ElementRank(emb.parent);
+      int coarse_rank = old_pncmesh->ElementRank(emb.parent);
       int fine_rank = old_ranks[k];
 
       if (coarse_rank == MyRank && fine_rank != MyRank)
@@ -3146,14 +3161,14 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
       offd->SortColumnIndices();
    }
 
-   HypreParMatrix* R;
-   R = new HypreParMatrix(MyComm, dof_offsets[nrk], old_dof_offsets[nrk],
-                          dof_offsets, old_dof_offsets, diag, offd, cmap,
-                          true);
+   HypreParMatrix* new_R;
+   new_R = new HypreParMatrix(MyComm, dof_offsets[nrk], old_dof_offsets[nrk],
+                              dof_offsets, old_dof_offsets, diag, offd, cmap,
+                              true);
 
-   R->SetOwnerFlags(R->OwnsDiag(), R->OwnsOffd(), 1);
+   new_R->SetOwnerFlags(new_R->OwnsDiag(), new_R->OwnsOffd(), 1);
 
-   return R;
+   return new_R;
 }
 
 void ParFiniteElementSpace::Destroy()
@@ -3537,18 +3552,18 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
       shr_buf.UseDevice(true);
       shr_buf_offsets = nbr_ltdof.GetIMemory();
       {
-         Array<int> shr_ltdof(nbr_ltdof.GetJ(), nb_connections);
-         Array<int> unique_ltdof(shr_ltdof);
+         Array<int> shared_ltdof(nbr_ltdof.GetJ(), nb_connections);
+         Array<int> unique_ltdof(shared_ltdof);
          unique_ltdof.Sort();
          unique_ltdof.Unique();
          // Note: the next loop modifies the J array of nbr_ltdof
-         for (int i = 0; i < shr_ltdof.Size(); i++)
+         for (int i = 0; i < shared_ltdof.Size(); i++)
          {
-            shr_ltdof[i] = unique_ltdof.FindSorted(shr_ltdof[i]);
-            MFEM_ASSERT(shr_ltdof[i] != -1, "internal error");
+            shared_ltdof[i] = unique_ltdof.FindSorted(shared_ltdof[i]);
+            MFEM_ASSERT(shared_ltdof[i] != -1, "internal error");
          }
          Table unique_shr;
-         Transpose(shr_ltdof, unique_shr, unique_ltdof.Size());
+         Transpose(shared_ltdof, unique_shr, unique_ltdof.Size());
          unq_ltdof = Array<int>(unique_ltdof, unique_ltdof.Size());
          unq_shr_i = Array<int>(unique_shr.GetI(), unique_shr.Size()+1);
          unq_shr_j = Array<int>(unique_shr.GetJ(), unique_shr.Size_of_connections());

fem/pnonlinearform.cpp

@@ -90,8 +90,8 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 
    const int N = ess_tdof_list.Size();
    const auto idx = ess_tdof_list.Read();
-   auto Y = y.ReadWrite();
-   MFEM_FORALL(i, N, Y[idx[i]] = 0.0; );
+   auto Y_RW = y.ReadWrite();
+   MFEM_FORALL(i, N, Y_RW[idx[i]] = 0.0; );
 }
 
 const SparseMatrix &ParNonlinearForm::GetLocalGradient(const Vector &x) const

fem/prestriction.hpp

@@ -23,34 +23,348 @@ namespace mfem
 
 class ParFiniteElementSpace;
 
-/// Operator that extracts Face degrees of freedom in parallel.
-/** Objects of this type are typically created and owned by FiniteElementSpace
-    objects, see FiniteElementSpace::GetFaceRestriction(). */
-class ParL2FaceRestriction : public L2FaceRestriction
+/// Operator that extracts Face degrees of freedom for NCMesh in parallel.
+/** Objects of this type are typically created and owned by
+    ParFiniteElementSpace objects, see
+    ParFiniteElementSpace::GetFaceRestriction(). */
+class ParNCH1FaceRestriction : public H1FaceRestriction
 {
+protected:
+   const FaceType type;
+   InterpolationManager interpolations;
+   mutable Vector x_interp;
+
 public:
-   ParL2FaceRestriction(const ParFiniteElementSpace&, ElementDofOrdering,
+   /** @brief Constructs an ParNCH1FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs */
+   ParNCH1FaceRestriction(const ParFiniteElementSpace &fes,
+                          ElementDofOrdering ordering,
+                          FaceType type);
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering.
+       @param[in,out] y The L-vector degrees of freedom. */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
+       for the gathering: E-vector to L-vector, and the interpolators from
+       coarse to fine face for master non-comforming faces.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+};
+
+/// Operator that extracts Face degrees of freedom in parallel.
+/** Objects of this type are typically created and owned by
+    ParFiniteElementSpace objects, see
+    ParFiniteElementSpace::GetFaceRestriction(). */
+class ParL2FaceRestriction : virtual public L2FaceRestriction
+{
+protected:
+   /** @brief Constructs an ParL2FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+       @param[in] build    Request the ParL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from ParL2FaceRestriction. */
+   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
+                        ElementDofOrdering ordering,
+                        FaceType type,
+                        L2FaceValues m,
+                        bool build);
+
+public:
+   /** @brief Constructs an ParL2FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2 */
+   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
+                        ElementDofOrdering ordering,
                         FaceType type,
                         L2FaceValues m = L2FaceValues::DoubleValued);
-   void Mult(const Vector &x, Vector &y) const;
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this L2FaceRestriction. */
-   virtual void FillI(SparseMatrix &mat, const bool keep_nbr_block = false) const;
+       given by this ParL2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
+   void FillI(SparseMatrix &mat,
+              const bool keep_nbr_block = false) const override;
+
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this L2FaceRestriction. @a mat contains the interior dofs
+       given by this ParL2FaceRestriction. @a mat contains the interior dofs
        contribution, the @a face_mat contains the shared dofs contribution.*/
-   virtual void FillI(SparseMatrix &mat, SparseMatrix &face_mat) const;
+   void FillI(SparseMatrix &mat,
+              SparseMatrix &face_mat) const;
+
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this L2FaceRestriction, and the values of ea_data.
+       pattern given by this ParL2FaceRestriction, and the values of ea_data.
        @a mat contains the interior dofs contribution, the @a face_mat contains
        the shared dofs contribution.*/
-   virtual void FillJAndData(const Vector &ea_data,
-                             SparseMatrix &mat,
-                             SparseMatrix &face_mat) const;
+   void FillJAndData(const Vector &ea_data,
+                     SparseMatrix &mat,
+                     SparseMatrix &face_mat) const;
 
-   virtual void FillJAndData(const Vector &ea_data,
-                             SparseMatrix &mat,
-                             const bool keep_nbr_block = false) const;
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this ParL2FaceRestriction, and the values of
+       fea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     const bool keep_nbr_block = false) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
+       for the gathering: E-vector to L-vector.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x 2 x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void DoubleValuedConformingMult(const Vector& x, Vector& y) const override;
+};
+
+/// Operator that extracts Face degrees of freedom for NCMesh in parallel.
+/** Objects of this type are typically created and owned by
+    ParFiniteElementSpace objects, see
+    ParFiniteElementSpace::GetFaceRestriction(). */
+class ParNCL2FaceRestriction
+   : public NCL2FaceRestriction, public ParL2FaceRestriction
+{
+public:
+   /** @brief Constructs an ParNCL2FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2 */
+   ParNCL2FaceRestriction(const ParFiniteElementSpace& fes,
+                          ElementDofOrdering ordering,
+                          FaceType type,
+                          L2FaceValues m = L2FaceValues::DoubleValued);
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+   /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
+       pattern given by this ParNCL2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillI(SparseMatrix &mat,
+              const bool keep_nbr_block = false) const override;
+
+   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
+       given by this ParNCL2FaceRestriction. @a mat contains the interior dofs
+       contribution, the @a face_mat contains the shared dofs contribution.
+
+       @warning This method is not implemented yet. */
+   void FillI(SparseMatrix &mat,
+              SparseMatrix &face_mat) const;
+
+   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
+       pattern given by this ParNCL2FaceRestriction, and the values of ea_data.
+       @a mat contains the interior dofs contribution, the @a face_mat contains
+       the shared dofs contribution.
+
+       @warning This method is not implemented yet. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     SparseMatrix &face_mat) const;
+
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this ParNCL2FaceRestriction, and the values
+       of ea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     const bool keep_nbr_block = false) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
+       for the gathering: E-vector to L-vector, and the interpolators from
+       coarse to fine face for master non-comforming faces.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with nonconforming faces and when:
+       L2FaceValues m == L2FaceValues::SingleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void SingleValuedNonconformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with nonconforming faces and when:
+       L2FaceValues m == L2FaceValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     (face_dofs x vdim x 2 x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void DoubleValuedNonconformingMult(const Vector& x, Vector& y) const override;
 };
 
 }

fem/quadinterpolator_face.cpp

@@ -21,18 +21,21 @@ namespace mfem
 static void GetSigns(const FiniteElementSpace &fes, const FaceType type,
                      Array<bool> &signs)
 {
-   const int dim = fes.GetMesh()->SpaceDimension();
-   int e1, e2;
-   int inf1, inf2;
+   const Mesh &mesh = *fes.GetMesh();
+   const int dim = mesh.SpaceDimension();
    int face_id;
    int f_ind = 0;
-   for (int f = 0; f < fes.GetNF(); ++f)
+   for (int f = 0; f < mesh.GetNumFacesWithGhost(); ++f)
    {
-      fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-      fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-      face_id = inf1 / 64;
-      if ( (type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-           (type==FaceType::Boundary && e2<0 && inf2<0) )
+      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
+      face_id = face.element[0].local_face_id;
+      if (face.IsNonconformingCoarse())
+      {
+         // We skip nonconforming coarse-fine faces as they are treated
+         // by the corresponding nonconforming fine-coarse faces.
+         continue;
+      }
+      else if ( face.IsOfFaceType(type) )
       {
          if (dim==2)
          {

fem/restriction.hpp

@@ -41,7 +41,7 @@ protected:
    const int nedofs;
    Array<int> offsets;
    Array<int> indices;
-   Array<int> gatherMap;
+   Array<int> gather_map;
 
 public:
    ElementRestriction(const FiniteElementSpace&, ElementDofOrdering);
@@ -172,125 +172,719 @@ class H1FaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
-   const int nf;
+   const int nf; // Number of faces of the requested type
    const int vdim;
    const bool byvdim;
-   const int ndofs;
-   const int dof;
-   const int nfdofs;
-   Array<int> scatter_indices;
-   Array<int> offsets;
-   Array<int> gather_indices;
+   const int face_dofs; // Number of dofs on each face
+   const int elem_dofs; // Number of dofs in each element
+   const int nfdofs; // Total number of face E-vector dofs
+   const int ndofs; // Total number of dofs
+   Array<int> scatter_indices; // Scattering indices for element 1 on each face
+   Array<int> gather_offsets; // offsets for the gathering indices of each dof
+   Array<int> gather_indices; // gathering indices for each dof
 
-public:
-   /** @brief Constructor for a H1FaceRestriction.
+   /** @brief Construct an H1FaceRestriction.
 
-       @param[in] fes The FiniteElementSpace on which this H1FaceRestriction
-                      operates.
-       @param[in] ordering The requested output ordering of the
-                           H1FaceRestriction, either Native or Lexicographic.
-       @param[in] type The requested type of faces on which this operator
-                       extracts the degrees of freedom, either Interior or
-                       Boundary.
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific element ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] build    Request the NCL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from H1FaceRestriction.
    */
+   H1FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type,
+                     bool build);
+public:
+   /** @brief Construct an H1FaceRestriction.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific element ordering
+       @param[in] type     Request internal or boundary faces dofs */
    H1FaceRestriction(const FiniteElementSpace& fes,
                      const ElementDofOrdering ordering,
                      const FaceType type);
 
-   /** @brief Extract the face degrees of freedom from @a x into @a y.
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
 
-       @param[in]  x The L-vector of degrees of freedom.
-       @param[out] y The degrees of freedom on the face, corresponding to a face
-                     E-vector.
-   */
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
    void Mult(const Vector &x, Vector &y) const override;
 
-   /** @brief Add the face degrees of freedom @a x to the element degrees of
-       freedom @a y.
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
 
-       @param[in]     x The face degrees of freedom on the face.
-       @param[in,out] y The L-vector of degrees of freedom to which we add the
-                        face degrees of freedom.
-   */
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
    void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
+       for the gathering: E-vector to L-vector.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+protected:
+   mutable Array<int> face_map; // Used in the computation of GetFaceDofs
+
+   /** @brief Verify that H1FaceRestriction is build from an H1 FESpace.
+
+       @param[in] ordering The FESpace element ordering.
+   */
+   void CheckFESpace(const ElementDofOrdering ordering);
+
+   /** @brief Set the scattering indices of elem1, and increment the offsets for
+       the face described by the @a face.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+       @param[in] ordering Request a specific element ordering.
+    */
+   void SetFaceDofsScatterIndices(const Mesh::FaceInformation &face,
+                                  const int face_index,
+                                  const ElementDofOrdering ordering);
+
+   /** @brief Set the gathering indices of elem1 for the interior face described
+       by the @a face.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+       @param[in] ordering Request a specific element ordering.
+    */
+   void SetFaceDofsGatherIndices(const Mesh::FaceInformation &face,
+                                 const int face_index,
+                                 const ElementDofOrdering ordering);
 };
 
-/// Operator that extracts Face degrees of freedom on L2 FiniteElementSpaces.
+/// Operator that extracts Face degrees of freedom for L2 spaces.
 /** Objects of this type are typically created and owned by FiniteElementSpace
     objects, see FiniteElementSpace::GetFaceRestriction(). */
 class L2FaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
-   const int nf;
-   const int ne;
-   const int vdim;
+   const int nf; // Number of faces of the requested type
+   const int ne; // Number of elements
+   const int vdim; // vdim
    const bool byvdim;
-   const int ndofs;
-   const int dof;
-   const int elemDofs;
+   const int face_dofs; // Number of dofs on each face
+   const int elem_dofs; // Number of dofs in each element
+   const int nfdofs; // Total number of dofs on the faces
+   const int ndofs; // Total number of dofs
+   const FaceType type;
    const L2FaceValues m;
-   const int nfdofs;
-   Array<int> scatter_indices1;
-   Array<int> scatter_indices2;
-   Array<int> offsets;
-   Array<int> gather_indices;
+   Array<int> scatter_indices1; // Scattering indices for element 1 on each face
+   Array<int> scatter_indices2; // Scattering indices for element 2 on each face
+   Array<int> gather_offsets; // offsets for the gathering indices of each dof
+   Array<int> gather_indices; // gathering indices for each dof
 
-   L2FaceRestriction(const FiniteElementSpace&,
-                     const FaceType,
-                     const L2FaceValues m = L2FaceValues::DoubleValued);
+   /** @brief Constructs an L2FaceRestriction.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+       @param[in] build    Request the NCL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from L2FaceRestriction.
+   */
+   L2FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type,
+                     const L2FaceValues m,
+                     bool build);
 
 public:
-   L2FaceRestriction(const FiniteElementSpace&,
-                     const ElementDofOrdering,
-                     const FaceType,
+   /** @brief Constructs an L2FaceRestriction.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2 */
+   L2FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type,
                      const L2FaceValues m = L2FaceValues::DoubleValued);
 
-   /** @brief Extract the face degrees of freedom from @a x into @a y.
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
 
-       @param[in]  x The L-vector of degrees of freedom.
-       @param[out] y The degrees of freedom on the face, corresponding to a face
-                     E-vector.
-   */
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf)
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf)
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
    void Mult(const Vector &x, Vector &y) const override;
 
-   /** @brief Add the face degrees of freedom @a x to the element degrees of
-       freedom @a y.
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
 
-       @param[in]     x The face degrees of freedom on the face.
-       @param[in,out] y The L-vector of degrees of freedom to which we add the
-                        face degrees of freedom.
-   */
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf)
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf)
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
    void AddMultTranspose(const Vector &x, Vector &y) const override;
 
-   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this L2FaceRestriction. */
+   /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
+       pattern given by this L2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
    virtual void FillI(SparseMatrix &mat, const bool keep_nbr_block = false) const;
 
-   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this L2FaceRestriction, and the values of ea_data. */
-   virtual void FillJAndData(const Vector &ea_data,
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this L2FaceRestriction, and the values of
+       fea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
+   virtual void FillJAndData(const Vector &fea_data,
                              SparseMatrix &mat,
                              const bool keep_nbr_block = false) const;
 
-   /// This methods adds the DG face matrices to the element matrices.
-   void AddFaceMatricesToElementMatrices(Vector &fea_data,
-                                         Vector &ea_data) const;
+   /** @brief This methods adds the DG face matrices to the element matrices.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf
+                           On each face the first and second local matrices
+                           correspond to the contributions of elem1 and elem2 on
+                           themselves respectively.
+       @param[in,out] ea_data The dense matrices representing the element local
+                              contributions for each element to which will be
+                              added the face contributions.
+                              The format is: dofs x dofs x ne, where dofs is the
+                              number of dofs per element and ne the number of
+                              elements. */
+   virtual void AddFaceMatricesToElementMatrices(const Vector &fea_data,
+                                                 Vector &ea_data) const;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
+       for the gathering: E-vector to L-vector.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+protected:
+   mutable Array<int> face_map; // Used in the computation of GetFaceDofs
+
+   /** @brief Verify that L2FaceRestriction is build from an L2 FESpace.
+
+       @param[in] ordering The FESpace element ordering.
+   */
+   void CheckFESpace(const ElementDofOrdering ordering);
+
+   /** @brief Set the scattering indices of elem1, and increment the offsets for
+       the face described by the @a face. The ordering of the face dofs of elem1
+       is lexicographic relative to elem1.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void SetFaceDofsScatterIndices1(const Mesh::FaceInformation &face,
+                                   const int face_index);
+
+   /** @brief Permute and set the scattering indices of elem2, and increment the
+       offsets for the face described by the @a face. The permutation orders the
+       dofs of elem2 lexicographically as the ones of elem1.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void PermuteAndSetFaceDofsScatterIndices2(const Mesh::FaceInformation &face,
+                                             const int face_index);
+
+   /** @brief Permute and set the scattering indices of elem2 for the shared
+       face described by the @a face. The permutation orders the dofs of elem2 as
+       the ones of elem1.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void PermuteAndSetSharedFaceDofsScatterIndices2(
+      const Mesh::FaceInformation &face,
+      const int face_index);
+
+   /** @brief Set the scattering indices of elem2 for the boundary face
+       described by the @a face.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void SetBoundaryDofsScatterIndices2(const Mesh::FaceInformation &face,
+                                       const int face_index);
+
+   /** @brief Set the gathering indices of elem1 for the interior face described
+       by the @a face.
+
+       Note: This function modifies the offsets.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void SetFaceDofsGatherIndices1(const Mesh::FaceInformation &face,
+                                  const int face_index);
+
+   /** @brief Permute and set the gathering indices of elem2 for the interior
+       face described by the @a face. The permutation orders the dofs of elem2 as
+       the ones of elem1.
+
+       Note: This function modifies the offsets.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void PermuteAndSetFaceDofsGatherIndices2(const Mesh::FaceInformation &face,
+                                            const int face_index);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::SingleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void SingleValuedConformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x 2 x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   virtual void DoubleValuedConformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::SingleValued
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void SingleValuedConformingAddMultTranspose(const Vector& x, Vector& y) const;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::DoubleValued
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x 2 x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void DoubleValuedConformingAddMultTranspose(const Vector& x, Vector& y) const;
 };
 
-// Return the face degrees of freedom returned in Lexicographic order.
-void GetFaceDofs(const int dim, const int face_id,
-                 const int dof1d, Array<int> &faceMap);
+/** This struct stores which side is the master nonconforming side and the
+    index of the interpolator, see InterpolationManager class below. */
+struct InterpConfig
+{
+   uint32_t is_non_conforming : 1;
+   uint32_t master_side : 1;
+   uint32_t index : 30;
 
-// Convert from Native ordering to lexicographic ordering
+   // default constructor, create a conforming face with index 0.
+   InterpConfig() = default;
+
+   // Non-conforming face
+   InterpConfig(int master_side, int nc_index)
+      : is_non_conforming(1), master_side(master_side), index(nc_index)
+   { }
+
+   InterpConfig(const InterpConfig&) = default;
+
+   InterpConfig &operator=(const InterpConfig &rhs) = default;
+};
+
+/** @brief This class manages the storage and computation of the interpolations
+    from master (coarse) face to slave (fine) face.
+*/
+class InterpolationManager
+{
+protected:
+   const FiniteElementSpace &fes;
+   const ElementDofOrdering ordering;
+   Array<InterpConfig> interp_config; // interpolator index for each face
+   Vector interpolators; // face_dofs x face_dofs x num_interpolators
+   int nc_cpt; // Counter for interpolators, and used as index.
+
+   /** The interpolators are associated to a key of containing the address of
+       PointMatrix and a local face identifier. */
+   using Key = std::pair<const DenseMatrix*,int>;
+   /// The temporary map used to store the different interpolators.
+   using Map = std::map<Key, std::pair<int,const DenseMatrix*>>;
+   Map interp_map; // The temporary map that stores the interpolators.
+
+public:
+   InterpolationManager() = delete;
+
+   /** @brief main constructor.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs
+    */
+   InterpolationManager(const FiniteElementSpace &fes,
+                        ElementDofOrdering ordering,
+                        FaceType type);
+
+   /** @brief Register the face with @a face and index @a face_index as a
+       conforming face for the interpolation of the degrees of freedom.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void RegisterFaceConformingInterpolation(const Mesh::FaceInformation &face,
+                                            int face_index);
+
+   /** @brief Register the face with @a face and index @a face_index as a
+       conforming face for the interpolation of the degrees of freedom.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void RegisterFaceCoarseToFineInterpolation(const Mesh::FaceInformation &face,
+                                              int face_index);
+
+   /** @brief Transform the interpolation matrix map into a contiguous memory
+       structure. */
+   void LinearizeInterpolatorMapIntoVector();
+
+   /// @brief Return the total number of interpolators.
+   int GetNumInterpolators() const
+   {
+      return nc_cpt;
+   }
+
+   /** @brief Return an mfem::Vector containing the interpolators in the
+       following format: face_dofs x face_dofs x num_interpolators. */
+   const Vector& GetInterpolators() const
+   {
+      return interpolators;
+   }
+
+   /** @brief Return an array containing the interpolation configuration for
+       each face registered with RegisterFaceConformingInterpolation and
+       RegisterFaceCoarseToFineInterpolation. */
+   const Array<InterpConfig>& GetFaceInterpConfig() const
+   {
+      return interp_config;
+   }
+
+private:
+   /** @brief Returns the interpolation operator from a master (coarse) face to
+       a slave (fine) face.
+
+       @param[in] face The face information of the current face.
+       @param[in] ptMat The PointMatrix describing the position and orientation
+                        of the fine face in the coarse face. This PointMatrix is
+                        usually obtained from the mesh through the method
+                        GetNCFacesPtMat.
+       @param[in] ordering  Request a specific element ordering.
+       @return The dense matrix corresponding to the interpolation of the face
+               degrees of freedom of the master (coarse) face to the slave
+               (fine) face. */
+   const DenseMatrix* GetCoarseToFineInterpolation(
+      const Mesh::FaceInformation &face,
+      const DenseMatrix* ptMat);
+};
+
+/** @brief Operator that extracts face degrees of freedom for L2 nonconforming
+    spaces.
+
+    In order to support face restrictions on nonconforming meshes, this
+    operator interpolates master (coarse) face degrees of freedom onto the
+    slave (fine) face. This allows face integrators to treat nonconforming
+    faces just as regular conforming faces. */
+class NCL2FaceRestriction : virtual public L2FaceRestriction
+{
+protected:
+   InterpolationManager interpolations;
+   mutable Vector x_interp;
+
+   /** @brief Constructs an NCL2FaceRestriction, this is a specialization of a
+       L2FaceRestriction for nonconforming meshes.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+       @param[in] build    Request the NCL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from NCL2FaceRestriction.
+   */
+   NCL2FaceRestriction(const FiniteElementSpace& fes,
+                       const ElementDofOrdering ordering,
+                       const FaceType type,
+                       const L2FaceValues m,
+                       bool build);
+public:
+   /** @brief Constructs an NCL2FaceRestriction, this is a specialization of a
+       L2FaceRestriction for nonconforming meshes.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+   */
+   NCL2FaceRestriction(const FiniteElementSpace& fes,
+                       const ElementDofOrdering ordering,
+                       const FaceType type,
+                       const L2FaceValues m = L2FaceValues::DoubleValued);
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+   /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
+       pattern given by this NCL2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillI(SparseMatrix &mat,
+              const bool keep_nbr_block = false) const override;
+
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this NCL2FaceRestriction, and the values of
+       ea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     const bool keep_nbr_block = false) const override;
+
+   /** @brief This methods adds the DG face matrices to the element matrices.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first and second local matrices
+                           correspond to the contributions of elem1 and elem2 on
+                           themselves respectively.
+       @param[in,out] ea_data The dense matrices representing the element local
+                              contributions for each element to which will be
+                              added the face contributions.
+                              The format is: dofs x dofs x ne, where dofs is the
+                              number of dofs per element and ne the number of
+                              elements.
+
+       @warning This method is not implemented yet. */
+   void AddFaceMatricesToElementMatrices(const Vector &fea_data,
+                                         Vector &ea_data) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
+       for the gathering: E-vector to L-vector, and the interpolators from
+       coarse to fine face for master non-comforming faces.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with nonconforming faces and when:
+       L2FaceValues m == L2FaceValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     (face_dofs x vdim x 2 x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   virtual void DoubleValuedNonconformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs. Should only be used when:
+       L2FaceValues m == L2FaceValues::SingleValued
+
+       @param[in] x The dofs vector that needs coarse dofs to be express in term
+                    of the coarse basis, the result is stored in x_interp.
+   */
+   void SingleValuedNonconformingTransposeInterpolation(const Vector& x) const;
+
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs. Should only be used when:
+       L2FaceValues m == L2FaceValues::DoubleValued
+
+       @param[in] x The dofs vector that needs coarse dofs to be express in term
+                    of the coarse basis, the result is stored in x_interp.
+   */
+   void DoubleValuedNonconformingTransposeInterpolation(const Vector& x) const;
+};
+
+/** @brief Return the face map that extracts the degrees of freedom for the
+    requested local face of a quad or hex, returned in Lexicographic order.
+
+    @param[in] dim The dimension of the space
+    @param[in] face_id The local face identifier
+    @param[in] dof1d The 1D number of degrees of freedom for each dimension
+    @param[out] face_map The map that maps each face dof to an element dof
+*/
+void GetFaceDofs(const int dim, const int face_id,
+                 const int dof1d, Array<int> &face_map);
+
+/** @brief Convert a dof face index from Native ordering to lexicographic
+    ordering for quads and hexes.
+
+    @param[in] dim The dimension of the element, 2 for quad, 3 for hex
+    @param[in] face_id The local face identifier
+    @param[in] size1d The 1D number of degrees of freedom for each dimension
+    @param[in] index The native index on the face
+    @return The lexicographic index on the face
+*/
 int ToLexOrdering(const int dim, const int face_id, const int size1d,
                   const int index);
 
-// Permute dofs or quads on a face for e2 to match with the ordering of e1
+/** @brief Compute the dof face index of elem2 corresponding to the given dof
+    face index.
+
+    @param[in] dim The dimension of the element, 2 for quad, 3 for hex
+    @param[in] face_id1 The local face identifier of elem1
+    @param[in] face_id2 The local face identifier of elem2
+    @param[in] orientation The orientation of elem2 relative to elem1 on the
+                           face
+    @param[in] size1d The 1D number of degrees of freedom for each dimension
+    @param[in] index The dof index on elem1
+    @return The dof index on elem2 facing the dof on elem1
+*/
 int PermuteFaceL2(const int dim, const int face_id1,
                   const int face_id2, const int orientation,
                   const int size1d, const int index);
 
 }
 
-#endif //MFEM_RESTRICTION
+#endif // MFEM_RESTRICTION

fem/staticcond.cpp

@@ -284,7 +284,7 @@ void StaticCondensation::Finalize()
 }
 
 void StaticCondensation::EliminateReducedTrueDofs(
-   const Array<int> &ess_rtdof_list, Matrix::DiagonalPolicy dpolicy)
+   const Array<int> &ess_rtdof_list_, Matrix::DiagonalPolicy dpolicy)
 {
    if (!Parallel() || S) // not parallel or not finalized
    {
@@ -292,16 +292,16 @@ void StaticCondensation::EliminateReducedTrueDofs(
       {
          S_e = new SparseMatrix(S->Height());
       }
-      for (int i = 0; i < ess_rtdof_list.Size(); i++)
+      for (int i = 0; i < ess_rtdof_list_.Size(); i++)
       {
-         S->EliminateRowCol(ess_rtdof_list[i], *S_e, dpolicy);
+         S->EliminateRowCol(ess_rtdof_list_[i], *S_e, dpolicy);
       }
    }
    else // parallel and finalized
    {
 #ifdef MFEM_USE_MPI
       MFEM_ASSERT(pS_e.Ptr() == NULL, "essential b.c. already eliminated");
-      pS_e.EliminateRowsCols(pS, ess_rtdof_list);
+      pS_e.EliminateRowsCols(pS, ess_rtdof_list_);
 #endif
    }
 }

fem/staticcond.hpp

@@ -199,14 +199,14 @@ public:
 
    /** Restrict a list of true FE space dofs to a list of reduced/trace true FE
        space dofs. */
-   void ConvertListToReducedTrueDofs(const Array<int> &ess_tdof_list,
-                                     Array<int> &ess_rtdof_list) const
+   void ConvertListToReducedTrueDofs(const Array<int> &ess_tdof_list_,
+                                     Array<int> &ess_rtdof_list_) const
    {
       Array<int> ess_tdof_marker, ess_rtdof_marker;
-      FiniteElementSpace::ListToMarker(ess_tdof_list, fes->GetTrueVSize(),
+      FiniteElementSpace::ListToMarker(ess_tdof_list_, fes->GetTrueVSize(),
                                        ess_tdof_marker);
       ConvertMarkerToReducedTrueDofs(ess_tdof_marker, ess_rtdof_marker);
-      FiniteElementSpace::MarkerToList(ess_rtdof_marker, ess_rtdof_list);
+      FiniteElementSpace::MarkerToList(ess_rtdof_marker, ess_rtdof_list_);
    }
 
    /** Given a solution of the reduced system 'sc_sol' and the RHS 'b' for the

fem/tbilinearform.hpp

@@ -274,18 +274,18 @@ public:
    {
       y = 0.0;
 
-      solVecLayout_type solVecLayout(this->solVecLayout);
-      solFESpace solFES(this->solFES);
+      solVecLayout_type solVecLayoutLoc(this->solVecLayout);
+      solFESpace solFESLoc(this->solFES);
 
       TTensor3<dofs,vdim,BE,vcomplex_t> xy_dof;
 
       const int NE = mesh.GetNE();
       for (int el = 0; el < NE; el += TE)
       {
-         solFES.SetElement(el);
+         solFESLoc.SetElement(el);
 
-         solFES.VectorExtract(solVecLayout, x, xy_dof.layout, xy_dof);
-         solFES.VectorAssemble(xy_dof.layout, xy_dof, solVecLayout, y);
+         solFESLoc.VectorExtract(solVecLayoutLoc, x, xy_dof.layout, xy_dof);
+         solFESLoc.VectorAssemble(xy_dof.layout, xy_dof, solVecLayoutLoc, y);
       }
    }
 
@@ -346,8 +346,8 @@ public:
    {
       typedef TTensor3<dofs,vdim,BE,vcomplex_t> vdof_data_t;
 
-      solVecLayout_t solVecLayout(this->solVecLayout);
-      solFESpace solFES(this->solFES);
+      solVecLayout_t solVecLayoutLoc(this->solVecLayout);
+      solFESpace solFESLoc(this->solFES);
 
       const int NE = mesh.GetNE();
       // TODO: How do we make sure that this array is aligned properly, AND
@@ -358,8 +358,8 @@ public:
       sx.MakeDataOwner();
       for (int el = 0; el < NE; el += TE)
       {
-         solFES.SetElement(el);
-         solFES.VectorExtract(solVecLayout, x, vdof_data_t::layout, vsx);
+         solFESLoc.SetElement(el);
+         solFESLoc.VectorExtract(solVecLayoutLoc, x, vdof_data_t::layout, vsx);
          vsx += vdof_data_t::size;
       }
    }
@@ -397,9 +397,9 @@ public:
    void AssembleMatrix(SparseMatrix &M) const
    {
       Trans_t T(mesh, meshEval);
-      solFESpace solFES(this->solFES);
-      solShapeEval solEval(this->solEval);
-      solVecLayout_t solVecLayout(this->solVecLayout);
+      solFESpace solFESLoc(this->solFES);
+      solShapeEval solEvalLoc(this->solEval);
+      solVecLayout_t solVecLayoutLoc(this->solVecLayout);
       coeff_eval_t wQ(int_rule, coeff);
 
       const int NE = mesh.GetNE();
@@ -429,12 +429,12 @@ public:
             TMatrix<dofs,dofs,vcomplex_t> M_loc;
             S_spec::ElementMatrix::Compute(
                asm_qpt_data[k].layout, asm_qpt_data[k], M_loc.layout, M_loc,
-               solEval);
+               solEvalLoc);
 
-            solFES.SetElement(el_k);
+            solFESLoc.SetElement(el_k);
             for (int bi = 0; bi < vdim; bi++)
             {
-               solFES.AssembleBlock(bi, bi, solVecLayout, M_loc, M);
+               solFESLoc.AssembleBlock(bi, bi, solVecLayoutLoc, M_loc, M);
             }
          }
       }
@@ -445,7 +445,7 @@ public:
    void AssembleMatrix(DenseTensor &M) const
    {
       Trans_t T(mesh, meshEval);
-      solShapeEval solEval(this->solEval);
+      solShapeEval solEvalLoc(this->solEval);
       coeff_eval_t wQ(int_rule, coeff);
 
       const int NE = mesh.GetNE();
@@ -476,7 +476,7 @@ public:
             TMatrix<dofs,dofs,vcomplex_t> M_loc;
             S_spec::ElementMatrix::Compute(
                asm_qpt_data[k].layout, asm_qpt_data[k], M_loc.layout, M_loc,
-               solEval);
+               solEvalLoc);
 
             for (int s = 0; s < SS && el_k+s < NE; s++)
             {
@@ -498,7 +498,7 @@ public:
    void AssembleBilinearForm(BilinearForm &a) const
    {
       Trans_t T(mesh, meshEval);
-      solShapeEval solEval(this->solEval);
+      solShapeEval solEvalLoc(this->solEval);
       coeff_eval_t wQ(int_rule, coeff);
 
       Array<int> vdofs;
@@ -533,7 +533,7 @@ public:
             TMatrix<dofs,dofs,vcomplex_t> M_loc;
             S_spec::ElementMatrix::Compute(
                asm_qpt_data[k].layout, asm_qpt_data[k], M_loc.layout, M_loc,
-               solEval);
+               solEvalLoc);
 
             if (dof_map) // switch from tensor-product ordering
             {
@@ -600,19 +600,19 @@ public:
       // For now, when vdim > 1, assume block-diagonal matrix with the same
       // diagonal block for all components.
       // M is assumed to be (dof x dof x NE).
-      solVecLayout_t solVecLayout(this->solVecLayout);
+      solVecLayout_t solVecLayoutLoc(this->solVecLayout);
       const int NE = mesh.GetNE();
       for (int el = 0; el < NE; el++)
       {
          TTensor3<dofs,vdim,1,AutoSIMD<complex_t,1,1> > x_dof, y_dof;
 
          solFES.SetElement(el);
-         solFES.VectorExtract(solVecLayout, x, x_dof.layout, x_dof);
+         solFES.VectorExtract(solVecLayoutLoc, x, x_dof.layout, x_dof);
          Mult_AB<false>(TMatrix<dofs,dofs>::layout,
                         M(el).Data(),
                         x_dof.layout.merge_23(), x_dof,
                         y_dof.layout.merge_23(), y_dof);
-         solFES.VectorAssemble(y_dof.layout, y_dof, solVecLayout, y);
+         solFES.VectorAssemble(y_dof.layout, y_dof, solVecLayoutLoc, y);
       }
    }
 };

fem/tmop.cpp

@@ -2311,13 +2311,17 @@ AdaptivityEvaluator::~AdaptivityEvaluator()
 #endif
 }
 
-void TMOP_Integrator::ReleasePADeviceMemory()
+void TMOP_Integrator::ReleasePADeviceMemory(bool copy_to_host)
 {
    if (PA.enabled)
    {
-      PA.H.GetMemory().DeleteDevice();
-      PA.H0.GetMemory().DeleteDevice();
-      PA.Jtr.GetMemory().DeleteDevice();
+      PA.H.GetMemory().DeleteDevice(copy_to_host);
+      PA.H0.GetMemory().DeleteDevice(copy_to_host);
+      if (!copy_to_host && !PA.Jtr.GetMemory().HostIsValid())
+      {
+         PA.Jtr_needs_update = true;
+      }
+      PA.Jtr.GetMemory().DeleteDevice(copy_to_host);
    }
 }
 

fem/tmop.hpp

@@ -1551,7 +1551,7 @@ public:
 
    /// Release the device memory of large PA allocations. This will copy device
    /// memory back to the host before releasing.
-   void ReleasePADeviceMemory();
+   void ReleasePADeviceMemory(bool copy_to_host = true);
 
    /// Prescribe a set of integration rules; relevant for mixed meshes.
    /** This function has priority over SetIntRule(), if both are called. */

fem/transfer.cpp

@@ -903,7 +903,8 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
                                    const FiniteElementSpace& hFESpace_)
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize())
 {
-   if (lFESpace_.FEColl() == hFESpace_.FEColl())
+   bool isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
+   if (lFESpace_.FEColl() == hFESpace_.FEColl() && !isvar_order)
    {
       OperatorPtr P(Operator::ANY_TYPE);
       hFESpace_.GetTransferOperator(lFESpace_, P);
@@ -912,8 +913,11 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
    }
    else if (lFESpace_.GetMesh()->GetNE() > 0
             && hFESpace_.GetMesh()->GetNE() > 0
+            && lFESpace_.GetVDim() == 1
+            && hFESpace_.GetVDim() == 1
             && dynamic_cast<const TensorBasisElement*>(lFESpace_.GetFE(0))
             && dynamic_cast<const TensorBasisElement*>(hFESpace_.GetFE(0))
+            && !isvar_order
             && (hFESpace_.FEColl()->GetContType() ==
                 mfem::FiniteElementCollection::CONTINUOUS ||
                 hFESpace_.FEColl()->GetContType() ==
@@ -945,6 +949,7 @@ PRefinementTransferOperator::PRefinementTransferOperator(
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize()), lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
+   isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
 }
 
 PRefinementTransferOperator::~PRefinementTransferOperator() {}
@@ -969,7 +974,7 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
       DofTransformation * doftrans_l = lFESpace.GetElementDofs(i, l_dofs);
 
       const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
-      if (geom != cached_geom)
+      if (geom != cached_geom || isvar_order)
       {
          h_fe = hFESpace.GetFE(i);
          l_fe = lFESpace.GetFE(i);
@@ -1026,7 +1031,7 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
       DofTransformation * doftrans_l = lFESpace.GetElementDofs(i, l_dofs);
 
       const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
-      if (geom != cached_geom)
+      if (geom != cached_geom || isvar_order)
       {
          h_fe = hFESpace.GetFE(i);
          l_fe = lFESpace.GetFE(i);
@@ -1424,20 +1429,36 @@ void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
    elem_restrict_lex_l->MultTranspose(localL, y);
 }
 
-#ifdef MFEM_USE_MPI
-TrueTransferOperator::TrueTransferOperator(const
-                                           ParFiniteElementSpace& lFESpace_,
-                                           const ParFiniteElementSpace& hFESpace_)
+
+TrueTransferOperator::TrueTransferOperator(const FiniteElementSpace& lFESpace_,
+                                           const FiniteElementSpace& hFESpace_)
    : Operator(hFESpace_.GetTrueVSize(), lFESpace_.GetTrueVSize()),
      lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
    localTransferOperator = new TransferOperator(lFESpace_, hFESpace_);
 
-   tmpL.SetSize(lFESpace_.GetVSize());
-   tmpH.SetSize(hFESpace_.GetVSize());
+   P = lFESpace.GetProlongationMatrix();
+   R = hFESpace.IsVariableOrder() ? hFESpace.GetHpRestrictionMatrix() :
+       hFESpace.GetRestrictionMatrix();
 
-   hFESpace.GetRestrictionMatrix()->BuildTranspose();
+   // P and R can be both null
+   // P can be null and R not null
+   // If P is not null it is assumed that R is not null as well
+   if (P) { MFEM_VERIFY(R, "Both P and R have to be not NULL") }
+
+   if (P)
+   {
+      tmpL.SetSize(lFESpace_.GetVSize());
+      tmpH.SetSize(hFESpace_.GetVSize());
+      R->EnsureMultTranspose();
+   }
+   // P can be null and R not null
+   else if (R)
+   {
+      tmpH.SetSize(hFESpace_.GetVSize());
+      R->EnsureMultTranspose();
+   }
 }
 
 TrueTransferOperator::~TrueTransferOperator()
@@ -1447,17 +1468,40 @@ TrueTransferOperator::~TrueTransferOperator()
 
 void TrueTransferOperator::Mult(const Vector& x, Vector& y) const
 {
-   lFESpace.GetProlongationMatrix()->Mult(x, tmpL);
-   localTransferOperator->Mult(tmpL, tmpH);
-   hFESpace.GetRestrictionMatrix()->Mult(tmpH, y);
+   if (P)
+   {
+      P->Mult(x, tmpL);
+      localTransferOperator->Mult(tmpL, tmpH);
+      R->Mult(tmpH, y);
+   }
+   else if (R)
+   {
+      localTransferOperator->Mult(x, tmpH);
+      R->Mult(tmpH, y);
+   }
+   else
+   {
+      localTransferOperator->Mult(x, y);
+   }
 }
 
 void TrueTransferOperator::MultTranspose(const Vector& x, Vector& y) const
 {
-   hFESpace.GetRestrictionMatrix()->MultTranspose(x, tmpH);
-   localTransferOperator->MultTranspose(tmpH, tmpL);
-   lFESpace.GetProlongationMatrix()->MultTranspose(tmpL, y);
+   if (P)
+   {
+      R->MultTranspose(x, tmpH);
+      localTransferOperator->MultTranspose(tmpH, tmpL);
+      P->MultTranspose(tmpL, y);
+   }
+   else if (R)
+   {
+      R->MultTranspose(x, tmpH);
+      localTransferOperator->MultTranspose(tmpH, y);
+   }
+   else
+   {
+      localTransferOperator->MultTranspose(x, y);
+   }
 }
-#endif
 
 } // namespace mfem

fem/transfer.hpp

@@ -387,6 +387,7 @@ class PRefinementTransferOperator : public Operator
 private:
    const FiniteElementSpace& lFESpace;
    const FiniteElementSpace& hFESpace;
+   bool isvar_order;
 
 public:
    /// @brief Constructs a transfer operator from \p lFESpace to \p hFESpace
@@ -452,14 +453,15 @@ public:
    virtual void MultTranspose(const Vector& x, Vector& y) const override;
 };
 
-#ifdef MFEM_USE_MPI
 /// @brief Matrix-free transfer operator between finite element spaces working
 /// on true degrees of freedom
 class TrueTransferOperator : public Operator
 {
 private:
-   const ParFiniteElementSpace& lFESpace;
-   const ParFiniteElementSpace& hFESpace;
+   const FiniteElementSpace& lFESpace;
+   const FiniteElementSpace& hFESpace;
+   const Operator * P = nullptr;
+   const SparseMatrix * R = nullptr;
    TransferOperator* localTransferOperator;
    mutable Vector tmpL;
    mutable Vector tmpH;
@@ -467,8 +469,8 @@ private:
 public:
    /// @brief Constructs a transfer operator working on true degrees of freedom
    /// from \p lFESpace to \p hFESpace
-   TrueTransferOperator(const ParFiniteElementSpace& lFESpace_,
-                        const ParFiniteElementSpace& hFESpace_);
+   TrueTransferOperator(const FiniteElementSpace& lFESpace_,
+                        const FiniteElementSpace& hFESpace_);
 
    /// Destructor
    ~TrueTransferOperator();
@@ -484,7 +486,6 @@ public:
        the true dof vector \p y corresponding to the coarse space. */
    virtual void MultTranspose(const Vector& x, Vector& y) const override;
 };
-#endif
 
 } // namespace mfem
 

general/binaryio.hpp

@@ -58,16 +58,19 @@ void AppendBytes(std::vector<char> &vec, const T &val)
    vec.insert(vec.end(), ptr, ptr + sizeof(T));
 }
 
-/// Given a buffer @a buf of length @a nbytes, encode the data in base-64
-/// format, and write the encoded data to the output stream @a out.
+/// @brief Given a buffer @a bytes of length @a nbytes, encode the data in
+/// base-64 format, and write the encoded data to the output stream @a out.
 void WriteBase64(std::ostream &out, const void *bytes, size_t nbytes);
 
-/// Decode @a len base-64 encoded characters in the buffer @a src, and store the
-/// resulting decoded data in @a buf. @a buf will be resized as needed.
+/// @brief Decode @a len base-64 encoded characters in the buffer @a src, and
+/// store the resulting decoded data in @a buf. @a buf will be resized as
+/// needed.
 void DecodeBase64(const char *src, size_t len, std::vector<char> &buf);
 
-/// Return the number of characters needed to encode @a nbytes in base-64. This
-/// is equal to 4*nbytes/3, rounded up to the nearest multiple of 4.
+/// @brief Return the number of characters needed to encode @a nbytes in
+/// base-64.
+///
+/// This is equal to 4*nbytes/3, rounded up to the nearest multiple of 4.
 size_t NumBase64Chars(size_t nbytes);
 
 } // namespace mfem::bin_io

general/cuda.hpp

@@ -19,6 +19,7 @@
 #define MFEM_CUDA_BLOCKS 256
 
 #ifdef MFEM_USE_CUDA
+#define MFEM_USE_CUDA_OR_HIP
 #define MFEM_DEVICE __device__
 #define MFEM_LAMBDA __host__
 #define MFEM_HOST_DEVICE __host__ __device__

general/forall.hpp

@@ -42,6 +42,19 @@ const int MAX_Q1D = 14;
 #define MFEM_UNROLL(N)
 #endif
 
+// MFEM_GPU_FORALL: "parallel for" executed with CUDA or HIP based on the MFEM
+// build-time configuration (MFEM_USE_CUDA or MFEM_USE_HIP). If neither CUDA nor
+// HIP is enabled, this macro is a no-op.
+#if defined(MFEM_USE_CUDA)
+#define MFEM_GPU_FORALL(i, N,...) CuWrap1D(N, [=] MFEM_DEVICE      \
+                                       (int i) {__VA_ARGS__})
+#elif defined(MFEM_USE_HIP)
+#define MFEM_GPU_FORALL(i, N,...) HipWrap1D(N, [=] MFEM_DEVICE     \
+                                        (int i) {__VA_ARGS__})
+#else
+#define MFEM_GPU_FORALL(i, N,...) do { } while (false)
+#endif
+
 // Implementation of MFEM's "parallel for" (forall) device/host kernel
 // interfaces supporting RAJA, CUDA, OpenMP, and sequential backends.
 

general/hash.hpp

@@ -66,6 +66,16 @@ struct Hashed4
  *
  *  All items in the container can also be accessed sequentially using the
  *  provided iterator.
+ *
+ *  Notes:
+ *   The data structure and implementation is based on a BlockArray<T> which
+ *   provides an efficient item storage that avoids heap fragmentation, and
+ *   index-based item access. The hash table implemented on top of the
+ *   BlockArray provides fast associative (key -> value) access by grouping
+ *   items into bins (buckets) of O(1) size.
+ *   - "id" denotes the index of an item in the underlying BlockArray<T>,
+ *   - "idx" denotes the index of a bin, determined by hashing a key with
+ *     the function `Hash`.
  */
 template<typename T>
 class HashTable : public BlockArray<T>
@@ -74,68 +84,212 @@ protected:
    typedef BlockArray<T> Base;
 
 public:
+   /** @brief Main constructor of the HashTable class.
+
+       @param[in] block_size The size of the storage blocks of the underlying
+                             BlockArray<T>.
+       @param[in] init_hash_size The initial size of the hash table. Must be
+                                 a power of 2. */
    HashTable(int block_size = 16*1024, int init_hash_size = 32*1024);
-   HashTable(const HashTable& other); // deep copy
+   /// @brief Deep copy
+   HashTable(const HashTable& other);
    ~HashTable();
 
-   /// Get item whose parents are 'p1', 'p2'... Create it if it doesn't exist.
+   /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Default
+       construct an item of type T if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    T* Get(int p1, int p2);
+
+   /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2',
+       'p3', 'p4'. The key 'p4' is optional. Default construct an item of type T
+       if no value corresponds to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The index "id" of the key in the BlockArray<T>.
+
+      @warning This method should only be called if T inherits from Hashed4. */
    T* Get(int p1, int p2, int p3, int p4 = -1 /* p4 optional */);
 
    /// Get id of item whose parents are p1, p2... Create it if it doesn't exist.
+   /** @brief Get the "id" of an item, this "id" corresponding to the index of the
+       item in the underlying BlockArray<T> object. Default construct an item
+       and id if no value corresponds to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    int GetId(int p1, int p2);
+
+   /** @brief Get the "id" of an item, this "id" corresponding to the index of the
+       item in the underlying BlockArray<T> object. Default construct an item
+       and id if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed4. */
    int GetId(int p1, int p2, int p3, int p4 = -1);
 
    /// Find item whose parents are p1, p2... Return NULL if it doesn't exist.
+   /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Return
+       nullptr if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The item associated to the key (p1,p2).
+
+       @warning This method should only be called if T inherits from Hashed2. */
    T* Find(int p1, int p2);
+
+   /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2',
+       'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value
+       correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The item associated to the key (p1,p2,p3,p4).
+
+       @warning This method should only be called if T inherits from Hashed4. */
    T* Find(int p1, int p2, int p3, int p4 = -1);
 
+   /** @brief Item const accessor with key (or parents) the pair 'p1', 'p2'.
+       Return nullptr if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The item associated to the key (p1,p2).
+
+      @warning This method should only be called if T inherits from Hashed2. */
    const T* Find(int p1, int p2) const;
+
+   /** @brief Item const accessor with key (or parents) the quadruplet 'p1',
+       'p2', 'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value
+       correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The item associated to the key (p1,p2,p3,p4).
+
+       @warning This method should only be called if T inherits from Hashed4. */
    const T* Find(int p1, int p2, int p3, int p4 = -1) const;
 
    /// Find id of item whose parents are p1, p2... Return -1 if it doesn't exist.
+   /** @brief Find the "id" of an item, this "id" corresponding to the index of
+       the item in the underlying BlockArray<T> object. Default construct an
+       item and id if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    int FindId(int p1, int p2) const;
+
+   /** @brief Find the "id" of an item, this "id" corresponding to the index of
+       the item in the underlying BlockArray<T> object. Default construct an
+       item and id if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed4. */
    int FindId(int p1, int p2, int p3, int p4 = -1) const;
 
-   /// Return the number of elements currently stored in the HashTable.
+   /// @brief Return the number of elements currently stored in the HashTable.
    int Size() const { return Base::Size() - unused.Size(); }
 
-   /// Return the total number of ids (used and unused) in the HashTable.
+   /// @brief Return the total number of ids (used and unused) in the HashTable.
    int NumIds() const { return Base::Size(); }
 
-   /// Return the number of free/unused ids in the HashTable.
+   /// @brief Return the number of free/unused ids in the HashTable.
    int NumFreeIds() const { return unused.Size(); }
 
-   /// Return true if item 'id' exists in (is used by) the container.
-   /** It is assumed that 0 <= id < NumIds(). */
+   /** @brief Return true if item 'id' exists in (is used by) the container.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+
+       @warning It is assumed that 0 <= id < NumIds(). */
    bool IdExists(int id) const { return (Base::At(id).next != -2); }
 
-   /// Remove an item from the hash table.
-   /** Its id will be reused by newly added items. */
+   /** @brief Remove an item from the hash table.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+
+       @warning Its id will be reused by newly added items. */
    void Delete(int id);
 
-   /// Remove all items.
+   /// @brief Remove all items.
    void DeleteAll();
 
-   /// Allocate an item at 'id'. Enlarge the underlying BlockArray if necessary.
-   /** This is a special purpose method used when loading data from a file.
-       Does nothing if the slot 'id' has already been allocated. */
+   /** @brief Allocate an item at 'id'. Enlarge the underlying BlockArray if
+       necessary.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+
+       @warning This is a special purpose method used when loading data from a
+       file. Does nothing if the slot 'id' has already been allocated. */
    void Alloc(int id, int p1, int p2);
 
-   /// Reinitialize the internal list of unallocated items.
-   /** This is a special purpose method used when loading data from a file. */
+   /** @brief Reinitialize the internal list of unallocated items.
+
+       @warning This is a special purpose method used when loading data from a file. */
    void UpdateUnused();
 
-   /// Make an item hashed under different parent IDs.
+   /** @brief Change the key associated with an item.
+
+       In other words, makes an item hashed under different parent IDs.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] new_p1 First part of the new key.
+       @param[in] new_p2 Second part of the new key.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    void Reparent(int id, int new_p1, int new_p2);
+
+   /** @brief Change the key associated with an item.
+
+       In other words, makes an item hashed under different parent IDs.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] new_p1 First part of the new key.
+       @param[in] new_p2 Second part of the new key.
+       @param[in] new_p3 Third part of the new key.
+       @param[in] new_p4 Fourth part of the new key (optional).
+
+       @warning This method should only be called if T inherits from Hashed4. */
    void Reparent(int id, int new_p1, int new_p2, int new_p3, int new_p4 = -1);
 
-   /// Return total size of allocated memory (tables plus items), in bytes.
+   /// @brief Return total size of allocated memory (tables plus items), in bytes.
    long MemoryUsage() const;
 
-   /// Write details of the memory usage to the mfem output stream.
+   /// @brief Write details of the memory usage to the mfem output stream.
    void PrintMemoryDetail() const;
 
+   /// @brief Print a histogram of bin sizes for debugging purposes.
+   void PrintStats() const;
+
    class iterator : public Base::iterator
    {
    protected:
@@ -183,33 +337,114 @@ public:
    const_iterator cend() const { return const_iterator(); }
 
 protected:
+   /** The hash table: each bin is a linked list of items. For each non-empty
+       bin, this arrays stores the 'id' of the first item in the list, or -1
+       if the bin is empty. */
    int* table;
+
+   /** mask = table_size-1. Used for fast modulo operation in Hash(), to wrap
+       the raw hashed index around the current table size (which must be a power
+       of two). */
    int mask;
+
+   /** List of deleted items in the BlockArray<T>. New items are created with
+       these ids first, before they are appended to the block array. */
    Array<int> unused;
 
-   // hash functions (NOTE: the constants are arbitrary)
-   inline int Hash(int p1, int p2) const
-   { return (984120265*p1 + 125965121*p2) & mask; }
+   /** @brief hash function for Hashed2 items.
 
-   inline int Hash(int p1, int p2, int p3) const
-   { return (984120265*p1 + 125965121*p2 + 495698413*p3) & mask; }
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The hash key "idx" identifying a bin/bucket.
+
+       NOTE: the constants are arbitrary
+       @warning This method should only be called if T inherits from Hashed2. */
+   inline int Hash(size_t p1, size_t p2) const
+   { return (984120265ul*p1 + 125965121ul*p2) & mask; }
+
+   /** @brief hash function for Hashed4 items.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @return The hash key "idx" identifying a bin/bucket.
+
+       NOTE: The constants are arbitrary.
+       NOTE: p4 is not hashed nor stored as p1, p2, p3 identify a face uniquely.
+       @warning This method should only be called if T inherits from Hashed4. */
+   inline int Hash(size_t p1, size_t p2, size_t p3) const
+   { return (984120265ul*p1 + 125965121ul*p2 + 495698413ul*p3) & mask; }
 
    // Delete() and Reparent() use one of these:
+   /// @brief Hash function for items of type T that inherit from Hashed2.
    inline int Hash(const Hashed2& item) const
    { return Hash(item.p1, item.p2); }
 
+   /// @brief Hash function for items of type T that inherit from Hashed4.
    inline int Hash(const Hashed4& item) const
    { return Hash(item.p1, item.p2, item.p3); }
 
+   /** @brief Search the index of the item associated to the key (p1,p2)
+       starting from the item with index @a id.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    int SearchList(int id, int p1, int p2) const;
+
+   /** @brief Search the index of the item associated to the key (p1,p2,p3,(p4))
+       starting from the item with index @a id.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed4. */
    int SearchList(int id, int p1, int p2, int p3) const;
 
+   /** @brief Insert the item 'id' into bin 'idx'.
+
+       @param[in] idx The bin/bucket index.
+       @param[in] id The index of the item in the BlockArray<T>.
+       @param[in] item The item to insert at the begining of the linked list.
+
+       @warning The method only works with bin 'idx' and does not check the
+                overall fill factor of the hash table. If appropriate,
+                use CheckRehash() for that. */
    inline void Insert(int idx, int id, T &item);
+
+   /** @brief Unlink an item @a id from the linked list of bin @a idx.
+
+       @param[in] idx The bin/bucket index.
+       @param[in] id The index of the item in the BlockArray<T>.
+
+       @warning The method aborts if the item is not found. */
    void Unlink(int idx, int id);
 
-   /// Check table load factor and resize if necessary
+   /** @brief Check table fill factor and resize if necessary.
+
+       The method checks the average size of the bins (i.e., the fill factor).
+       If the fill factor is > 2, the table is enlarged (see DoRehash()). */
    inline void CheckRehash();
+
+   /** @brief Double the size of the hash table (i.e., double the number of bins)
+       and reinsert all items into the new bins.
+
+       NOTE: Rehashing is computationally expensive (O(N) in the number of items),
+       but since it is only done rarely (when the number of items doubles),
+       the amortized complexity of inserting an item is still O(1). */
    void DoRehash();
+
+   /** @brief Return the size of the bin "idx".
+
+       @param[in] idx The index of the bin.
+       @return The size of the bin. */
+   int BinSize(int idx) const;
 };
 
 
@@ -591,6 +826,7 @@ void HashTable<T>::Alloc(int id, int p1, int p2)
       item.p2 = p2;
 
       Insert(Hash(p1, p2), id, item);
+      CheckRehash();
    }
 }
 
@@ -649,6 +885,48 @@ void HashTable<T>::PrintMemoryDetail() const
              << " + " << unused.MemoryUsage();
 }
 
+template<typename T>
+int HashTable<T>::BinSize(int idx) const
+{
+   int count = 0;
+   int id = table[idx];
+   while (id >= 0)
+   {
+      const T& item = Base::At(id);
+      id = item.next;
+      count++;
+   }
+   return count;
+}
+
+template<typename T>
+void HashTable<T>::PrintStats() const
+{
+   int table_size = mask+1;
+   mfem::out << "Hash table size: " << table_size << "\n";
+   mfem::out << "Item count: " << Size() << "\n";
+   mfem::out << "BlockArray size: " << Base::Size() << "\n";
+
+   const int H = 16;
+   int hist[H];
+
+   for (int i = 0; i < H; i++) { hist[i] = 0; }
+
+   for (int i = 0; i < table_size; i++)
+   {
+      int bs = BinSize(i);
+      if (bs >= H) { bs = H-1; }
+      hist[bs]++;
+   }
+
+   mfem::out << "Bin size histogram:\n";
+   for (int i = 0; i < H; i++)
+   {
+      mfem::out << "  size " << i << ": "
+                << hist[i] << " bins" << std::endl;
+   }
+}
+
 
 template <typename int_type_const_iter>
 HashFunction &HashFunction::EncodeAndHashInts(int_type_const_iter begin,

general/hip.hpp

@@ -19,6 +19,7 @@
 #define MFEM_HIP_BLOCKS 256
 
 #ifdef MFEM_USE_HIP
+#define MFEM_USE_CUDA_OR_HIP
 #define MFEM_DEVICE __device__
 #define MFEM_LAMBDA __host__ __device__
 #define MFEM_HOST_DEVICE __host__ __device__

general/mem_manager.cpp

@@ -470,7 +470,10 @@ public:
    void *HtoD(void *dst, const void *src, size_t bytes)
    { return HipMemcpyHtoD(dst, src, bytes); }
    void *DtoD(void* dst, const void* src, size_t bytes)
-   { return HipMemcpyDtoD(dst, src, bytes); }
+   // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so
+   // instead we use hipMemcpyAsync to get similar behavior.
+   // for more info see: https://github.com/mfem/mfem/pull/2780
+   { return HipMemcpyDtoDAsync(dst, src, bytes); }
    void *DtoH(void *dst, const void *src, size_t bytes)
    { return HipMemcpyDtoH(dst, src, bytes); }
 };
@@ -593,7 +596,10 @@ public:
       return CuMemcpyDtoD(dst, src, bytes);
 #endif
 #ifdef MFEM_USE_HIP
-      return HipMemcpyDtoD(dst, src, bytes);
+      // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so
+      // instead we use hipMemcpyAsync to get similar behavior.
+      // for more info see: https://github.com/mfem/mfem/pull/2780
+      return HipMemcpyDtoDAsync(dst, src, bytes);
 #endif
       // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }

general/mem_manager.hpp

@@ -18,7 +18,7 @@
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
 #ifdef MFEM_USE_MPI
-#include <HYPRE_config.h> // HYPRE_USING_CUDA
+#include <HYPRE_config.h> // HYPRE_USING_GPU
 #endif
 
 namespace mfem
@@ -934,12 +934,12 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
    if (!(base.flags & REGISTERED))
    {
       if (
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
          // If the following condition is true then MemoryManager::Exists()
          // should also be true:
          IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
 #else
-         // When HYPRE_USING_CUDA is defined we always register the 'base' if
+         // When HYPRE_USING_GPU is defined we always register the 'base' if
          // the MemoryManager::Exists():
          MemoryManager::Exists()
 #endif

linalg/CMakeLists.txt

@@ -15,6 +15,7 @@ list(APPEND SRCS
   blockoperator.cpp
   blockvector.cpp
   complex_operator.cpp
+  complex_densemat.cpp
   constraints.cpp
   densemat.cpp
   symmat.cpp
@@ -34,6 +35,7 @@ list(APPEND HDRS
   blockoperator.hpp
   blockvector.hpp
   complex_operator.hpp
+  complex_densemat.hpp
   constraints.hpp
   densemat.hpp
   dinvariants.hpp

linalg/complex_densemat.cpp

@@ -0,0 +1,366 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "complex_densemat.hpp"
+#include <complex>
+
+#ifdef MFEM_USE_LAPACK
+extern "C" void
+zgetrf_(int *, int *, std::complex<double> *, int *, int *, int *);
+extern "C" void
+zgetrs_(char *, int *, int *, std::complex<double> *, int *, int *,
+        std::complex<double> *, int *, int *);
+extern "C" void
+zgetri_(int *N, std::complex<double> *A, int *LDA, int *IPIV,
+        std::complex<double> *WORK,
+        int *LWORK, int *INFO);
+#endif
+
+namespace mfem
+{
+
+DenseMatrix & ComplexDenseMatrix::real()
+{
+   MFEM_ASSERT(Op_Real_, "ComplexDenseMatrix has no real part!");
+   return dynamic_cast<DenseMatrix &>(*Op_Real_);
+}
+
+DenseMatrix & ComplexDenseMatrix::imag()
+{
+   MFEM_ASSERT(Op_Imag_, "ComplexDenseMatrix has no imaginary part!");
+   return dynamic_cast<DenseMatrix &>(*Op_Imag_);
+}
+
+const DenseMatrix & ComplexDenseMatrix::real() const
+{
+   MFEM_ASSERT(Op_Real_, "ComplexDenseMatrix has no real part!");
+   return dynamic_cast<const DenseMatrix &>(*Op_Real_);
+}
+
+const DenseMatrix & ComplexDenseMatrix::imag() const
+{
+   MFEM_ASSERT(Op_Imag_, "ComplexDenseMatrix has no imaginary part!");
+   return dynamic_cast<const DenseMatrix &>(*Op_Imag_);
+}
+
+DenseMatrix * ComplexDenseMatrix::GetSystemMatrix() const
+{
+   int h = height/2;
+   int w = width/2;
+   DenseMatrix * A = new DenseMatrix(2*h,2*w);
+   double * data = A->Data();
+   double * data_r = nullptr;
+   double * data_i = nullptr;
+
+   // assuming Hermitian convension
+   *A = 0.;
+   if (hasRealPart())
+   {
+      data_r = real().Data();
+      for (int j = 0; j<w; j++)
+      {
+         for (int i = 0; i<h; i++)
+         {
+            data[i+j*height] = data_r[i+j*h];
+            data[i+h+(j+h)*height] = data_r[i+j*h];
+         }
+      }
+   }
+   if (hasImagPart())
+   {
+      data_i = imag().Data();
+      for (int j = 0; j<w; j++)
+      {
+         for (int i = 0; i<h; i++)
+         {
+            data[i+h+j*height] = data_i[i+j*h];
+            data[i+(j+h)*height] = -data_i[i+j*h];
+         }
+      }
+   }
+   return A;
+}
+
+ComplexDenseMatrix * ComplexDenseMatrix::ComputeInverse()
+{
+   MFEM_VERIFY(height == width, "Matrix has to be square");
+
+   // complex data
+   int h = height/2;
+   int w = width/2;
+   std::complex<double> * data = new std::complex<double>[h*w];
+
+   // copy data
+   if (hasRealPart() && hasImagPart())
+   {
+      double * data_r = real().Data();
+      double * data_i = imag().Data();
+      for (int i = 0; i < h*w; i++)
+      {
+         data[i] = std::complex<double> (data_r[i], data_i[i]);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("ComplexDenseMatrix has either only real or imag part");
+   }
+
+#ifdef MFEM_USE_LAPACK
+   int   *ipiv = new int[w];
+   int    lwork = -1;
+   std::complex<double> qwork, *work;
+   int    info;
+
+   zgetrf_(&w, &w, data, &w, ipiv, &info);
+   if (info)
+   {
+      mfem_error("DenseMatrix::Invert() : Error in ZGETRF");
+   }
+
+   zgetri_(&w, data, &w, ipiv, &qwork, &lwork, &info);
+
+   lwork = (int) qwork.real();
+   work = new std::complex<double>[lwork];
+
+   zgetri_(&w, data, &w, ipiv, work, &lwork, &info);
+
+   if (info)
+   {
+      mfem_error("DenseMatrix::Invert() : Error in ZGETRI");
+   }
+
+   delete [] work;
+   delete [] ipiv;
+
+#else
+   // compiling without LAPACK
+   int c, i, j, n = w;
+   double a, b;
+   Array<int> piv(n);
+   std::complex<double> ac,bc;
+
+   for (c = 0; c < n; c++)
+   {
+      a = std::abs(data[c+c*h]);
+      i = c;
+      for (j = c + 1; j < n; j++)
+      {
+         b = std::abs(data[j+c*h]);
+         if (a < b)
+         {
+            a = b;
+            i = j;
+         }
+      }
+      if (a == 0.0)
+      {
+         mfem_error("DenseMatrix::Invert() : singular matrix");
+      }
+      piv[c] = i;
+      for (j = 0; j < n; j++)
+      {
+         mfem::Swap<std::complex<double>>(data[c+j*h], data[i+j*h]);
+      }
+
+      ac = data[c+c*h] = 1.0 / data[c+c*h];
+      for (j = 0; j < c; j++)
+      {
+         data[c+j*h] *= ac;
+      }
+      for (j++; j < n; j++)
+      {
+         data[c+j*h] *= ac;
+      }
+      for (i = 0; i < c; i++)
+      {
+         data[i+c*h] = ac * (bc = -data[i+c*h]);
+         for (j = 0; j < c; j++)
+         {
+            data[i+j*h] += bc * data[c+j*h];
+         }
+         for (j++; j < n; j++)
+         {
+            data[i+j*h] += bc * data[c+j*h];
+         }
+      }
+      for (i++; i < n; i++)
+      {
+         data[i+c*h] = ac * (bc = -data[i+c*h]);
+         for (j = 0; j < c; j++)
+         {
+            data[i+j*h] += bc * data[c+j*h];
+         }
+         for (j++; j < n; j++)
+         {
+            data[i+j*h] += bc * data[c+j*h];
+         }
+      }
+   }
+
+   for (c = n - 1; c >= 0; c--)
+   {
+      j = piv[c];
+      for (i = 0; i < n; i++)
+      {
+         mfem::Swap<std::complex<double>>(data[i+c*h], data[i+j*h]);
+      }
+   }
+
+#endif
+
+   DenseMatrix * C_r = new DenseMatrix(h);
+   DenseMatrix * C_i = new DenseMatrix(h);
+
+   double * datac_r = C_r->Data();
+   double * datac_i = C_i->Data();
+
+   for (int i = 0; i < h*w; i++)
+   {
+      datac_r[i] = data[i].real();
+      datac_i[i] = data[i].imag();
+   }
+
+   return new ComplexDenseMatrix(C_r,C_i,true,true);
+
+}
+
+ComplexDenseMatrix * Mult(const ComplexDenseMatrix &A,
+                          const ComplexDenseMatrix &B)
+{
+   // C = C_r + i C_i = (A_r + i * A_i) * (B_r + i * B_i)
+   //                 = A_r * B_r - A_i B_i + i (A_r * B_i + A_i * B_r)
+
+   int h = A.Height()/2;
+   int w = B.Width()/2;
+
+   MFEM_VERIFY(A.Width() == B.Height(), "Incompatible matrix dimenions");
+
+   //only real case (imag is null)
+   DenseMatrix * C_r = nullptr;
+   DenseMatrix * C_i = nullptr;
+   if ((A.hasRealPart() && B.hasRealPart()) ||
+       (A.hasImagPart() && B.hasImagPart()))
+   {
+      C_r = new DenseMatrix(h,w);
+   }
+   if ((A.hasRealPart() && B.hasImagPart()) ||
+       (A.hasImagPart() && B.hasRealPart()))
+   {
+      C_i = new DenseMatrix(h,w);
+   }
+
+   MFEM_VERIFY(C_r || C_i, "Both real and imag parts are null");
+
+   if (A.hasRealPart() && B.hasRealPart())
+   {
+      Mult(A.real(), B.real(),*C_r);
+   }
+   if (A.hasImagPart() && B.hasImagPart())
+   {
+      if (A.hasRealPart() && B.hasRealPart())
+      {
+         AddMult_a(-1.,A.imag(), B.imag(),*C_r);
+      }
+      else
+      {
+         Mult(A.imag(), B.imag(),*C_r);
+      }
+   }
+
+   if (A.hasRealPart() && B.hasImagPart())
+   {
+      Mult(A.real(), B.imag(),*C_i);
+   }
+
+   if (A.hasImagPart() && B.hasRealPart())
+   {
+      if (A.hasRealPart() && B.hasImagPart())
+      {
+         AddMult(A.imag(), B.real(),*C_i);
+      }
+      else
+      {
+         Mult(A.imag(), B.real(),*C_i);
+      }
+   }
+
+   return new ComplexDenseMatrix(C_r,C_i,true,true);
+}
+
+ComplexDenseMatrix * MultAtB(const ComplexDenseMatrix &A,
+                             const ComplexDenseMatrix &B)
+{
+   // C = C_r + i C_i = (A_r^t - i * A_i^t) * (B_r + i * B_i)
+   //                     = A_r^t * B_r + A_i^t * B_i + i (A_r^t * B_i - A_i^t * B_r)
+
+   int h = A.Width()/2;
+   int w = B.Width()/2;
+
+   MFEM_VERIFY(A.Height() == B.Height(), "Incompatible matrix dimenions");
+
+   //only real case (imag is null)
+   DenseMatrix * C_r = nullptr;
+   DenseMatrix * C_i = nullptr;
+   if ((A.hasRealPart() && B.hasRealPart()) ||
+       (A.hasImagPart() && B.hasImagPart()))
+   {
+      C_r = new DenseMatrix(h,w);
+   }
+   if ((A.hasRealPart() && B.hasImagPart()) ||
+       (A.hasImagPart() && B.hasRealPart()))
+   {
+      C_i = new DenseMatrix(h,w);
+   }
+
+   MFEM_VERIFY(C_r || C_i, "Both real and imag parts are null");
+
+   if (A.hasRealPart() && B.hasRealPart())
+   {
+      MultAtB(A.real(), B.real(),*C_r);
+   }
+   if (A.hasImagPart() && B.hasImagPart())
+   {
+      if (A.hasRealPart() && B.hasRealPart())
+      {
+         DenseMatrix tempC_r(h,w);
+         MultAtB(A.imag(), B.imag(),tempC_r);
+         (*C_r) += tempC_r;
+      }
+      else
+      {
+         MultAtB(A.imag(), B.imag(),*C_r);
+      }
+   }
+
+   if (A.hasRealPart() && B.hasImagPart())
+   {
+      MultAtB(A.real(), B.imag(),*C_i);
+   }
+
+   if (A.hasImagPart() && B.hasRealPart())
+   {
+      if (A.hasRealPart() && B.hasImagPart())
+      {
+         DenseMatrix tempC_i(h,w);
+         MultAtB(A.imag(), B.real(),tempC_i);
+         (*C_i) -= tempC_i;
+      }
+      else
+      {
+         MultAtB(A.imag(), B.real(),*C_i);
+      }
+   }
+
+   return new ComplexDenseMatrix(C_r,C_i,true,true);
+}
+
+
+} // mfem namespace

linalg/complex_densemat.hpp

@@ -0,0 +1,65 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_COMPLEX_DENSEMAT
+#define MFEM_COMPLEX_DENSEMAT
+
+#include "complex_operator.hpp"
+
+namespace mfem
+{
+
+/** @brief Specialization of the ComplexOperator built from a pair of Dense
+    Matrices.
+
+    The purpose of this specialization is to support the inverse of a
+    ComplexDenseMatrix and various MatMat operations
+
+    See ComplexOperator documentation for more information.
+    Note: Only the Hermitian convention is supported
+ */
+class ComplexDenseMatrix : public ComplexOperator
+{
+
+public:
+   ComplexDenseMatrix(DenseMatrix * A_Real, DenseMatrix * A_Imag,
+                      bool ownReal, bool ownImag)
+      : ComplexOperator(A_Real, A_Imag, ownReal, ownImag)
+   { }
+
+   virtual DenseMatrix & real();
+   virtual DenseMatrix & imag();
+
+   virtual const DenseMatrix & real() const;
+   virtual const DenseMatrix & imag() const;
+
+   /** Combine the blocks making up this complex operator into a single
+       DenseMatrix. Note that this combined operator requires roughly
+       twice the memory of the block structured operator. */
+   DenseMatrix * GetSystemMatrix() const;
+
+   virtual Type GetType() const { return Complex_DenseMat; }
+
+   ComplexDenseMatrix * ComputeInverse();
+
+};
+
+/// Matrix matrix multiplication.  A = B * C.
+ComplexDenseMatrix * Mult(const ComplexDenseMatrix &B,
+                          const ComplexDenseMatrix &C);
+
+/// Multiply the Complex transpose of a matrix A with a matrix B:   Ah*B
+ComplexDenseMatrix * MultAtB(const ComplexDenseMatrix &A,
+                             const ComplexDenseMatrix &B);
+
+} // namespace mfem
+
+#endif // MFEM_COMPLEX_DENSEMAT

linalg/hypre.cpp

@@ -65,7 +65,7 @@ bool CanShallowCopy(const Memory<T> &src, MemoryClass mc)
 inline void HypreParVector::_SetDataAndSize_()
 {
    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
    SetDataAndSize(hypre_VectorData(x_loc),
                   internal::to_int(hypre_VectorSize(x_loc)));
 #else
@@ -112,7 +112,7 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
 #endif
    double tmp = 0.0;
    hypre_VectorData(x_loc) = &tmp;
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_VectorMemoryLocation(x_loc) =
       is_device_ptr ? HYPRE_MEMORY_DEVICE : HYPRE_MEMORY_HOST;
 #else
@@ -257,7 +257,7 @@ void HypreParVector::HypreRead() const
    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
    hypre_VectorData(x_loc) =
       const_cast<double*>(data.Read(GetHypreMemoryClass(), size));
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_VectorMemoryLocation(x_loc) = HYPRE_MEMORY_DEVICE;
 #endif
 }
@@ -266,7 +266,7 @@ void HypreParVector::HypreReadWrite()
 {
    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
    hypre_VectorData(x_loc) = data.ReadWrite(GetHypreMemoryClass(), size);
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_VectorMemoryLocation(x_loc) = HYPRE_MEMORY_DEVICE;
 #endif
 }
@@ -275,7 +275,7 @@ void HypreParVector::HypreWrite()
 {
    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
    hypre_VectorData(x_loc) = data.Write(GetHypreMemoryClass(), size);
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_VectorMemoryLocation(x_loc) = HYPRE_MEMORY_DEVICE;
 #endif
 }
@@ -289,7 +289,7 @@ void HypreParVector::WrapMemoryRead(const Memory<double> &mem)
    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
    hypre_VectorData(x_loc) =
       const_cast<double*>(mem.Read(GetHypreMemoryClass(), size));
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_VectorMemoryLocation(x_loc) = HYPRE_MEMORY_DEVICE;
 #endif
    data.MakeAlias(mem, 0, size);
@@ -303,7 +303,7 @@ void HypreParVector::WrapMemoryReadWrite(Memory<double> &mem)
    data.Delete();
    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
    hypre_VectorData(x_loc) = mem.ReadWrite(GetHypreMemoryClass(), size);
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_VectorMemoryLocation(x_loc) = HYPRE_MEMORY_DEVICE;
 #endif
    data.MakeAlias(mem, 0, size);
@@ -317,7 +317,7 @@ void HypreParVector::WrapMemoryWrite(Memory<double> &mem)
    data.Delete();
    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
    hypre_VectorData(x_loc) = mem.Write(GetHypreMemoryClass(), size);
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_VectorMemoryLocation(x_loc) = HYPRE_MEMORY_DEVICE;
 #endif
    data.MakeAlias(mem, 0, size);
@@ -885,7 +885,7 @@ HypreParMatrix::HypreParMatrix(
    hypre_CSRMatrixJ(A->diag) = diag_j;
    hypre_CSRMatrixData(A->diag) = diag_data;
    hypre_CSRMatrixNumNonzeros(A->diag) = diag_i[local_num_rows];
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_CSRMatrixMemoryLocation(A->diag) = HYPRE_MEMORY_HOST;
 #endif
    hypre_CSRMatrixSetRownnz(A->diag);
@@ -895,7 +895,7 @@ HypreParMatrix::HypreParMatrix(
    hypre_CSRMatrixJ(A->offd) = offd_j;
    hypre_CSRMatrixData(A->offd) = offd_data;
    hypre_CSRMatrixNumNonzeros(A->offd) = offd_i[local_num_rows];
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_CSRMatrixMemoryLocation(A->offd) = HYPRE_MEMORY_HOST;
 #endif
    hypre_CSRMatrixSetRownnz(A->offd);
@@ -1070,7 +1070,7 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
    hypre_CSRMatrixI(A->diag)    = i_diag;
    hypre_CSRMatrixJ(A->diag)    = j_diag;
    hypre_CSRMatrixData(A->diag) = mem_diag.data;
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_CSRMatrixMemoryLocation(A->diag) = HYPRE_MEMORY_HOST;
 #endif
    hypre_CSRMatrixSetRownnz(A->diag);
@@ -1079,7 +1079,7 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
    hypre_CSRMatrixI(A->offd)    = i_offd;
    hypre_CSRMatrixJ(A->offd)    = j_offd;
    hypre_CSRMatrixData(A->offd) = mem_offd.data;
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    hypre_CSRMatrixMemoryLocation(A->offd) = HYPRE_MEMORY_HOST;
 #endif
    hypre_CSRMatrixSetRownnz(A->offd);
@@ -1306,7 +1306,7 @@ hypre_ParCSRMatrix* HypreParMatrix::StealData()
    MFEM_ASSERT(diagOwner == offdOwner, "");
    MFEM_ASSERT(ParCSROwner, "");
    hypre_ParCSRMatrix *R = A;
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    if (diagOwner == -1) { HostReadWrite(); }
    else { HypreReadWrite(); }
 #endif
@@ -1418,8 +1418,8 @@ void HypreParMatrix::GetDiag(Vector &diag) const
 {
    const int size = Height();
    diag.SetSize(size);
-#ifdef HYPRE_USING_CUDA
-   if (Device::Allows(Backend::CUDA_MASK))
+#ifdef HYPRE_USING_GPU
+   if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
       MFEM_ASSERT(A->diag->memory_location == HYPRE_MEMORY_DEVICE, "");
       double *d_diag = diag.Write();
@@ -2447,7 +2447,7 @@ void HypreParMatrix::Destroy()
 
    if (A == NULL) { return; }
 
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    if (ParCSROwner && (diagOwner < 0 || offdOwner < 0))
    {
       // Put the "host" or "hypre" pointers in {i,j,data} of A->{diag,offd}, so
@@ -2590,7 +2590,7 @@ HypreParMatrix * ParMult(const HypreParMatrix *A, const HypreParMatrix *B,
                          bool own_matrix)
 {
    hypre_ParCSRMatrix * ab;
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    ab = hypre_ParCSRMatMat(*A, *B);
 #else
    ab = hypre_ParMatmul(*A,*B);
@@ -2611,7 +2611,7 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
 {
    hypre_ParCSRMatrix * rap;
 
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    // FIXME: this way of computing Pt A P can completely eliminate zero rows
    //        from the sparsity pattern of the product which prevents
    //        EliminateZeroRows() from working correctly. This issue is observed
@@ -2658,7 +2658,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
 {
    hypre_ParCSRMatrix * rap;
 
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    {
       hypre_ParCSRMatrix *Q = hypre_ParCSRMatMat(*A,*P);
       rap = hypre_ParCSRTMatMat(*Rt,Q);
@@ -3253,9 +3253,9 @@ void HypreSmoother::SetOperator(const Operator &op)
    }
    if (l1_norms && pos_l1_norms)
    {
-#ifdef HYPRE_USING_CUDA
+#if defined(HYPRE_USING_GPU)
       double *d_l1_norms = l1_norms;  // avoid *this capture
-      CuWrap1D(height, [=] MFEM_DEVICE (int i)
+      MFEM_GPU_FORALL(i, height,
       {
          d_l1_norms[i] = std::abs(d_l1_norms[i]);
       });
@@ -4458,7 +4458,7 @@ HypreBoomerAMG::HypreBoomerAMG(const HypreParMatrix &A) : HypreSolver(&A)
 
 void HypreBoomerAMG::SetDefaultOptions()
 {
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
    // AMG coarsening options:
    int coarsen_type = 10;   // 10 = HMIS, 8 = PMIS, 6 = Falgout, 0 = CLJP
    int agg_levels   = 1;    // number of aggressive coarsening levels
@@ -4727,8 +4727,8 @@ void HypreBoomerAMG::RecomputeRBMs()
 
 void HypreBoomerAMG::SetElasticityOptions(ParFiniteElementSpace *fespace)
 {
-#ifdef HYPRE_USING_CUDA
-   MFEM_ABORT("this method is not supported in hypre built with CUDA");
+#ifdef HYPRE_USING_GPU
+   MFEM_ABORT("this method is not supported in hypre built with GPU support");
 #endif
 
    // Save the finite element space to support multiple calls to SetOperator()
@@ -4908,7 +4908,7 @@ void HypreAMS::Init(ParFiniteElementSpace *edge_fespace)
    int rlx_sweeps       = 1;
    double rlx_weight    = 1.0;
    double rlx_omega     = 1.0;
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
    int amg_coarsen_type = 10;
    int amg_agg_levels   = 1;
    int amg_rlx_type     = 8;
@@ -5159,7 +5159,7 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace)
    int rlx_sweeps       = 1;
    double rlx_weight    = 1.0;
    double rlx_omega     = 1.0;
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
    int rlx_type         = 2;
    int amg_coarsen_type = 10;
    int amg_agg_levels   = 1;

linalg/hypre.hpp

@@ -32,9 +32,16 @@
 #error "MFEM does not work with HYPRE's complex numbers support"
 #endif
 
+#if defined(HYPRE_USING_GPU) && \
+    !(defined(HYPRE_USING_CUDA) || defined(HYPRE_USING_HIP))
+#error "Unsupported GPU build of HYPRE! Only CUDA and HIP builds are supported."
+#endif
 #if defined(HYPRE_USING_CUDA) && !defined(MFEM_USE_CUDA)
 #error "MFEM_USE_CUDA=YES is required when HYPRE is built with CUDA!"
 #endif
+#if defined(HYPRE_USING_HIP) && !defined(MFEM_USE_HIP)
+#error "MFEM_USE_HIP=YES is required when HYPRE is built with HIP!"
+#endif
 
 #include "sparsemat.hpp"
 #include "hypre_parcsr.hpp"
@@ -74,7 +81,7 @@ inline int to_int(HYPRE_Int i)
 /// The MemoryClass used by Hypre objects.
 inline constexpr MemoryClass GetHypreMemoryClass()
 {
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
    return MemoryClass::HOST;
 #elif defined(HYPRE_USING_UNIFIED_MEMORY)
    return MemoryClass::MANAGED;
@@ -86,7 +93,7 @@ inline constexpr MemoryClass GetHypreMemoryClass()
 /// The MemoryType used by MFEM when allocating arrays for Hypre objects.
 inline MemoryType GetHypreMemoryType()
 {
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
    return Device::GetHostMemoryType();
 #elif defined(HYPRE_USING_UNIFIED_MEMORY)
    return MemoryType::MANAGED;
@@ -919,7 +926,7 @@ public:
    enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, l1GStr = 4, lumpedJacobi = 5,
                GS = 6, OPFS = 10, Chebyshev = 16, Taubin = 1001, FIR = 1002
              };
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
    static constexpr Type default_type = l1GS;
 #else
    static constexpr Type default_type = l1Jacobi;

linalg/linalg.hpp

@@ -19,6 +19,7 @@
 #include "matrix.hpp"
 #include "sparsemat.hpp"
 #include "complex_operator.hpp"
+#include "complex_densemat.hpp"
 #include "blockvector.hpp"
 #include "blockmatrix.hpp"
 #include "blockoperator.hpp"

linalg/operator.hpp

@@ -265,7 +265,8 @@ public:
       PETSC_MATGENERIC, ///< ID for class PetscParMatrix, unspecified format.
       Complex_Operator, ///< ID for class ComplexOperator.
       MFEM_ComplexSparseMat, ///< ID for class ComplexSparseMatrix.
-      Complex_Hypre_ParCSR   ///< ID for class ComplexHypreParMatrix.
+      Complex_Hypre_ParCSR,   ///< ID for class ComplexHypreParMatrix.
+      Complex_DenseMat  ///< ID for class ComplexDenseMatrix
    };
 
    /// Return the type ID of the Operator class.

linalg/petsc.cpp

@@ -3399,6 +3399,7 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts)
                                                              hvec_coords->Size(),false);
 
          // likely elasticity -> we attach rigid-body modes as near-null space information to the local matrices
+         // and to the global matrix
          if (vdim == sdim)
          {
             MatNullSpace nnsp;
@@ -3413,7 +3414,15 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts)
             ierr = VecCreateMPIWithArray(comm,sdim,hvec_coords->Size(),
                                          hvec_coords->GlobalSize(),data_coords,&pvec_coords);
             CCHKERRQ(comm,ierr);
-            ierr = MatISGetLocalMat(pA,&lA); CCHKERRQ(PETSC_COMM_SELF,ierr);
+            ierr = MatGetNearNullSpace(pA,&nnsp); CCHKERRQ(comm,ierr);
+            if (!nnsp)
+            {
+               ierr = MatNullSpaceCreateRigidBody(pvec_coords,&nnsp);
+               CCHKERRQ(comm,ierr);
+               ierr = MatSetNearNullSpace(pA,nnsp); CCHKERRQ(comm,ierr);
+               ierr = MatNullSpaceDestroy(&nnsp); CCHKERRQ(comm,ierr);
+            }
+            ierr = MatISGetLocalMat(pA,&lA); CCHKERRQ(comm,ierr);
             ierr = MatCreateVecs(lA,&lvec_coords,NULL); CCHKERRQ(PETSC_COMM_SELF,ierr);
             ierr = VecSetBlockSize(lvec_coords,sdim); CCHKERRQ(PETSC_COMM_SELF,ierr);
             ierr = MatGetLocalToGlobalMapping(pA,&l2g,NULL); CCHKERRQ(comm,ierr);

linalg/sparsemat.cpp

@@ -23,55 +23,75 @@
 #include <limits>
 #include <cstring>
 
+#if defined(MFEM_USE_CUDA)
+#define MFEM_cu_or_hip(stub) cu##stub
+#define MFEM_Cu_or_Hip(stub) Cu##stub
+#define MFEM_CU_or_HIP(stub) CU##stub
+#define MFEM_CUDA_or_HIP(stub) CUDA##stub
+
+#if CUSPARSE_VERSION >=  11400
+#define MFEM_GPUSPARSE_ALG CUSPARSE_SPMV_CSR_ALG1
+#else // CUSPARSE_VERSION >= 11400
+#define MFEM_GPUSPARSE_ALG CUSPARSE_CSRMV_ALG1
+#endif // CUSPARSE_VERSION >= 11400
+
+#elif defined(MFEM_USE_HIP)
+#define MFEM_cu_or_hip(stub) hip##stub
+#define MFEM_Cu_or_Hip(stub) Hip##stub
+#define MFEM_CU_or_HIP(stub) HIP##stub
+#define MFEM_CUDA_or_HIP(stub) HIP##stub
+
+// https://hipsparse.readthedocs.io/en/latest/usermanual.html#hipsparsespmvalg-t
+#define MFEM_GPUSPARSE_ALG HIPSPARSE_CSRMV_ALG1
+#endif // defined(MFEM_USE_CUDA)
+
 namespace mfem
 {
 
 using namespace std;
 
-#ifdef MFEM_USE_CUDA
+#ifdef MFEM_USE_CUDA_OR_HIP
 int SparseMatrix::SparseMatrixCount = 0;
-cusparseHandle_t SparseMatrix::handle = nullptr;
+// doxygen doesn't like the macro-assisted typename so let's skip parsing it:
+// \cond false
+MFEM_cu_or_hip(sparseHandle_t) SparseMatrix::handle = nullptr;
+// \endcond
 size_t SparseMatrix::bufferSize = 0;
 void * SparseMatrix::dBuffer = nullptr;
-#  if CUSPARSE_VERSION >=  11400
-#    define MFEM_CUSPARSE_ALG CUSPARSE_SPMV_CSR_ALG1
-#  else
-#    define MFEM_CUSPARSE_ALG CUSPARSE_CSRMV_ALG1
-#  endif // CUSPARSE_VERSION >= 11400
-#endif // MFEM_USE_CUDA
+#endif // MFEM_USE_CUDA_OR_HIP
 
-void SparseMatrix::InitCuSparse()
+void SparseMatrix::InitGPUSparse()
 {
-   // Initialize cuSPARSE library
-#ifdef MFEM_USE_CUDA
-   if (Device::Allows(Backend::CUDA_MASK))
+   // Initialize cuSPARSE/hipSPARSE library
+#ifdef MFEM_USE_CUDA_OR_HIP
+   if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
-      if (!handle) { cusparseCreate(&handle); }
-      useCuSparse=true;
+      if (!handle) { MFEM_cu_or_hip(sparseCreate)(&handle); }
+      useGPUSparse=true;
       SparseMatrixCount++;
    }
    else
    {
-      useCuSparse=false;
+      useGPUSparse=false;
    }
-#endif
+#endif // MFEM_USE_CUDA_OR_HIP
 }
 
-void SparseMatrix::ClearCuSparse()
+void SparseMatrix::ClearGPUSparse()
 {
-#ifdef MFEM_USE_CUDA
+#ifdef MFEM_USE_CUDA_OR_HIP
    if (initBuffers)
    {
-#if CUDA_VERSION >= 10010
-      cusparseDestroySpMat(matA_descr);
-      cusparseDestroyDnVec(vecX_descr);
-      cusparseDestroyDnVec(vecY_descr);
+#if CUDA_VERSION >= 10010 || defined(MFEM_USE_HIP)
+      MFEM_cu_or_hip(sparseDestroySpMat)(matA_descr);
+      MFEM_cu_or_hip(sparseDestroyDnVec)(vecX_descr);
+      MFEM_cu_or_hip(sparseDestroyDnVec)(vecY_descr);
 #else
       cusparseDestroyMatDescr(matA_descr);
-#endif
+#endif // CUDA_VERSION >= 10010 || defined(MFEM_USE_HIP)
       initBuffers = false;
    }
-#endif
+#endif // MFEM_USE_CUDA_OR_HIP
 }
 
 SparseMatrix::SparseMatrix(int nrows, int ncols)
@@ -97,7 +117,7 @@ SparseMatrix::SparseMatrix(int nrows, int ncols)
    NodesMem = new RowNodeAlloc;
 #endif
 
-   InitCuSparse();
+   InitGPUSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
@@ -116,7 +136,7 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
    NodesMem = NULL;
 #endif
 
-   InitCuSparse();
+   InitGPUSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
@@ -149,7 +169,7 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
       }
    }
 
-   InitCuSparse();
+   InitGPUSparse();
 }
 
 SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
@@ -172,7 +192,7 @@ SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
       I[i] = i * rowsize;
    }
 
-   InitCuSparse();
+   InitGPUSparse();
 }
 
 SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph,
@@ -240,7 +260,7 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph,
    At = NULL;
    isSorted = mat.isSorted;
 
-   InitCuSparse();
+   InitGPUSparse();
 }
 
 SparseMatrix::SparseMatrix(const Vector &v)
@@ -269,7 +289,7 @@ SparseMatrix::SparseMatrix(const Vector &v)
       A[r] = v[r];
    }
 
-   InitCuSparse();
+   InitGPUSparse();
 }
 
 SparseMatrix& SparseMatrix::operator=(const SparseMatrix &rhs)
@@ -310,7 +330,7 @@ void SparseMatrix::SetEmpty()
 #endif
    isSorted = false;
 
-   ClearCuSparse();
+   ClearGPUSparse();
 }
 
 int SparseMatrix::RowSize(const int i) const
@@ -660,65 +680,100 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
 
    // Skip if matrix has no non-zeros
    if (nnz == 0) {return;}
-   if (Device::Allows(Backend::CUDA_MASK) && useCuSparse)
+   if ((Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) && useGPUSparse)
    {
-#ifdef MFEM_USE_CUDA
+#ifdef MFEM_USE_CUDA_OR_HIP
       const double alpha = a;
       const double beta  = 1.0;
 
       // Setup descriptors
       if (!initBuffers)
       {
-#if CUDA_VERSION >= 10010
+#if CUDA_VERSION >= 10010 || defined(MFEM_USE_HIP)
          // Setup matrix descriptor
-         cusparseCreateCsr(&matA_descr,Height(), Width(), J.Capacity(),
-                           const_cast<int *>(d_I),
-                           const_cast<int *>(d_J), const_cast<double *>(d_A), CUSPARSE_INDEX_32I,
-                           CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F);
+         MFEM_cu_or_hip(sparseCreateCsr)(
+            &matA_descr,Height(),
+            Width(),
+            J.Capacity(),
+            const_cast<int *>(d_I),
+            const_cast<int *>(d_J),
+            const_cast<double *>(d_A),
+            MFEM_CU_or_HIP(SPARSE_INDEX_32I),
+            MFEM_CU_or_HIP(SPARSE_INDEX_32I),
+            MFEM_CU_or_HIP(SPARSE_INDEX_BASE_ZERO),
+            MFEM_CUDA_or_HIP(_R_64F));
 
          // Create handles for input/output vectors
-         cusparseCreateDnVec(&vecX_descr, x.Size(), const_cast<double *>(d_x),
-                             CUDA_R_64F);
-         cusparseCreateDnVec(&vecY_descr, y.Size(), d_y, CUDA_R_64F);
+         MFEM_cu_or_hip(sparseCreateDnVec)(&vecX_descr,
+                                           x.Size(),
+                                           const_cast<double *>(d_x),
+                                           MFEM_CUDA_or_HIP(_R_64F));
+         MFEM_cu_or_hip(sparseCreateDnVec)(&vecY_descr, y.Size(), d_y,
+                                           MFEM_CUDA_or_HIP(_R_64F));
 #else
          cusparseCreateMatDescr(&matA_descr);
          cusparseSetMatIndexBase(matA_descr, CUSPARSE_INDEX_BASE_ZERO);
          cusparseSetMatType(matA_descr, CUSPARSE_MATRIX_TYPE_GENERAL);
-#endif
+#endif // CUDA_VERSION >= 10010 || defined(MFEM_USE_HIP)
          initBuffers = true;
       }
       // Allocate kernel space. Buffer is shared between different sparsemats
       size_t newBufferSize = 0;
 
-      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
-                              matA_descr,
-                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
-                              MFEM_CUSPARSE_ALG, &newBufferSize);
+      MFEM_cu_or_hip(sparseSpMV_bufferSize)(
+         handle,
+         MFEM_CU_or_HIP(SPARSE_OPERATION_NON_TRANSPOSE),
+         &alpha,
+         matA_descr,
+         vecX_descr,
+         &beta,
+         vecY_descr,
+         MFEM_CUDA_or_HIP(_R_64F),
+         MFEM_GPUSPARSE_ALG,
+         &newBufferSize);
 
       // Check if we need to resize
       if (newBufferSize > bufferSize)
       {
          bufferSize = newBufferSize;
-         if (dBuffer != nullptr) { CuMemFree(dBuffer); }
-         CuMemAlloc(&dBuffer, bufferSize);
+         if (dBuffer != nullptr) { MFEM_Cu_or_Hip(MemFree)(dBuffer); }
+         MFEM_Cu_or_Hip(MemAlloc)(&dBuffer, bufferSize);
       }
 
-#if CUDA_VERSION >= 10010
+#if CUDA_VERSION >= 10010 || defined(MFEM_USE_HIP)
       // Update input/output vectors
-      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
-      cusparseDnVecSetValues(vecY_descr, d_y);
+      MFEM_cu_or_hip(sparseDnVecSetValues)(vecX_descr,
+                                           const_cast<double *>(d_x));
+      MFEM_cu_or_hip(sparseDnVecSetValues)(vecY_descr, d_y);
 
       // Y = alpha A * X + beta * Y
-      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
-                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, MFEM_CUSPARSE_ALG, dBuffer);
+      MFEM_cu_or_hip(sparseSpMV)(
+         handle,
+         MFEM_CU_or_HIP(SPARSE_OPERATION_NON_TRANSPOSE),
+         &alpha,
+         matA_descr,
+         vecX_descr,
+         &beta,
+         vecY_descr,
+         MFEM_CUDA_or_HIP(_R_64F),
+         MFEM_GPUSPARSE_ALG,
+         dBuffer);
 #else
-      cusparseDcsrmv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-                     Height(), Width(), J.Capacity(),
-                     &alpha, matA_descr,
-                     const_cast<double *>(d_A), const_cast<int *>(d_I), const_cast<int *>(d_J),
-                     const_cast<double *>(d_x), &beta, d_y);
-#endif // CUDA_VERSION >= 10010
-#endif // MFEM_USE_CUDA
+      cusparseDcsrmv(handle,
+                     CUSPARSE_OPERATION_NON_TRANSPOSE,
+                     Height(),
+                     Width(),
+                     J.Capacity(),
+                     &alpha,
+                     matA_descr,
+                     const_cast<double *>(d_A),
+                     const_cast<int *>(d_I),
+                     const_cast<int *>(d_J),
+                     const_cast<double *>(d_x),
+                     &beta,
+                     d_y);
+#endif // CUDA_VERSION >= 10010 || defined(MFEM_USE_HIP)
+#endif // MFEM_USE_CUDA_OR_HIP
    }
    else
    {
@@ -736,7 +791,7 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
 
    }
 
-#else
+#else // MFEM_USE_LEGACY_OPENMP
    const double *Ap = A, *xp = x.GetData();
    double *yp = y.GetData();
    const int *Jp = J, *Ip = I;
@@ -752,7 +807,7 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
       }
       yp[i] += a * d;
    }
-#endif
+#endif // MFEM_USE_LEGACY_OPENMP
 }
 
 void SparseMatrix::MultTranspose(const Vector &x, Vector &y) const
@@ -772,12 +827,13 @@ void SparseMatrix::AddMultTranspose(const Vector &x, Vector &y,
 
    if (!Finalized())
    {
-      double *yp = y.GetData();
+      double *yp = y.HostReadWrite();
+      const double *xp = x.HostRead();
       // The matrix is not finalized, but multiplication is still possible
       for (int i = 0; i < height; i++)
       {
          RowNode *row = Rows[i];
-         double b = a * x(i);
+         double b = a * xp[i];
          for ( ; row != NULL; row = row->Prev)
          {
             yp[row->Column] += row->Value * b;
@@ -792,8 +848,9 @@ void SparseMatrix::AddMultTranspose(const Vector &x, Vector &y,
    }
    else
    {
-      MFEM_VERIFY(Device::IsDisabled(), "transpose action on device is not "
-                  "enabled; see BuildTranspose() for details.");
+      MFEM_VERIFY(!Device::Allows(~Backend::CPU_MASK), "transpose action with "
+                  "this backend is not enabled; see EnsureMultTranspose() for "
+                  "details.");
       for (int i = 0; i < height; i++)
       {
          const double xi = a * x[i];
@@ -821,6 +878,14 @@ void SparseMatrix::ResetTranspose() const
    At = NULL;
 }
 
+void SparseMatrix::EnsureMultTranspose() const
+{
+   if (Device::Allows(~Backend::CPU_MASK))
+   {
+      BuildTranspose();
+   }
+}
+
 void SparseMatrix::PartMult(
    const Array<int> &rows, const Vector &x, Vector &y) const
 {
@@ -998,8 +1063,9 @@ void SparseMatrix::AbsMultTranspose(const Vector &x, Vector &y) const
    }
    else
    {
-      MFEM_VERIFY(Device::IsDisabled(), "transpose action on device is not "
-                  "enabled; see BuildTranspose() for details.");
+      MFEM_VERIFY(!Device::Allows(~Backend::CPU_MASK), "transpose action with "
+                  "this backend is not enabled; see EnsureMultTranspose() for "
+                  "details.");
       for (int i = 0; i < height; i++)
       {
          const double xi = x[i];
@@ -3259,7 +3325,7 @@ void SparseMatrix::Destroy()
 #endif
    delete At;
 
-   ClearCuSparse();
+   ClearGPUSparse();
 }
 
 int SparseMatrix::ActualWidth() const
@@ -3995,4 +4061,29 @@ void SparseMatrix::Swap(SparseMatrix &other)
    mfem::Swap(isSorted, other.isSorted);
 }
 
+SparseMatrix::~SparseMatrix()
+{
+   Destroy();
+#ifdef MFEM_USE_CUDA_OR_HIP
+   if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
+   {
+      if (SparseMatrixCount==1)
+      {
+         if (handle)
+         {
+            MFEM_cu_or_hip(sparseDestroy)(handle);
+            handle = nullptr;
+         }
+         if (dBuffer)
+         {
+            MFEM_Cu_or_Hip(MemFree)(dBuffer);
+            dBuffer = nullptr;
+            bufferSize = 0;
+         }
+      }
+      SparseMatrixCount--;
+   }
+#endif // MFEM_USE_CUDA_OR_HIP
+}
+
 }

linalg/sparsemat.hpp

@@ -22,6 +22,11 @@
 #include "../general/globals.hpp"
 #include "densemat.hpp"
 
+#if defined(MFEM_USE_HIP)
+#include <hipsparse.h>
+#endif
+
+
 namespace mfem
 {
 
@@ -81,27 +86,41 @@ protected:
    void Destroy();   // Delete all owned data
    void SetEmpty();  // Init all entries with empty values
 
-   bool useCuSparse{true}; // Use cuSPARSE if available
+   bool useGPUSparse = true; // Use cuSPARSE or hipSPARSE if available
 
-   // Initialize cuSPARSE
-   void InitCuSparse();
+   // Initialize cuSPARSE/hipSPARSE
+   void InitGPUSparse();
 
-#ifdef MFEM_USE_CUDA
-   cusparseStatus_t status;
-   static cusparseHandle_t handle;
-   cusparseMatDescr_t descr=0;
+#ifdef MFEM_USE_CUDA_OR_HIP
+   // common for hipSPARSE and cuSPARSE
    static int SparseMatrixCount;
    static size_t bufferSize;
    static void *dBuffer;
-   mutable bool initBuffers{false};
+   mutable bool initBuffers = false;
+
+#if defined(MFEM_USE_CUDA)
+   cusparseStatus_t status;
+   static cusparseHandle_t handle;
+   cusparseMatDescr_t descr = 0;
+
 #if CUDA_VERSION >= 10010
    mutable cusparseSpMatDescr_t matA_descr;
    mutable cusparseDnVecDescr_t vecX_descr;
    mutable cusparseDnVecDescr_t vecY_descr;
-#else
+#else // CUDA_VERSION >= 10010
    mutable cusparseMatDescr_t matA_descr;
-#endif
-#endif
+#endif // CUDA_VERSION >= 10010
+
+#else // defined(MFEM_USE_CUDA)
+   hipsparseStatus_t status;
+   static hipsparseHandle_t handle;
+   hipsparseMatDescr_t descr = 0;
+
+   mutable hipsparseSpMatDescr_t matA_descr;
+   mutable hipsparseDnVecDescr_t vecX_descr;
+   mutable hipsparseDnVecDescr_t vecY_descr;
+#endif // defined(MFEM_USE_CUDA)
+#endif // MFEM_USE_CUDA_OR_HIP
 
 public:
    /// Create an empty SparseMatrix.
@@ -109,7 +128,7 @@ public:
    {
       SetEmpty();
 
-      InitCuSparse();
+      InitGPUSparse();
    }
 
    /** @brief Create a sparse matrix with flexible sparsity structure using a
@@ -151,8 +170,15 @@ public:
    /// Create a SparseMatrix with diagonal @a v, i.e. A = Diag(v)
    SparseMatrix(const Vector & v);
 
-   // Runtime option to use cuSPARSE. Only valid when using a CUDA backend.
-   void UseCuSparse(bool useCuSparse_ = true) { useCuSparse = useCuSparse_;}
+   /** @brief Runtime option to use cuSPARSE or hipSPARSE. Only valid when using
+       a CUDA or HIP backend.
+
+       @note This option is enabled by default, so typically one would use this
+       method to disable the use of cuSPARSE/hipSPARSE. */
+   void UseGPUSparse(bool useGPUSparse_ = true) { useGPUSparse = useGPUSparse_;}
+   /// Deprecated equivalent of UseGPUSparse().
+   MFEM_DEPRECATED
+   void UseCuSparse(bool useCuSparse_ = true) { UseGPUSparse(useCuSparse_); }
 
    /// Assignment operator: deep copy
    SparseMatrix& operator=(const SparseMatrix &rhs);
@@ -169,9 +195,12 @@ public:
    /// Clear the contents of the SparseMatrix.
    void Clear() { Destroy(); SetEmpty(); }
 
-   /** @brief Clear the CuSparse descriptors.
+   /** @brief Clear the cuSPARSE/hipSPARSE descriptors.
        This must be called after releasing the device memory of A. */
-   void ClearCuSparse();
+   void ClearGPUSparse();
+   /// Deprecated equivalent of ClearGPUSparse().
+   MFEM_DEPRECATED
+   void ClearCuSparse() { ClearGPUSparse(); }
 
    /// Check if the SparseMatrix is empty.
    bool Empty() const { return (A == NULL) && (Rows == NULL); }
@@ -317,30 +346,45 @@ public:
                          const double a = 1.0) const;
 
    /** @brief Build and store internally the transpose of this matrix which will
-       be used in the methods AddMultTranspose() and MultTranspose(). */
+       be used in the methods AddMultTranspose(), MultTranspose(), and
+       AbsMultTranspose(). */
    /** If this method has been called, the internal transpose matrix will be
        used to perform the action of the transpose matrix in AddMultTranspose(),
-       and MultTranspose().
+       MultTranspose(), and AbsMultTranspose().
 
        Warning: any changes in this matrix will invalidate the internal
        transpose. To rebuild the transpose, call ResetTranspose() followed by a
        call to this method. If the internal transpose is already built, this
        method has no effect.
 
-       When any non-default backend is enabled, i.e. Device::IsEnabled() is
-       true, the methods AddMultTranspose(), and MultTranspose(), require the
-       internal transpose to be built. If that is not the case (i.e. the
-       internal transpose is not built), these methods will raise an error with
-       an appropriate message pointing to this method. When using the default
-       backend, calling this method is optional.
+       When any non-serial-CPU backend is enabled, i.e. the call
+       Device::Allows(~ Backend::CPU_MASK) returns true, the above methods
+       require the internal transpose to be built. If that is not the case (i.e.
+       the internal transpose is not built), these methods will raise an error
+       with an appropriate message pointing to EnsureMultTranspose(). When using
+       any backend from Backend::CPU_MASK, calling this method is optional.
 
-       This method can only be used when the sparse matrix is finalized. */
+       This method can only be used when the sparse matrix is finalized.
+
+       @sa EnsureMultTranspose(), ResetTranspose(). */
    void BuildTranspose() const;
 
    /** Reset (destroy) the internal transpose matrix. See BuildTranspose() for
        more details. */
    void ResetTranspose() const;
 
+   /** @brief Ensures that the matrix is capable of performing MultTranspose(),
+       AddMultTranspose(), and AbsMultTranspose(). */
+   /** For non-serial-CPU backends (e.g. GPU, OpenMP), multiplying by the
+       transpose requires that the internal transpose matrix be already built.
+       When such a backend is enabled, this function will build the internal
+       transpose matrix, see BuildTranspose().
+
+       For the serial CPU backends, the internal transpose is not required, and
+       this function is a no-op. This allows for significant memory savings
+       when the internal transpose matrix is not required. */
+   void EnsureMultTranspose() const;
+
    void PartMult(const Array<int> &rows, const Vector &x, Vector &y) const;
    void PartAddMult(const Array<int> &rows, const Vector &x, Vector &y,
                     const double a=1.0) const;
@@ -628,30 +672,7 @@ public:
    void Swap(SparseMatrix &other);
 
    /// Destroys sparse matrix.
-   virtual ~SparseMatrix()
-   {
-      Destroy();
-#ifdef MFEM_USE_CUDA
-      if (useCuSparse)
-      {
-         if (SparseMatrixCount==1)
-         {
-            if (handle)
-            {
-               cusparseDestroy(handle);
-               handle = nullptr;
-            }
-            if (dBuffer)
-            {
-               CuMemFree(dBuffer);
-               dBuffer = nullptr;
-               bufferSize = 0;
-            }
-         }
-         SparseMatrixCount--;
-      }
-#endif
-   }
+   virtual ~SparseMatrix();
 
    Type GetType() const { return MFEM_SPARSEMAT; }
 };

mesh/mesh.cpp

@@ -1096,7 +1096,7 @@ FaceElementTransformations *Mesh::GetBdrFaceTransformations(int BdrElemNo)
    FaceElementTransformations *tr;
    int fn = GetBdrFace(BdrElemNo);
 
-   // Check if the face is interior, shared, or non-conforming.
+   // Check if the face is interior, shared, or nonconforming.
    if (FaceIsTrueInterior(fn) || faces_info[fn].NCFace >= 0)
    {
       return NULL;
@@ -1127,6 +1127,269 @@ int Mesh::GetBdrFace(int BdrElemNo) const
    return fn;
 }
 
+Mesh::FaceInformation Mesh::GetFaceInformation(int f) const
+{
+   FaceInformation face;
+   int e1, e2;
+   int inf1, inf2;
+   int ncface;
+   GetFaceElements(f, &e1, &e2);
+   GetFaceInfos(f, &inf1, &inf2, &ncface);
+   face.element[0].index = e1;
+   face.element[0].location = ElementLocation::Local;
+   face.element[0].orientation = inf1%64;
+   face.element[0].local_face_id = inf1/64;
+   face.element[1].local_face_id = inf2/64;
+   face.ncface = ncface;
+   face.point_matrix = nullptr;
+   // The following figures out face.location, face.conformity,
+   // face.element[1].index, and face.element[1].orientation.
+   if (f < GetNumFaces()) // Non-ghost face
+   {
+      if (e2>=0)
+      {
+         if (ncface==-1)
+         {
+            face.tag = FaceInfoTag::LocalConforming;
+            face.topology = FaceTopology::Conforming;
+            face.element[1].location = ElementLocation::Local;
+            face.element[0].conformity = ElementConformity::Coincident;
+            face.element[1].conformity = ElementConformity::Coincident;
+            face.element[1].index = e2;
+            face.element[1].orientation = inf2%64;
+         }
+         else // ncface >= 0
+         {
+            face.tag = FaceInfoTag::LocalSlaveNonconforming;
+            face.topology = FaceTopology::Nonconforming;
+            face.element[1].location = ElementLocation::Local;
+            face.element[0].conformity = ElementConformity::Coincident;
+            face.element[1].conformity = ElementConformity::Superset;
+            face.element[1].index = e2;
+            MFEM_ASSERT(inf2%64==0, "unexpected slave face orientation.");
+            face.element[1].orientation = inf2%64;
+            face.point_matrix = nc_faces_info[ncface].PointMatrix;
+         }
+      }
+      else // e2<0
+      {
+         if (ncface==-1)
+         {
+            if (inf2<0)
+            {
+               face.tag = FaceInfoTag::Boundary;
+               face.topology = FaceTopology::Boundary;
+               face.element[1].location = ElementLocation::NA;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::NA;
+               face.element[1].index = -1;
+               face.element[1].orientation = -1;
+            }
+            else // inf2 >= 0
+            {
+               face.tag = FaceInfoTag::SharedConforming;
+               face.topology = FaceTopology::Conforming;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::Coincident;
+               face.element[1].location = ElementLocation::FaceNbr;
+               face.element[1].index = -1 - e2;
+               face.element[1].orientation = inf2%64;
+            }
+         }
+         else // ncface >= 0
+         {
+            if (inf2 < 0)
+            {
+               face.tag = FaceInfoTag::MasterNonconforming;
+               face.topology = FaceTopology::Nonconforming;
+               face.element[1].location = ElementLocation::NA;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::Subset;
+               face.element[1].index = -1;
+               face.element[1].orientation = -1;
+            }
+            else
+            {
+               face.tag = FaceInfoTag::SharedSlaveNonconforming;
+               face.topology = FaceTopology::Nonconforming;
+               face.element[1].location = ElementLocation::FaceNbr;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::Superset;
+               face.element[1].index = -1 - e2;
+               face.element[1].orientation = inf2%64;
+            }
+            face.point_matrix = nc_faces_info[ncface].PointMatrix;
+         }
+      }
+   }
+   else // Ghost face
+   {
+      if (e1==-1)
+      {
+         face.tag = FaceInfoTag::GhostMaster;
+         face.topology = FaceTopology::NA;
+         face.element[1].location = ElementLocation::NA;
+         face.element[0].conformity = ElementConformity::NA;
+         face.element[1].conformity = ElementConformity::NA;
+         face.element[1].index = -1;
+         face.element[1].orientation = -1;
+      }
+      else
+      {
+         face.tag = FaceInfoTag::GhostSlave;
+         face.topology = FaceTopology::Nonconforming;
+         face.element[1].location = ElementLocation::FaceNbr;
+         face.element[0].conformity = ElementConformity::Superset;
+         face.element[1].conformity = ElementConformity::Coincident;
+         face.element[1].index = -1 - e2;
+         face.element[1].orientation = inf2%64;
+         face.point_matrix = nc_faces_info[ncface].PointMatrix;
+      }
+   }
+   return face;
+}
+
+Mesh::FaceInformation::operator Mesh::FaceInfo() const
+{
+   FaceInfo res {-1, -1, -1, -1, -1};
+   switch (tag)
+   {
+      case FaceInfoTag::LocalConforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         res.NCFace = ncface;
+         break;
+      case FaceInfoTag::LocalSlaveNonconforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         res.NCFace = ncface;
+         break;
+      case FaceInfoTag::Boundary:
+         res.Elem1No = element[0].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         break;
+      case FaceInfoTag::SharedConforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = -1 - element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         break;
+      case FaceInfoTag::MasterNonconforming:
+         res.Elem1No = element[0].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         break;
+      case FaceInfoTag::SharedSlaveNonconforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = -1 - element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         break;
+      case FaceInfoTag::GhostMaster:
+         break;
+      case FaceInfoTag::GhostSlave:
+         res.Elem1No = element[0].index;
+         res.Elem2No = -1 - element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         break;
+   }
+   return res;
+}
+
+std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
+{
+   os << "face topology=";
+   switch (info.topology)
+   {
+      case Mesh::FaceTopology::Boundary:
+         os << "Boundary";
+         break;
+      case Mesh::FaceTopology::Conforming:
+         os << "Conforming";
+         break;
+      case Mesh::FaceTopology::Nonconforming:
+         os << "Non-conforming";
+         break;
+      case Mesh::FaceTopology::NA:
+         os << "NA";
+         break;
+   }
+   os << "element[0].location=";
+   switch (info.element[0].location)
+   {
+      case Mesh::ElementLocation::Local:
+         os << "Local";
+         break;
+      case Mesh::ElementLocation::FaceNbr:
+         os << "FaceNbr";
+         break;
+      case Mesh::ElementLocation::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[1].location=";
+   switch (info.element[1].location)
+   {
+      case Mesh::ElementLocation::Local:
+         os << "Local";
+         break;
+      case Mesh::ElementLocation::FaceNbr:
+         os << "FaceNbr";
+         break;
+      case Mesh::ElementLocation::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[0].conformity=";
+   switch (info.element[0].conformity)
+   {
+      case Mesh::ElementConformity::Coincident:
+         os << "Coincident";
+         break;
+      case Mesh::ElementConformity::Superset:
+         os << "Superset";
+         break;
+      case Mesh::ElementConformity::Subset:
+         os << "Subset";
+         break;
+      case Mesh::ElementConformity::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[1].conformity=";
+   switch (info.element[1].conformity)
+   {
+      case Mesh::ElementConformity::Coincident:
+         os << "Coincident";
+         break;
+      case Mesh::ElementConformity::Superset:
+         os << "Superset";
+         break;
+      case Mesh::ElementConformity::Subset:
+         os << "Subset";
+         break;
+      case Mesh::ElementConformity::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[0].index=" << info.element[0].index << std::endl
+      << "element[1].index=" << info.element[1].index << std::endl
+      << "element[0].local_face_id=" << info.element[0].local_face_id << std::endl
+      << "element[1].local_face_id=" << info.element[1].local_face_id << std::endl
+      << "element[0].orientation=" << info.element[0].orientation << std::endl
+      << "element[1].orientation=" << info.element[1].orientation << std::endl
+      << "ncface=" << info.ncface << std::endl;
+   return os;
+}
+
 void Mesh::GetFaceElements(int Face, int *Elem1, int *Elem2) const
 {
    *Elem1 = faces_info[Face].Elem1No;
@@ -5090,26 +5353,32 @@ int Mesh::GetNumFaces() const
    return 0;
 }
 
-static int CountFacesByType(const Mesh &mesh, const FaceType type)
+int Mesh::GetNumFacesWithGhost() const
 {
-   int e1, e2;
-   int inf1, inf2;
-   int nf = 0;
-   for (int f = 0; f < mesh.GetNumFaces(); ++f)
-   {
-      mesh.GetFaceElements(f, &e1, &e2);
-      mesh.GetFaceInfos(f, &inf1, &inf2);
-      if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-          (type==FaceType::Boundary && e2<0 && inf2<0) ) { nf++; }
-   }
-   return nf;
+   return faces_info.Size();
 }
 
 int Mesh::GetNFbyType(FaceType type) const
 {
    const bool isInt = type==FaceType::Interior;
    int &nf = isInt ? nbInteriorFaces : nbBoundaryFaces;
-   if (nf<0) { nf = CountFacesByType(*this, type); }
+   if (nf<0)
+   {
+      nf = 0;
+      for (int f = 0; f < GetNumFacesWithGhost(); ++f)
+      {
+         FaceInformation face = GetFaceInformation(f);
+         if ( face.IsOfFaceType(type) )
+         {
+            if (face.IsNonconformingCoarse())
+            {
+               // We don't count nonconforming coarse faces.
+               continue;
+            }
+            nf++;
+         }
+      }
+   }
    return nf;
 }
 
@@ -10152,61 +10421,6 @@ void Mesh::PrintBdrVTU(std::string fname,
    PrintVTU(fname, format, high_order_output, compression_level, true);
 }
 
-template <typename T>
-void WriteBinaryOrASCII(std::ostream &out, std::vector<char> &buf, const T &val,
-                        const char *suffix, VTKFormat format)
-{
-   if (format == VTKFormat::ASCII) { out << val << suffix; }
-   else { bin_io::AppendBytes(buf, val); }
-}
-
-// Ensure ASCII output of uint8_t to stream is integer rather than character
-template <>
-void WriteBinaryOrASCII<uint8_t>(std::ostream &out, std::vector<char> &buf,
-                                 const uint8_t &val, const char *suffix,
-                                 VTKFormat format)
-{
-   if (format == VTKFormat::ASCII) { out << static_cast<int>(val) << suffix; }
-   else { bin_io::AppendBytes(buf, val); }
-}
-
-template <>
-void WriteBinaryOrASCII<double>(std::ostream &out, std::vector<char> &buf,
-                                const double &val, const char *suffix,
-                                VTKFormat format)
-{
-   if (format == VTKFormat::BINARY32)
-   {
-      bin_io::AppendBytes<float>(buf, float(val));
-   }
-   else if (format == VTKFormat::BINARY)
-   {
-      bin_io::AppendBytes(buf, val);
-   }
-   else
-   {
-      out << val << suffix;
-   }
-}
-
-template <>
-void WriteBinaryOrASCII<float>(std::ostream &out, std::vector<char> &buf,
-                               const float &val, const char *suffix,
-                               VTKFormat format)
-{
-   if (format == VTKFormat::BINARY) { bin_io::AppendBytes<double>(buf, val); }
-   else if (format == VTKFormat::BINARY32) { bin_io::AppendBytes(buf, val); }
-   else { out << val << suffix; }
-}
-
-void WriteBase64WithSizeAndClear(std::ostream &out, std::vector<char> &buf,
-                                 int compression_level)
-{
-   WriteVTKEncodedCompressed(out, buf.data(), buf.size(), compression_level);
-   out << '\n';
-   buf.clear();
-}
-
 void Mesh::PrintVTU(std::ostream &out, int ref, VTKFormat format,
                     bool high_order_output, int compression_level,
                     bool bdr_elements)

mesh/mesh.hpp

@@ -91,6 +91,65 @@ protected:
    Array<Element *> boundary;
    Array<Element *> faces;
 
+   /** @brief This structure stores the low level information necessary to
+       interpret the configuration of elements on a specific face. This
+       information can be accessed using methods like GetFaceElements(),
+       GetFaceInfos(), FaceIsInterior(), etc.
+
+       For accessing higher level deciphered information look at
+       Mesh::FaceInformation, and its accessor Mesh::GetFaceInformation().
+
+       Each face contains information on the indices, local reference faces,
+       orientations, and potential nonconformity for the two neighboring
+       elements on a face.
+       Each face can either be an interior, boundary, or shared interior face.
+       Each interior face is shared by two elements referred as Elem1 and Elem2.
+       For boundary faces only the information on Elem1 is relevant.
+       Shared interior faces correspond to faces where Elem1 and Elem2 are
+       distributed on different MPI ranks.
+       Regarding conformity, three cases are distinguished, conforming faces,
+       nonconforming slave faces, and nonconforming master faces. Master and
+       slave referring to the coarse and fine elements respectively on a
+       nonconforming face.
+       Nonconforming slave faces always have the slave element as Elem1 and
+       the master element as Elem2. On the other side, nonconforming master
+       faces always have the master element as Elem1, and one of the slave
+       element as Elem2. Except for ghost nonconforming slave faces, where
+       Elem1 is the master side and Elem2 is the slave side.
+
+       The indices of Elem1 and Elem2 can be indirectly extracted from
+       FaceInfo::Elem1No and FaceInfo::Elem2No, read the note below for special
+       cases on the index of Elem2.
+
+       The local face identifiers are deciphered from FaceInfo::Elem1Inf and
+       FaceInfo::Elem2Inf through the formula: LocalFaceIndex = ElemInf/64,
+       the semantic of the computed local face identifier can be found in
+       fem/geom.cpp. The local face identifier corresponds to an index
+       in the Constants<Geometry>::Edges arrays for 2D element geometries, and
+       to an index in the Constants<Geometry>::FaceVert arrays for 3D element
+       geometries.
+
+       The orientation of each element relative to a face is obtained through
+       the formula: Orientation = ElemInf%64, the semantic of the orientation
+       can also be found in fem/geom.cpp. The orientation corresponds to
+       an index in the Constants<Geometry>::Orient arrays, providing the
+       sequence of vertices identifying the orientation of an edge/face. By
+       convention the orientation of Elem1 is always set to 0, serving as the
+       reference orientation. The orientation of Elem2 relatively to Elem1 is
+       therefore determined just by using the orientation of Elem2. An important
+       special case is the one of nonconforming faces, the orientation should
+       be composed with the PointMatrix, which also contains orientation
+       information. A special treatment should be done for 2D, the orientation
+       in the PointMatrix is not included, therefore when applying the
+       PointMatrix transformation, the PointMatrix should be flipped, except for
+       shared nonconforming slave faces where the transformation can be applied
+       as is.
+
+       Another special case is the case of shared nonconforming faces. Ghost
+       faces use a different design based on so called "ghost" faces.
+       Ghost faces, as their name suggest are very well hidden, and they
+       usually have a separate interface from "standard" faces.
+   */
    struct FaceInfo
    {
       // Inf = 64 * LocalFaceIndex + FaceOrientation
@@ -104,12 +163,12 @@ protected:
    //
    // A local face is one generated from a local element and has index i in
    // faces_info such that i < GetNumFaces(). Also, Elem1No always refers to the
-   // element (slave or master, in the non-conforming case) that generated the
+   // element (slave or master, in the nonconforming case) that generated the
    // face.
    // Classification of a local (non-ghost) face based on its FaceInfo:
    // - Elem2No >= 0 --> local interior face; can be either:
    //    - NCFace == -1 --> conforming face, or
-   //    - NCFace >= 0 --> non-conforming slave face; Elem2No is the index of
+   //    - NCFace >= 0 --> nonconforming slave face; Elem2No is the index of
    //      the master volume element; Elem2Inf%64 is 0, see the note in
    //      Mesh::GenerateNCFaceInfo().
    // - Elem2No < 0 --> local "boundary" face; can be one of:
@@ -118,14 +177,14 @@ protected:
    //       - Elem2Inf >= 0 --> shared face where element 2 is a face-neighbor
    //         element with index -1-Elem2No. This state is initialized by
    //         ParMesh::ExchangeFaceNbrData().
-   //    - NCFace >= 0 --> non-conforming face; can be one of:
-   //       - Elem2Inf < 0 --> master non-conforming face, interior or shared;
+   //    - NCFace >= 0 --> nonconforming face; can be one of:
+   //       - Elem2Inf < 0 --> master nonconforming face, interior or shared;
    //         In this case, Elem2No is -1; see GenerateNCFaceInfo().
-   //       - Elem2Inf >= 0 --> shared slave non-conforming face where element 2
+   //       - Elem2Inf >= 0 --> shared slave nonconforming face where element 2
    //         is the master face-neighbor element with index -1-Elem2No; see
    //         ParNCMesh::GetFaceNeighbors().
    //
-   // A ghost face is a non-conforming face that is generated by a non-local,
+   // A ghost face is a nonconforming face that is generated by a non-local,
    // i.e. ghost, element. A ghost face has index i in faces_info such that
    // i >= GetNumFaces().
    // Classification of a ghost (non-local) face based on its FaceInfo:
@@ -211,7 +270,7 @@ public:
    Array<int> bdr_attributes;
 
    NURBSExtension *NURBSext; ///< Optional NURBS mesh extension.
-   NCMesh *ncmesh;           ///< Optional non-conforming mesh extension.
+   NCMesh *ncmesh;           ///< Optional nonconforming mesh extension.
    Array<GeometricFactors*> geom_factors; ///< Optional geometric factors.
    Array<FaceGeometricFactors*>
    face_geom_factors; ///< Optional face geometric factors.
@@ -651,7 +710,7 @@ public:
 
    int AddVertex(double x, double y = 0.0, double z = 0.0);
    int AddVertex(const double *coords);
-   /// Mark vertex @a i as non-conforming, with parent vertices @a p1 and @a p2.
+   /// Mark vertex @a i as nonconforming, with parent vertices @a p1 and @a p2.
    void AddVertexParents(int i, int p1, int p2);
 
    int AddSegment(int v1, int v2, int attr = 1);
@@ -880,13 +939,19 @@ public:
    /// Return the number of faces (3D), edges (2D) or vertices (1D).
    int GetNumFaces() const;
 
-   /// Returns the number of faces according to the requested type.
-   /** If type==Boundary returns only the "true" number of boundary faces
-       contrary to GetNBE() that returns "fake" boundary faces associated to
-       visualization for GLVis.
-       Similarly, if type==Interior, the "fake" boundary faces associated to
-       visualization are counted as interior faces. */
-   int GetNFbyType(FaceType type) const;
+   /** @brief Return the number of faces (3D), edges (2D) or vertices (1D)
+       including ghost faces. */
+   int GetNumFacesWithGhost() const;
+
+   /** @brief Returns the number of faces according to the requested type, does
+       not count master nonconforming faces.
+
+       If type==Boundary returns only the number of true boundary faces
+       contrary to GetNBE() that returns all "boundary" elements which may
+       include actual interior faces.
+       Similarly, if type==Interior, only the true interior faces are counted
+       excluding all master nonconforming faces. */
+   virtual int GetNFbyType(FaceType type) const;
 
    /// Utility function: sum integers from all processors (Allreduce).
    virtual long ReduceInt(int value) const { return value; }
@@ -1170,8 +1235,9 @@ public:
    ///    mask & 4 - Loc1, mask & 8 - Loc2, mask & 16 - Face.
    /// These mask values are defined in the ConfigMasks enum type as part of the
    /// FaceElementTransformations class in fem/eltrans.hpp.
-   FaceElementTransformations *GetFaceElementTransformations(int FaceNo,
-                                                             int mask = 31);
+   virtual FaceElementTransformations *GetFaceElementTransformations(
+      int FaceNo,
+      int mask = 31);
 
    FaceElementTransformations *GetInteriorFaceTransformations (int FaceNo)
    {
@@ -1189,6 +1255,172 @@ public:
    {
       return (faces_info[FaceNo].Elem2No >= 0);
    }
+
+   /** This enumerated type describes the three main face topologies:
+       - Boundary, for faces on the boundary of the computational domain,
+       - Conforming, for conforming faces interior to the computational domain,
+       - Nonconforming, for nonconforming faces interior to the computational
+         domain. */
+   enum class FaceTopology { Boundary,
+                             Conforming,
+                             Nonconforming,
+                             NA
+                           };
+
+   /** This enumerated type describes the location of the two elements sharing a
+       face, Local meaning that the element is local to the MPI rank, FaceNbr
+       meaning that the element is distributed on a different MPI rank, this
+       typically means that methods with FaceNbr should be used to access the
+       relevant information, e.g., ParFiniteElementSpace::GetFaceNbrElementVDofs.
+    */
+   enum class ElementLocation { Local, FaceNbr, NA };
+
+   /** This enumerated type describes the topological relation of an element to
+       a face:
+       - Coincident meaning that the element's face is topologically equal to
+         the mesh face.
+       - Superset meaning that the element's face is topologically coarser than
+         the mesh face, i.e., the element's face contains the mesh face.
+       - Subset meaning that the element's face is topologically finer than the
+         mesh face, i.e., the element's face is contained in the mesh face.
+       Superset and Subset are only relevant for nonconforming faces.
+       Master nonconforming faces have a conforming element on one side, and a
+       fine element on the other side. Slave nonconforming faces have a
+       conforming element on one side, and a coarse element on the other side.
+    */
+   enum class ElementConformity { Coincident, Superset, Subset, NA };
+
+   /** This enumerated type describes the corresponding FaceInfo internal
+       representation (encoded cases), c.f. FaceInfo's documentation:
+       Classification of a local (non-ghost) face based on its FaceInfo:
+         - Elem2No >= 0 --> local interior face; can be either:
+            - NCFace == -1 --> LocalConforming,
+            - NCFace >= 0 --> LocalSlaveNonconforming,
+         - Elem2No < 0 --> local "boundary" face; can be one of:
+            - NCFace == -1 --> conforming face; can be either:
+               - Elem2Inf < 0 --> Boundary,
+               - Elem2Inf >= 0 --> SharedConforming,
+            - NCFace >= 0 --> nonconforming face; can be one of:
+               - Elem2Inf < 0 --> MasterNonconforming (shared or not shared),
+               - Elem2Inf >= 0 --> SharedSlaveNonconforming.
+       Classification of a ghost (non-local) face based on its FaceInfo:
+         - Elem1No == -1 --> GhostMaster (includes other unused ghost faces),
+         - Elem1No >= 0 --> GhostSlave.
+    */
+   enum class FaceInfoTag { Boundary,
+                            LocalConforming,
+                            LocalSlaveNonconforming,
+                            SharedConforming,
+                            SharedSlaveNonconforming,
+                            MasterNonconforming,
+                            GhostSlave,
+                            GhostMaster
+                          };
+
+   /** @brief This structure is used as a human readable output format that
+       decipheres the information contained in Mesh::FaceInfo when using the
+       Mesh::GetFaceInformation() method.
+
+       The element indices in this structure don't need further processing,
+       contrary to the ones obtained through Mesh::GetFacesElements and can
+       directly be used, e.g., Elem1 and Elem2 indices.
+       Likewise the orientations for Elem1 and Elem2 already take into account
+       special cases and can be used as is.
+   */
+   struct FaceInformation
+   {
+      FaceTopology topology;
+
+      struct
+      {
+         ElementLocation location;
+         ElementConformity conformity;
+         int index;
+         int local_face_id;
+         int orientation;
+      } element[2];
+
+      FaceInfoTag tag;
+      int ncface;
+      const DenseMatrix* point_matrix;
+
+      /** @brief Return true if the face is a local interior face which is NOT
+          a master nonconforming face. */
+      bool IsLocal() const
+      {
+         return element[1].location == Mesh::ElementLocation::Local;
+      }
+
+      /** @brief Return true if the face is a shared interior face which is NOT
+          a master nonconforming face. */
+      bool IsShared() const
+      {
+         return element[1].location == Mesh::ElementLocation::FaceNbr;
+      }
+
+      /** @brief return true if the face is an interior face to the computaion
+          domain, either a local or shared interior face (not a boundary face)
+          which is NOT a master nonconforming face.
+       */
+      bool IsInterior() const
+      {
+         return topology == FaceTopology::Conforming ||
+                topology == FaceTopology::Nonconforming;
+      }
+
+      /** @brief Return true if the face is a boundary face. */
+      bool IsBoundary() const
+      {
+         return topology == FaceTopology::Boundary;
+      }
+
+      /// @brief Return true if the face is of the same type as @a type.
+      bool IsOfFaceType(FaceType type) const
+      {
+         switch (type)
+         {
+            case FaceType::Interior:
+               return IsInterior();
+            case FaceType::Boundary:
+               return IsBoundary();
+            default:
+               return false;
+         }
+      }
+
+      /// @brief Return true if the face is a conforming face.
+      bool IsConforming() const
+      {
+         return topology == FaceTopology::Conforming;
+      }
+
+      /// @brief Return true if the face is a nonconforming fine face.
+      bool IsNonconformingFine() const
+      {
+         return topology == FaceTopology::Nonconforming &&
+                (element[0].conformity == ElementConformity::Superset ||
+                 element[1].conformity == ElementConformity::Superset);
+      }
+
+      /// @brief Return true if the face is a nonconforming coarse face.
+      /** Note that ghost nonconforming master faces cannot be clearly
+          identified as such with the currently available information, so this
+          method will return false for such faces. */
+      bool IsNonconformingCoarse() const
+      {
+         return topology == FaceTopology::Nonconforming &&
+                element[1].conformity == ElementConformity::Subset;
+      }
+
+      /// @brief cast operator from FaceInformation to FaceInfo.
+      operator Mesh::FaceInfo() const;
+   };
+
+   /** This method aims to provide face information in a deciphered format, i.e.
+       Mesh::FaceInformation, compared to the raw encoded information returned
+       by Mesh::GetFaceElements() and Mesh::GetFaceInfos(). */
+   FaceInformation GetFaceInformation(int f) const;
+
    void GetFaceElements (int Face, int *Elem1, int *Elem2) const;
    void GetFaceInfos (int Face, int *Inf1, int *Inf2) const;
    void GetFaceInfos (int Face, int *Inf1, int *Inf2, int *NCFace) const;
@@ -1329,7 +1561,7 @@ public:
 
    /** Refine selected mesh elements. Refinement type can be specified for each
        element. The function can do conforming refinement of triangles and
-       tetrahedra and non-conforming refinement (i.e., with hanging-nodes) of
+       tetrahedra and nonconforming refinement (i.e., with hanging-nodes) of
        triangles, quadrilaterals and hexahedra. If 'nonconforming' = -1,
        suitable refinement method is selected automatically (namely, conforming
        refinement for triangles). Use nonconforming = 0/1 to force the method.
@@ -1381,9 +1613,9 @@ public:
    void DegreeElevate(int rel_degree, int degree = 16);
    ///@}
 
-   /** Make sure that a quad/hex mesh is considered to be non-conforming (i.e.,
+   /** Make sure that a quad/hex mesh is considered to be nonconforming (i.e.,
        has an associated NCMesh object). Simplex meshes can be both conforming
-       (default) or non-conforming. */
+       (default) or nonconforming. */
    void EnsureNCMesh(bool simplices_nonconforming = false);
 
    bool Conforming() const { return ncmesh == NULL; }
@@ -1710,6 +1942,9 @@ inline void ShiftRight(int &a, int &b, int &c)
    a = c;  c = b;  b = t;
 }
 
+/// @brief Print function for Mesh::FaceInformation.
+std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info);
+
 }
 
 #endif

mesh/ncmesh.cpp

@@ -942,7 +942,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
       //     +-----------+           *--X
       //    0             1
 
-      if (ref_type == 1) // split along X axis
+      if (ref_type == Refinement::X) // split along X axis
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid23 = GetMidEdgeNode(no[2], no[3]);
@@ -962,7 +962,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[4], no[5], no[6], no[7], mid45, mid67);
          CheckAnisoFace(no[3], no[2], no[1], no[0], mid23, mid01);
       }
-      else if (ref_type == 2) // split along Y axis
+      else if (ref_type == Refinement::Y) // split along Y axis
       {
          int mid12 = GetMidEdgeNode(no[1], no[2]);
          int mid30 = GetMidEdgeNode(no[3], no[0]);
@@ -982,7 +982,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[5], no[6], no[7], no[4], mid56, mid74);
          CheckAnisoFace(no[0], no[3], no[2], no[1], mid30, mid12);
       }
-      else if (ref_type == 4) // split along Z axis
+      else if (ref_type == Refinement::Z) // split along Z axis
       {
          int mid04 = GetMidEdgeNode(no[0], no[4]);
          int mid15 = GetMidEdgeNode(no[1], no[5]);
@@ -1002,7 +1002,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[6], no[2], no[3], no[7], mid26, mid37);
          CheckAnisoFace(no[7], no[3], no[0], no[4], mid37, mid04);
       }
-      else if (ref_type == 3) // XY split
+      else if (ref_type == Refinement::XY) // XY split
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1041,7 +1041,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckIsoFace(no[3], no[2], no[1], no[0], mid23, mid12, mid01, mid30, midf0);
          CheckIsoFace(no[4], no[5], no[6], no[7], mid45, mid56, mid67, mid74, midf5);
       }
-      else if (ref_type == 5) // XZ split
+      else if (ref_type == Refinement::XZ) // XZ split
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid23 = GetMidEdgeNode(no[2], no[3]);
@@ -1080,7 +1080,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckIsoFace(no[0], no[1], no[5], no[4], mid01, mid15, mid45, mid04, midf1);
          CheckIsoFace(no[2], no[3], no[7], no[6], mid23, mid37, mid67, mid26, midf3);
       }
-      else if (ref_type == 6) // YZ split
+      else if (ref_type == Refinement::YZ) // YZ split
       {
          int mid12 = GetMidEdgeNode(no[1], no[2]);
          int mid30 = GetMidEdgeNode(no[3], no[0]);
@@ -1119,7 +1119,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckIsoFace(no[1], no[2], no[6], no[5], mid12, mid26, mid56, mid15, midf2);
          CheckIsoFace(no[3], no[0], no[4], no[7], mid30, mid04, mid74, mid37, midf4);
       }
-      else if (ref_type == 7) // full isotropic refinement
+      else if (ref_type == Refinement::XYZ) // full isotropic refinement
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1189,7 +1189,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          MFEM_ABORT("invalid refinement type.");
       }
 
-      if (ref_type != 7) { Iso = false; }
+      if (ref_type != Refinement::XYZ) { Iso = false; }
    }
    else if (el.Geom() == Geometry::PRISM)
    {
@@ -1209,7 +1209,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
 
       if (ref_type < 4) // XY refinement (split in 4 wedges)
       {
-         ref_type = 3; // for consistence
+         ref_type = Refinement::XY; // for consistence
 
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1239,7 +1239,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[1], no[2], no[5], no[4], mid12, mid45);
          CheckAnisoFace(no[2], no[0], no[3], no[5], mid20, mid53);
       }
-      else if (ref_type == 4) // Z refinement only (split in 2 wedges)
+      else if (ref_type == Refinement::Z) // Z refinement only (split in 2 wedges)
       {
          int mid03 = GetMidEdgeNode(no[0], no[3]);
          int mid14 = GetMidEdgeNode(no[1], no[4]);
@@ -1259,7 +1259,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
       }
       else if (ref_type > 4) // full isotropic refinement (split in 8 wedges)
       {
-         ref_type = 7; // for consistence
+         ref_type = Refinement::XYZ; // for consistence
 
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1318,7 +1318,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          MFEM_ABORT("invalid refinement type.");
       }
 
-      if (ref_type != 7) { Iso = false; }
+      if (ref_type != Refinement::XYZ) { Iso = false; }
    }
    else if (el.Geom() == Geometry::TETRAHEDRON)
    {
@@ -1335,7 +1335,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
       //     +------------+     *--X
       //    0              1
 
-      ref_type = 7; // for consistence
+      ref_type = Refinement::XYZ; // for consistence
 
       int mid01 = GetMidEdgeNode(no[0], no[1]);
       int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1412,7 +1412,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
    {
       ref_type &= 0x3; // ignore Z bit
 
-      if (ref_type == 1) // X split
+      if (ref_type == Refinement::X) // X split
       {
          int mid01 = nodes.GetId(no[0], no[1]);
          int mid23 = nodes.GetId(no[2], no[3]);
@@ -1423,7 +1423,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          child[1] = NewQuadrilateral(mid01, no[1], no[2], mid23,
                                      attr, fa[0], fa[1], fa[2], -1);
       }
-      else if (ref_type == 2) // Y split
+      else if (ref_type == Refinement::Y) // Y split
       {
          int mid12 = nodes.GetId(no[1], no[2]);
          int mid30 = nodes.GetId(no[3], no[0]);
@@ -1434,7 +1434,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          child[1] = NewQuadrilateral(mid30, mid12, no[2], no[3],
                                      attr, -1, fa[1], fa[2], fa[3]);
       }
-      else if (ref_type == 3) // iso split
+      else if (ref_type == Refinement::XY) // iso split
       {
          int mid01 = nodes.GetId(no[0], no[1]);
          int mid12 = nodes.GetId(no[1], no[2]);
@@ -1460,11 +1460,11 @@ void NCMesh::RefineElement(int elem, char ref_type)
          MFEM_ABORT("Invalid refinement type.");
       }
 
-      if (ref_type != 3) { Iso = false; }
+      if (ref_type != Refinement::XY) { Iso = false; }
    }
    else if (el.Geom() == Geometry::TRIANGLE)
    {
-      ref_type = 3; // for consistence
+      ref_type = Refinement::XY; // for consistence
 
       // isotropic split - the only ref_type available for triangles
       int mid01 = nodes.GetId(no[0], no[1]);
@@ -1478,7 +1478,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
    }
    else if (el.Geom() == Geometry::SEGMENT)
    {
-      ref_type = 1; // for consistence
+      ref_type = Refinement::X; // for consistence
 
       int mid = nodes.GetId(no[0], no[1]);
       child[0] = NewSegment(no[0], mid, attr, fa[0], -1);
@@ -1624,43 +1624,62 @@ void NCMesh::DerefineElement(int elem)
       }
    }
 
-   int fa[6];
-   int rt1 = el.ref_type - 1;
+   int faces_attribute[6];
+   int ref_type_key = el.ref_type - 1;
 
    for (int i = 0; i < 8; i++) { el.node[i] = -1; }
 
    // retrieve original corner nodes and face attributes from the children
    if (el.Geom() == Geometry::CUBE)
    {
-      for (int i = 0; i < 8; i++)
+      // Sets corner nodes from childs
+      constexpr int nb_cube_childs = 8;
+      for (int i = 0; i < nb_cube_childs; i++)
       {
-         Element &ch = elements[child[hex_deref_table[rt1][i]]];
+         const int child_local_index = hex_deref_table[ref_type_key][i];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          el.node[i] = ch.node[i];
       }
-      for (int i = 0; i < 6; i++)
+      // Sets faces attributes from childs' faces
+      constexpr int nb_cube_faces = 6;
+      for (int i = 0; i < nb_cube_faces; i++)
       {
-         Element &ch = elements[child[hex_deref_table[rt1][i + 8]]];
+         const int child_local_index = hex_deref_table[ref_type_key]
+                                       [i + nb_cube_childs];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::PRISM)
    {
-      MFEM_ASSERT(prism_deref_table[rt1][0] != -1, "invalid prism refinement");
-      for (int i = 0; i < 6; i++)
+      MFEM_ASSERT(prism_deref_table[ref_type_key][0] != -1,
+                  "invalid prism refinement");
+      constexpr int nb_prism_childs = 6;
+      for (int i = 0; i < nb_prism_childs; i++)
       {
-         Element &ch = elements[child[prism_deref_table[rt1][i]]];
+         const int child_local_index = prism_deref_table[ref_type_key][i];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          el.node[i] = ch.node[i];
       }
       el.node[6] = el.node[7] = -1;
 
-      for (int i = 0; i < 5; i++)
+      constexpr int nb_prism_faces = 5;
+      for (int i = 0; i < nb_prism_faces; i++)
       {
-         Element &ch = elements[child[prism_deref_table[rt1][i + 6]]];
+         const int child_local_index = prism_deref_table[ref_type_key]
+                                       [i + nb_prism_childs];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::TETRAHEDRON)
@@ -1671,43 +1690,55 @@ void NCMesh::DerefineElement(int elem)
          Element& ch2 = elements[child[(i+1) & 0x3]];
          el.node[i] = ch1.node[i];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch2.node[fv[0]], ch2.node[fv[1]],
-                            ch2.node[fv[2]], ch2.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch2.node[fv[0]], ch2.node[fv[1]],
+                                         ch2.node[fv[2]], ch2.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::SQUARE)
    {
-      for (int i = 0; i < 4; i++)
+      constexpr int nb_square_childs = 4;
+      for (int i = 0; i < nb_square_childs; i++)
       {
-         Element &ch = elements[child[quad_deref_table[rt1][i]]];
+         const int child_local_index = quad_deref_table[ref_type_key][i];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          el.node[i] = ch.node[i];
       }
-      for (int i = 0; i < 4; i++)
+      constexpr int nb_square_faces = 4;
+      for (int i = 0; i < nb_square_faces; i++)
       {
-         Element &ch = elements[child[quad_deref_table[rt1][i + 4]]];
+         const int child_local_index = quad_deref_table[ref_type_key]
+                                       [i + nb_square_childs];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::TRIANGLE)
    {
-      for (int i = 0; i < 3; i++)
+      constexpr int nb_triangle_childs = 3;
+      for (int i = 0; i < nb_triangle_childs; i++)
       {
          Element& ch = elements[child[i]];
          el.node[i] = ch.node[i];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::SEGMENT)
    {
-      for (int i = 0; i < 2; i++)
+      constexpr int nb_segment_childs = 2;
+      for (int i = 0; i < nb_segment_childs; i++)
       {
          int ni = elements[child[i]].node[i];
          el.node[i] = ni;
-         fa[i] = faces.Find(ni, ni, ni, ni)->attribute;
+         faces_attribute[i] = faces.Find(ni, ni, ni, ni)->attribute;
       }
    }
    else
@@ -1731,7 +1762,7 @@ void NCMesh::DerefineElement(int elem)
       FreeElement(child[i]);
    }
 
-   RegisterFaces(elem, fa);
+   RegisterFaces(elem, faces_attribute);
 
    // delete unused faces
    childFaces.Sort();
@@ -1901,7 +1932,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
    {
       if (el.rank >= 0) // skip elements beyond the ghost layer in parallel
       {
-         if (el.rank == MyRank)
+         if (!IsGhost(el))
          {
             leaf_elements.Append(elem);
          }
@@ -1922,7 +1953,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
          el.index = -1;
       }
    }
-   else
+   else // Refined element
    {
       // in non-leaf elements, the 'rank' and 'index' members have no meaning
       el.rank = -1;
@@ -1930,7 +1961,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
 
       // recurse to subtrees; try to order leaf elements along a space-filling
       // curve by changing the order the children are visited at each level
-      if (el.Geom() == Geometry::SQUARE && el.ref_type == 3)
+      if (el.Geom() == Geometry::SQUARE && el.ref_type == Refinement::XY)
       {
          for (int i = 0; i < 4; i++)
          {
@@ -1939,7 +1970,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
             CollectLeafElements(el.child[ch], st, ghosts, counter);
          }
       }
-      else if (el.Geom() == Geometry::CUBE && el.ref_type == 7)
+      else if (el.Geom() == Geometry::CUBE && el.ref_type == Refinement::XYZ)
       {
          for (int i = 0; i < 8; i++)
          {
@@ -1948,7 +1979,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
             CollectLeafElements(el.child[ch], st, ghosts, counter);
          }
       }
-      else // no SFC tables yet for remaining cases
+      else // no space filling curve tables yet for remaining cases
       {
          for (int i = 0; i < 8; i++)
          {
@@ -1965,7 +1996,8 @@ void NCMesh::UpdateLeafElements()
 {
    Array<int> ghosts;
 
-   // collect leaf elements from all roots
+   // collect leaf elements in leaf_elements and ghosts elements in ghosts from
+   // all roots
    leaf_elements.SetSize(0);
    for (int i = 0, counter = 0; i < root_state.Size(); i++)
    {
@@ -2355,7 +2387,7 @@ void NCMesh::GetMeshComponents(Mesh &mesh) const
             else
             {
                MFEM_ASSERT(nc_elem.geom == Geometry::SEGMENT, "");
-               auto* point = (Segment*) mesh.NewElement(Geometry::POINT);
+               auto* point = (mfem::Point*) mesh.NewElement(Geometry::POINT);
                point->SetAttribute(face->attribute);
                point->GetVertices()[0] = nodes[node[fv[0]]].vert_index;
                mesh.boundary.Append(point);
@@ -2738,7 +2770,7 @@ void NCMesh::TraverseQuadFace(int vn0, int vn1, int vn2, int vn3,
 
          // reorder the point matrix according to slave face orientation
          PointMatrix pm_r;
-         sl.local = ReorderFacePointMat(vn0, vn1, vn2, vn3, elem, pm, pm_r);;
+         sl.local = ReorderFacePointMat(vn0, vn1, vn2, vn3, elem, pm, pm_r);
          sl.matrix = matrix_map.GetIndex(pm_r);
 
          eface[0] = eface[2] = fa;

mesh/ncmesh.hpp

@@ -34,11 +34,14 @@ namespace mfem
     in the X, Y and Z directions, respectively (Z is ignored for quads). */
 struct Refinement
 {
+   enum : char { X = 1, Y = 2, Z = 4, XY = 3, XZ = 5, YZ = 6, XYZ = 7 };
    int index; ///< Mesh element number
    char ref_type; ///< refinement XYZ bit mask (7 = full isotropic)
 
    Refinement() = default;
-   Refinement(int index, int type = 7) : index(index), ref_type(type) {}
+
+   Refinement(int index, int type = Refinement::XYZ)
+      : index(index), ref_type(type) {}
 };
 
 
@@ -134,11 +137,16 @@ public:
 
    virtual ~NCMesh();
 
+   /// Return the dimension of the NCMesh.
    int Dimension() const { return Dim; }
+   /// Return the space dimension of the NCMesh.
    int SpaceDimension() const { return spaceDim; }
 
+   /// Return the number of vertices in the NCMesh.
    int GetNVertices() const { return NVertices; }
+   /// Return the number of edges in the NCMesh.
    int GetNEdges() const { return NEdges; }
+   /// Return the number of (2D) faces in the NCMesh.
    int GetNFaces() const { return NFaces; }
    virtual int GetNGhostElements() const { return 0; }
 
@@ -531,8 +539,34 @@ protected: // implementation
    Table element_vertex; ///< leaf-element to vertex table, see FindSetNeighbors
 
 
+   /// Update the leaf elements indices in leaf_elements
    void UpdateLeafElements();
+
+   /** @brief This method assigns indices to vertices (Node::vert_index) that
+       will be seen by the Mesh class and the rest of MFEM.
+
+       We must be careful to:
+       1. Stay compatible with the conforming code, which expects top-level
+          (original) vertices to be indexed first, otherwise GridFunctions
+          defined on a conforming mesh would no longer be valid when the
+          mesh is converted to an NC mesh.
+
+       2. Make sure serial NCMesh is compatible with the parallel ParNCMesh,
+          so it is possible to read parallel partial solutions in serial code
+          (e.g., serial GLVis). This means handling ghost elements, if present.
+
+       3. Assign vertices in a globally consistent order for parallel meshes:
+          if two vertices i,j are shared by two ranks r1,r2, and i<j on r1,
+          then i<j on r2 as well. This is true for top-level vertices but also
+          for the remaining shared vertices thanks to the globally consistent
+          SFC ordering of the leaf elements. This property reduces communication
+          and simplifies ParNCMesh. */
    void UpdateVertices(); ///< update Vertex::index and vertex_nodeId
+
+   /** Collect the leaf elements in leaf_elements, and the ghost elements in
+       ghosts. Compute and set the element indices of @a elements. On quad and
+       hex refined elements tries to order leaf elements along a space-filling
+       curve according to the given @a state variable. */
    void CollectLeafElements(int elem, int state, Array<int> &ghosts,
                             int &counter);
 
@@ -542,11 +576,17 @@ protected: // implementation
        Mesh::GetGeckoElementOrdering. */
    void InitRootState(int root_count);
 
+   /** Compute the Geometry::Type present in the root elements (coarse elements)
+       and set @a Geoms bitmask accordingly. */
    void InitGeomFlags();
 
+   /// Return true if the mesh contains prism elements.
    bool HavePrisms() const { return Geoms & (1 << Geometry::PRISM); }
+
+   /// Return true if the mesh contains tetrahedral elements.
    bool HaveTets() const   { return Geoms & (1 << Geometry::TETRAHEDRON); }
 
+   /// Return true if the Element @a el is a ghost element.
    bool IsGhost(const Element &el) const { return el.rank != MyRank; }
 
 
@@ -558,9 +598,14 @@ protected: // implementation
 
    Table derefinements; ///< possible derefinements, see GetDerefinementTable
 
+   /** Refine the element @a elem with the refinement @a ref_type
+       (c.f. Refinement::enum) */
    void RefineElement(int elem, char ref_type);
+
+   /// Derefine the element @a elem, does nothing on leaf elements.
    void DerefineElement(int elem);
 
+   // Add an Element @a el to the NCMesh, optimized to reuse freed elements.
    int AddElement(const Element &el)
    {
       if (free_element_ids.Size())
@@ -572,6 +617,8 @@ protected: // implementation
       }
       return elements.Append(el);
    }
+
+   // Free the element with index @a id.
    void FreeElement(int id)
    {
       free_element_ids.Append(id);
@@ -776,6 +823,22 @@ protected: // implementation
       }
    };
 
+   /** @brief The PointMatrix stores the coordinates of the slave face using the
+       master face coordinate as reference.
+
+       In 2D, the point matrix has the orientation of the parent
+       edge, so its columns need to be flipped when applying it, see
+       ApplyLocalSlaveTransformation.
+
+       In 3D, the orientation part of Elem2Inf is encoded in the point
+       matrix.
+
+       The following transformation gives the relation betwen the
+       reference quad face coordinates (xi, eta) in [0,1]^2, and the fine quad
+       face coordinates (x, y):
+       x = a0*(1-xi)*(1-eta) + a1*xi*(1-eta) + a2*xi*eta + a3*(1-xi)*eta
+       y = b0*(1-xi)*(1-eta) + b1*xi*(1-eta) + b2*xi*eta + b3*(1-xi)*eta
+   */
    struct PointMatrix
    {
       int np;

mesh/pmesh.cpp

@@ -2859,12 +2859,34 @@ void ParMesh::GetGhostFaceTransformation(
    }
 }
 
+FaceElementTransformations *ParMesh::GetFaceElementTransformations(
+   int FaceNo,
+   int mask)
+{
+   if (FaceNo < GetNumFaces())
+   {
+      return Mesh::GetFaceElementTransformations(FaceNo, mask);
+   }
+   else
+   {
+      const bool fill2 = mask & 10; // Elem2 and/or Loc2
+      return GetSharedFaceTransformationsByLocalIndex(FaceNo, fill2);
+   }
+}
+
 FaceElementTransformations *ParMesh::
 GetSharedFaceTransformations(int sf, bool fill2)
 {
    int FaceNo = GetSharedFace(sf);
 
+   return GetSharedFaceTransformationsByLocalIndex(FaceNo, fill2);
+}
+
+FaceElementTransformations *ParMesh::
+GetSharedFaceTransformationsByLocalIndex(int FaceNo, bool fill2)
+{
    FaceInfo &face_info = faces_info[FaceNo];
+   MFEM_VERIFY(face_info.Elem2Inf >= 0, "The face must be shared.");
 
    bool is_slave = Nonconforming() && IsSlaveFace(face_info);
    bool is_ghost = Nonconforming() && FaceNo >= GetNumFaces();
@@ -3010,6 +3032,13 @@ int ParMesh::GetSharedFace(int sface) const
    }
 }
 
+int ParMesh::GetNFbyType(FaceType type) const
+{
+   MFEM_VERIFY(have_face_nbr_data,
+               "ExchangeFaceNbrData() should be called before using GetNFbyType");
+   return Mesh::GetNFbyType(type);
+}
+
 // shift cyclically 3 integers a, b, c, so that the smallest of
 // order[a], order[b], order[c] is first
 static inline

mesh/pmesh.hpp

@@ -95,7 +95,7 @@ protected:
 
    // Mark all tets to ensure consistency across MPI tasks; also mark the
    // shared and boundary triangle faces using the consistently marked tets.
-   virtual void MarkTetMeshForRefinement(DSTable &v_to_v);
+   void MarkTetMeshForRefinement(DSTable &v_to_v) override;
 
    /// Return a number(0-1) identifying how the given edge has been split
    int GetEdgeSplittings(Element *edge, const DSTable &v_to_v, int *middle);
@@ -132,23 +132,23 @@ protected:
    void ExchangeFaceNbrData(Table *gr_sface, int *s2l_face);
 
    /// Refine a mixed 2D mesh uniformly.
-   virtual void UniformRefinement2D();
+   void UniformRefinement2D() override;
 
    /// Refine a mixed 3D mesh uniformly.
-   virtual void UniformRefinement3D();
+   void UniformRefinement3D() override;
 
-   virtual void NURBSUniformRefinement();
+   void NURBSUniformRefinement() override;
 
    /// This function is not public anymore. Use GeneralRefinement instead.
-   virtual void LocalRefinement(const Array<int> &marked_el, int type = 3);
+   void LocalRefinement(const Array<int> &marked_el, int type = 3) override;
 
    /// This function is not public anymore. Use GeneralRefinement instead.
-   virtual void NonconformingRefinement(const Array<Refinement> &refinements,
-                                        int nc_limit = 0);
+   void NonconformingRefinement(const Array<Refinement> &refinements,
+                                int nc_limit = 0) override;
 
-   virtual bool NonconformingDerefinement(Array<double> &elem_error,
-                                          double threshold, int nc_limit = 0,
-                                          int op = 1);
+   bool NonconformingDerefinement(Array<double> &elem_error,
+                                  double threshold, int nc_limit = 0,
+                                  int op = 1) override;
 
    void RebalanceImpl(const Array<int> *partition);
 
@@ -278,9 +278,9 @@ public:
        See @a Mesh::MakeSimplicial for more details. */
    static ParMesh MakeSimplicial(ParMesh &orig_mesh);
 
-   virtual void Finalize(bool refine = false, bool fix_orientation = false);
+   void Finalize(bool refine = false, bool fix_orientation = false) override;
 
-   virtual void SetAttributes();
+   void SetAttributes() override;
 
    MPI_Comm GetComm() const { return MyComm; }
    int GetNRanks() const { return NRanks; }
@@ -342,8 +342,8 @@ public:
    void ExchangeFaceNbrData();
    void ExchangeFaceNbrNodes();
 
-   virtual void SetCurvature(int order, bool discont = false, int space_dim = -1,
-                             int ordering = 1);
+   void SetCurvature(int order, bool discont = false, int space_dim = -1,
+                     int ordering = 1) override;
 
    int GetNFaceNeighbors() const { return face_nbr_group.Size(); }
    int GetNFaceNeighborElements() const { return face_nbr_elements.Size(); }
@@ -357,12 +357,56 @@ public:
        with indices offset by the local number of elements. */
    Table *GetFaceToAllElementTable() const;
 
-   /** Get the FaceElementTransformations for the given shared face (edge 2D).
+   /// Returns (a pointer to an object containing) the following data:
+   ///
+   /// 1) Elem1No - the index of the first element that contains this face this
+   ///    is the element that has the same outward unit normal vector as the
+   ///    face;
+   ///
+   /// 2) Elem2No - the index of the second element that contains this face this
+   ///    element has outward unit normal vector as the face multiplied with -1;
+   ///
+   /// 3) Elem1, Elem2 - pointers to the ElementTransformation's of the first
+   ///    and the second element respectively;
+   ///
+   /// 4) Face - pointer to the ElementTransformation of the face;
+   ///
+   /// 5) Loc1, Loc2 - IntegrationPointTransformation's mapping the face
+   ///    coordinate system to the element coordinate system (both in their
+   ///    reference elements). Used to transform IntegrationPoints from face to
+   ///    element. More formally, let:
+   ///       TL1, TL2 be the transformations represented by Loc1, Loc2,
+   ///       TE1, TE2 - the transformations represented by Elem1, Elem2,
+   ///       TF - the transformation represented by Face, then
+   ///       TF(x) = TE1(TL1(x)) = TE2(TL2(x)) for all x in the reference face.
+   ///
+   /// 6) FaceGeom - the base geometry for the face.
+   ///
+   /// The mask specifies which fields in the structure to return:
+   ///    mask & 1 - Elem1, mask & 2 - Elem2
+   ///    mask & 4 - Loc1, mask & 8 - Loc2, mask & 16 - Face.
+   /// These mask values are defined in the ConfigMasks enum type as part of the
+   /// FaceElementTransformations class in fem/eltrans.hpp.
+   FaceElementTransformations *GetFaceElementTransformations(
+      int FaceNo,
+      int mask = 31) override;
+
+   /** Get the FaceElementTransformations for the given shared face (edge 2D)
+       using the shared face index @a sf. @a fill2 specify if the information
+       for elem2 of the face should be computed or not.
        In the returned object, 1 and 2 refer to the local and the neighbor
        elements, respectively. */
    FaceElementTransformations *
    GetSharedFaceTransformations(int sf, bool fill2 = true);
 
+   /** Get the FaceElementTransformations for the given shared face (edge 2D)
+       using the face index @a FaceNo. @a fill2 specify if the information
+       for elem2 of the face should be computed or not.
+       In the returned object, 1 and 2 refer to the local and the neighbor
+       elements, respectively. */
+   FaceElementTransformations *
+   GetSharedFaceTransformationsByLocalIndex(int FaceNo, bool fill2 = true);
+
    ElementTransformation *
    GetFaceNbrElementTransformation(int i)
    {
@@ -381,11 +425,21 @@ public:
    /// Return the local face index for the given shared face.
    int GetSharedFace(int sface) const;
 
+   /** @brief Returns the number of local faces according to the requested type,
+       does not count master non-conforming faces.
+
+       If type==Boundary returns only the number of true boundary faces
+       contrary to GetNBE() that returns all "boundary" elements which may
+       include actual interior faces.
+       Similarly, if type==Interior, only the true interior faces (including
+       shared faces) are counted excluding all master non-conforming faces. */
+   int GetNFbyType(FaceType type) const override;
+
    /// See the remarks for the serial version in mesh.hpp
-   MFEM_DEPRECATED virtual void ReorientTetMesh();
+   MFEM_DEPRECATED void ReorientTetMesh() override;
 
    /// Utility function: sum integers from all processors (Allreduce).
-   virtual long ReduceInt(int value) const;
+   long ReduceInt(int value) const override;
 
    /** Load balance the mesh by equipartitioning the global space-filling
        sequence of elements. Works for nonconforming meshes only. */
@@ -401,23 +455,23 @@ public:
 
    /** Print the part of the mesh in the calling processor adding the interface
        as boundary (for visualization purposes) using the mfem v1.0 format. */
-   virtual void Print(std::ostream &out = mfem::out) const;
+   void Print(std::ostream &out = mfem::out) const override;
 
    /// Save the ParMesh to files (one for each MPI rank). The files will be
    /// given suffixes according to the MPI rank. The mesh will be written to the
    /// files using ParMesh::Print. The given @a precision will be used for ASCII
    /// output.
-   virtual void Save(const char *fname, int precision=16) const;
+   void Save(const char *fname, int precision=16) const override;
 
 #ifdef MFEM_USE_ADIOS2
    /** Print the part of the mesh in the calling processor using adios2 bp
        format. */
-   virtual void Print(adios2stream &out) const;
+   void Print(adios2stream &out) const override;
 #endif
 
    /** Print the part of the mesh in the calling processor adding the interface
        as boundary (for visualization purposes) using Netgen/Truegrid format .*/
-   virtual void PrintXG(std::ostream &out = mfem::out) const;
+   void PrintXG(std::ostream &out = mfem::out) const override;
 
    /** Write the mesh to the stream 'out' on Process 0 in a form suitable for
        visualization: the mesh is written as a disjoint mesh and the shared
@@ -435,15 +489,15 @@ public:
    /** Print the mesh in parallel PVTU format. The PVTU and VTU files will be
        stored in the directory specified by @a pathname. If the directory does
        not exist, it will be created. */
-   virtual void PrintVTU(std::string pathname,
-                         VTKFormat format=VTKFormat::ASCII,
-                         bool high_order_output=false,
-                         int compression_level=0,
-                         bool bdr=false);
+   void PrintVTU(std::string pathname,
+                 VTKFormat format=VTKFormat::ASCII,
+                 bool high_order_output=false,
+                 int compression_level=0,
+                 bool bdr=false) override;
 
    /// Parallel version of Mesh::Load().
-   virtual void Load(std::istream &input, int generate_edges = 0,
-                     int refine = 1, bool fix_orientation = true);
+   void Load(std::istream &input, int generate_edges = 0,
+             int refine = 1, bool fix_orientation = true) override;
 
    /// Returns the minimum and maximum corners of the mesh bounding box. For
    /// high-order meshes, the geometry is refined first "ref" times.
@@ -457,11 +511,11 @@ public:
    void Swap(ParMesh &other);
 
    /// Print various parallel mesh stats
-   virtual void PrintInfo(std::ostream &out = mfem::out);
+   void PrintInfo(std::ostream &out = mfem::out) override;
 
-   virtual int FindPoints(DenseMatrix& point_mat, Array<int>& elem_ids,
-                          Array<IntegrationPoint>& ips, bool warn = true,
-                          InverseElementTransformation *inv_trans = NULL);
+   int FindPoints(DenseMatrix& point_mat, Array<int>& elem_ids,
+                  Array<IntegrationPoint>& ips, bool warn = true,
+                  InverseElementTransformation *inv_trans = NULL) override;
 
    /// Debugging method
    void PrintSharedEntities(const char *fname_prefix) const;

mesh/pumi.cpp

@@ -360,6 +360,10 @@ void PumiMesh::ReadSCORECMesh(apf::Mesh2* apf_mesh, apf::Numbering* v_num_loc,
    NumOfElements = countOwned(apf_mesh,Dim);
    elements.SetSize(NumOfElements);
 
+   // Look for the gmsh physical entity tag
+   const char* gmshTagName = "gmsh_physical_entity";
+   apf::MeshTag* gmshPhysEnt = apf_mesh->findTag(gmshTagName);
+
    // Read elements from SCOREC Mesh
    itr = apf_mesh->begin(Dim);
    unsigned int j=0;
@@ -368,8 +372,12 @@ void PumiMesh::ReadSCORECMesh(apf::Mesh2* apf_mesh, apf::Numbering* v_num_loc,
       // Get vertices
       apf::Downward verts;
       apf_mesh->getDownward(ent,0,verts); // num_vert
-      // Get attribute Tag vs Geometry
+      // Get attribute Tag from gmsh if it exists
       int attr = 1;
+      if ( gmshPhysEnt )
+      {
+         apf_mesh->getIntTag(ent,gmshPhysEnt,&attr);
+      }
 
       int geom_type = apf_mesh->getType(ent);
       elements[j] = NewElement(geom_type);
@@ -511,6 +519,10 @@ ParPumiMesh::ParPumiMesh(MPI_Comm comm, apf::Mesh2* apf_mesh,
    NumOfElements = countOwned(apf_mesh,Dim);
    elements.SetSize(NumOfElements);
 
+   // Look for the gmsh physical entity tag
+   const char* gmshTagName = "gmsh_physical_entity";
+   apf::MeshTag* gmshPhysEnt = apf_mesh->findTag(gmshTagName);
+
    // Read elements from SCOREC Mesh
    itr = apf_mesh->begin(Dim);
    for (int j = 0; (ent = apf_mesh->iterate(itr)); j++)
@@ -518,9 +530,14 @@ ParPumiMesh::ParPumiMesh(MPI_Comm comm, apf::Mesh2* apf_mesh,
       // Get vertices
       apf::Downward verts;
       apf_mesh->getDownward(ent,0,verts);
+      // Get attribute Tag from gmsh if it exists
+      int attr = 1;
+      if ( gmshPhysEnt )
+      {
+         apf_mesh->getIntTag(ent,gmshPhysEnt,&attr);
+      }
 
       // Get attribute Tag vs Geometry
-      int attr = 1;
       int geom_type = apf_mesh->getType(ent);
       elements[j] = NewElement(geom_type);
       ReadPumiElement(ent, verts, attr, v_num_loc, elements[j]);

mesh/vtk.cpp

@@ -600,4 +600,51 @@ const char *VTKByteOrder()
 
 }
 
+// Ensure ASCII output of uint8_t to stream is integer rather than character
+template <>
+void WriteBinaryOrASCII<uint8_t>(std::ostream &out, std::vector<char> &buf,
+                                 const uint8_t &val, const char *suffix,
+                                 VTKFormat format)
+{
+   if (format == VTKFormat::ASCII) { out << static_cast<int>(val) << suffix; }
+   else { bin_io::AppendBytes(buf, val); }
+}
+
+template <>
+void WriteBinaryOrASCII<double>(std::ostream &out, std::vector<char> &buf,
+                                const double &val, const char *suffix,
+                                VTKFormat format)
+{
+   if (format == VTKFormat::BINARY32)
+   {
+      bin_io::AppendBytes<float>(buf, float(val));
+   }
+   else if (format == VTKFormat::BINARY)
+   {
+      bin_io::AppendBytes(buf, val);
+   }
+   else
+   {
+      out << ZeroSubnormal(val) << suffix;
+   }
+}
+
+template <>
+void WriteBinaryOrASCII<float>(std::ostream &out, std::vector<char> &buf,
+                               const float &val, const char *suffix,
+                               VTKFormat format)
+{
+   if (format == VTKFormat::BINARY) { bin_io::AppendBytes<double>(buf, val); }
+   else if (format == VTKFormat::BINARY32) { bin_io::AppendBytes(buf, val); }
+   else { out << ZeroSubnormal(val) << suffix; }
+}
+
+void WriteBase64WithSizeAndClear(std::ostream &out, std::vector<char> &buf,
+                                 int compression_level)
+{
+   WriteVTKEncodedCompressed(out, buf.data(), buf.size(), compression_level);
+   out << '\n';
+   buf.clear();
+}
+
 } // namespace mfem

mesh/vtk.hpp

@@ -13,23 +13,34 @@
 #define MFEM_VTK
 
 #include "../fem/geom.hpp"
+#include "../general/binaryio.hpp"
 
 namespace mfem
 {
 
 // Helpers for reading and writing VTK format
 
-// VTK element types defined at: https://git.io/JvZLm
+/// @brief Helper class for converting between MFEM and VTK geometry types.
+///
+/// Note: The VTK element types defined are at: https://git.io/JvZLm
 struct VTKGeometry
 {
+   /// @name VTK geometry types
+   ///@{
    static const int POINT = 1;
+
+   /// @name Low-order (linear, straight-sided) VTK geometric types
+   ///@{
    static const int SEGMENT = 3;
    static const int TRIANGLE = 5;
    static const int SQUARE = 9;
    static const int TETRAHEDRON = 10;
    static const int CUBE = 12;
    static const int PRISM = 13;
+   ///@}
 
+   /// @name Legacy quadratic VTK geometric types
+   ///@{
    static const int QUADRATIC_SEGMENT = 21;
    static const int QUADRATIC_TRIANGLE = 22;
    static const int BIQUADRATIC_SQUARE = 28;
@@ -37,49 +48,138 @@ struct VTKGeometry
    static const int TRIQUADRATIC_CUBE = 29;
    static const int QUADRATIC_PRISM = 26;
    static const int BIQUADRATIC_QUADRATIC_PRISM = 32;
+   ///@}
 
+   /// @name Arbitrary-order VTK geometric types
+   ///@{
    static const int LAGRANGE_SEGMENT = 68;
    static const int LAGRANGE_TRIANGLE = 69;
    static const int LAGRANGE_SQUARE = 70;
    static const int LAGRANGE_TETRAHEDRON = 71;
    static const int LAGRANGE_CUBE = 72;
    static const int LAGRANGE_PRISM = 73;
+   ///@}
+   ///@}
 
+   /// Permutation from MFEM's prism ordering to VTK's prism ordering.
    static const int PrismMap[6];
+
+   /// @brief Permutation from MFEM's vertex ordering to VTK's vertex ordering.
+   /// @note If the MFEM and VTK orderings are the same, the vertex permutation
+   /// will be NULL.
    static const int *VertexPermutation[Geometry::NUM_GEOMETRIES];
 
+   /// Map from MFEM's Geometry::Type to linear VTK geometries.
    static const int Map[Geometry::NUM_GEOMETRIES];
+   /// Map from MFEM's Geometry::Type to legacy quadratic VTK geometries/
    static const int QuadraticMap[Geometry::NUM_GEOMETRIES];
+   /// Map from MFEM's Geometry::Type to arbitrary-order Lagrange VTK geometries
    static const int HighOrderMap[Geometry::NUM_GEOMETRIES];
 
+   /// Given a VTK geometry type, return the corresponding MFEM Geometry::Type.
    static Geometry::Type GetMFEMGeometry(int vtk_geom);
+   /// @brief Does the given VTK geometry type describe an arbitrary-order
+   /// Lagrange element?
    static bool IsLagrange(int vtk_geom);
+   /// @brief Does the given VTK geometry type describe a legacy quadratic
+   /// element?
    static bool IsQuadratic(int vtk_geom);
+   /// @brief For the given VTK geometry type and number of points, return the
+   /// order of the element.
    static int GetOrder(int vtk_geom, int npoints);
 };
 
+/// Data array format for VTK and VTU files.
 enum class VTKFormat
 {
+   /// Data arrays will be written in ASCII format.
    ASCII,
+   /// Data arrays will be written in binary format. Floating point numbers will
+   /// be be output with 64 bits of precision.
    BINARY,
+   /// Data arrays will be written in binary format. Floating point numbers will
+   /// be be output with 32 bits of precision.
    BINARY32
 };
 
-/// Create the VTK element connectivity array for a given element geometry and
-/// refinement level. Converts node numbers from MFEM to VTK ordering.
+/// @brief Create the VTK element connectivity array for a given element
+/// geometry and refinement level.
+///
+/// The output array @a con will be such that, for the @a ith VTK node index,
+/// con[i] will contain the index of the corresponding node in MFEM ordering.
 void CreateVTKElementConnectivity(Array<int> &con, Geometry::Type geom,
                                   int ref);
 
-/// Outputs encoded binary data in the format needed by VTK. The binary data
-/// will be base 64 encoded, and compressed if @a compression_level is not
-/// zero. The proper header will be prepended to the data.
+/// @brief Outputs encoded binary data in the base 64 format needed by VTK.
+///
+/// The binary data will be base 64 encoded, and compressed if @a
+/// compression_level is not zero. The proper header will be prepended to the
+/// data.
 void WriteVTKEncodedCompressed(std::ostream &out, const void *bytes,
                                uint32_t nbytes, int compression_level);
 
+/// @brief Return the VTK node index of the barycentric point @a b in a
+/// triangle with refinement level @a ref.
+///
+/// The barycentric index @a b has three components, satisfying b[0] + b[1] +
+/// b[2] == ref.
 int BarycentricToVTKTriangle(int *b, int ref);
 
+/// Determine the byte order and return either "BigEndian" or "LittleEndian"
 const char *VTKByteOrder();
 
+/// @brief Write either ASCII data to the stream or binary data to the buffer
+/// depending on the given format.
+///
+/// If @a format is VTK::ASCII, write the canonical ASCII representation of @a
+/// val to the output stream. Subnormal floating point numbers are rounded to
+/// zero. Otherwise, append its raw binary data to the byte buffer @a buf.
+///
+/// Note that there are specializations for @a uint8_t (to write as a numeric
+/// value rather than a character), and for @a float and @a double values to use
+/// the precision specified by @a format.
+template <typename T>
+void WriteBinaryOrASCII(std::ostream &out, std::vector<char> &buf, const T &val,
+                        const char *suffix, VTKFormat format)
+{
+   if (format == VTKFormat::ASCII) { out << val << suffix; }
+   else { bin_io::AppendBytes(buf, val); }
+}
+
+/// @brief Specialization of @ref WriteBinaryOrASCII for @a uint8_t to ensure
+/// ASCII output is numeric (rather than interpreting @a val as a character.)
+template <>
+void WriteBinaryOrASCII<uint8_t>(std::ostream &out, std::vector<char> &buf,
+                                 const uint8_t &val, const char *suffix,
+                                 VTKFormat format);
+
+/// @brief Specialization of @ref WriteBinaryOrASCII for @a double.
+///
+/// If @a format is equal to VTKFormat::BINARY32, @a val is converted to a @a
+/// float and written as 32 bits. Subnormals are rounded to zero in ASCII
+/// output.
+template <>
+void WriteBinaryOrASCII<double>(std::ostream &out, std::vector<char> &buf,
+                                const double &val, const char *suffix,
+                                VTKFormat format);
+
+/// @brief Specialization of @ref WriteBinaryOrASCII<T> for @a float.
+///
+/// If @a format is equal to VTKFormat::BINARY, @a val is converted to a @a
+/// double and written as 64 bits. Subnormals are rounded to zero in ASCII
+/// output.
+template <>
+void WriteBinaryOrASCII<float>(std::ostream &out, std::vector<char> &buf,
+                               const float &val, const char *suffix,
+                               VTKFormat format);
+
+/// @brief Encode in base 64 (and potentially compress) the given data, write it
+/// to the output stream (with a header) and clear the buffer.
+///
+/// @sa WriteVTKEncodedCompressed.
+void WriteBase64WithSizeAndClear(std::ostream &out, std::vector<char> &buf,
+                                 int compression_level);
+
 } // namespace mfem
 
 #endif

miniapps/autodiff/CMakeLists.txt

@@ -46,8 +46,8 @@ set(PARADIFF_COMMON_FILES
 	EXTRA_SOURCES ${PARADIFF_COMMON_SOURCES} ${SEQADIFF_COMMON_SOURCES}
 	EXTRA_HEADERS ${PARADIFF_COMMON_HEADERS} ${SEQADIFF_COMMON_HEADERS})
 
-message(STATUS "PARADIFF_COMMON_FILES: ${PARADIFF_COMMON_FILES}")
-message(STATUS "SEQADIFF_COMMON_FILES: ${SEQADIFF_COMMON_FILES}")
+# message(STATUS "PARADIFF_COMMON_FILES: ${PARADIFF_COMMON_FILES}")
+# message(STATUS "SEQADIFF_COMMON_FILES: ${SEQADIFF_COMMON_FILES}")
 
 add_mfem_miniapp(paradiff
      MAIN par_example.cpp

miniapps/common/pfem_extras.cpp

@@ -310,7 +310,7 @@ void VisualizeMesh(socketstream &sock, const char *vishost, int visport,
 }
 
 void VisualizeField(socketstream &sock, const char *vishost, int visport,
-                    ParGridFunction &gf, const char *title,
+                    const ParGridFunction &gf, const char *title,
                     int x, int y, int w, int h, const char *keys, bool vec)
 {
    ParMesh &pmesh = *gf.ParFESpace()->GetParMesh();

miniapps/common/pfem_extras.hpp

@@ -197,7 +197,7 @@ void VisualizeMesh(socketstream &sock, const char *vishost, int visport,
 /// specified host and port. Set the visualization window title, and optionally,
 /// its geometry.
 void VisualizeField(socketstream &sock, const char *vishost, int visport,
-                    ParGridFunction &gf, const char *title,
+                    const ParGridFunction &gf, const char *title,
                     int x = 0, int y = 0, int w = 400, int h = 400,
                     const char *keys = NULL, bool vec = false);
 

miniapps/gslib/findpts.cpp

@@ -30,6 +30,7 @@
 //    findpts -m ../../data/rt-2d-p4-tri.mesh -o 4
 //    findpts -m ../../data/inline-tri.mesh -o 3
 //    findpts -m ../../data/inline-quad.mesh -o 3
+//    findpts -m ../../data/inline-quad.mesh -o 3 -hr -pr
 //    findpts -m ../../data/inline-tet.mesh -o 3
 //    findpts -m ../../data/inline-hex.mesh -o 3
 //    findpts -m ../../data/inline-wedge.mesh -o 3
@@ -41,6 +42,63 @@
 using namespace mfem;
 using namespace std;
 
+// Experimental - required for visualizing functions on p-refined spaces.
+GridFunction* ProlongToMaxOrder(const GridFunction *x, const int fieldtype)
+{
+   const FiniteElementSpace *fespace = x->FESpace();
+   Mesh *mesh = fespace->GetMesh();
+   const FiniteElementCollection *fec = fespace->FEColl();
+
+   // find the max order in the space
+   int max_order = 1;
+   for (int i = 0; i < mesh->GetNE(); i++)
+   {
+      max_order = std::max(fespace->GetElementOrder(i), max_order);
+   }
+
+   // create a visualization space of max order for all elements
+   FiniteElementCollection *fecInt = NULL;
+   if (fieldtype == 0)
+   {
+      fecInt = new H1_FECollection(max_order, mesh->Dimension());
+   }
+   else if (fieldtype == 1)
+   {
+      fecInt = new L2_FECollection(max_order, mesh->Dimension());
+   }
+   FiniteElementSpace *spaceInt = new FiniteElementSpace(mesh, fecInt);
+
+   IsoparametricTransformation T;
+   DenseMatrix I;
+
+   GridFunction *xInt = new GridFunction(spaceInt);
+
+   // interpolate solution vector in the larger space
+   for (int i = 0; i < mesh->GetNE(); i++)
+   {
+      Geometry::Type geom = mesh->GetElementGeometry(i);
+      T.SetIdentityTransformation(geom);
+
+      Array<int> dofs;
+      fespace->GetElementDofs(i, dofs);
+      Vector elemvect, vectInt;
+      x->GetSubVector(dofs, elemvect);
+
+      const auto *fe = fec->GetFE(geom, fespace->GetElementOrder(i));
+      const auto *feInt = fecInt->GetFE(geom, max_order);
+
+      feInt->GetTransferMatrix(*fe, T, I);
+      spaceInt->GetElementDofs(i, dofs);
+      vectInt.SetSize(dofs.Size());
+
+      I.Mult(elemvect, vectInt);
+      xInt->SetSubVector(dofs, vectInt);
+   }
+
+   xInt->MakeOwner(fecInt);
+   return xInt;
+}
+
 // Scalar function to project
 double field_func(const Vector &x)
 {
@@ -66,6 +124,8 @@ int main (int argc, char *argv[])
    bool visualization    = true;
    int fieldtype         = 0;
    int ncomp             = 1;
+   bool hrefinement      = false;
+   bool prefinement      = false;
 
    // Parse command-line options.
    OptionsParser args(argc, argv);
@@ -84,6 +144,13 @@ int main (int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&hrefinement, "-hr", "--h-refinement", "-no-hr",
+                  "--no-h-refinement",
+                  "Do random h refinements to mesh.");
+   args.AddOption(&prefinement, "-pr", "--p-refinement", "-no-pr",
+                  "--no-p-refinement",
+                  "Do random p refinements to solution field.");
+
    args.Parse();
    if (!args.Good())
    {
@@ -105,6 +172,7 @@ int main (int argc, char *argv[])
    Vector pos_min, pos_max;
    MFEM_VERIFY(mesh_poly_deg > 0, "The order of the mesh must be positive.");
    mesh.GetBoundingBox(pos_min, pos_max, mesh_poly_deg);
+   if (hrefinement || prefinement) { mesh.EnsureNCMesh(); }
    cout << "--- Generating equidistant point for:\n"
         << "x in [" << pos_min(0) << ", " << pos_max(0) << "]\n"
         << "y in [" << pos_min(1) << ", " << pos_max(1) << "]\n";
@@ -113,6 +181,9 @@ int main (int argc, char *argv[])
       cout << "z in [" << pos_min(2) << ", " << pos_max(2) << "]\n";
    }
 
+   // Random h-refinements to mesh
+   if (hrefinement) { mesh.RandomRefinement(0.5); }
+
    // Curve the mesh based on the chosen polynomial degree.
    H1_FECollection fecm(mesh_poly_deg, dim);
    FiniteElementSpace fespace(&mesh, &fecm, dim);
@@ -153,10 +224,29 @@ int main (int argc, char *argv[])
    FiniteElementSpace sc_fes(&mesh, fec, ncomp);
    GridFunction field_vals(&sc_fes);
 
+   // Random p-refinements to the solution field
+   if (prefinement)
+   {
+      for (int e = 0; e < mesh.GetNE(); e++)
+      {
+         if (rand() % 2 == 0)
+         {
+            int element_order = sc_fes.GetElementOrder(e);
+            sc_fes.SetElementOrder(e, element_order + 1);
+         }
+      }
+      sc_fes.Update(false);
+      field_vals.Update();
+   }
+
    // Project the GridFunction using VectorFunctionCoefficient.
    VectorFunctionCoefficient F(vec_dim, F_exact);
    field_vals.ProjectCoefficient(F);
 
+   GridFunction *field_vals_pref = prefinement ?
+                                   ProlongToMaxOrder(&field_vals, fieldtype) :
+                                   &field_vals;
+
    // Display the mesh and the field through glvis.
    if (visualization)
    {
@@ -172,7 +262,7 @@ int main (int argc, char *argv[])
       else
       {
          sout.precision(8);
-         sout << "solution\n" << mesh << field_vals;
+         sout << "solution\n" << mesh << *field_vals_pref;
          if (dim == 2) { sout << "keys RmjA*****\n"; }
          if (dim == 3) { sout << "keys mA\n"; }
          sout << flush;
@@ -192,8 +282,8 @@ int main (int argc, char *argv[])
       for (int i = 0; i < ir.GetNPoints(); i++)
       {
          const IntegrationPoint &ip = ir.IntPoint(i);
-         vxyz(i)           = 100*pos_min(0) + ip.x * (pos_max(0)-pos_min(0));
-         vxyz(pts_cnt + i) = 100*pos_min(1) + ip.y * (pos_max(1)-pos_min(1));
+         vxyz(i)           = pos_min(0) + ip.x * (pos_max(0)-pos_min(0));
+         vxyz(pts_cnt + i) = pos_min(1) + ip.y * (pos_max(1)-pos_min(1));
       }
    }
    else
@@ -252,6 +342,7 @@ int main (int argc, char *argv[])
    // Free the internal gslib data.
    finder.FreeData();
 
+   if (prefinement) { delete field_vals_pref; }
    delete fec;
 
    return 0;

miniapps/gslib/pfindpts.cpp

@@ -30,6 +30,7 @@
 //    mpirun -np 2 pfindpts -m ../../data/rt-2d-p4-tri.mesh -o 4
 //    mpirun -np 2 pfindpts -m ../../data/inline-tri.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/inline-quad.mesh -o 3
+//    mpirun -np 2 pfindpts -m ../../data/inline-quad.mesh -o 3 -hr
 //    mpirun -np 2 pfindpts -m ../../data/inline-tet.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/inline-hex.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/inline-wedge.mesh -o 3
@@ -75,6 +76,7 @@ int main (int argc, char *argv[])
    int fieldtype         = 0;
    int ncomp             = 1;
    bool search_on_rank_0 = false;
+   bool hrefinement      = false;
 
    // Parse command-line options.
    OptionsParser args(argc, argv);
@@ -98,6 +100,10 @@ int main (int argc, char *argv[])
    args.AddOption(&search_on_rank_0, "-sr0", "--search-on-r0", "-no-sr0",
                   "--no-search-on-r0",
                   "Enable search only on rank 0 (disable to search points on all tasks).");
+   args.AddOption(&hrefinement, "-hr", "--h-refinement", "-no-hr",
+                  "--no-h-refinement",
+                  "Do random h refinements to mesh.");
+
    args.Parse();
    if (!args.Good())
    {
@@ -134,10 +140,14 @@ int main (int argc, char *argv[])
    }
 
    // Distribute the mesh.
+   if (hrefinement) { mesh->EnsureNCMesh(); }
    ParMesh pmesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    for (int lev = 0; lev < rp_levels; lev++) { pmesh.UniformRefinement(); }
 
+   // Random h-refinements to mesh
+   if (hrefinement) { pmesh.RandomRefinement(0.5); }
+
    // Curve the mesh based on the chosen polynomial degree.
    H1_FECollection fecm(mesh_poly_deg, dim);
    ParFiniteElementSpace pfespace(&pmesh, &fecm, dim);

miniapps/meshing/CMakeLists.txt

@@ -99,13 +99,13 @@ if (MFEM_USE_MPI)
 
   # Add parallel tests.
   if (MFEM_ENABLE_TESTING)
-    add_test(NAME pmesh-optimizer_np=4
+    add_test(NAME pmesh-optimizer_np=${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pmesh-optimizer> -no-vis
       -m ${CMAKE_CURRENT_SOURCE_DIR}/icf.mesh
       ${MPIEXEC_POSTFLAGS})
 
-    add_test(NAME pminimal-surface_np=4
+    add_test(NAME pminimal-surface_np=${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pminimal-surface> -no-vis
       ${MPIEXEC_POSTFLAGS})