minor edits

Fix warning from assigned and unused variables

.github/workflows/builds-and-tests.yml

@@ -162,7 +162,7 @@ jobs:
 
     - name: get hypre
       if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -171,7 +171,7 @@ jobs:
 
     - name: get hypre (Windows)
       if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -190,7 +190,7 @@ jobs:
 
     - name: install metis
       if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
       with:
         archive: ${{ env.METIS_ARCHIVE }}
         dir: ${{ env.METIS_TOP_DIR }}
@@ -217,7 +217,7 @@ jobs:
 
     # MFEM build and test
     - name: build
-      uses: mfem/github-actions/build-mfem@v2.3
+      uses: mfem/github-actions/build-mfem@v2.4
       env:
         VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
       with:
@@ -263,13 +263,15 @@ jobs:
       if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
       run: |
         CTEST_CONFIG="Release"
-        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
+        cd ${{ env.MFEM_TOP_DIR }}/build && \
+        ctest --output-on-failure -C ${CTEST_CONFIG} || \
+        ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
       shell: bash
 
     # Code coverage (process and upload reports)
     - name: codecov
       if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.2
+      uses: mfem/github-actions/upload-coverage@v2.4
       with:
         name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
         project_dir: ${{ env.MFEM_TOP_DIR }}

.github/workflows/mfem-analysis.yml

@@ -57,7 +57,7 @@ jobs:
 
     - name: Get Hypre
       if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -72,14 +72,14 @@ jobs:
 
     - name: Install Metis
       if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
       with:
         archive: ${{ env.METIS_ARCHIVE }}
         dir: ${{ env.METIS_TOP_DIR }}
 
     # MFEM build and test
     - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.4
       with:
         os: ${{ runner.os }}
         target: opt

.github/workflows/mfem-sanitizer.yml

@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+name: "Sanitizer"
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches:
+      - master
+      - next
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  Serial:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
+
+    - name: MFEM Checkout
+      uses: actions/checkout@v3
+      with:
+        path: mfem
+
+    - name: MFEM Build
+      uses: mfem/github-actions/build-mfem@v2.4
+      with:
+        os: ${{ runner.os }}
+        target: opt
+        mpi: seq
+        hypre-dir: unused-hypre-dir
+        metis-dir: unused-metis-dir
+        mfem-dir: mfem
+        build-system: make
+        library-only: false
+        config-options:
+          CXX="clang++-14"
+          CXXFLAGS="-g -O1 -std=c++11
+                    -fsanitize=address
+                    -fno-omit-frame-pointer
+                    -fsanitize-address-use-after-scope"
+
+    - name: MFEM Info
+      working-directory: mfem
+      run: make info
+
+    - name: MFEM Sanitize
+      working-directory: mfem
+      run:
+        ASAN_OPTIONS="detect_leaks=1,
+                      strict_init_order=1,
+                      strict_string_checks=1,
+                      check_initialization_order=1,
+                      detect_stack_use_after_return=1"
+        make test

.gitignore

@@ -29,6 +29,7 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
+config/user.cmake
 config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
@@ -112,6 +113,12 @@ examples/ex25p-*.*
 examples/ex28_*
 examples/ex28p_*
 examples/flux.*
+examples/dsol.*
+examples/cond.*
+examples/cond_j.*
+examples/cond_mesh.*
+examples/port_mesh.*
+examples/port_mode.*
 
 examples/amgx/ex1
 examples/amgx/ex1p
@@ -214,6 +221,7 @@ miniapps/meshing/pmesh-fitting
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
+miniapps/meshing/mesh-quality
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -316,12 +324,28 @@ miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*
 
+miniapps/hdiv-linear-solver/darcy
+miniapps/hdiv-linear-solver/grad_div
+
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh
 
 miniapps/multidomain/multidomain
 miniapps/hooke/hooke
 
+miniapps/dpg/diffusion
+miniapps/dpg/pdiffusion
+miniapps/dpg/convection-diffusion
+miniapps/dpg/pconvection-diffusion
+miniapps/dpg/acoustics
+miniapps/dpg/pacoustics
+miniapps/dpg/maxwell
+miniapps/dpg/pmaxwell
+miniapps/dpg/ParaView
+
+miniapps/spde/generate_random_field
+miniapps/spde/ParaView
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -334,6 +358,8 @@ tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
 tests/unit/debug_device_tests
+tests/unit/parallel_in_serial.mesh
+tests/unit/parallel_in_serial.gf
 
 # Benchmark binaries
 tests/benchmarks/bench_ceed

.gitlab/configs/report-build-and-test.yml

@@ -22,12 +22,10 @@
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date
@@ -57,12 +55,10 @@
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/configs/setup-baseline.yml

@@ -47,12 +47,10 @@ setup_baseline:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/configs/setup-build-and-test.yml

@@ -35,13 +35,11 @@ setup:
       (
         date
         echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+        # try every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/mfem-data.lock'"
         date
@@ -69,12 +67,10 @@ setup:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/lassen-build-and-test.yml

@@ -14,14 +14,14 @@ stages:
   - build_and_test
   - report
 
-opt_mpi_cuda_xl_16_1_1_8:
+opt_mpi_cuda_xl_16_1_1_12:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 opt_mpi_cuda_hypre_cuda_xl:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 # Jobs report

.gitlab/quartz-baseline.yml

@@ -51,6 +51,8 @@ cleanup:
   script:
     - echo "BUILD_ROOT=${BUILD_ROOT}"
     - rm -rf "${BUILD_ROOT}" || true
+    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
+    - make -C "${CI_PROJECT_DIR}" distclean
 
 report_baseline:
   extends: [.on_quartz]
@@ -66,12 +68,10 @@ report_baseline:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date
@@ -82,12 +82,14 @@ report_baseline:
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
         cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
-          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
-        fi
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
+        # We create an autotest-email.html file, because that's how we signal
+        # that there was an error / diff (temporary).
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+           [[ -f ${rundir}/${BASELINE_TEST}-${SYS_TYPE}.diff ]]; then
+          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
+        fi
         msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
         if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
           git pull && \

.gitlab/quartz-build-and-test.yml

@@ -27,39 +27,39 @@ allocate_resource:
   timeout: 6h
 
 # GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_6_1_0:
+debug_ser_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
+    SPEC: "%gcc@10.3.1 +debug~mpi"
   extends: .build_and_test_on_quartz
 
-debug_par_gcc_6_1_0:
+debug_par_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
+    SPEC: "%gcc@10.3.1 +debug+mpi"
   extends: .build_and_test_on_quartz
 
-opt_ser_gcc_6_1_0:
+opt_ser_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
+    SPEC: "%gcc@10.3.1 ~mpi"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0:
+opt_par_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0"
+    SPEC: "%gcc@10.3.1"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_sundials:
+opt_par_gcc_10_sundials:
   variables:
-    SPEC: "%gcc@6.1.0 +sundials"
+    SPEC: "%gcc@10.3.1 +sundials"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_petsc:
+opt_par_gcc_10_petsc:
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_pumi:
+opt_par_gcc_10_pumi:
   variables:
-    SPEC: "%gcc@6.1.0 +pumi"
+    SPEC: "%gcc@10.3.1 +pumi"
   extends: .build_and_test_on_quartz
 
 # Release

.gitlab/scripts/baseline

@@ -42,47 +42,34 @@ fi
 # post
 mkdir ${artifacts_path}
 
-if [[ -s ${glob_err} ]]
-then
-  echo "ERROR during ${BASELINE_TEST} execution";
-  echo "Here is the ${glob_err} file content";
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  exit 1;
-elif [[ ! -f ${base_patch} && ! -f ${base_out} ]]
-then
-  echo "Something went WRONG in ${BASELINE_TEST}:";
-  echo "Either ${base_patch} or ${base_out} should exists";
-  exit 1;
-elif [[ -f ${base_patch} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-
+status=0
 if [[ -f ${BASELINE_TEST}.out ]]; then
   cp ${BASELINE_TEST}.out ${artifacts_path}
 fi
-
+if [[ -s ${glob_err} ]]; then
+  echo "ERROR during ${BASELINE_TEST} execution"
+  echo "Here is the ${glob_err} file content"
+  cat ${glob_err}
+  cp ${glob_err} ${artifacts_path}/${glob_err}
+  status=1
+fi
+if [[ -f ${base_patch} ]]; then
+  echo "${BASELINE_TEST}: Differences found, patch generated"
+  cp ${base_patch} ${artifacts_path}/${base_patch}
+elif [[ -f ${base_out} ]]; then
+  echo "${BASELINE_TEST}: Differences found, replacement file generated"
+  cp ${base_out} ${artifacts_path}/${base_out}
+fi
 # base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]
-then
+if [[ -f ${base_diff} ]]; then
   echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
   cat ${base_diff}
   cp ${base_diff} ${artifacts_path}/${base_diff}
-  # We create a .err file, because that's how we signal that there was a diff.
-  cp ${base_diff} ${artifacts_path}/gitlab-${BASELINE_TEST}-${MACHINE_NAME}.err
+  status=1
 fi
-
-if [[ ! -s ${base_diff} ]]
-then
+if [[ $status -eq 0 ]]; then
   echo "${BASELINE_TEST}: PASSED"
-  true
 else
   echo "${BASELINE_TEST}: FAILED"
-  false
 fi
+exit $status

CHANGELOG

@@ -11,22 +11,85 @@
 Version 4.5.3 (development)
 ===========================
 
+New and updated examples and miniapps
+-------------------------------------
+- Added a new example code, Example 36/36p, to demonstrate the solution of
+  the obstacle problem with a new finite element method.
+
+- Added a new miniapp, Mesh Quality, for evaluating mesh quality using size,
+  skewness, and aspect-ratio computed from the Jacobian of the transformation.
+
+- Added a new miniapp for interface and boundary fitting to implicit domains
+  defined using level-set functions. See miniapps/meshing/pmesh-fitting.cpp
+
+- Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes serial
+  and parallel examples for diffusion, convection-diffusion, acoustics and
+  Maxwell equations. The miniapp includes new classes such as (Par)DPGWeakForm,
+  (Par)ComplexDPGWeakForm and (Complex)BlockStaticCondensation. Three new
+  integrators are added in support of DPG systems: TraceIntegrator,
+  NormalTraceIntegrator and TangentTraceIntegrator.
+
+- Added new SubMesh examples demonstrating source terms and boundary conditions
+  transferred from SubMesh objects.
+
+- Added a new H(div) solvers miniapp in miniapps/hdiv-linear-solver,
+  demonstrating the use of a matrix-free saddle-point solver methodology,
+  suitable for high-order discretizations and for GPU acceleration. Examples
+  illustrating the solution of Darcy and grad-div problems are included.
+
+- Added a random refinement option to the mesh-explorer miniapp to assist users
+  in experimenting with nonconforming meshes.
+
+- Moved the distance solver methods from miniapps/shifted to miniapps/common.
+
+Meshing improvements
+--------------------
 - Added new methods in the Mesh class to set and get attributes on NURBS patches
   and patch boundaries.
 
-New and updated examples and miniapps
--------------------------------------
-- Added a miniapp pmesh-fitting in miniapps/meshing for interface and boundary fitting to implicit domains defined using level-set functions.
+- Added HIP support to the SUNDIALS interface.
 
-- Moved the distance solver methods from miniapps/shifted to miniapps/common. 
+- TMOP improvement: added asymptotically-balanced compound metrics 90, 94, 328,
+  338. Added the tmop-metric-magnitude tool for tracking how metrics change
+  under geometric perturbations.
 
 Discretization improvements
 ---------------------------
 - Face restriction operators for Nedelec and Raviart-Thomas finite element
   spaces are now supported through the ConformingFaceRestriction class.
 
+- SubMesh and ParSubMesh have been extended to support the transfer of
+  Nedelec and Raviart-Thomas finite element spaces.
+
 - VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
 
+- Added support for p-refined meshes in FindPointsGSLIB.
+
+Linear and nonlinear solvers
+----------------------------
+- Updated interface to MUMPS direct solver to support multiple right-hand
+  sides, block low-rank compression, builds using 64-bit integers, and other
+  improvements.
+
+- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
+  This interface provides a serial (OpenMP shared memory) version of Pardiso for
+  use with SparseMatrix. This complements the existing parallel (MPI distributed
+  memory) version already available through the CPardiso MFEM integration.
+
+Integrations, testing and documentation
+---------------------------------------
+- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
+  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
+
+Miscellaneous
+-------------
+- Improved lambda body debugging with the addition of mfem::forall functions.
+  These functions can take the place of the MFEM_FORALL macros, which have been
+  preserved for backwards compatibility.
+
+- Reorganized files for bilinear form, linear form, and nonlinear form integrators
+  in the fem/integ/ subdirectory.
+
 
 Version 4.5.2, released on March 23, 2023
 =========================================

CMakeLists.txt

@@ -82,7 +82,7 @@ if (MFEM_USE_CONDUIT OR
    #  * find_package(PETSc REQUIRED)
    set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK)
+if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
@@ -317,6 +317,9 @@ if (MFEM_USE_SUNDIALS)
   if (MFEM_USE_CUDA)
     list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
   endif()
+  if (MFEM_USE_HIP)
+    list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
+  endif()
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
@@ -333,6 +336,7 @@ endif()
 if (MFEM_USE_MUMPS)
   if (MFEM_USE_MPI)
     find_package(MUMPS REQUIRED mumps_common pord)
+    set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})
   else()
     message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
   endif()
@@ -466,12 +470,18 @@ if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
 endif()
 
+# MKL CPardiso
 if (MFEM_USE_MKL_CPARDISO)
   if (MFEM_USE_MPI)
    find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
   endif()
 endif()
 
+# MKL Pardiso
+if (MFEM_USE_MKL_PARDISO)
+   find_package(MKL_PARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64)
+endif()
+
 # PARELAG
 if (MFEM_USE_PARELAG)
    find_package(PARELAG REQUIRED)
@@ -521,8 +531,8 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")

CONTRIBUTING.md

@@ -121,6 +121,7 @@ The MFEM source code has the following structure:
   ├── fem
   │   ├── ceed
   │   ├── fe
+  │   ├── integ
   │   ├── lor
   │   ├── moonolith
   │   ├── qinterp
@@ -136,6 +137,7 @@ The MFEM source code has the following structure:
   │   ├── common
   │   ├── electromagnetics
   │   ├── gslib
+  │   ├── hdiv-linear-solver
   │   ├── hooke
   │   ├── meshing
   │   ├── mtop
@@ -209,7 +211,7 @@ device/host memory manager.
 - The main device-relevant classes and sources are:
   + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
   + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](https://docs.mfem.org/html/forall_8hpp.html) macro
+  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
   + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
 
 #### Utilities, building and documentation

INSTALL

@@ -628,9 +628,13 @@ The specific libraries and their options are:
   both MPI and hypre.
   If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
   for CUDA.
-  URL: http://computation.llnl.gov/projects/sundials/sundials-software
+  If MFEM_USE_HIP is enabled, we expect that SUNDIALS is built with support
+  for HIP.
+  URL: http://computing.llnl.gov/projects/sundials/sundials-software
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.
+  Versions: SUNDIALS >= 5.0.0,
+            SUNDIALS >= 5.4.0 for CUDA support, and
+            SUNDIALS >= 5.7.0 for HIP support.
 
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html

config/cmake/MFEMConfig.cmake.in

@@ -55,6 +55,8 @@ set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
 set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
 set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
+set(MFEM_USE_MKL_CPARDISO @MFEM_USE_MKL_CPARDISO@)
+set(MFEM_USE_MKL_PARDISO @MFEM_USE_MKL_PARDISO@)
 set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)

config/cmake/config.hpp.in

@@ -80,96 +80,101 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC
 
+// Which library functions to use in class StopWatch for measuring time.
+// For a list of the available options, see INSTALL.
+// If not defined, an option is selected automatically.
+#cmakedefine MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+
+// Enable MFEM functionality based on the SUNDIALS libraries.
+#cmakedefine MFEM_USE_SUNDIALS
+
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE
 
 // Enable MFEM functionality based on the SuperLU_DIST library.
 #cmakedefine MFEM_USE_SUPERLU
+#cmakedefine MFEM_USE_SUPERLU5
 
 // Enable MFEM functionality based on the MUMPS library.
 #cmakedefine MFEM_USE_MUMPS
+#cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK
 
-// Enable functionality based on the Ginkgo library
+// Enable functionality based on the Ginkgo library.
 #cmakedefine MFEM_USE_GINKGO
 
-// Enable MFEM functionality based on the AmgX library
+// Enable MFEM functionality based on the AmgX library.
 #cmakedefine MFEM_USE_AMGX
 
-// Enable MFEM functionality based on the GnuTLS library
+// Enable secure socket streams based on the GNUTLS library.
 #cmakedefine MFEM_USE_GNUTLS
 
-// Enable MFEM functionality based on the GSLIB library
-#cmakedefine MFEM_USE_GSLIB
-
-// Enable MFEM functionality based on the NetCDF library
-#cmakedefine MFEM_USE_NETCDF
-
-// Enable MFEM functionality based on the PETSc library
-#cmakedefine MFEM_USE_PETSC
-
-// Enable MFEM functionality based on the SLEPc library
-#cmakedefine MFEM_USE_SLEPC
-
-// Enable MFEM functionality based on the Sidre library
+// Enable Sidre support.
 #cmakedefine MFEM_USE_SIDRE
 
-// Enable the use of SIMD in the high performance templated classes
+// Enable the use of SIMD in the high performance templated classes.
 #cmakedefine MFEM_USE_SIMD
 
-// Enable MFEM functionality based on the FMS library
+// Enable FMS support.
 #cmakedefine MFEM_USE_FMS
 
-// Enable MFEM functionality based on Conduit
+// Enable Conduit support.
 #cmakedefine MFEM_USE_CONDUIT
 
-// Enable MFEM functionality based on the PUMI library
+// Enable functionality based on the NetCDF library (reading CUBIT files).
+#cmakedefine MFEM_USE_NETCDF
+
+// Enable functionality based on the PETSc library.
+#cmakedefine MFEM_USE_PETSC
+
+// Enable functionality based on the SLEPc library.
+#cmakedefine MFEM_USE_SLEPC
+
+// Enable functionality based on the MPFR library.
+#cmakedefine MFEM_USE_MPFR
+
+// Enable MFEM functionality based on the PUMI library.
 #cmakedefine MFEM_USE_PUMI
 
-// Enable MFEM functionality based on the Moonolith library
+// Enable Moonolith-based general interpolation between finite element spaces.
 #cmakedefine MFEM_USE_MOONOLITH
 
-// Enable MFEM functionality based on the HiOp library
+// Enable MFEM functionality based on the HIOP library.
 #cmakedefine MFEM_USE_HIOP
 
-// Build the GPU/CUDA-enabled version of the MFEM library.
+// Enable MFEM functionality based on the GSLIB library.
+#cmakedefine MFEM_USE_GSLIB
+
+// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
 
-// Build the HIP-enabled version of the MFEM library.
+// Build the AMD GPU/HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 #cmakedefine MFEM_USE_HIP
 
-// Enable MFEM functionality based on the RAJA library
+// Enable functionality based on the RAJA library.
 #cmakedefine MFEM_USE_RAJA
 
-// Enable MFEM functionality based on the OCCA library
+// Enable functionality based on the OCCA library.
 #cmakedefine MFEM_USE_OCCA
 
-// Enable MFEM functionality based on the libCEED library
+// Enable functionality based on the libCEED library.
 #cmakedefine MFEM_USE_CEED
 
-// Enable MFEM functionality based on the Umpire library
-#cmakedefine MFEM_USE_UMPIRE
-
-// Enable MFEM functionality based on the ADIOS2 library
-#cmakedefine MFEM_USE_ADIOS2
-
-// Enable MFEM functionality based on the Caliper library
+// Enable functionality based on the Caliper library.
 #cmakedefine MFEM_USE_CALIPER
 
-// Enable MFEM functionality based on the Algoim library
+// Enable functionality based on the Algoim library.
 #cmakedefine MFEM_USE_ALGOIM
 
-// Which library functions to use in class StopWatch for measuring time.
-// For a list of the available options, see INSTALL.
-// If not defined, an option is selected automatically.
-#define MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+// Enable functionality based on the Umpire library.
+#cmakedefine MFEM_USE_UMPIRE
 
-// Enable MFEM functionality based on the SUNDIALS libraries.
-#cmakedefine MFEM_USE_SUNDIALS
+// Enable IO functionality based on the ADIOS2 library.
+#cmakedefine MFEM_USE_ADIOS2
 
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
@@ -181,13 +186,16 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO
 
-// Use forward mode for automatic differentiation
+// Enable interface to the MKL Pardiso library.
+#cmakedefine MFEM_USE_MKL_PARDISO
+
+// Use forward mode for automatic differentiation.
 #cmakedefine MFEM_USE_ADFORWARD
 
-// Enable the use of the CoDiPack library for AD
+// Enable the use of the CoDiPack library for AD.
 #cmakedefine MFEM_USE_CODIPACK
 
-// Enable MFEM functionality based on the Google Benchmark library.
+// Enable functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK
 
 // Enable Enzyme for AD

config/cmake/modules/FindMKL_PARDISO.cmake

@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - MKL_PARDISO_FOUND
+#   - MKL_PARDISO_LIBRARIES
+#   - MKL_PARDISO_INCLUDE_DIRS
+
+if(NOT MKL_LIBRARY_DIR)
+  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
+  set(MKL_LIBRARY_DIR "lib/intel64")
+endif()
+
+include(MfemCmakeUtilities)
+mfem_find_package(MKL_PARDISO MKL_PARDISO
+    MKL_PARDISO_DIR "include" mkl_pardiso.h ${MKL_LIBRARY_DIR} mkl_core
+  "Paths to headers required by MKL Pardiso." "Libraries required by MKL PARDISO."
+  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
+  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential)

config/cmake/modules/FindMUMPS.cmake

@@ -11,8 +11,9 @@
 
 # Sets the following variables:
 #   - MUMPS_FOUND
-#   - MUMPS_INCLUDE_DIRS
 #   - MUMPS_LIBRARIES
+#   - MUMPS_INCLUDE_DIRS
+#   - MUMPS_VERSION
 
 include(MfemCmakeUtilities)
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
@@ -21,3 +22,18 @@ mfem_find_package(MUMPS MUMPS MUMPS_DIR
   "Libraries required by MUMPS."
   ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
   ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
+
+if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
+  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
+          ${CMAKE_CURRENT_BINARY_DIR}/config
+          ${CMAKE_CURRENT_SOURCE_DIR}/config/get_mumps_version.cpp
+          CMAKE_FLAGS -DINCLUDE_DIRECTORIES:STRING=${MUMPS_INCLUDE_DIRS}
+          RUN_OUTPUT_VARIABLE MUMPS_VERSION_OUTPUT)
+  if ((MUMPS_VERSION_RUN_RESULT EQUAL 0) AND MUMPS_VERSION_OUTPUT)
+    string(STRIP "${MUMPS_VERSION_OUTPUT}" MUMPS_VERSION)
+    set(MUMPS_VERSION ${MUMPS_VERSION} CACHE STRING "MUMPS version." FORCE)
+    message(STATUS "Found MUMPS version ${MUMPS_VERSION}")
+  else()
+    message(FATAL_ERROR "Unable to determine MUMPS version.")
+  endif()
+endif()

config/cmake/modules/FindSUNDIALS.cmake

@@ -22,8 +22,8 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
     "include" nvector/nvector_serial.h "lib" sundials_nvecserial
   ADD_COMPONENT NVector_Cuda
     "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_ParHyp
-    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
+  ADD_COMPONENT NVector_Hip
+    "include" nvector/nvector_hip.h "lib" sundials_nvechip
   ADD_COMPONENT NVector_Parallel
     "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
   ADD_COMPONENT NVector_MPIPlusX

config/config.hpp.in

@@ -30,10 +30,10 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)
 
-// The absolute path of the MFEM source prefix
+// The absolute path of the MFEM source prefix.
 // #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
 
-// The absolute path of the MFEM installation prefix
+// The absolute path of the MFEM installation prefix.
 // #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
 
 // Description of the git commit used to build MFEM.
@@ -91,7 +91,7 @@
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE
 
-// Enable MFEM functionality based on the SuperLU library.
+// Enable MFEM functionality based on the SuperLU_DIST library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5
 
@@ -102,40 +102,40 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
 
-// Enable MFEM features based on the Ginkgo library
+// Enable MFEM features based on the Ginkgo library.
 // #define MFEM_USE_GINKGO
 
 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX
 
-// Enable secure socket streams based on the GNUTLS library
+// Enable secure socket streams based on the GNUTLS library.
 // #define MFEM_USE_GNUTLS
 
-// Enable Sidre support
+// Enable Sidre support.
 // #define MFEM_USE_SIDRE
 
-// Enable the use of SIMD in the high performance templated classes
+// Enable the use of SIMD in the high performance templated classes.
 // #define MFEM_USE_SIMD
 
-// Enable FMS support
+// Enable FMS support.
 // #define MFEM_USE_FMS
 
-// Enable Conduit support
+// Enable Conduit support.
 // #define MFEM_USE_CONDUIT
 
-// Enable functionality based on the NetCDF library (reading CUBIT files)
+// Enable functionality based on the NetCDF library (reading CUBIT files).
 // #define MFEM_USE_NETCDF
 
-// Enable functionality based on the PETSc library
+// Enable functionality based on the PETSc library.
 // #define MFEM_USE_PETSC
 
-// Enable functionality based on the SLEPc library
+// Enable functionality based on the SLEPc library.
 // #define MFEM_USE_SLEPC
 
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
-// Enable MFEM functionality based on the PUMI library
+// Enable MFEM functionality based on the PUMI library.
 // #define MFEM_USE_PUMI
 
 // Enable Moonolith-based general interpolation between finite element spaces.
@@ -144,7 +144,7 @@
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP
 
-// Enable MFEM functionality based on the GSLIB library
+// Enable MFEM functionality based on the GSLIB library.
 // #define MFEM_USE_GSLIB
 
 // Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
@@ -186,10 +186,13 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO
 
-// Use forward mode for automatic differentiation
+// Enable interface to the MKL Pardiso library.
+// #define MFEM_USE_MKL_PARDISO
+
+// Use forward mode for automatic differentiation.
 // #define MFEM_USE_ADFORWARD
 
-// Enable the use of the CoDiPack library for AD
+// Enable the use of the CoDiPack library for AD.
 // #define MFEM_USE_CODIPACK
 
 // Enable functionality based on the Google Benchmark library.

config/config.mk.in

@@ -57,6 +57,7 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
+MFEM_USE_MKL_PARDISO   = @MFEM_USE_MKL_PARDISO@
 MFEM_USE_MOONOLITH     = @MFEM_USE_MOONOLITH@
 MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@

config/defaults.cmake

@@ -60,6 +60,7 @@ option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
+option(MFEM_USE_MKL_PARDISO "Enable MKL Pardiso" OFF)
 option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
@@ -134,16 +135,18 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
     "Additional packages required by ParMETIS.")
 
-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_8.1.2" CACHE PATH
     "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
-set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
+set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
+    "LAPACK" "BLAS" CACHE STRING
     "Additional packages required by SuperLU_DIST.")
 
-set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
+set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
     "Path to the MUMPS library.")
-# Packages required by MUMPS, depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
+# MUMPS may also depend on "OpenMP", depending on how it was compiled.
+set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
+    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
     "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -226,6 +229,8 @@ set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
 set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
+set(MKL_PARDISO_DIR "" CACHE STRING "MKL installation path.")
+
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")

config/defaults.mk

@@ -160,6 +160,7 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_MKL_PARDISO   = NO
 MFEM_USE_MOONOLITH     = NO
 MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
@@ -266,6 +267,9 @@ endif
 ifeq ($(MFEM_USE_CUDA),YES)
    SUNDIALS_LIB += -lsundials_nveccuda
 endif
+ifeq ($(MFEM_USE_HIP),YES)
+   SUNDIALS_LIB += -lsundials_nvechip
+endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
@@ -284,10 +288,10 @@ ifeq ($(MFEM_USE_SUPERLU5),YES)
    SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
       -lsuperlu_dist_5.1.0
 else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_8.1.2
    SUPERLU_OPT = -I$(SUPERLU_DIR)/include
    SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
-      -lsuperlu_dist -lblas
+      -lsuperlu_dist $(LAPACK_LIB)
 endif
 
 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
@@ -311,7 +315,7 @@ MPI_FORTRAN_LIB = -lmpifort
 # MPI_FORTRAN_LIB += -lgfortran
 
 # MUMPS library configuration
-MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
+MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
 MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
  -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
@@ -484,7 +488,6 @@ ifdef GOTCHA_DIR
    CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
 endif
 
-
 # BLITZ library configuration
 BLITZ_DIR = @MFEM_DIR@/../blitz
 BLITZ_OPT = -I$(BLITZ_DIR)/include
@@ -539,6 +542,14 @@ MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
    -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
    -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
 
+# MKL Pardiso library configuration
+MKL_PARDISO_DIR ?=
+MKL_LIBRARY_SUBDIR ?= lib
+MKL_PARDISO_OPT = -I$(MKL_PARDISO_DIR)/include
+MKL_PARDISO_LIB = $(XLINKER)-rpath,$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+   -L$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+
 # PARELAG library configuration
 PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src

data/disk.mesh

@@ -1,109 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-2
-
-elements
-5
-1 3 4 5 6 7
-1 3 0 1 5 4
-1 3 1 2 6 5
-1 3 3 7 6 2
-1 3 0 4 7 3
-
-boundary
-4
-1 1 0 1
-1 1 2 3
-1 1 1 2
-1 1 3 0
-
-edges
-12
-0 0 1
-0 4 5
-0 7 6
-0 3 2
-1 1 2
-1 5 6
-1 4 7
-1 0 3
-2 0 4
-2 1 5
-2 2 6
-2 3 7
-
-vertices
-8
-
-knotvectors
-3
-2 3 0 0 0 1 1 1
-2 3 0 0 0 1 1 1
-2 3 0 0 0 1 1 1
-
-weights
-1
-1
-1
-1
-1
-1
-1
-1
-0.70710678118655
-1
-1
-0.70710678118655
-0.70710678118655
-1
-1
-0.70710678118655
-1
-1
-1
-1
-1
-0.85355339059327
-0.85355339059327
-0.85355339059327
-0.85355339059327
-
-FiniteElementSpace
-FiniteElementCollection: NURBS2
-VDim: 2
-Ordering: 1
-
--0.70710678118 -0.70710678118
-0.70710678118 -0.70710678118
-0.70710678118 0.70710678118
--0.70710678118 0.70710678118
--0.35355339059 -0.35355339059
-0.35355339059 -0.35355339059
-0.35355339059 0.35355339059
--0.35355339059 0.35355339059
-0 -1.41421356236
-0 -0.35355339059
-0 0.35355339059
-0 1.41421356236
-1.41421356236 0
-0.35355339059 0
--0.35355339059 0
--1.41421356236 0
--0.530330085885 -0.530330085885
-0.530330085885 -0.530330085885
-0.530330085885 0.530330085885
--0.530330085885 0.530330085885
-0 0
-0 -0.883883476475
-0.883883476475 0
-0 0.883883476475
--0.883883476475 0

data/fichera-quad-mixed.mesh

@@ -0,0 +1,48 @@
+MFEM mesh v1.0
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+6
+1 3 0 1 4 3
+1 3 2 3 6 5
+1 2 3 4 8
+1 2 4 7 8
+1 2 7 6 8
+1 2 6 3 8
+
+boundary
+8
+1 1 0 1
+2 1 1 4
+3 1 4 7
+4 1 7 6
+5 1 6 5
+6 1 5 2
+7 1 2 3
+8 1 3 0
+
+vertices
+9
+2
+0.5 0
+1 0
+0 0.5
+0.5 0.5
+1 0.5
+0 1
+0.5 1
+1 1
+0.75 0.75

data/fichera-quad.mesh

@@ -0,0 +1,44 @@
+MFEM mesh v1.0
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+3
+1 3 0 1 4 3
+1 3 2 3 6 5
+1 3 3 4 7 6
+
+boundary
+8
+1 1 0 1
+2 1 1 4
+3 1 4 7
+4 1 7 6
+5 1 6 5
+6 1 5 2
+7 1 2 3
+8 1 3 0
+
+vertices
+8
+2
+0.5 0
+1 0
+0 0.5
+0.5 0.5
+1 0.5
+0 1
+0.5 1
+1 1

data/llnl-p3.mesh

@@ -0,0 +1,322 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+2
+
+elements
+26
+1 2 1 18 0
+1 3 1 3 19 18
+2 3 3 6 20 19
+1 3 6 9 21 20
+2 3 9 12 22 21
+1 3 12 15 23 22
+2 2 23 15 24
+1 2 1 4 3
+2 3 4 7 6 3
+1 3 7 10 9 6
+2 3 10 13 12 9
+1 3 13 16 15 12
+2 3 16 25 24 15
+1 3 2 5 4 1
+1 3 5 8 7 4
+1 3 8 11 10 7
+1 3 11 14 13 10
+1 3 14 17 16 13
+1 2 25 16 17
+1 3 18 19 27 26
+2 3 19 20 28 27
+1 3 20 21 29 28
+2 3 21 22 30 29
+1 3 22 23 31 30
+2 3 23 24 32 31
+1 3 24 25 33 32
+
+boundary
+18
+1 1 28 27
+2 1 30 29
+3 1 32 31
+4 1 0 1
+4 1 1 2
+4 1 2 5
+4 1 5 8
+4 1 8 11
+4 1 11 14
+4 1 14 17
+4 1 17 25
+4 1 25 33
+4 1 33 32
+4 1 31 30
+4 1 29 28
+4 1 27 26
+4 1 26 18
+4 1 18 0
+
+vertices
+34
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P3
+VDim: 2
+Ordering: 1
+
+0 0
+0.53125 0
+1 0
+0.53125 0.09375
+0.5625 0.09375
+1 0.09375
+0.53125 0.21875
+0.6875 0.21875
+1 0.1875
+0.53125 0.25
+0.71875 0.25
+1 0.25
+0.53125 0.375
+0.84375 0.375
+1 0.34375
+0.53125 0.40625
+0.875 0.40625
+1 0.40625
+0 0.53125
+0.09375 0.53125
+0.21875 0.53125
+0.25 0.53125
+0.375 0.53125
+0.40625 0.53125
+0.53125 0.53125
+1 0.53125
+0 1
+0.09375 1
+0.21875 1
+0.25 1
+0.375 1
+0.40625 1
+0.53125 1
+1 1
+
+0.33175106835972 0.094168845750364
+0.094168845750364 0.33175106835972
+-5.1759634627347e-17 0.14683388869532
+6.5255471622478e-17 0.38441611130468
+0.14683388869532 6.0713766400335e-17
+0.38441611130468 8.2458945395444e-19
+0.53125 0.025911862710939
+0.53125 0.067838137289061
+0.34721731046049 0.13433915461926
+0.13433915461926 0.34721731046049
+0.025911862710939 0.53125
+0.067838137289061 0.53125
+0.53125 0.12829915028125
+0.53125 0.18420084971875
+0.39979807890035 0.24774225329947
+0.24774225329947 0.39979807890035
+0.12829915028125 0.53125
+0.18420084971875 0.53125
+0.53125 0.22738728757031
+0.53125 0.24136271242969
+0.41294327101031 0.27609302796952
+0.27609302796952 0.41294327101031
+0.22738728757031 0.53125
+0.24136271242969 0.53125
+0.53125 0.28454915028125
+0.53125 0.34045084971875
+0.46552403945017 0.38949612664974
+0.38949612664974 0.46552403945017
+0.28454915028125 0.53125
+0.34045084971875 0.53125
+0.53125 0.38363728757031
+0.53125 0.39761271242969
+0.47866923156014 0.41784690131979
+0.41784690131979 0.47866923156014
+0.38363728757031 0.53125
+0.39761271242969 0.53125
+0.53125 0.44079915028125
+0.53125 0.49670084971875
+0.44079915028125 0.53125
+0.49670084971875 0.53125
+0.53988728757031 0.025911862710939
+0.55386271242969 0.067838137289061
+0.53988728757031 0.09375
+0.55386271242969 0.09375
+0.59704915028125 0.12829915028125
+0.65295084971875 0.18420084971875
+0.57443643785157 0.21875
+0.64431356214843 0.21875
+0.69613728757031 0.22738728757031
+0.71011271242969 0.24136271242969
+0.58307372542188 0.25
+0.66692627457812 0.25
+0.75329915028125 0.28454915028125
+0.80920084971875 0.34045084971875
+0.61762287570313 0.375
+0.75737712429687 0.375
+0.85238728757031 0.38363728757031
+0.86636271242969 0.39761271242969
+0.62626016327344 0.40625
+0.77998983672656 0.40625
+0.90954915028125 0.44079915028125
+0.96545084971875 0.49670084971875
+0.6608093135547 0.53125
+0.8704406864453 0.53125
+1 0.025911862710939
+1 0.067838137289061
+0.68342202598438 0.09375
+0.87907797401562 0.09375
+0.6608093135547 0
+0.8704406864453 0
+1 0.11966186271094
+1 0.16158813728906
+0.77387287570313 0.21011271242969
+0.91362712429687 0.19613728757031
+1 0.20477457514063
+1 0.23272542485937
+0.79648558813282 0.25
+0.92226441186718 0.25
+1 0.27591186271094
+1 0.31783813728906
+0.88693643785157 0.36636271242969
+0.95681356214843 0.35238728757031
+1 0.36102457514063
+1 0.38897542485937
+0.90954915028125 0.40625
+0.96545084971875 0.40625
+1 0.44079915028125
+1 0.49670084971875
+0.09375 0.6608093135547
+0.09375 0.8704406864453
+0.025911862710939 1
+0.067838137289061 1
+0 0.6608093135547
+0 0.8704406864453
+0.21875 0.6608093135547
+0.21875 0.8704406864453
+0.12829915028125 1
+0.18420084971875 1
+0.25 0.6608093135547
+0.25 0.8704406864453
+0.22738728757031 1
+0.24136271242969 1
+0.375 0.6608093135547
+0.375 0.8704406864453
+0.28454915028125 1
+0.34045084971875 1
+0.40625 0.6608093135547
+0.40625 0.8704406864453
+0.38363728757031 1
+0.39761271242969 1
+0.53125 0.6608093135547
+0.53125 0.8704406864453
+0.44079915028125 1
+0.49670084971875 1
+1 0.6608093135547
+1 0.8704406864453
+0.6608093135547 1
+0.8704406864453 1
+0.14782497614169 0.14782497614169
+0.3364183509774 0.10629113008478
+0.3439701728879 0.12590539815918
+0.10629113008478 0.3364183509774
+0.12590539815918 0.3439701728879
+0.36175027742635 0.16568300020856
+0.38526511193449 0.21639840771017
+0.16568300020856 0.36175027742635
+0.21639840771017 0.38526511193449
+0.40343132064181 0.2555782146968
+0.40931002926885 0.2682570665722
+0.2555782146968 0.40343132064181
+0.2682570665722 0.40931002926885
+0.42747623797617 0.30743687355882
+0.45099107248431 0.35815228106044
+0.30743687355882 0.42747623797617
+0.35815228106044 0.45099107248431
+0.46915728119164 0.39733208804706
+0.47503598981867 0.41001093992246
+0.39733208804706 0.46915728119164
+0.41001093992246 0.47503598981867
+0.47232443490112 0.47232443490112
+0.54166666666667 0.0625
+0.57886271242969 0.12829915028125
+0.61931356214843 0.18420084971875
+0.54943643785157 0.12829915028125
+0.56488728757031 0.18420084971875
+0.65056356214843 0.22738728757031
+0.66067627457812 0.24136271242969
+0.57682372542188 0.22738728757031
+0.58068643785157 0.24136271242969
+0.69192627457812 0.28454915028125
+0.73237712429687 0.34045084971875
+0.59262287570313 0.28454915028125
+0.60807372542188 0.34045084971875
+0.76362712429687 0.38363728757031
+0.77373983672656 0.39761271242969
+0.62001016327344 0.38363728757031
+0.62387287570313 0.39761271242969
+0.80498983672656 0.44079915028125
+0.8454406864453 0.49670084971875
+0.6358093135547 0.44079915028125
+0.65126016327344 0.49670084971875
+0.87282797401562 0.025911862710939
+0.8766906864453 0.067838137289061
+0.6670593135547 0.025911862710939
+0.67717202598438 0.067838137289061
+0.88862712429687 0.12204915028125
+0.90407797401562 0.16783813728906
+0.70842202598438 0.12591186271094
+0.74887287570313 0.17795084971875
+0.91601441186718 0.21102457514063
+0.91987712429687 0.23511271242969
+0.78012287570313 0.22113728757031
+0.79023558813282 0.23897542485937
+0.93181356214843 0.27829915028125
+0.94726441186718 0.32408813728906
+0.82148558813282 0.28216186271094
+0.86193643785157 0.33420084971875
+0.95920084971875 0.36727457514063
+0.96306356214843 0.39136271242969
+0.89318643785157 0.37738728757031
+0.90329915028125 0.39522542485937
+0.95833333333333 0.44791666666667
+0.025911862710939 0.6608093135547
+0.067838137289061 0.6608093135547
+0.025911862710939 0.8704406864453
+0.067838137289061 0.8704406864453
+0.12829915028125 0.6608093135547
+0.18420084971875 0.6608093135547
+0.12829915028125 0.8704406864453
+0.18420084971875 0.8704406864453
+0.22738728757031 0.6608093135547
+0.24136271242969 0.6608093135547
+0.22738728757031 0.8704406864453
+0.24136271242969 0.8704406864453
+0.28454915028125 0.6608093135547
+0.34045084971875 0.6608093135547
+0.28454915028125 0.8704406864453
+0.34045084971875 0.8704406864453
+0.38363728757031 0.6608093135547
+0.39761271242969 0.6608093135547
+0.38363728757031 0.8704406864453
+0.39761271242969 0.8704406864453
+0.44079915028125 0.6608093135547
+0.49670084971875 0.6608093135547
+0.44079915028125 0.8704406864453
+0.49670084971875 0.8704406864453
+0.6608093135547 0.6608093135547
+0.8704406864453 0.6608093135547
+0.6608093135547 0.8704406864453
+0.8704406864453 0.8704406864453

doc/CodeDocumentation.conf.in

@@ -795,6 +795,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
+                         @MFEM_SOURCE_DIR@/miniapps/hdiv-linear-solver \
                          @MFEM_SOURCE_DIR@/miniapps/hooke \
                          @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
                          @MFEM_SOURCE_DIR@/miniapps/hooke/materials \
@@ -810,7 +811,10 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/shifted \
                          @MFEM_SOURCE_DIR@/miniapps/solvers \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/spde \
+                         @MFEM_SOURCE_DIR@/miniapps/dpg \
+                         @MFEM_SOURCE_DIR@/miniapps/dpg/util
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

doc/CodeDocumentation.dox

@@ -39,7 +39,7 @@ namespace mfem {
  * - Device
  * - Memory
  * - MemoryManager
- * - MFEM_FORALL macro in forall.hpp
+ * - mfem::forall functions in forall.hpp
  *
  * <H3>Example codes</H3>
  * - <a class="el" href="ex0_8cpp_source.html">Example 0</a>: simplest example, nodal H1 FEM for the Laplace problem
@@ -105,6 +105,8 @@ namespace mfem {
  * - <a class="el" href="ex32p_8cpp_source.html">Example 32p</a>: parallel anisotropic Maxwell eigensolver
  * - <a class="el" href="ex33_8cpp_source.html">Example 33</a>: nodal H1 FEM for the fractional Laplacian problem
  * - <a class="el" href="ex33p_8cpp_source.html">Example 33p</a>: parallel nodal H1 FEM for the fractional Laplacian problem
+ * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
+ * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -186,6 +188,7 @@ namespace mfem {
  * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
  * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
  * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
+ * - <a class="el" href="mesh-quality_8cpp_source.html">Mesh Quality</a>: visualize and check mesh quality
  * - <a class="el" href="trimmer_8cpp_source.html">Trimmer</a>: trim elements from existing meshes
  * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
  * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
@@ -198,12 +201,15 @@ namespace mfem {
  * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
  * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
  * - <a class="el" href="extrapolate_8cpp_source.html">Extrapolation</a>: PDE-based extrapolation of finite element functions
- * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
+ * - <a class="el" href="block-solvers_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
  * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
  * - <a class="el" href="par__example_8cpp_source.html">Parallel AD</a>: Parallel p-Laplacian example
  * - <a class="el" href="seq__example_8cpp_source.html">Serial AD</a>: Serial p-Laplacian example
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
+ * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

examples/CMakeLists.txt

@@ -40,6 +40,8 @@ list(APPEND ALL_EXE_SRCS
   ex30.cpp
   ex31.cpp
   ex33.cpp
+  ex34.cpp
+  ex36.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -77,6 +79,9 @@ if (MFEM_USE_MPI)
     ex31p.cpp
     ex32p.cpp
     ex33p.cpp
+    ex34p.cpp
+    ex35p.cpp
+    ex36p.cpp
     )
 endif()
 
@@ -119,9 +124,10 @@ if (MFEM_ENABLE_TESTING)
   # Add CUDA/HIP tests.
   set(DEVICE_EXAMPLES
       # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
       # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
+      ex34p ex35p)
   set(MFEM_TEST_DEVICE)
   if (MFEM_USE_CUDA)
     set(MFEM_TEST_DEVICE "cuda")
@@ -161,6 +167,15 @@ if (MFEM_ENABLE_TESTING)
       $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
       ${MPIEXEC_POSTFLAGS})
   endif()
+
+  # If MUMPS is enabled, add a test run that uses it.
+  if (MFEM_USE_MUMPS)
+    add_test(NAME ex25p_mumps_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex25p> "-no-vis" "--mumps-solver"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
 endif()
 
 # Include the examples/amgx directory if AmgX is enabled

examples/contact/DirichletObstacleProblem.cpp

@@ -1,141 +0,0 @@
-#include "mfem.hpp"
-#include "Problems.hpp"
-#include "IPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-
-
-double fRhs(const Vector &pt);
-double obstacle(const Vector &pt);
-double dmanufacturedFun(const Vector &pt);
-
-int main(int argc, char *argv[])
-{
-  int FEorder = 1; // order of the finite elements
-  int linSolver = 0;
-  int maxIPMiters = 30;
-  bool iAmRoot = true;
-  int ref_levels = 3; 
-  OptionsParser args(argc, argv);
-  args.AddOption(&FEorder, "-o", "--order",\
-		  "Order of the finite elements.");
-  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-       "IP-Newton linear system solution strategy.");
-  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-		  "Maximum number of IPM iterations");
-  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
-		  "Mesh Refinement");
-  
-  args.Parse();
-  if(!args.Good())
-  {
-    args.PrintUsage(cout);
-    return 1;
-  }
-  else
-  {
-    if( iAmRoot )
-    {
-      args.PrintOptions(cout);
-    }
-  }
-
-  const char *meshFile = "../../data/inline-quad.mesh";
-  Mesh *mesh = new Mesh(meshFile, 1, 1);
-  int dim = mesh->Dimension(); // geometric dimension of the domain
-  for (int l = 0; l < ref_levels; l++)
-  {
-      mesh->UniformRefinement();
-  }
-
-  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
-  Array<int> ess_tdof_list;
-  if (mesh->bdr_attributes.Size())
-  {
-     Array<int> ess_bdr(mesh->bdr_attributes.Max());
-     ess_bdr = 1;
-     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-  }
-  
-  double DC_val = 0.0;
-  int dimD = Vh->GetTrueVSize();
-  Vector x0(dimD); x0 = DC_val;
-  Vector xf(dimD); xf = 0.0;
-
-  ObstacleProblem problem(Vh, x0, &fRhs, &obstacle, ess_tdof_list);
-  
-
-  InteriorPointSolver optimizer(&problem); 
-  optimizer.SetTol(1.e-7);
-  optimizer.SetLinearSolver(linSolver);
-  optimizer.SetMaxIter(maxIPMiters);
-  optimizer.Mult(x0, xf);
-  
-  double Einitial = problem.E(x0);
-  double Efinal = problem.E(xf);
-  cout << "Energy objective at initial point = " << Einitial << endl;
-  cout << "Energy objective at optimizer = " << Efinal << endl;
-
-  GridFunction d_gf(Vh);
-
-  d_gf = xf;
-  
-  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
-  GridFunction dm_gf(Vh);
-  dm_gf.ProjectCoefficient(dm_fc);
-
-  
-  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
-  paraview_dc.SetPrefixPath("ParaView");
-  paraview_dc.SetLevelsOfDetail(FEorder);
-  paraview_dc.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc.SetHighOrderOutput(true);
-  paraview_dc.SetCycle(0);
-  paraview_dc.SetTime(0.0);
-  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
-  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
-  paraview_dc.Save();
-  
-
-  delete Vh;
-  delete fec;
-  delete mesh;
-  return 0;
-}
-
-
-
-double dmanufacturedFun(const Vector &pt)
-{
-  double alpha = 16.5;
-  return sin(M_PI * pt(1)) * (sin(M_PI * pt(0)) - alpha * pow(pt(0) * (1. - pt(0)), 2));
-}
-
-
-// f(x) forcing term... which enters the objective energy functional
-// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
-// of f(x). f(x) is such that in the absence of bound-constraints then
-// the solution of the optimization problem satisfies the PDE
-// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
-// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
-
-double fRhs(const Vector &pt)
-{
-  double alpha = 16.5;
-  double fx;
-  fx = pow(M_PI, 2) * sin(M_PI * pt(0));
-  fx += alpha * (2. * pow(pt(0), 2) + 2. * pow(1.-pt(0), 2) - 8. * pt(0) * (1.-pt(0)));
-  fx += pow(M_PI, 2) * sin(M_PI * pt(0)) * dmanufacturedFun(pt);
-  fx *= sin(M_PI * pt(1));
-  return fx;
-}
-
-double obstacle(const Vector &pt)
-{
-  return 0.0;
-}

examples/contact/DirichletObstacleProblemVariant.cpp

@@ -1,156 +0,0 @@
-#include "mfem.hpp"
-#include "Problems.hpp"
-#include "IPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-
-
-double fRhs(const Vector &pt);
-double obstacle(const Vector &pt);
-double dmanufacturedFun(const Vector &pt);
-
-int main(int argc, char *argv[])
-{
-  int FEorder = 1; // order of the finite elements
-  int linSolver = 0;
-  int maxIPMiters = 30;
-  bool iAmRoot = true;
-  int ref_levels = 3; 
-  OptionsParser args(argc, argv);
-  args.AddOption(&FEorder, "-o", "--order",\
-		  "Order of the finite elements.");
-  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-       "IP-Newton linear system solution strategy.");
-  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-		  "Maximum number of IPM iterations");
-  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
-		  "Mesh Refinement");
-  
-  args.Parse();
-  if(!args.Good())
-  {
-    args.PrintUsage(cout);
-    return 1;
-  }
-  else
-  {
-    if( iAmRoot )
-    {
-      args.PrintOptions(cout);
-    }
-  }
-
-  const char *meshFile = "../../data/inline-quad.mesh";
-  Mesh *mesh = new Mesh(meshFile, 1, 1);
-  int dim = mesh->Dimension(); // geometric dimension of the domain
-  for (int l = 0; l < ref_levels; l++)
-  {
-      mesh->UniformRefinement();
-  }
-
-  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
-  Array<int> ess_tdof_list;
-  if (mesh->bdr_attributes.Size())
-  {
-     Array<int> ess_bdr(mesh->bdr_attributes.Max());
-     ess_bdr = 1;
-     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-  }
-  
-  double DC_val = 0.06;
-  Vector x0DC(Vh->GetTrueVSize()); x0DC = DC_val;
-  int dimD = Vh->GetTrueVSize() - ess_tdof_list.Size();
-  Vector x0(dimD); x0 = 0.0;
-  Vector xf(dimD); xf = 0.0;
-
-  ObstacleProblemVariant problem(Vh, x0DC, &fRhs, &obstacle, ess_tdof_list);
-
-  InteriorPointSolver optimizer(&problem); 
-  optimizer.SetTol(1.e-7);
-  optimizer.SetLinearSolver(linSolver);
-  optimizer.SetMaxIter(maxIPMiters);
-  optimizer.Mult(x0, xf);
-  
-  double Einitial = problem.E(x0);
-  double Efinal = problem.E(xf);
-  cout << "Energy objective at initial point = " << Einitial << endl;
-  cout << "Energy objective at optimizer = " << Efinal << endl;
-
-  Array<int> noness_tdof_list;
-  noness_tdof_list.SetSize(dimD);
-  int i = 0;
-  for(int j = 0; j < Vh->GetTrueVSize(); j++)
-  {
-     if(ess_tdof_list.Find(j) == -1)
-     {
-        noness_tdof_list[i] = j;
-        i += 1;
-     }
-  }
-
-
-  GridFunction d_gf(Vh);
-  d_gf.Set(1.0, x0DC);
-  d_gf.SetSubVector(noness_tdof_list, xf);
-
-
-
-  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
-  GridFunction dm_gf(Vh);
-  dm_gf.ProjectCoefficient(dm_fc);
-
-  
-  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
-  paraview_dc.SetPrefixPath("ParaView");
-  paraview_dc.SetLevelsOfDetail(FEorder);
-  paraview_dc.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc.SetHighOrderOutput(true);
-  paraview_dc.SetCycle(0);
-  paraview_dc.SetTime(0.0);
-  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
-  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
-  paraview_dc.Save();
- 
-
-  delete Vh;
-  delete fec;
-  delete mesh;
-  return 0;
-}
-
-
-
-double dmanufacturedFun(const Vector &pt)
-{
-  double alpha = 16.5;
-  return sin(M_PI * pt(1)) * (sin(M_PI * pt(0)) - alpha * pow(pt(0) * (1. - pt(0)), 2));
-}
-
-
-// f(x) forcing term... which enters the objective energy functional
-// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
-// of f(x). f(x) is such that in the absence of bound-constraints then
-// the solution of the optimization problem satisfies the PDE
-// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
-// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
-
-double fRhs(const Vector &pt)
-{
-  double alpha = 16.5;
-  double fx;
-  fx = pow(M_PI, 2) * sin(M_PI * pt(0));
-  fx += alpha * (2. * pow(pt(0), 2) + 2. * pow(1.-pt(0), 2) - 8. * pt(0) * (1.-pt(0)));
-  fx += pow(M_PI, 2) * sin(M_PI * pt(0)) * dmanufacturedFun(pt);
-  fx *= sin(M_PI * pt(1));
-  return fx;
-}
-
-double obstacle(const Vector &pt)
-{
-  return 0.0;
-}

examples/contact/IPsolver.cpp

@@ -1,6 +1,6 @@
 #include "mfem.hpp"
 #include "IPsolver.hpp"
-#include "Problems.hpp"
+#include "problems.hpp"
 #include <fstream>
 #include <iostream>
 #include <cstdlib>
@@ -10,10 +10,11 @@ using namespace mfem;
 
 
 
-InteriorPointSolver::InteriorPointSolver(GeneralOptProblem * Problem) : optProblem(Problem), block_offsetsumlz(5), block_offsetsuml(4), block_offsetsx(3),
-Huu(nullptr), Hum(nullptr), Hmu(nullptr), Hmm(nullptr), Wmm(nullptr), D(nullptr), Ju(nullptr), Jm(nullptr), JuT(nullptr), JmT(nullptr), Huucl(nullptr), HLuu(nullptr), saveLogBarrierIterates(false)
+InteriorPointSolver::InteriorPointSolver(OptProblem * Problem, ParFiniteElementSpace *Vhin) 
+: problem(Problem), block_offsetsumlz(5), block_offsetsuml(4), block_offsetsx(3),
+ saveLogBarrierIterates(false), Vh(Vhin)
 {
-  rel_tol  = 1.e-2;
+  tol  = 1.e-2;
   max_iter = 20;
   mu_k     = 1.0;
 
@@ -32,7 +33,6 @@ Huu(nullptr), Hum(nullptr), Hmu(nullptr), Hmm(nullptr), Wmm(nullptr), D(nullptr)
   kMu     = 0.2;
   thetaMu = 1.5;
 
-  // TO DO -- include the filter
 
   thetaMax = 1.e6; // maximum constraint violation
   // data for the second order correction
@@ -44,9 +44,9 @@ Huu(nullptr), Hum(nullptr), Hmu(nullptr), Hmm(nullptr), Wmm(nullptr), D(nullptr)
 
   kEps   = 1.e1;
 
-  dimU = optProblem->GetDimU();
-  dimM = optProblem->GetDimM();
-  dimC = optProblem->GetDimC();
+  dimU = problem->GetDimU();
+  dimM = problem->GetDimM();
+  dimC = problem->GetDimC();
   ckSoc.SetSize(dimC);
   
   block_offsetsumlz[0] = 0;
@@ -60,10 +60,11 @@ Huu(nullptr), Hum(nullptr), Hmu(nullptr), Hmm(nullptr), Wmm(nullptr), D(nullptr)
   for(int i = 0; i < block_offsetsx.Size(); i++)    { block_offsetsx[i]   = block_offsetsuml[i] ; }
 
   // lower-bound for the inequality constraint m >= ml
-  ml = optProblem->Getml();
+  ml = problem->Getml();
   
-  lk.SetSize(dimC);  lk  = 0.0;
+  lk.SetSize(dimC);   lk  = 0.0;
   zlk.SetSize(dimM); zlk = 0.0;
+  mf.SetSize(dimM);   mf = 0.0;
 
   linSolver = 0;
   MyRank = 0;
@@ -102,18 +103,19 @@ void InteriorPointSolver::Mult(const Vector &x0, Vector &xf)
 {
   BlockVector x0block(block_offsetsx); x0block = 0.0;
   x0block.GetBlock(0).Set(1.0, x0);
-  // To do: give options for user specificiation of initialization m0
-  x0block.GetBlock(1) = 100.;
+  // hard coded initialization :(
+  x0block.GetBlock(1) = 1.0;
   x0block.GetBlock(1).Add(1.0, ml);
   BlockVector xfblock(block_offsetsx); xfblock = 0.0;
   Mult(x0block, xfblock);
   xf.Set(1.0, xfblock.GetBlock(0));
+  mf.Set(1.0, xfblock.GetBlock(1));
 }
 
 void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
 {
   converged = false;
-  
+  IPNewtonKrylovIters.open("IPNewtonKrylovIters.dat", ios::out | ios::trunc);  
   BlockVector xk(block_offsetsx), xhat(block_offsetsx); xk = 0; xhat = 0.0;
   BlockVector Xk(block_offsetsumlz), Xhat(block_offsetsumlz); Xk = 0.0; Xhat = 0.0;
   BlockVector Xhatuml(block_offsetsuml); Xhatuml = 0.0;
@@ -158,14 +160,21 @@ void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
 
   for(jOpt = 0; jOpt < max_iter; jOpt++)
   {
-    mfem::out << "interior-point solve step " << jOpt << endl;
+    if(iAmRoot)
+    {
+      cout << "interior-point solve step " << jOpt << endl;
+    }
     // A-2. Check convergence of overall optimization problem
     printOptimalityError = false;
     Eevalmu0 = E(xk, lk, zlk, printOptimalityError);
-    if(Eevalmu0 < rel_tol)
+    if(Eevalmu0 < tol)
     {
       converged = true;
-      mfem::out << "solved optimization problem :)\n";
+      if(iAmRoot)
+      {
+	      IPNewtonKrylovIters.close();
+        cout << "solved optimization problem :)\n";
+      }
       break;
     }
     
@@ -178,9 +187,12 @@ void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
       Eeval = E(xk, lk, zlk, mu_k, printOptimalityError);
       if(Eeval < kEps * mu_k)
       {
-        mfem::out << "solved barrier subproblem, for mu = " << mu_k << endl;
+        if(iAmRoot)
+        {
+          cout << "solved barrier subproblem :), for mu = " << mu_k << endl;
+        }
         // A-3.1. Recompute the barrier parameter
-        mu_k  = max(rel_tol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
+        mu_k  = max(tol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
         // A-3.2. Re-initialize the filter
         F1.DeleteAll();
         F2.DeleteAll();
@@ -193,7 +205,10 @@ void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
     
     // A-4. Compute the search direction
     // solve for (uhat, mhat, lhat)
-    mfem::out << "\n** A-4. IP-Newton solve **\n";
+    if(iAmRoot)
+    {
+      cout << "\n** A-4. IP-Newton solve **\n";
+    }
     zlhat = 0.0; Xhatuml = 0.0;
     // why do we have Xhatuml ....???
     // TO DO: remove Xhatuml in favor of passing Xhat
@@ -217,13 +232,19 @@ void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
 
 
     // A-5. Backtracking line search.
-    mfem::out << "\n** A-5. Linesearch **\n";
-    mfem::out << "mu = " << mu_k << endl;
-    
+    if(iAmRoot)
+    {
+      cout << "\n** A-5. Linesearch **\n";
+      cout << "mu = " << mu_k << endl;
+    }
     lineSearch(Xk, Xhat, mu_k);
-    
+
     if(lineSearchSuccess)
     {
+      if(iAmRoot)
+      {
+        cout << "lineSearch successful :)\n";
+      }
       if(!switchCondition || !sufficientDecrease)
       {
         F1.Append( (1. - gTheta) * thx0);
@@ -239,15 +260,22 @@ void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
     }
     else
     {
-      mfem::out << "lineSearch not successful :(\n";
-      mfem::out << "attempting feasibility restoration with theta = " << thx0 << endl;
-      mfem::out << "no feasibility restoration implemented, exiting now \n";
+      if(iAmRoot)
+      {
+        cout << "lineSearch not successful :(\n";
+        cout << "attempting feasibility restoration with theta = " << thx0 << endl;
+        cout << "no feasibility restoration implemented, exiting now \n";
+      }
       break;
+      //cout << "feasibility restoration!!! :( :( :(\n";
+      //problem->feasibilityRestoration(x, 1.e-12);
+      //      break;
     }
     //
-    if(jOpt + 1 == max_iter) 
+    if(jOpt + 1 == max_iter && iAmRoot) 
     { 
-      mfem::out << "maximum optimization iterations :(\n";
+      cout << "maximum optimization iterations :(\n";
+      IPNewtonKrylovIters.close();
     }
   }
   // done with optimization routine, just reassign data to xf reference so
@@ -262,8 +290,8 @@ void InteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &z
   // WARNING: Huu, Hum, Hmu, Hmm should all be Hessian terms of the Lagrangian, currently we 
   //          them by Hessian terms of the objective function and neglect the Hessian of l^T c
 
-  Huu = optProblem->Duuf(x); Hum = optProblem->Dumf(x);
-  Hmu = optProblem->Dmuf(x); Hmm = optProblem->Dmmf(x);
+  Huu = problem->Duuf(x); Hum = problem->Dumf(x);
+  Hmu = problem->Dmuf(x); Hmm = problem->Dmmf(x);
 
   Vector DiagLogBar(dimM); DiagLogBar = 0.0;
   for(int ii = 0; ii < dimM; ii++)
@@ -283,40 +311,30 @@ void InteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &z
     diagStream.close();
   } 
 
-
-  D = new SparseMatrix(DiagLogBar);
-  
+  delete Wmm;
   if(Hmm != nullptr)
   {
-    Wmm = new SparseMatrix(*Hmm);
-    Wmm->Add(1.0, *D);
+    SparseMatrix * D = new SparseMatrix(DiagLogBar);
+    Wmm = Add(*Hmm, *D);
+    delete D;
   }
   else
   {
-    Wmm = D;
-  }
-
-  Ju = optProblem->Duc(x); JuT = Transpose(*Ju);
-  Jm = optProblem->Dmc(x); JmT = Transpose(*Jm);
- 
-  Huucl = optProblem->lDuuc(x, l);
-  if(Huucl != nullptr)
-  {
-    HLuu = Add(*Huucl, *Huu);
-    Ak.SetBlock(0, 0, HLuu);
-  } 
-  else
-  {
-    Ak.SetBlock(0, 0, Huu);
+    Wmm = new SparseMatrix(DiagLogBar);
   }
 
+  delete JuT;
+  delete JmT;
+  Ju = problem->Duc(x); JuT = Transpose(*Ju);
+  Jm = problem->Dmc(x); JmT = Transpose(*Jm);
+  
   //         IP-Newton system matrix
   //    Ak = [[H_(u,u)  H_(u,m)   J_u^T]
   //          [H_(m,u)  W_(m,m)   J_m^T]
   //          [ J_u      J_m       0  ]]
 
-  Ak.SetBlock(0, 2, JuT);
-  Ak.SetBlock(1, 1, Wmm); Ak.SetBlock(1, 2, JmT);
+  Ak.SetBlock(0, 0, Huu);                         Ak.SetBlock(0, 2, JuT);
+                          Ak.SetBlock(1, 1, Wmm); Ak.SetBlock(1, 2, JmT);
   Ak.SetBlock(2, 0,  Ju); Ak.SetBlock(2, 1,  Jm);
 
   if(Hum != nullptr) { Ak.SetBlock(0, 1, Hum); Ak.SetBlock(1, 0, Hmu); }
@@ -348,7 +366,7 @@ void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, V
   }
   if(!socSolve) 
   {
-    optProblem->c(x, b.GetBlock(2));
+    problem->c(x, b.GetBlock(2));
   }
   else
   {
@@ -357,7 +375,6 @@ void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, V
   b *= -1.0; 
   Xhat = 0.0;
 
-
   #ifdef MFEM_USE_SUITESPARSE
     // Direct solve for IP-Newton saddle-point system
     // A = [ [ Huu   0    Ju^T]
@@ -391,14 +408,14 @@ void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, V
     {
       // Direct solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
       // where Wmm = D for contact problems
-      SparseMatrix * Huuloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0)));
-      SparseMatrix * Wmmloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1)));
-      SparseMatrix * Juloc  = dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0)));
-      SparseMatrix * JuTloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2)));
-      Vector DVec(dimM); DVec = 0.0;
+      SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+      SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+      SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+      SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+      Vector Dvec(dimM); Dvec = 0.0;
       Vector one(dimM); one = 1.0;
-      D->Mult(one, DVec);
-      SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, DVec);     // Ju^T D Ju
+      Wmmloc->Mult(one, Dvec);
+      SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
       SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
 
       
@@ -425,25 +442,104 @@ void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, V
       Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
       Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
 
+      delete Wmmloc;
+      delete Huuloc;
       delete JuTDJu;
+      delete Juloc;
       delete Areduced;
     }
   #else
     MFEM_VERIFY(linSolver > 1, "linSolver = 0, 1 require MFEM_USE_SUITESPARSE=YES");
   #endif
-  if(linSolver > 1)
+  if (linSolver == 2 || linSolver == 3)
   { 
     // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
     // where Wmm = D for contact problems
     // here the iterative solver is a Jacobi-preconditioned CG-solve
-    SparseMatrix * Huuloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0)));
-    SparseMatrix * Wmmloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1)));
-    SparseMatrix * Juloc  = dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0)));
-    SparseMatrix * JuTloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2)));
-    Vector DVec(dimM); DVec = 0.0;
+    SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+    SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+    SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+    SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+    // Vector Dvec(dimM); Dvec = 0.0;
+    // Vector one(dimM); one = 1.0;
+    // Wmmloc->Mult(one, Dvec);
+
+    // SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
+    SparseMatrix *JuTDJu   = RAP(*Juloc,*Wmmloc,*Juloc);     // Ju^T D Ju
+    SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+    /* prepare the reduced rhs */
+    // breduced = bu + Ju^T (bm + Wmm bl)
+    Vector breduced(dimU); breduced = 0.0;
+    Vector tempVec(dimM); tempVec = 0.0;
+    Wmmloc->Mult(b.GetBlock(2), tempVec);
+    tempVec.Add(1.0, b.GetBlock(1));
+    JuTloc->Mult(tempVec, breduced);
+    breduced.Add(1.0, b.GetBlock(0));
+    
+    /* set up an iterative solver */
+    int globalNumRows = dimU;
+    HYPRE_BigInt rowStarts[2];
+    rowStarts[0] = 0;
+    rowStarts[1] = dimU;
+    HypreParMatrix * Ahypre = new HypreParMatrix(MPI_COMM_WORLD, globalNumRows, rowStarts, Areduced);
+    // CGSolver Asolver(MPI_COMM_WORLD);
+    HyprePCG Asolver(MPI_COMM_WORLD);
+    HypreBoomerAMG * Aprec = new HypreBoomerAMG(*Ahypre);
+    Aprec->SetPrintLevel(0);
+    if(linSolver == 3)
+    {
+      Aprec->SetElasticityOptions(Vh);
+    }
+    Aprec->SetSystemsOptions(3,false);
+
+    Asolver.SetOperator(*Ahypre);
+    Asolver.SetPrintLevel(2);
+    Asolver.SetMaxIter(1000);
+    // Asolver.SetResidualConvergenceOptions();
+    Asolver.SetTol(1.e-6);
+    Asolver.SetPreconditioner(*Aprec);
+    // Asolver.SetResidualConvergenceOptions();
+
+    Asolver.Mult(breduced, Xhat.GetBlock(0));
+    int num_iterations;
+    Asolver.GetNumIterations(num_iterations);
+    cgnum_iterations.Append(num_iterations);
+    // int numNewtonKrylovIters = -1;
+    // numNewtonKrylovIters = Asolver.GetNumIterations();
+    // IPNewtonKrylovIters << numNewtonKrylovIters << endl;
+
+    delete Aprec;
+    delete Ahypre;
+    
+    // now propagate solved uhat to obtain mhat and lhat
+    // xm = Ju xu - bl
+    Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+    Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+    // // xl = Wmm xm - bm
+    Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+    Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+    
+
+    delete Wmmloc;
+    delete Huuloc;
+    delete JuTDJu;
+    delete Juloc;
+    delete Areduced;
+  }
+  else if(linSolver > 2)
+  { 
+    // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+    // where Wmm = D for contact problems
+    // here the iterative solver is a Jacobi-preconditioned CG-solve
+    SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+    SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+    SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+    SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+    Vector Dvec(dimM); Dvec = 0.0;
     Vector one(dimM); one = 1.0;
-    D->Mult(one, DVec);
-    SparseMatrix *JuTDJu   = Mult_AtDA(*Juloc, DVec);     // Ju^T D Ju
+    Wmmloc->Mult(one, Dvec);
+    SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
     SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
 
     /* prepare the reduced rhs */
@@ -455,32 +551,16 @@ void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, V
     JuTloc->Mult(tempVec, breduced);
     breduced.Add(1.0, b.GetBlock(0));
     
-    if (linSolver == 2)
-    {
-      /* Jacobi preconditioned conjugate-gradient solve */
-      DSmoother AreducedPrec((SparseMatrix &)(*Areduced));
-      CGSolver AreducedSolver;
-      AreducedSolver.SetOperator(*Areduced);
-      AreducedSolver.SetAbsTol(1.e-12);
-      AreducedSolver.SetRelTol(1.e-8);
-      AreducedSolver.SetMaxIter(500);
-      AreducedSolver.SetPreconditioner(AreducedPrec);
-      AreducedSolver.SetPrintLevel(1);
-      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
-    }
-    else
-    {
-      /* Gauss-Seidel preconditioned GMRES solve */
-      GSSmoother AreducedPrec((SparseMatrix &)(*Areduced));
-      GMRESSolver AreducedSolver;
-      AreducedSolver.SetOperator(*Areduced);
-      AreducedSolver.SetAbsTol(1.e-12);
-      AreducedSolver.SetRelTol(1.e-8);
-      AreducedSolver.SetMaxIter(500);
-      AreducedSolver.SetPreconditioner(AreducedPrec);
-      AreducedSolver.SetPrintLevel(1);
-      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
-    }
+    /* set up an iterative solver */
+    GSSmoother AreducedPrec((SparseMatrix &)(*Areduced));
+    GMRESSolver AreducedSolver;
+    AreducedSolver.SetOperator(*Areduced);
+    AreducedSolver.SetAbsTol(1.e-12);
+    AreducedSolver.SetRelTol(1.e-8);
+    AreducedSolver.SetMaxIter(500);
+    AreducedSolver.SetPreconditioner(AreducedPrec);
+    AreducedSolver.SetPrintLevel(1);
+    AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
     
     // now propagate solved uhat to obtain mhat and lhat
     // xm = Ju xu - bl
@@ -491,7 +571,10 @@ void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, V
     Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
     Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
 
+    delete Wmmloc;
+    delete Huuloc;
     delete JuTDJu;
+    delete Juloc;
     delete Areduced;
   }
     
@@ -501,19 +584,6 @@ void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, V
   {
     zlhat(ii) = -1.*(zl(ii) + (zl(ii) * Xhat(ii + dimU) - mu) / (x(ii + dimU) - ml(ii)) );
   }
-
-  // free memory
-  if(Hmm != nullptr)
-  {
-    delete Wmm;
-  }
-  if( Huucl != nullptr)
-  {
-    delete HLuu; HLuu = nullptr;
-  }
-  delete D;
-  delete JuT;
-  delete JmT;
 }
 
 // here Xhat, X will be BlockVectors w.r.t. the 4 partitioning X = (u, m, l, zl)
@@ -557,23 +627,25 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
 
   Dxphi(x0, mu, Dxphi0);
   Dxphi0_xhat = InnerProduct(Dxphi0, xhat);
+  double xhat_L2norm = sqrt(InnerProduct(xhat, xhat));
+  double Dxphi_L2norm = sqrt(InnerProduct(Dxphi0, Dxphi0));
   descentDirection = Dxphi0_xhat < 0. ? true : false;
   if(descentDirection)
   {
-    mfem::out << "is a descent direction for the log-barrier objective\n";
+    cout << "is a descent direction for the log-barrier objective\n";
   }
   else
   {
-    mfem::out << "is not a descent direction for the log-barrier objective\n";
+    cout << "is not a descent direction for the log-barrier objective\n";
   }
-  mfem::out << "Dxphi^T xhat / (|| Dxphi||_2 * || xhat ||_2) = " << Dxphi0_xhat / (xhat.Norml2() * Dxphi0.Norml2()) << endl;
+  cout << "Dxphi^T xhat / (|| Dxphi ||_2 * || xhat ||_2) = " << Dxphi0_xhat / (xhat_L2norm * Dxphi_L2norm) << endl;
   thx0 = theta(x0);
   phx0 = phi(x0, mu);
 
   lineSearchSuccess = false;
   for(int i = 0; i < maxBacktrack; i++)
   {
-    mfem::out << "\n--------- alpha = " << alpha << " ---------\n";
+    cout << "\n--------- alpha = " << alpha << " ---------\n";
 
     // ----- A-5.2. Compute trial point: xtrial = x0 + alpha_i xhat
     xtrial.Set(1.0, x0);
@@ -582,11 +654,10 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
     // ------ A-5.3. if not in filter region go to A.5.4 otherwise go to A-5.5.
     thxtrial = theta(xtrial);
     phxtrial = phi(xtrial, mu);
-
     filterCheck(thxtrial, phxtrial);    
     if(!inFilterRegion)
     {
-      mfem::out << "not in filter region :)\n";
+      cout << "not in filter region :)\n";
       // ------ A.5.4: Check sufficient decrease
       if(!descentDirection)
       {
@@ -596,9 +667,11 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
       {
         switchCondition = (alpha * pow(abs(Dxphi0_xhat), sPhi) > delta * pow(thx0, sTheta)) ? true : false;
       }
-      mfem::out << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
-      mfem::out << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
-      mfem::out << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
+      cout << "alpha |Dxphi(x0)^T xhat|^sPhi = " << alpha * pow(abs(Dxphi0_xhat), sPhi) << endl;
+      cout << "delta * theta(x0)^sTheta = " << delta * pow(thx0, sTheta) << endl;
+      cout << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
+      cout << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
+      cout << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
       
       // Case I      
       if(thx0 <= thetaMin && switchCondition)
@@ -606,7 +679,7 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
         sufficientDecrease = phxtrial <= phx0 + eta * alpha * Dxphi0_xhat ? true : false;
         if(sufficientDecrease)
         {
-          mfem::out << "Accepted step length -- sufficient decrease in log-barrier objective.\n"; 
+          if(iAmRoot) { cout << "A-5.4. Case I -- accepted step length.\n"; } 
           // accept the trial step
           lineSearchSuccess = true;
           break;
@@ -616,7 +689,7 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
       {
         if(thxtrial <= (1. - gTheta) * thx0 || phxtrial <= phx0 - gPhi * thx0)
         {
-          mfem::out << "Accepted step length -- decrease in either constraint violation or log-barrier objective.\n"; 
+          if(iAmRoot) { cout << "A-5.4. Case II -- accepted step length.\n"; } 
           // accept the trial step
           lineSearchSuccess = true;
           break;
@@ -625,9 +698,9 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
       // A-5.5: Initialize the second-order correction
       if((!(thx0 < thxtrial)) && i == 0)
       {
-        mfem::out << "second order correction\n";
-        optProblem->c(xtrial, ckSoc);
-        optProblem->c(x0, ck0);
+        cout << "second order correction\n";
+        problem->c(xtrial, ckSoc);
+        problem->c(x0, ck0);
         ckSoc.Add(alphaMax, ck0);
         // A-5.6 Compute the second-order correction.
         IPNewtonSolve(x0, l0, z0, zhatsoc, Xhatumlsoc, mu, true);
@@ -638,7 +711,7 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
     }
     else
     {
-      mfem::out << "in filter region\n";
+      cout << "in filter region :(\n"; 
     }
 
     // include more if needed
@@ -647,7 +720,6 @@ void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double
   } 
 }
 
-
 void InteriorPointSolver::projectZ(const Vector &x, Vector &z, double mu)
 {
   double zi;
@@ -691,7 +763,7 @@ double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vecto
   DxL(x, l, zl, gradL);
   E1 = gradL.Normlinf();
 
-  optProblem->c(x, cx);
+  problem->c(x, cx);
   E2 = cx.Normlinf();
 
   for(int ii = 0; ii < dimM; ii++) 
@@ -705,12 +777,12 @@ double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vecto
   ll1 =  l.Norml1();
   sc = max(sMax, zl1 / (double(dimM)) ) / sMax;
   sd = max(sMax, (ll1 + zl1) / (double(dimC + dimM))) / sMax;
-  if(print)
+  if(iAmRoot && print)
   {
-    mfem::out << "evaluating optimality error for mu = " << mu << endl;
-    mfem::out << "stationarity measure = "    << E1 / sd << endl;
-    mfem::out << "feasibility measure  = "    << E2      << endl;
-    mfem::out << "complimentarity measure = " << E3 / sc << endl;
+    cout << "evaluating optimality error for mu = " << mu << endl;
+    cout << "stationarity measure = "    << E1 / sd << endl;
+    cout << "feasibility measure  = "    << E2      << endl;
+    cout << "complimentarity measure = " << E3 / sc << endl;
   }
   return max(max(E1 / sd, E2), E3 / sc);
 }
@@ -723,14 +795,14 @@ double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vecto
 double InteriorPointSolver::theta(const BlockVector &x)
 {
   Vector cx(dimC); cx = 0.0;
-  optProblem->c(x, cx);
-  return cx.Norml2();
+  problem->c(x, cx);
+  return sqrt(InnerProduct(cx, cx));
 }
 
 // log-barrier objective
 double InteriorPointSolver::phi(const BlockVector &x, double mu)
 {
-  double fx = optProblem->CalcObjective(x); 
+  double fx = problem->CalcObjective(x); 
   double logBarrierLoc = 0.0;
   for(int i = 0; i < dimM; i++) 
   { 
@@ -742,25 +814,22 @@ double InteriorPointSolver::phi(const BlockVector &x, double mu)
 }
 
 
-
-
 // gradient of log-barrier objective with respect to x = (u, m)
 void InteriorPointSolver::Dxphi(const BlockVector &x, double mu, BlockVector &y)
 {
-  optProblem->CalcObjectiveGrad(x, y);
+  problem->CalcObjectiveGrad(x, y);
   for(int i = 0; i < dimM; i++) 
   { 
-    y(dimU + i) -= mu / (x(dimU + i));
+    y(dimU + i) -= mu / (x(dimU + i) - ml(i));
   } 
 }
 
-
 // Lagrangian function evaluation
 // L(x, l, zl) = f(x) + l^T c(x) - zl^T m
 double InteriorPointSolver::L(const BlockVector &x, const Vector &l, const Vector &zl)
 {
-  double fx = optProblem->CalcObjective(x);
-  Vector cx(dimC); optProblem->c(x, cx);
+  double fx = problem->CalcObjective(x);
+  Vector cx(dimC); problem->c(x, cx);
   return (fx + InnerProduct(cx, l) - InnerProduct(x.GetBlock(1), zl));
 }
 
@@ -768,16 +837,16 @@ void InteriorPointSolver::DxL(const BlockVector &x, const Vector &l, const Vecto
 {
   // evaluate the gradient of the objective with respect to the primal variables x = (u, m)
   BlockVector gradxf(block_offsetsx); gradxf = 0.0;
-  optProblem->CalcObjectiveGrad(x, gradxf);
+  problem->CalcObjectiveGrad(x, gradxf);
   
   SparseMatrix *Jacu, *Jacm, *JacuT, *JacmT;
-  Jacu = optProblem->Duc(x); Jacm = optProblem->Dmc(x);
+  Jacu = problem->Duc(x); Jacm = problem->Dmc(x);
   JacuT = Transpose(*Jacu);
   JacmT = Transpose(*Jacm);
   JacuT->Mult(l, y.GetBlock(0));
   JacmT->Mult(l, y.GetBlock(1));
-  delete JacuT; 
-  delete JacmT;
+  delete Jacu; delete JacuT;
+  delete Jacm; delete JacmT;
   y.Add(1.0, gradxf);
   (y.GetBlock(1)).Add(-1.0, zl);
 }
@@ -790,7 +859,7 @@ bool InteriorPointSolver::GetConverged() const
 
 void InteriorPointSolver::SetTol(double Tol)
 {
-  rel_tol = Tol;
+  tol = Tol;
 }
 
 void InteriorPointSolver::SetMaxIter(int max_it)
@@ -818,6 +887,17 @@ void InteriorPointSolver::SetLinearSolver(int LinSolver)
 
 InteriorPointSolver::~InteriorPointSolver() 
 {
+  delete Wmm;
+  delete Huu;
+  delete Hum;
+  delete Hmu;
+  delete Hmm;
+  delete Hum;
+  delete Ju;
+  delete Jm;
+  delete JuT;
+  delete JmT;
+
   F1.DeleteAll();
   F2.DeleteAll();
   block_offsetsx.DeleteAll();

examples/contact/IPsolver.hpp

@@ -1,5 +1,5 @@
 #include "mfem.hpp"
-#include "Problems.hpp"
+#include "problems.hpp"
 #include <fstream>
 #include <iostream>
 
@@ -13,11 +13,11 @@ using namespace mfem;
 class InteriorPointSolver
 {
 protected:
-    GeneralOptProblem* optProblem;
-    double rel_tol;
+    OptProblem* problem;
+    double tol;
     int  max_iter;
     double mu_k; // \mu_k
-    Vector lk, zlk;
+    Vector lk, zlk, mf;
 
     double sMax, kSig, tauMin, eta, thetaMin, delta, sTheta, sPhi, kMu, thetaMu;
     double thetaMax, kSoc, gTheta, gPhi, kEps;
@@ -37,8 +37,16 @@ protected:
     Vector ml;
 
     Vector ckSoc;
-    SparseMatrix * Huu, * Hum, * Hmu, * Hmm, * Wmm, *D, * Ju, * Jm, * JuT, * JmT;
-    SparseMatrix * Huucl, *HLuu;    
+    SparseMatrix * Huu = nullptr; 
+    SparseMatrix * Hum = nullptr; 
+    SparseMatrix * Hmu = nullptr; 
+    SparseMatrix * Hmm = nullptr; 
+    SparseMatrix * Wmm = nullptr; 
+    SparseMatrix * Ju  = nullptr; 
+    SparseMatrix * Jm  = nullptr; 
+    SparseMatrix * JuT = nullptr; 
+    SparseMatrix * JmT  = nullptr;;
+    
     int jOpt;
     bool converged;
     
@@ -48,11 +56,24 @@ protected:
     bool saveLogBarrierIterates;
 
     int linSolver;
+    std::ofstream IPNewtonKrylovIters;
+
+    ParFiniteElementSpace *Vh;
+    Array<int> cgnum_iterations;
+
+
+    // not sure if this data is needed or if it can 
+    // all be accounted for in the problem class
+    // which variables have equality constraints
+    //Array<int> eqConstrainedVariables;
+    //Array<double> eqConstrainedValues;
+
+
 
 public:
-    InteriorPointSolver(GeneralOptProblem*);
+    InteriorPointSolver(OptProblem*, ParFiniteElementSpace *);
     void Mult(const BlockVector& , BlockVector&); // used when the user wants to be aware of bound-constrained variable m >= ml
-    void Mult(const Vector&, Vector &); // useful when the user doesn't need to know about bound-constrained variable   m >= ml, e.g., when m is a slack variable
+    void Mult(const Vector&, Vector &); // useful when the user doesn't need to know about bound-constrained variable   m >= ml
     double MaxStepSize(Vector& , Vector& , Vector& , double);
     double MaxStepSize(Vector& , Vector& , double);
     void FormIPNewtonMat(BlockVector& , Vector& , Vector& , BlockOperator &);
@@ -74,6 +95,8 @@ public:
     void SetBarrierParameter(double);    
     void SaveLogBarrierHessianIterates(bool);
     void SetLinearSolver(int);
+    Vector GetBoundConstrainedVariable() {return mf;}
+    Array<int> & GetCGIterNumbers() {return cgnum_iterations;}
     virtual ~InteriorPointSolver();
 };
 

examples/contact/ObstacleProblem.cpp

@@ -1,125 +0,0 @@
-#include "mfem.hpp"
-#include "Problems.hpp"
-#include "IPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-
-
-double dmanufacturedFun(const Vector &);
-double fRhs(const Vector &);
-double obstacle(const Vector &);
-
-int main(int argc, char *argv[])
-{
-  int FEorder = 1; // order of the finite elements
-  int linSolver = 0;
-  int maxIPMiters = 30;
-  bool iAmRoot = true;
-  
-  OptionsParser args(argc, argv);
-  args.AddOption(&FEorder, "-o", "--order",\
-		  "Order of the finite elements.");
-  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-       "IP-Newton linear system solution strategy.");
-  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-		  "Maximum number of IPM iterations");
-  
-  args.Parse();
-  if(!args.Good())
-  {
-    args.PrintUsage(cout);
-    return 1;
-  }
-  else
-  {
-    if( iAmRoot )
-    {
-      args.PrintOptions(cout);
-    }
-  }
-
-  const char *meshFile = "../../data/inline-quad.mesh";
-  Mesh *mesh = new Mesh(meshFile, 1, 1);
-  int dim = mesh->Dimension(); // geometric dimension of the domain
-  {
-     int ref_levels = 3;
-     for (int l = 0; l < ref_levels; l++)
-     {
-        mesh->UniformRefinement();
-     }
-  }
-
-  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
-  ObstacleProblem problem(Vh, &fRhs, &obstacle);
-  
-  int dimD = problem.GetDimD();
-  Vector x0(dimD); x0 = 0.0;
-  Vector xf(dimD); xf = 0.0;
-
-  InteriorPointSolver optimizer(&problem); 
-  optimizer.SetTol(1.e-7);
-  optimizer.SetLinearSolver(linSolver);
-  optimizer.SetMaxIter(maxIPMiters);
-  optimizer.Mult(x0, xf);
-
-  double Einitial = problem->E(x0);
-  double Efinal = problem->E(xf);
-  cout << "Energy objective at initial point = " << Einitial << endl;
-  cout << "Energy objective at QP optimizer = " << Efinal << endl;
-
-
-  GridFunction d_gf(Vh);
-
-  d_gf = xf;
-
-  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
-  GridFunction dm_gf(Vh);
-  dm_gf.ProjectCoefficient(dm_fc);
-  
-  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
-  paraview_dc.SetPrefixPath("ParaView");
-  paraview_dc.SetLevelsOfDetail(FEorder);
-  paraview_dc.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc.SetHighOrderOutput(true);
-  paraview_dc.SetCycle(0);
-  paraview_dc.SetTime(0.0);
-  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
-  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
-  paraview_dc.Save();
-  
-  
-  delete Vh;
-  delete fec;
-  delete mesh;
-  return 0;
-}
-
-
-double dmanufacturedFun(const Vector &x)
-{
-  return cos(2*M_PI*x(0)) + 0.2 - 2.0*(pow(x(0),3) - 1.5*pow(x(0),2));
-}
-
-
-// f(x) forcing term... which enters the objective energy functional
-// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
-// of f(x). f(x) is such that in the absence of bound-constraints then
-// the solution of the optimization problem satisfies the PDE
-// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
-// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
-double fRhs(const Vector &x)
-{
-  double fx = 0.;
-  fx = 0.2 - 2.0 * (pow(x(0),3)- 1.5*pow(x(0),2.) - 6 * x(0) + 3.) + (1. + pow(2.*M_PI,2))*cos(2.*M_PI*x(0));
-  return fx;
-}
-
-double obstacle(const Vector &x)
-{
-  return 0.0;
-}

examples/contact/ParObstacleProblem.cpp

@@ -1,129 +0,0 @@
-//                         Obstacle Problem
-//
-//
-// Compile with: make ParObstacleProblem
-//
-// Sample runs: mpirun -np 4 ./ParObstacleProblem
-//
-//
-// Description: This example code demonstrates the use of MFEM to solve the
-//              bound-constrained energy minimization problem
-//
-//                      minimize (||∇u||² + ||u||²) subject to u ≥ ϕ in H¹.
-
-#include "mfem.hpp"
-#include "ParProblems.hpp"
-#include "ParIPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-double dmanufacturedFun(const Vector &);
-double fRhs(const Vector &);
-
-int main(int argc, char *argv[])
-{
-  // Initialize MPI
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   int FEorder = 1; // order of the finite elements
-   int linSolver = 2;
-   int maxIPMiters = 30;
-   int ref_levels = 3;
-   OptionsParser args(argc, argv);
-   args.AddOption(&FEorder, "-o", "--order",\
-	 	  "Order of the finite elements.");
-   args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-        "IP-Newton linear system solution strategy.");
-   args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-	 	  "Maximum number of IPM iterations");
-   args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
-		  "Mesh Refinement");
-  
-   args.Parse();
-   if(!args.Good())
-   {
-     args.PrintUsage(cout);
-     return 1;
-   }
-   else
-   {
-     if(Mpi::Root())
-     {  
-       args.PrintOptions(cout);
-     }
-   }
-
-   const char *meshFile = "../../data/inline-quad.mesh";
-   Mesh mesh(meshFile, 1, 1);
-   int dim = mesh.Dimension(); // geometric dimension of the meshed domain
-   {
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-
-   FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-   ParFiniteElementSpace   *Vh  = new ParFiniteElementSpace(&pmesh, fec);
-
-   ParObstacleProblem problem(Vh,Vh,&fRhs);
-  
-   int dimD = problem.GetDimD();
-   Vector x0(dimD); x0 = 100.0;
-   Vector xf(dimD); xf = 0.0;
-
-   ParInteriorPointSolver optimizer(&problem); 
-   optimizer.SetTol(1.e-8);
-   optimizer.SetLinearSolveTol(1.e-10);
-   optimizer.SetLinearSolver(linSolver);
-   optimizer.SetMaxIter(maxIPMiters);
-   optimizer.Mult(x0, xf);
-
-   ParGridFunction d_gf(Vh);
-
-   d_gf.SetFromTrueDofs(xf);
-
-
-   FunctionCoefficient dm_fc(dmanufacturedFun); // manufactured solution
-   ParGridFunction dm_gf(Vh);
-   dm_gf.ProjectCoefficient(dm_fc);
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream exact_sock(vishost, visport);
-   exact_sock.precision(8);
-   exact_sock << "parallel " << num_procs << " " << myid << "\n";
-   exact_sock << "solution\n" << pmesh << dm_gf
-              << "window_title 'Manufactured solution'" << flush;
-
-   socketstream sol_sock(vishost, visport);
-   sol_sock.precision(8);
-   sol_sock << "parallel " << num_procs << " " << myid << "\n";
-   sol_sock << "solution\n" << pmesh << d_gf 
-            << "window_title 'Numerical solution'" << flush;
-
-   delete Vh;
-   delete fec;
-   return 0;
-}
-
-
-double dmanufacturedFun(const Vector &x)
-{
-  return cos(2*M_PI*x(0)) + 0.2 - 2.0*(pow(x(0),3) - 1.5*pow(x(0),2));
-}
-
-double fRhs(const Vector &x)
-{
-  double fx = 0.;
-  fx = 0.2 - 2.0 * (pow(x(0),3)- 1.5*pow(x(0),2.) - 6 * x(0) + 3.) + (1. + pow(2.*M_PI,2))*cos(2.*M_PI*x(0));
-  return fx;
-}

examples/contact/ParProblems.cpp

@@ -1,212 +0,0 @@
-#include "mfem.hpp"
-#include "ParProblems.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-ParGeneralOptProblem::ParGeneralOptProblem(ParFiniteElementSpace * fesU_, ParFiniteElementSpace * fesM_) 
-: fesU(fesU_), fesM(fesM_) 
-{ 
-   dimU = fesU->GetTrueVSize();
-   dimM = fesM->GetTrueVSize();
-   dimC = fesM->GetTrueVSize();
-}
-
-void ParGeneralOptProblem::CalcObjectiveGrad(const BlockVector &x, BlockVector &y) const
-{
-   Duf(x, y.GetBlock(0));
-   Dmf(x, y.GetBlock(1));
-}
-
-ParGeneralOptProblem::~ParGeneralOptProblem()
-{
-   block_offsetsx.DeleteAll();
-}
-
-
-// min E(d) s.t. g(d) >= 0
-// min_(d,s) E(d) s.t. c(d,s) := g(d) - s = 0, s >= 0
-ParOptProblem::ParOptProblem(ParFiniteElementSpace * fesU_, 
-                                     ParFiniteElementSpace * fesM_)
- : ParGeneralOptProblem(fesU_, fesM_), block_offsetsx(3)
-{
-   block_offsetsx[0] = 0;
-   block_offsetsx[1] = dimU;
-   block_offsetsx[2] = dimM;
-   block_offsetsx.PartialSum();
-   ml.SetSize(dimM); ml = 0.0;
-   Vector negIdentDiag(dimM);
-   negIdentDiag = -1.0;
-   SparseMatrix * diag = new SparseMatrix(negIdentDiag);
-   Ih = new HypreParMatrix(fesM->GetComm(), fesM->GlobalTrueVSize(),
-                                             fesM->GetTrueDofOffsets(), diag);
-   HypreStealOwnership(*Ih, *diag);
-   delete diag;
-}
-
-double ParOptProblem::CalcObjective(const BlockVector &x) const { return E(x.GetBlock(0)); }
-
-void ParOptProblem::Duf(const BlockVector &x, Vector &y) const { DdE(x.GetBlock(0), y); }
-
-void ParOptProblem::Dmf(const BlockVector &x, Vector &y) const { y = 0.0; }
-
-HypreParMatrix * ParOptProblem::Duuf(const BlockVector &x) 
-{ 
-   return DddE(x.GetBlock(0)); 
-}
-
-HypreParMatrix * ParOptProblem::Dumf(const BlockVector &x) { return nullptr; }
-
-HypreParMatrix * ParOptProblem::Dmuf(const BlockVector &x) { return nullptr; }
-
-HypreParMatrix * ParOptProblem::Dmmf(const BlockVector &x) { return nullptr; }
-
-void ParOptProblem::c(const BlockVector &x, Vector &y) const // c(u,m) = g(u) - m 
-{
-   g(x.GetBlock(0), y);
-   y.Add(-1.0, x.GetBlock(1));  
-}
-
-HypreParMatrix * ParOptProblem::Duc(const BlockVector &x) 
-{ 
-   return Ddg(x.GetBlock(0)); 
-}
-
-HypreParMatrix * ParOptProblem::Dmc(const BlockVector &x) 
-{ 
-   return Ih;
-} 
-
-ParOptProblem::~ParOptProblem() 
-{
-   delete Ih;
-}
-
-
-// Obstacle Problem, no essential boundary conditions enforced
-// Hessian of energy term is K + M (stiffness + mass)
-ParObstacleProblem::ParObstacleProblem(ParFiniteElementSpace *fesU_, 
-                                       ParFiniteElementSpace *fesM_, 
-                                       double (*fSource)(const Vector &)) : 
-                                       ParOptProblem(fesU_,fesM_), f(dimU), psi(dimU), J(nullptr)
-{
-   Kform = new ParBilinearForm(fesU);
-   Kform->AddDomainIntegrator(new MassIntegrator);
-   Kform->AddDomainIntegrator(new DiffusionIntegrator);
-   Kform->Assemble();
-   Kform->Finalize();
-   Kform->FormSystemMatrix(ess_tdof_list, K);
-
-   FunctionCoefficient fcoeff(fSource);
-   fform = new ParLinearForm(fesU);
-   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
-   fform->Assemble();
-   Vector F(dimU);
-   fform->ParallelAssemble(F);
-   f.SetSize(dimU);
-   f.Set(1.0, F);
-
-   psi = 0.0;
-   
-   Vector iDiag(dimU); iDiag = 1.0;
-   SparseMatrix * Jacg = new SparseMatrix(iDiag);
-   
-   J = new HypreParMatrix(fesU->GetComm(),fesU->GlobalTrueVSize(),fesU->GetTrueDofOffsets(),Jacg);
-   HypreStealOwnership(*J, *Jacg);
-   delete Jacg;
-}
-
-// Obstacle Problem, essential boundary conditions enforced
-// Hessian of energy term is K (stiffness)
-ParObstacleProblem::ParObstacleProblem(ParFiniteElementSpace *fesU_, 
-                                       ParFiniteElementSpace *fesM_, 
-				       double (*fSource)(const Vector &),
-				       double (*obstacleSource)(const Vector &),
-				       Array<int> tdof_list, Vector &xDC) : ParOptProblem(fesU_,fesM_), f(dimU), psi(dimU), J(nullptr)
-{
-   // elastic energy functional terms	
-   ess_tdof_list = tdof_list;
-   Kform = new ParBilinearForm(fesU);
-   Kform->AddDomainIntegrator(new DiffusionIntegrator);
-   Kform->Assemble();
-   Kform->Finalize();
-   Kform->FormSystemMatrix(ess_tdof_list, K);
-
-   FunctionCoefficient fcoeff(fSource);
-   fform = new ParLinearForm(fesU);
-   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
-   fform->Assemble();
-   Vector F(dimU);
-   fform->ParallelAssemble(F);
-   f.SetSize(dimU);
-   f.Set(1.0, F);
-   Kform->EliminateVDofsInRHS(ess_tdof_list, xDC, f);
-   
-   // obstacle constraints --  
-   Vector iDiag(dimU); iDiag = 1.0;
-   for(int i = 0; i < ess_tdof_list.Size(); i++)
-   {
-     iDiag(ess_tdof_list[i]) = 0.0;
-   }
-   SparseMatrix * Jacg = new SparseMatrix(iDiag);
-
-   J = new HypreParMatrix(fesU->GetComm(),fesU->GlobalTrueVSize(),fesU->GetTrueDofOffsets(),Jacg);
-   HypreStealOwnership(*J, *Jacg);
-   delete Jacg;
-
-   FunctionCoefficient psi_fc(obstacleSource);
-   ParGridFunction psi_gf(fesU);
-   psi_gf.ProjectCoefficient(psi_fc);
-   psi.Set(1.0, (*psi_gf.GetTrueDofs()));
-   for(int i = 0; i < ess_tdof_list.Size(); i++)
-   {
-     psi(ess_tdof_list[i]) -= 1.e-8;
-   }
-}
-
-
-
-double ParObstacleProblem::E(const Vector &d) const
-{
-   Vector Kd(K.Height()); Kd = 0.0;
-   MFEM_VERIFY(d.Size() == K.Width(), "ParObstacleProblem::E - Inconsistent dimensions");
-   K.Mult(d, Kd);
-   return 0.5 * InnerProduct(MPI_COMM_WORLD, d, Kd) - InnerProduct(MPI_COMM_WORLD, f, d);
-}
-
-void ParObstacleProblem::DdE(const Vector &d, Vector &gradE) const
-{
-   gradE.SetSize(K.Height());
-   MFEM_VERIFY(d.Size() == K.Width(), "ParObstacleProblem::DdE - Inconsistent dimensions");
-   K.Mult(d, gradE);
-   MFEM_VERIFY(f.Size() == K.Height(), "ParObstacleProblem::DdE - Inconsistent dimensions");
-   gradE.Add(-1.0, f);
-}
-
-HypreParMatrix * ParObstacleProblem::DddE(const Vector &d)
-{
-   return &K; 
-}
-
-// g(d) = d >= \psi
-void ParObstacleProblem::g(const Vector &d, Vector &gd) const
-{
-   MFEM_VERIFY(d.Size() == J->Width(), "ParObstacleProblem::g - Inconsistent dimensions");
-   J->Mult(d, gd);
-   MFEM_VERIFY(gd.Size() == J->Height(), "ParObstacleProblem::g - Inconsistent dimensions");
-   gd.Add(-1.0, psi);
-}
-
-HypreParMatrix * ParObstacleProblem::Ddg(const Vector &d)
-{
-   return J;
-}
-
-ParObstacleProblem::~ParObstacleProblem()
-{
-   delete Kform;
-   delete fform;
-   delete J;
-}

examples/contact/ParProblems.hpp

@@ -1,107 +0,0 @@
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-#ifndef PARPROBLEM_DEFS
-#define PARPROBLEM_DEFS
-
-// abstract ParGeneralOptProblem class
-// of the form
-// min_(u,m) f(u,m) s.t. c(u,m)=0 and m>=ml
-// the primal variable (u, m) is represented as a BlockVector
-// think about supporting general lower and upper bounds (see HiOP user manual) 
-
-class ParGeneralOptProblem
-{
-protected:
-    int dimU, dimM, dimC;
-    ParFiniteElementSpace * fesU = nullptr;
-    ParFiniteElementSpace * fesM = nullptr;
-    Array<int> block_offsetsx;
-    Vector ml;
-public:
-    ParGeneralOptProblem(ParFiniteElementSpace * fesU_, ParFiniteElementSpace * fesM_); // constructor
-    virtual double CalcObjective(const BlockVector &) const = 0;
-    virtual void Duf(const BlockVector &, Vector &) const = 0;
-    virtual void Dmf(const BlockVector &, Vector &) const = 0;
-    void CalcObjectiveGrad(const BlockVector &, BlockVector &) const;
-    virtual HypreParMatrix * Duuf(const BlockVector &) = 0;
-    virtual HypreParMatrix * Dumf(const BlockVector &) = 0;
-    virtual HypreParMatrix * Dmuf(const BlockVector &) = 0;
-    virtual HypreParMatrix * Dmmf(const BlockVector &) = 0;
-    virtual HypreParMatrix * Duc(const BlockVector &) = 0;
-    virtual HypreParMatrix * Dmc(const BlockVector &) = 0;
-    // TO DO: include Hessian terms of constraint c
-    virtual void c(const BlockVector &, Vector &) const = 0;
-    int GetDimU() const { return dimU; };
-    int GetDimM() const { return dimM; }; 
-    int GetDimC() const { return dimC; };
-    ParFiniteElementSpace * GetfesU() {return fesU;}
-    ParFiniteElementSpace * GetfesM() {return fesM;}
-    Vector Getml() const { return ml; };
-    ~ParGeneralOptProblem(); // destructor
-};
-
-
-// abstract ContactProblem class
-// of the form
-// min_d e(d) s.t. g(d) >= 0
-class ParOptProblem : public ParGeneralOptProblem
-{
-protected:
-    Array<int> block_offsetsx;
-    HypreParMatrix * Ih;
-public:
-    ParOptProblem(ParFiniteElementSpace * fesU_, ParFiniteElementSpace * fesM_);        // constructor
-    double CalcObjective(const BlockVector &) const; // objective e
-    void Duf(const BlockVector &, Vector &) const;
-    void Dmf(const BlockVector &, Vector &) const;
-
-    HypreParMatrix * Duuf(const BlockVector &);
-    HypreParMatrix * Dumf(const BlockVector &);
-    HypreParMatrix * Dmuf(const BlockVector &);
-    HypreParMatrix * Dmmf(const BlockVector &);
-    HypreParMatrix * Duc(const BlockVector &);
-    HypreParMatrix * Dmc(const BlockVector &);
-
-    void c(const BlockVector &, Vector &) const;
-    virtual double E(const Vector &) const = 0;               // objective e(d) (energy function)
-    virtual void DdE(const Vector &, Vector &) const = 0;      // gradient of objective De / Dd
-    virtual HypreParMatrix * DddE(const Vector &) = 0;
-    // Hessian of objective D^2 e / D d^2
-    virtual HypreParMatrix * Ddg(const Vector &) = 0;
-    // Jacobian of inequality constraint Dg / Dd
-    virtual void g(const Vector &, Vector &) const = 0;       // inequality constraint g(d) >= 0 (gap function)
-    int GetDimD() const { return fesU->GetTrueVSize(); };
-    int GetDimS() const { return fesM->GetTrueVSize(); };
-    virtual ~ParOptProblem();
-};
-
-class ParObstacleProblem : public ParOptProblem
-{
-protected:
-   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d >= \psi
-   // stiffness matrix used to define objective
-   ParBilinearForm *Kform;
-   ParLinearForm   *fform;
-   Array<int> ess_tdof_list; // needed for calls to FormSystemMatrix
-   HypreParMatrix  K;
-   HypreParMatrix  *J;
-   ParFiniteElementSpace *Vh;
-   Vector f;
-   Vector psi;
-public :
-   ParObstacleProblem(ParFiniteElementSpace*, ParFiniteElementSpace*, double (*fSource)(const Vector &));
-   ParObstacleProblem(ParFiniteElementSpace*, ParFiniteElementSpace*, double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list, Vector &);
-   double E(const Vector &) const;
-   void DdE(const Vector &, Vector &) const;
-   HypreParMatrix* DddE(const Vector &);
-   void g(const Vector &, Vector &) const;
-   HypreParMatrix* Ddg(const Vector &);
-   virtual ~ParObstacleProblem();
-};
-
-#endif

examples/contact/ParSphericalObstacleProblem.cpp

@@ -1,173 +0,0 @@
-//                       Spherical Obstacle Problem
-//
-//
-// Compile with: make ParSphericalObstacleProblem
-//
-// Sample runs: mpirun -np 4 ./ParSphericalObstacleProblem -linSolver 0
-//              mpirun -np 4 ./ParSphericalObstacleProblem -linSolver 1
-//              mpirun -np 4 ./ParSphericalObstacleProblem -linSolver 2
-//
-//
-// Description: This example code demonstrates the use of MFEM to solve the
-//              bound-constrained energy minimization problem
-//
-//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
-
-#include "mfem.hpp"
-#include "ParProblems.hpp"
-#include "ParIPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-double fRhs(const Vector &);
-double spherical_obstacle(const Vector &);
-double exact_solution_obstacle(const Vector &);
-
-
-int main(int argc, char *argv[])
-{
-  // Initialize MPI
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   int FEorder = 1; // order of the finite elements
-   int linSolver = 2;
-   int maxIPMiters = 30;
-   int ref_levels = 3;
-   OptionsParser args(argc, argv);
-   args.AddOption(&FEorder, "-o", "--order",\
-	 	  "Order of the finite elements.");
-   args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-        "IP-Newton linear system solution strategy.");
-   args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-	 	  "Maximum number of IPM iterations");
-   args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
-		  "Mesh Refinement");
-  
-   args.Parse();
-   if(!args.Good())
-   {
-     args.PrintUsage(cout);
-     return 1;
-   }
-   else
-   {
-     if(myid == 0)
-     {  
-       args.PrintOptions(cout);
-     }
-   }
-
-   const char *meshFile = "../../data/disk.mesh";
-   Mesh mesh(meshFile, 1, 1);
-   int dim = mesh.Dimension(); // geometric dimension of the meshed domain
-   {
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   
-   FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-   ParFiniteElementSpace   *Vh  = new ParFiniteElementSpace(&pmesh, fec);
-   Array<int> boundary_dofs;
-   Vh->GetBoundaryTrueDofs(boundary_dofs);
-   int dimD = Vh->GetTrueVSize();
-   Vector xDC(dimD); xDC = 0.0;
-
-   ParObstacleProblem problem(Vh, Vh, &fRhs, &spherical_obstacle, boundary_dofs, xDC);
-   Vector x0(dimD); x0.Set(1.0, xDC);
-   Vector xf(dimD); xf = 0.0;
-
-   ParInteriorPointSolver optimizer(&problem); 
-   optimizer.SetTol(1.e-7);
-   optimizer.SetLinearSolveTol(1.e-10);
-   optimizer.SetLinearSolver(linSolver);
-   optimizer.SetMaxIter(maxIPMiters);
-   optimizer.Mult(x0, xf);
-
-   ParGridFunction d_gf(Vh);
-
-   d_gf.SetFromTrueDofs(xf);
-
-
-   FunctionCoefficient dtrue_fc(exact_solution_obstacle); // analytic solution
-   ParGridFunction dtrue_gf(Vh);
-   dtrue_gf.ProjectCoefficient(dtrue_fc);
-
-   double L2error = d_gf.ComputeL2Error(dtrue_fc);
-   if (myid == 0)
-   {
-     cout << "\n|| u_h - u ||_{L^2} = " << L2error << '\n' << endl;
-   } 
-   
-   ParaViewDataCollection paraview_dc("SphericalObstacleProblem", &pmesh);
-   paraview_dc.SetPrefixPath("ParaView");
-   paraview_dc.SetLevelsOfDetail(FEorder);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetTime(0.0);
-   paraview_dc.RegisterField("u(x,y) (analytic)", &dtrue_gf);
-   paraview_dc.RegisterField("u(x,y) (numerical)", &d_gf);
-   paraview_dc.Save();
-
-   delete Vh;
-   delete fec;
-   return 0;
-}
-
-
-double fRhs(const Vector &x)
-{
-  return 0.;
-}
-
-
-double spherical_obstacle(const Vector &pt)
-{
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double beta = 0.9;
-
-   double b = r0*beta;
-   double tmp = sqrt(r0*r0 - b*b);
-   double B = tmp + b*b/tmp;
-   double C = -b/tmp;
-
-   if (r > b)
-   {
-      return B + r * C;
-   }
-   else
-   {
-      return sqrt(r0*r0 - r*r);
-   }
-}
-
-double exact_solution_obstacle(const Vector &pt)
-{
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double a =  0.348982574111686;
-   double A = -0.340129705945858;
-
-   if (r > a)
-   {
-      return A * log(r);
-   }
-   else
-   {
-      return sqrt(r0*r0-r*r);
-   }
-} 
-

examples/contact/README.md

@@ -0,0 +1,17 @@
+# OneProcessAMGContact
+
+
+
+Be sure to edit the makefile so that it points to a parallel MFEM build
+
+specifically the MFEM_BUILD_DIR
+
+
+after building exQPContactBlockTL one can
+
+1. run the bash script scalingJobArray.bat via `source scalingJobArray.bat' which will populate the CG iterations required to solve
+      various linear systems into the data/ subdirectory
+2. run the python script data/process.py in order to put the scaling information into the single files algorithmicScaling_Elasticity.dat and algorithmicScaling_noElasticity.dat
+      in order to see the number of average AMG-CG iterations per optimization solve. 
+
+

examples/contact/SphericalObstacleProblem.cpp

@@ -1,175 +0,0 @@
-//                          Spherical Obstacle Problem
-//
-//
-// Compile with: make SphericalobstacleProblem
-//
-// Sample runs: ./SphericalobstacleProblem
-//
-//
-// Description: This example code demonstrates the use of MFEM to solve the
-//              bound-constrained energy minimization problem
-//
-//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
-
-
-#include "mfem.hpp"
-#include "Problems.hpp"
-#include "IPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-
-double fRhs(const Vector &);
-double spherical_obstacle(const Vector &);
-double exact_solution_obstacle(const Vector &);
-
-int main(int argc, char *argv[])
-{
-  int FEorder = 1;     // finite element order
-  int linSolver = 0;   // linear solver 0 (direct), 1 (iterative) or 2 (iterative)
-  int maxIPMiters = 30;
-  bool iAmRoot = true;
-  int ref_levels = 3;
-  
-  OptionsParser args(argc, argv);
-  args.AddOption(&FEorder, "-o", "--order",\
-		  "Order of the finite elements.");
-  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-       "IP-Newton linear system solution strategy.");
-  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-		  "Maximum number of IPM iterations");
-  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
-		  "Mesh Refinement");
-  
-  args.Parse();
-  if(!args.Good())
-  {
-    args.PrintUsage(cout);
-    return 1;
-  }
-  else
-  {
-    if( iAmRoot )
-    {
-      args.PrintOptions(cout);
-    }
-  }
-
-  const char *meshFile = "../../data/disk.mesh";
-  Mesh *mesh = new Mesh(meshFile, 1, 1);
-  int dim = mesh->Dimension(); // geometric dimension of the domain
-  {
-     for (int l = 0; l < ref_levels; l++)
-     {
-        mesh->UniformRefinement();
-     }
-  }
-  double h_min, h_max, kappa_min, kappa_max;
-  mesh->GetCharacteristics(h_min, h_max, kappa_min, kappa_max);
-
-  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
-  Array<int> ess_tdof_list;
-  if (mesh->bdr_attributes.Size())
-  {
-     Array<int> ess_bdr(mesh->bdr_attributes.Max());
-     ess_bdr = 1;
-     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-  }
-  
-  int dimD = Vh->GetTrueVSize();
-  Vector x0(dimD); x0 = 0.0;
-  Vector xf(dimD); xf = 0.0;
-  
-  ObstacleProblem problem(Vh, x0, &fRhs, &spherical_obstacle, ess_tdof_list);
-  InteriorPointSolver optimizer(&problem); 
-  
-  optimizer.SetTol(1.e-7);
-  optimizer.SetLinearSolver(linSolver);
-  optimizer.SetMaxIter(maxIPMiters);
-  optimizer.Mult(x0, xf);
-  
-  
-  double Einitial = problem.E(x0);
-  double Efinal = problem.E(xf);
-  cout << "Energy objective at initial point = " << Einitial << endl;
-  cout << "Energy objective at optimizer = " << Efinal << endl;
-
-  GridFunction d_gf(Vh);
-  d_gf = xf;
-
-  FunctionCoefficient dtrue_fc(exact_solution_obstacle); // exact solution
-  GridFunction dtrue_gf(Vh);
-  dtrue_gf.ProjectCoefficient(dtrue_fc);
-  
-  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
-  paraview_dc.SetPrefixPath("ParaView");
-  paraview_dc.SetLevelsOfDetail(FEorder);
-  paraview_dc.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc.SetHighOrderOutput(true);
-  paraview_dc.SetCycle(0);
-  paraview_dc.SetTime(0.0);
-  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
-  paraview_dc.RegisterField("d(x) (true)", &dtrue_gf);
-  paraview_dc.Save();
-  
-  FunctionCoefficient exact_coef(exact_solution_obstacle);
-  double L2_error = d_gf.ComputeL2Error(exact_coef);
-  cout << "||u - u_true||_L^2(Omega) = " << L2_error << ", hmax = " << h_max << ", hmin = " << h_min << endl;
-  
-  delete Vh;
-  delete fec;
-  delete mesh;
-  return 0;
-}
-
-
-double fRhs(const Vector &x)
-{
-  return 0.;
-}
-
-
-double spherical_obstacle(const Vector &pt)
-{
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double beta = 0.9;
-
-   double b = r0*beta;
-   double tmp = sqrt(r0*r0 - b*b);
-   double B = tmp + b*b/tmp;
-   double C = -b/tmp;
-
-   if (r > b)
-   {
-      return B + r * C;
-   }
-   else
-   {
-      return sqrt(r0*r0 - r*r);
-   }
-}
-
-double exact_solution_obstacle(const Vector &pt)
-{
-   double x = pt(0), y = pt(1);
-   double r = sqrt(x*x + y*y);
-   double r0 = 0.5;
-   double a =  0.348982574111686;
-   double A = -0.340129705945858;
-
-   if (r > a)
-   {
-      return A * log(r);
-   }
-   else
-   {
-      return sqrt(r0*r0-r*r);
-   }
-} 
-

examples/contact/TwoSidedDirichletObstacleProblem.cpp

@@ -1,143 +0,0 @@
-#include "mfem.hpp"
-#include "Problems.hpp"
-#include "IPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-
-
-double fRhs(const Vector &pt);
-double obstaclel(const Vector &pt);
-double obstacleu(const Vector &pt);
-double dmanufacturedFun(const Vector &pt);
-
-int main(int argc, char *argv[])
-{
-  int FEorder = 1; // order of the finite elements
-  int linSolver = 0;
-  int maxIPMiters = 30;
-  bool iAmRoot = true;
-  int ref_levels = 1; 
-  OptionsParser args(argc, argv);
-  args.AddOption(&FEorder, "-o", "--order",\
-		  "Order of the finite elements.");
-  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-       "IP-Newton linear system solution strategy.");
-  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-		  "Maximum number of IPM iterations");
-  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
-		  "Mesh Refinement");
-  
-  args.Parse();
-  if(!args.Good())
-  {
-    args.PrintUsage(cout);
-    return 1;
-  }
-  else
-  {
-    if( iAmRoot )
-    {
-      args.PrintOptions(cout);
-    }
-  }
-
-  const char *meshFile = "../../data/inline-quad.mesh";
-  Mesh *mesh = new Mesh(meshFile, 1, 1);
-  int dim = mesh->Dimension(); // geometric dimension of the domain
-  for (int l = 0; l < ref_levels; l++)
-  {
-      mesh->UniformRefinement();
-  }
-
-  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
-  Array<int> ess_tdof_list;
-  if (mesh->bdr_attributes.Size())
-  {
-     Array<int> ess_bdr(mesh->bdr_attributes.Max());
-     ess_bdr = 1;
-     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-  }
-  
-  double DC_val = 0.0;
-  int dimD = Vh->GetTrueVSize();
-
-  Vector x0(dimD); x0 = DC_val;
-  Vector xf(dimD); xf = 0.0;
-
-  ObstacleProblem problem(Vh, x0, &fRhs, &obstaclel, &obstacleu, ess_tdof_list);
-
-  InteriorPointSolver optimizer(&problem); 
-  optimizer.SetTol(1.e-7);
-  optimizer.SetLinearSolver(linSolver);
-  optimizer.SetMaxIter(maxIPMiters);
-  optimizer.Mult(x0, xf);
-  
-
-  GridFunction d_gf(Vh);
-
-  d_gf = xf;
-  
-  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
-  GridFunction dm_gf(Vh);
-  dm_gf.ProjectCoefficient(dm_fc);
-
-  
-  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
-  paraview_dc.SetPrefixPath("ParaView");
-  paraview_dc.SetLevelsOfDetail(FEorder);
-  paraview_dc.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc.SetHighOrderOutput(true);
-  paraview_dc.SetCycle(0);
-  paraview_dc.SetTime(0.0);
-  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
-  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
-  paraview_dc.Save();
-  
-
-  delete Vh;
-  delete fec;
-  delete mesh;
-  return 0;
-}
-
-
-
-double dmanufacturedFun(const Vector &pt)
-{
-  double alpha = 16.5;
-  return sin(M_PI * pt(1)) * (sin(M_PI * pt(0)) - alpha * pow(pt(0) * (1. - pt(0)), 2));
-}
-
-
-// f(x) forcing term... which enters the objective energy functional
-// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
-// of f(x). f(x) is such that in the absence of bound-constraints then
-// the solution of the optimization problem satisfies the PDE
-// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
-// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
-
-double fRhs(const Vector &pt)
-{
-  double alpha = 16.5;
-  double fx;
-  fx = pow(M_PI, 2) * sin(M_PI * pt(0));
-  fx += alpha * (2. * pow(pt(0), 2) + 2. * pow(1.-pt(0), 2) - 8. * pt(0) * (1.-pt(0)));
-  fx += pow(M_PI, 2) * sin(M_PI * pt(0)) * dmanufacturedFun(pt);
-  fx *= sin(M_PI * pt(1));
-  return fx;
-}
-
-double obstaclel(const Vector &pt)
-{
-  return 0.0;
-}
-
-double obstacleu(const Vector &pt)
-{
-  return 0.08;
-}

examples/contact/exQPContactBlockTL.cpp

@@ -1,16 +1,16 @@
-//                                Quadratic-Programming (QP) Contact example
+//                                Contact example
 //
-// Compile with: make exQPContactBlockTL
+// Compile with: make contact
 //
-// Sample runs:  ./exQPContactBlockTL
-
+// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
+// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
 
 #include <fstream>
 #include <iostream>
 #include <array>
 
 #include "mfem.hpp"
-#include "Problems.hpp"
+#include "problems.hpp"
 #include "IPsolver.hpp"
 
 using namespace std;
@@ -19,7 +19,9 @@ using namespace mfem;
 
 int main(int argc, char *argv[])
 {
-  int linSolver = 0;
+  Mpi::Init(argc, argv);
+  Hypre::Init();
+  int linSolver = 2;
   int maxIPMiters = 30;
   bool iAmRoot = true;
   int ref_levels = 0;
@@ -47,18 +49,18 @@ int main(int argc, char *argv[])
     }
   }
 	
-  Mesh * mesh1 = new Mesh("block1.mesh", 1, 1);
-  Mesh * mesh2 = new Mesh("rotatedblock2.mesh", 1, 1);
-  for(int i = 0; i < ref_levels; i++)
-  {
-     mesh1->UniformRefinement();
-     mesh2->UniformRefinement(); 
-  }
-
   // Create an instance of the nlp
-  ExContactBlockTL * contact = new ExContactBlockTL(mesh1, mesh2, 1);
+  ExContactBlockTL * contact = new ExContactBlockTL(ref_levels);
   int ndofs = contact->GetDimD();
   int nconstraints = contact->GetDimS();
+  std::ofstream problemDimStream;
+  problemDimStream.open("problemDim.dat", ios::out | ios::trunc);
+  problemDimStream << ndofs << endl;
+  problemDimStream.close();
+  std::ofstream problemDimConstraintsStream;
+  problemDimConstraintsStream.open("problemDimConstraints.dat", ios::out | ios::trunc);
+  problemDimConstraintsStream << nconstraints << endl;
+  problemDimConstraintsStream.close();
   
   // set up a QP-problem
   // E(d) = 1 / 2 d^T K d + f^T d
@@ -69,30 +71,18 @@ int main(int argc, char *argv[])
   //        that is one where the Dirichlet conditions hold... need to pull
   //        this data from contactBlockTL...
   Vector d0(ndofs); d0 = 0.0;
-  Array<int> ess_tdofs1 = contact->GetMesh1DirichletDofs();
-  Array<int> ess_tdofs2 = contact->GetMesh2DirichletDofs();
-  int sz1 = ess_tdofs1.Size();
-  int sz2 = ess_tdofs2.Size();
-  Array<int> DirichletDofs(sz1+sz2);
-  for (int i = 0; i<sz1; i++)
-  {
-    DirichletDofs[i] = ess_tdofs1[i];
-  }
-  for (int i = 0; i<sz2; i++)
-  {
-    DirichletDofs[i+sz1] = ess_tdofs2[i]+contact->GetVh1().GetTrueVSize();
-  }
-  GridFunction x1 = contact->GetMesh1GridFunction();
-  GridFunction x2 = contact->GetMesh2GridFunction();
-
+  Array<int> DirichletDofs = contact->GetDirichletDofs();
+  Array<double> DirichletVals = contact->GetDirichletVals();
   SparseMatrix *K;
   Vector f(ndofs); f = 0.0;
   contact->DdE(d0, f); K = contact->DddE(d0);
-  d0.SetVector(x1,0);
-  d0.SetVector(x2,x1.Size());
+  for(int i = 0; i < DirichletDofs.Size(); i++)
+  {
+    d0(DirichletDofs[i]) = DirichletVals[i];
+  }
   SparseMatrix *J;
   Vector g0(nconstraints); g0 = 0.0;
-  contact->g(d0, g0); J = contact->Ddg(d0);
+  J = contact->Ddg(d0); contact->g(d0, g0);
   Vector temp(nconstraints);
   J->Mult(d0, temp);
   g0.Add(-1.0, temp);
@@ -103,27 +93,35 @@ int main(int argc, char *argv[])
 
   double normJTei;
 
+  int reduced_nconstraints = 0; // find actual number of constraints
+  
+  
   Array<int> nonZeroRows;
   for(int i = 0; i < nconstraints; i++)
   {
-    Array<int> col_tmp;
-    Vector v_tmp; v_tmp = 0.0;
-    J->GetRow(i, col_tmp, v_tmp);
-    normJTei = v_tmp.Norml2();
+    ei(i) = 1.0;
+    J->MultTranspose(ei, JTei);
+    // nullify contributions from Dirichlet constrined dofs
+    for(int j = 0; j < DirichletDofs.Size(); j++)
+    {
+      JTei(DirichletDofs[j]) = 0.0;
+    }
+    normJTei = sqrt(InnerProduct(JTei, JTei));
     if (normJTei > 1.e-12)
     {
+      reduced_nconstraints += 1;
       nonZeroRows.Append(i);
     }
+    ei(i) = 0.0;
   }
-  mfem::out << J->Height() << " linearized constraints\n";
-  mfem::out << nonZeroRows.Size() << " (reduced) linearized constraints\n"; 
+  cout << "number of linearized constraints = " << reduced_nconstraints << endl; // 9 constraints 
   
   // remove zero rows of the gap function Jacobian and corresponding gap function entries
-  SparseMatrix * Jreduced = new SparseMatrix(nonZeroRows.Size(), ndofs);
-  Vector g0reduced(nonZeroRows.Size()); g0reduced = 0.0; 
+  SparseMatrix * Jreduced = new SparseMatrix(reduced_nconstraints, ndofs);
+  Vector g0reduced(reduced_nconstraints); g0reduced = 0.0; 
   
   
-  for(int i = 0; i < nonZeroRows.Size(); i++)
+  for(int i = 0; i < reduced_nconstraints; i++)
   {
     Array<int> col_tmp;
     Vector v_tmp; v_tmp = 0.0;
@@ -131,7 +129,7 @@ int main(int argc, char *argv[])
     
     /* obtain subset of columns of the given nonZero Jacobian row that are not Dirichlet constrained */
     bool freeDof;
-    Array<int> free_col_indicies;
+    Array<int> loc_indicies;
     for(int j = 0; j < col_tmp.Size(); j++)
     {
       freeDof = true;
@@ -144,102 +142,132 @@ int main(int argc, char *argv[])
       }
       if(freeDof)
       {
-        free_col_indicies.Append(j);
+        loc_indicies.Append(j);
       }
     }
 
-    Array<int> col_tmp_reduced(free_col_indicies.Size());
-    Vector v_tmp_reduced(free_col_indicies.Size());
-    for(int j = 0; j < free_col_indicies.Size(); j++)
+    Array<int> col_tmp_reduced(loc_indicies.Size());
+    Vector v_tmp_reduced(loc_indicies.Size());
+    for(int j = 0; j < loc_indicies.Size(); j++)
     {
-      col_tmp_reduced[j] = col_tmp[free_col_indicies[j]];
-      v_tmp_reduced(j)   = v_tmp(free_col_indicies[j]);
+      col_tmp_reduced[j] = col_tmp[loc_indicies[j]];
+      v_tmp_reduced(j)   = v_tmp(loc_indicies[j]);
     }
 
     Jreduced->SetRow(i, col_tmp_reduced, v_tmp_reduced);
     g0reduced(i) = g0(nonZeroRows[i]);
   }
 
-  QPOptProblem *QPContact = new QPOptProblem(*K, *Jreduced, f, g0reduced);
+
+  QPContactProblem *QPContact = new QPContactProblem(*K, *Jreduced, f, g0reduced);
   
-  InteriorPointSolver * QPContactOptimizer = new InteriorPointSolver(QPContact);
+  Mesh * mesh1 = new Mesh("meshes/block1.mesh", 1, 1);
+  Mesh * mesh2 = new Mesh("meshes/rotatedblock2.mesh", 1, 1);
+  for(int i = 0; i < ref_levels; i++)
+  {
+     mesh1->UniformRefinement();
+     mesh2->UniformRefinement(); 
+  }
+  
+  int numMeshes = 2;
+  Mesh *meshArray[numMeshes];
+  meshArray[0] = mesh1;
+  meshArray[1] = mesh2;
+  Mesh mesh(meshArray, numMeshes);
+  
+  ParMesh pmesh(MPI_COMM_WORLD, mesh);  
+  H1_FECollection fec(1, mesh.Dimension());
+  ParFiniteElementSpace fespace(&pmesh, &fec, mesh.Dimension(), Ordering::byVDIM);
+
+  InteriorPointSolver * QPContactOptimizer = new InteriorPointSolver(QPContact, &fespace);
   QPContactOptimizer->SetTol(1.e-6);
   QPContactOptimizer->SetLinearSolver(linSolver);
+  QPContactOptimizer->SetMaxIter(50);
   Vector x0(ndofs); x0 = 0.0;
-  x0.SetVector(x1,0);
-  x0.SetVector(x2,x1.Size());
-
+  for(int i = 0; i < DirichletDofs.Size(); i++)
+  {
+    x0(DirichletDofs[i]) = DirichletVals[i];
+  }
   Vector xf(ndofs); xf = 0.0;
   QPContactOptimizer->Mult(x0, xf);
   
-  MFEM_VERIFY(QPContactOptimizer->GetConverged(), "Interior point solver did not converge.");
   double Einitial = QPContact->E(x0);
   double Efinal = QPContact->E(xf);
   cout << "Energy objective at initial point = " << Einitial << endl;
   cout << "Energy objective at QP optimizer = " << Efinal << endl;
- 
+  QPContactOptimizer->GetCGIterNumbers().Print(mfem::out, 20);
+  MFEM_VERIFY(QPContactOptimizer->GetConverged(), "Interior point solver did not converge.");
   
   
-  int gdim = mesh1->Dimension();
-  FiniteElementCollection * fec = new H1_FECollection(1, gdim);
-  FiniteElementSpace * fespace1 = new FiniteElementSpace(mesh1, fec, gdim, Ordering::byVDIM);
-  FiniteElementSpace * fespace2 = new FiniteElementSpace(mesh2, fec, gdim, Ordering::byVDIM);
-   
-  GridFunction x1_gf(fespace1);
-  GridFunction x2_gf(fespace2);
+  //Mesh * mesh1 = new Mesh("meshes/block1.mesh", 1, 1);
+  //Mesh * mesh2 = new Mesh("meshes/rotatedblock2.mesh", 1, 1);
+  //for(int i = 0; i < ref_levels; i++)
+  //{
+  //   mesh1->UniformRefinement();
+  //   mesh2->UniformRefinement(); 
+  //}
+  //int gdim = mesh1->Dimension();
+  //FiniteElementCollection * fec = new H1_FECollection(1, gdim);
+  //FiniteElementSpace * fespace1 = new FiniteElementSpace(mesh1, fec, gdim, Ordering::byVDIM);
+  //FiniteElementSpace * fespace2 = new FiniteElementSpace(mesh2, fec, gdim, Ordering::byVDIM);
+  // 
+  //GridFunction x1_gf(fespace1);
+  //GridFunction x2_gf(fespace2);
 
-  int ndof1 = fespace1->GetTrueVSize();
-  int ndof2 = fespace2->GetTrueVSize();
-  int ndof  = ndof1 + ndof2;
-  for(int i = 0; i < ndof1; i++)
-  {
-    x1_gf(i) = xf(i);
-  }
-  for(int i = ndof1; i < ndof; i++)
-  {
-    x2_gf(i - ndof1) = xf(i);
-  }
+  //int ndof1 = fespace1->GetTrueVSize();
+  //int ndof2 = fespace2->GetTrueVSize();
+  //int ndof  = ndof1 + ndof2;
+  //for(int i = 0; i < ndof1; i++)
+  //{
+  //  x1_gf(i) = xf(i);
+  //}
+  //for(int i = ndof1; i < ndof; i++)
+  //{
+  //  x2_gf(i - ndof1) = xf(i);
+  //}
 
-  mesh1->SetNodalFESpace(fespace1);
-  mesh2->SetNodalFESpace(fespace2);
-  GridFunction *nodes1 = mesh1->GetNodes();
-  GridFunction *nodes2 = mesh2->GetNodes();
+  //mesh1->SetNodalFESpace(fespace1);
+  //mesh2->SetNodalFESpace(fespace2);
+  //GridFunction *nodes1 = mesh1->GetNodes();
+  //GridFunction *nodes2 = mesh2->GetNodes();
 
-  {
-    *nodes1 += x1_gf;
-    *nodes2 += x2_gf;
-  }
-  
+  //{
+  //  *nodes1 += x1_gf;
+  //  *nodes2 += x2_gf;
+  //}
+  //
 
-  ParaViewDataCollection paraview_dc1("QPContactBody1", mesh1);
-  paraview_dc1.SetPrefixPath("ParaView");
-  paraview_dc1.SetLevelsOfDetail(1);
-  paraview_dc1.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc1.SetHighOrderOutput(true);
-  paraview_dc1.SetCycle(0);
-  paraview_dc1.SetTime(0.0);
-  paraview_dc1.RegisterField("Body1", &x1_gf);
-  paraview_dc1.Save();
-  
-  ParaViewDataCollection paraview_dc2("QPContactBody2", mesh2);
-  paraview_dc2.SetPrefixPath("ParaView");
-  paraview_dc2.SetLevelsOfDetail(1);
-  paraview_dc2.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc2.SetHighOrderOutput(true);
-  paraview_dc2.SetCycle(0);
-  paraview_dc2.SetTime(0.0);
-  paraview_dc2.RegisterField("Body2", &x2_gf);
-  paraview_dc2.Save();
+  //ParaViewDataCollection paraview_dc1("QPContactBody1", mesh1);
+  //paraview_dc1.SetPrefixPath("ParaView");
+  //paraview_dc1.SetLevelsOfDetail(1);
+  //paraview_dc1.SetDataFormat(VTKFormat::BINARY);
+  //paraview_dc1.SetHighOrderOutput(true);
+  //paraview_dc1.SetCycle(0);
+  //paraview_dc1.SetTime(0.0);
+  //paraview_dc1.RegisterField("Body1", &x1_gf);
+  //paraview_dc1.Save();
+  //
+  //ParaViewDataCollection paraview_dc2("QPContactBody2", mesh2);
+  //paraview_dc2.SetPrefixPath("ParaView");
+  //paraview_dc2.SetLevelsOfDetail(1);
+  //paraview_dc2.SetDataFormat(VTKFormat::BINARY);
+  //paraview_dc2.SetHighOrderOutput(true);
+  //paraview_dc2.SetCycle(0);
+  //paraview_dc2.SetTime(0.0);
+  //paraview_dc2.RegisterField("Body2", &x2_gf);
+  //paraview_dc2.Save();
 
-  delete fespace1;
-  delete fespace2;
-  delete fec;
-  delete mesh1;
-  delete mesh2;
+  //delete fespace1;
+  //delete fespace2;
+  //delete fec;
+  //delete mesh1;
+  //delete mesh2;
   
   delete QPContact;
   delete QPContactOptimizer;
   
+  delete K;
+  delete J;
   delete Jreduced;
   delete contact;
   return 0;

examples/contact/makefile

@@ -1,125 +1,36 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../../
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/contact/,)
+MFEM_BUILD_DIR ?= ../..
+SRC = ./
+
 CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_EXAMPLES = ObstacleProblem SphericalObstacleProblem DirichletObstacleProblem exQPContactBlockTL
-PAR_EXAMPLES = ParObstacleProblem
-EXAMPLES = $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+# Remove built-in rule
+#%: %.cpp
 
-ifeq ($(MFEM_USE_SUITESPARSE),NO)
-$(SEQ_EXAMPLES):
-	$(error MFEM is not configured with SUITESPARSE)
-endif
-
-ifeq ($(MFEM_USE_MUMPS),NO)
-ifeq ($(MFEM_USE_MKL_CPARDISO), NO)
-$(PAR_EXAMPLES):
-	$(error MFEM is not configured with MUMPS or CPARDISO)
-endif
-endif
-
-all: $(EXAMPLES)
-
-ObstacleProblem: ObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) ObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
-
-SphericalObstacleProblem: SphericalObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) SphericalObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
-
-DirichletObstacleProblem: DirichletObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) DirichletObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
-
-DirichletObstacleProblemVariant: DirichletObstacleProblemVariant.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) DirichletObstacleProblemVariant.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
-
-TwoSidedDirichletObstacleProblem: TwoSidedDirichletObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) TwoSidedDirichletObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+exQPContactBlockTL: exQPContactBlockTL.o problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) exQPContactBlockTL.o problems.o IPsolver.o -o $@ $(MFEM_LIBS)
 
 
-exQPContactBlockTL: exQPContactBlockTL.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) exQPContactBlockTL.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
 
-ParTest: ParTest.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) ParTest.o -o $@ $(MFEM_LIBS)
+exQPContactBlockTL.o: exQPContactBlockTL.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
 
-
-ObstacleProblem.o: $(SRC)ObstacleProblem.cpp $(CONFIG_MK)
+problems.o: problems.cpp $(CONFIG_MK)
 	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
 
-SphericalObstacleProblem.o: $(SRC)SphericalObstacleProblem.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
-
-DirichletObstacleProblem.o: $(SRC)DirichletObstacleProblem.cpp $(CONFIG_MK)
+IPsolver.o: IPsolver.cpp $(CONFIG_MK)
 	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
 
-DirichletObstacleProblemVariant.o: $(SRC)DirichletObstacleProblemVariant.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
 
-TwoSidedDirichletObstacleProblem.o: $(SRC)TwoSidedDirichletObstacleProblem.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+.PHONY: clean
+clean:
+	rm -f *.o exQPContactBlockTL
 
-exQPContactBlockTL.o: $(SRC)exQPContactBlockTL.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
 
-Problems.o: $(SRC)Problems.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
-
-IPsolver.o: $(SRC)IPsolver.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
-
-ParObstacleProblem: ParObstacleProblem.o ParProblems.o ParIPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) ParObstacleProblem.o ParProblems.o ParIPsolver.o -o $@ $(MFEM_LIBS)
-
-ParSphericalObstacleProblem: ParSphericalObstacleProblem.o ParProblems.o ParIPsolver.o $(MFEM_LIB_FILE)
-	$(MFEM_CXX) $(MFEM_FLAGS) ParSphericalObstacleProblem.o ParProblems.o ParIPsolver.o -o $@ $(MFEM_LIBS)
-
-ParObstacleProblem.o: $(SRC)ParObstacleProblem.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
-
-ParSphericalObstacleProblem.o: $(SRC)ParSphericalObstacleProblem.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
-
-ParProblems.o: $(SRC)ParProblems.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
-
-ParIPsolver.o: $(SRC)ParIPsolver.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
-
-ParTest.o: $(SRC)ParTest.cpp $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
-
-clean: clean-build 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-# For out-of-source builds, link the data files from the source tree:
-ifneq ($(SRC),)
-DATA_FILES = block1.mesh rotatedblock2.mesh
-$(DATA_FILES): %: $(SRC)%
-	ln -sf $(<) .
-copy-data: | $(DATA_FILES)
-# For out-of-source builds, the test and sample runs for 'field-interp' need
-# data from the meshing miniapps directory:
-exQPContactBlockTL: | mesh-data
-.PHONY: mesh-data
-mesh-data:
-	$(MAKE) -C ./ copy-data
-endif

examples/contact/nodepair.hpp

@@ -849,9 +849,6 @@ void Assemble_Contact(const int m, const int npoints, const int ndofs,
       dg = 0.0;
       dg2 = 0.;
       NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
-      //x1.Print();
-      //xi2.Print();
-      //coords2.Print();
       g[s_conn[i]] = g_tmp; // should be unique
       Array<int> m_conn_i(4);
       m_conn.GetSubArray(4*i, 4, m_conn_i);
@@ -888,9 +885,13 @@ void Assemble_Contact(const int m, const int npoints, const int ndofs,
          dM_i[j] = j_idx[j];
          dM_j[j] = j_idx[j];
       }
-      //dg2.Print();
-      //dM[s_conn[i]].Print();
       dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
+      dM[s_conn[i]].Finalize();
+      dM[s_conn[i]].Threshold(0.0);
+      dM[s_conn[i]].SortColumnIndices();
    }
+   M.Finalize();
+   M.Threshold(0.0);
+   M.SortColumnIndices();
 };
 

examples/contact/obstacleProblem.cpp

@@ -1,119 +0,0 @@
-#include "mfem.hpp"
-#include "Problems.hpp"
-#include "IPsolver.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-
-
-double dmanufacturedFun(const Vector &);
-double fRhs(const Vector &);
-double obstacle(const Vector &);
-
-int main(int argc, char *argv[])
-{
-  int FEorder = 1; // order of the finite elements
-  int linSolver = 0;
-  int maxIPMiters = 30;
-  bool iAmRoot = true;
-  
-  OptionsParser args(argc, argv);
-  args.AddOption(&FEorder, "-o", "--order",\
-		  "Order of the finite elements.");
-  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
-       "IP-Newton linear system solution strategy.");
-  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
-		  "Maximum number of IPM iterations");
-  
-  args.Parse();
-  if(!args.Good())
-  {
-    args.PrintUsage(cout);
-    return 1;
-  }
-  else
-  {
-    if( iAmRoot )
-    {
-      args.PrintOptions(cout);
-    }
-  }
-
-  const char *meshFile = "../../data/inline-quad.mesh";
-  Mesh *mesh = new Mesh(meshFile, 1, 1);
-  int dim = mesh->Dimension(); // geometric dimension of the domain
-  {
-     int ref_levels = 3;
-     for (int l = 0; l < ref_levels; l++)
-     {
-        mesh->UniformRefinement();
-     }
-  }
-
-  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
-  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
-  ObstacleProblem problem(Vh, &fRhs, &obstacle);
-  
-  int dimD = problem.GetDimD();
-  Vector x0(dimD); x0 = 0.0;
-  Vector xf(dimD); xf = 0.0;
-
-  InteriorPointSolver optimizer(&problem); 
-  optimizer.SetTol(1.e-7);
-  optimizer.SetLinearSolver(linSolver);
-  optimizer.SetMaxIter(maxIPMiters);
-  optimizer.Mult(x0, xf);
-
-  GridFunction d_gf(Vh);
-
-  d_gf = xf;
-
-  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
-  GridFunction dm_gf(Vh);
-  dm_gf.ProjectCoefficient(dm_fc);
-  
-  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
-  paraview_dc.SetPrefixPath("ParaView");
-  paraview_dc.SetLevelsOfDetail(FEorder);
-  paraview_dc.SetDataFormat(VTKFormat::BINARY);
-  paraview_dc.SetHighOrderOutput(true);
-  paraview_dc.SetCycle(0);
-  paraview_dc.SetTime(0.0);
-  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
-  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
-  paraview_dc.Save();
-  
-  
-  delete Vh;
-  delete fec;
-  delete mesh;
-  return 0;
-}
-
-
-double dmanufacturedFun(const Vector &x)
-{
-  return cos(2*M_PI*x(0)) + 0.2 - 2.0*(pow(x(0),3) - 1.5*pow(x(0),2));
-}
-
-
-// f(x) forcing term... which enters the objective energy functional
-// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
-// of f(x). f(x) is such that in the absence of bound-constraints then
-// the solution of the optimization problem satisfies the PDE
-// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
-// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
-double fRhs(const Vector &x)
-{
-  double fx = 0.;
-  fx = 0.2 - 2.0 * (pow(x(0),3)- 1.5*pow(x(0),2.) - 6 * x(0) + 3.) + (1. + pow(2.*M_PI,2))*cos(2.*M_PI*x(0));
-  return fx;
-}
-
-double obstacle(const Vector &x)
-{
-  return 0.0;
-}

examples/contact/problems.hpp

@@ -2,7 +2,6 @@
 #include <fstream>
 #include <iostream>
 #include <set>
-
 using namespace std;
 using namespace mfem;
 
@@ -11,18 +10,19 @@ using namespace mfem;
 
 
 
-// abstract GeneralOptProblem class
-// for the problem
-// min_(u,m) f(u,m) 
-// such that c(u,m)=0 and m >= ml
-class GeneralOptProblem
+// abstract OptProblem class
+// of the form
+// min_(u,m) f(u,m) s.t. c(u,m)=0 and m>=ml
+// the primal variable (u, m) is represented as a BlockVector
+
+class OptProblem
 {
 protected:
     int dimU, dimM, dimC;
     Array<int> block_offsetsx;
     Vector ml;
 public:
-    GeneralOptProblem();
+    OptProblem();
     virtual double CalcObjective(const BlockVector &) const = 0;
     virtual void Duf(const BlockVector &, Vector &) const = 0;
     virtual void Dmf(const BlockVector &, Vector &) const = 0;
@@ -34,10 +34,7 @@ public:
     virtual void c(const BlockVector &, Vector &) const = 0;
     virtual SparseMatrix* Duc(const BlockVector &) = 0;
     virtual SparseMatrix* Dmc(const BlockVector &) = 0;
-    virtual SparseMatrix* lDuuc(const BlockVector &, const Vector &) = 0;
-    virtual SparseMatrix* lDumc(const BlockVector &, const Vector &) = 0;
-    virtual SparseMatrix* lDmuc(const BlockVector &, const Vector &) = 0;
-    virtual SparseMatrix* lDmmc(const BlockVector &, const Vector &) = 0;
+    // TO DO: include Hessian terms of constraint c
     // TO DO: include log-barrier lumped-mass and pass that
     // to the optimizer
     //virtual SparseMatrix* GetLogBarrierLumpedMass() = 0;
@@ -45,26 +42,23 @@ public:
     int GetDimM() const { return dimM; }; 
     int GetDimC() const { return dimC; };
     Vector Getml() const { return ml; };
-    ~GeneralOptProblem();
+    ~OptProblem();
 };
 
 
-// abstract OptProblem class
+// abstract ContactProblem class
 // of the form
 // min_d e(d) s.t. g(d) >= 0
-class OptProblem : public GeneralOptProblem
+// TO DO: add functionality for gap function Hessian apply 
+class ContactProblem : public OptProblem
 {
 protected:
     int dimD;
     int dimS;
     Array<int> block_offsetsx;
-    SparseMatrix * negIdentity;
-    SparseMatrix * zeroMatum;
-    SparseMatrix * zeroMatmu;
-    SparseMatrix * zeroMatmm;
 public:
-    //OptProblem(int, int);        // constructor
-    OptProblem();
+    //ContactProblem(int, int);        // constructor
+    ContactProblem();
     void InitializeParentData(int, int);
     double CalcObjective(const BlockVector &) const; // objective e
     void Duf(const BlockVector &, Vector &) const;
@@ -76,111 +70,107 @@ public:
     void c(const BlockVector &, Vector &) const;
     SparseMatrix* Duc(const BlockVector &);
     SparseMatrix* Dmc(const BlockVector &);
-    SparseMatrix* lDuuc(const BlockVector &, const Vector &);
-    SparseMatrix* lDumc(const BlockVector &, const Vector &);
-    SparseMatrix* lDmuc(const BlockVector &, const Vector &);
-    SparseMatrix* lDmmc(const BlockVector &, const Vector &);
     virtual double E(const Vector &) const = 0;           // objective e(d) (energy function)
     virtual void DdE(const Vector &, Vector &) const = 0; // gradient of objective De / Dd
     virtual SparseMatrix* DddE(const Vector &) = 0;       // Hessian of objective D^2 e / D d^2
     virtual void g(const Vector &, Vector &) const = 0;   // inequality constraint g(d) >= 0 (gap function)
     virtual SparseMatrix* Ddg(const Vector &) = 0;        // Jacobian of inequality constraint Dg / Dd
-    virtual SparseMatrix* lDddg(const Vector &, const Vector &) = 0;
     int GetDimD() const { return dimD; };
     int GetDimS() const { return dimS; };
-    virtual ~OptProblem();
+    virtual ~ContactProblem();
 };
 
 
-class ObstacleProblem : public OptProblem
+class ObstacleProblem : public ContactProblem
 {
 protected:
-   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d + \psi >= 0
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d >= 0
    // stiffness matrix used to define objective
    BilinearForm *Kform;
    LinearForm   *fform;
-   Array<int> ess_tdof_list;
-   SparseMatrix  *K;
+   Array<int> empty_tdof_list; // needed for calls to FormSystemMatrix
+   SparseMatrix  K;
    SparseMatrix  *J;
-   SparseMatrix  *Hcl;
    FiniteElementSpace *Vh;
    Vector f;
-   Vector psil;
-   Vector psiu;
-   bool twoBounds;
-   Vector xDC;
-   double Ce;
 public : 
-   ObstacleProblem(FiniteElementSpace*, double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &));
-   ObstacleProblem(FiniteElementSpace*, Vector&,  double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list);
-   ObstacleProblem(FiniteElementSpace*, Vector &, double (*fSource)(const Vector &), double (*obstacleSourcel)(const Vector &), double (*obstacleSourceu)(const Vector &), Array<int> tdof_list);
+   ObstacleProblem(FiniteElementSpace* , double (*fSource)(const Vector &));
    double E(const Vector &) const;
    void DdE(const Vector &, Vector &) const;
    SparseMatrix* DddE(const Vector &);
    void g(const Vector &, Vector &) const;
    SparseMatrix* Ddg(const Vector &);
-   SparseMatrix * lDddg(const Vector &, const Vector &);
+   // TO DO: include lumped-mass for the log-barrier term
+   //SparseMatrix* GetLogBarrierLumpedMass();
    virtual ~ObstacleProblem();
 };
 
-
-SparseMatrix * GenerateProjector(int n, Array<int> ess_tdof_list);
-
-
-class ObstacleProblemVariant : public OptProblem
+class DirichletObstacleProblem : public ContactProblem
 {
 protected:
    // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d + \psi >= 0
    // stiffness matrix used to define objective
    BilinearForm *Kform;
    LinearForm   *fform;
-   Array<int> ess_tdof_list;
-   Array<int> noness_tdof_list;
+   Array<int> ess_tdof_list; // needed for calls to FormSystemMatrix
    SparseMatrix  *K;
-   SparseMatrix  *RKP; // R K P = R K R^T
    SparseMatrix  *J;
-   SparseMatrix  *Hcl;
-   SparseMatrix *R;
    FiniteElementSpace *Vh;
    Vector f;
-   Vector psil;
+   Vector psi;
    Vector xDC;
-   double Ce;
 public : 
-   ObstacleProblemVariant(FiniteElementSpace*, Vector&,  double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list);
+   DirichletObstacleProblem(FiniteElementSpace*, Vector&,  double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list, bool);
    double E(const Vector &) const;
    void DdE(const Vector &, Vector &) const;
    SparseMatrix* DddE(const Vector &);
    void g(const Vector &, Vector &) const;
    SparseMatrix* Ddg(const Vector &);
-   SparseMatrix * lDddg(const Vector &, const Vector &);
-   virtual ~ObstacleProblemVariant();
+   virtual ~DirichletObstacleProblem();
 };
 
 
+// abstract out technology for removing null rows of the Jacobian from an existing contact problem
+class ReducedContactProblem : public ContactProblem
+{
+protected:
+   Array<int> activeConstraints;
+   Array<int> fixedDofs;
+   ContactProblem * contact;
+   int dimSin;
+public:
+   ReducedContactProblem(ContactProblem * contact, Array<int> activeConstraints, Array<int> fixedDofs);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   virtual ~ReducedContactProblem();
+};
 
 
-class QPOptProblem : public OptProblem
+class QPContactProblem : public ContactProblem
 {
 protected:
   SparseMatrix *K;
   SparseMatrix *J;
-  SparseMatrix *zeroMatdd;
   Vector f;
   Vector g0;
 public:
-  QPOptProblem(const SparseMatrix, const SparseMatrix, const Vector, const Vector);
+  QPContactProblem(const SparseMatrix, const SparseMatrix, const Vector, const Vector);
   double E(const Vector &) const;
   void DdE(const Vector &, Vector &) const;
   SparseMatrix* DddE(const Vector &);
   void g(const Vector &, Vector &) const;
   SparseMatrix* Ddg(const Vector &);
-  SparseMatrix * lDddg(const Vector &, const Vector &);
-  virtual ~QPOptProblem();
+  virtual ~QPContactProblem();
 };
 
 
-class ExContactBlockTL : public OptProblem
+typedef int Index;
+typedef double Number;
+
+class ExContactBlockTL : public ContactProblem
 {
 public:
    double E(const Vector &) const;
@@ -188,20 +178,127 @@ public:
    SparseMatrix* DddE(const Vector &);
    void g(const Vector &, Vector &) const;
    SparseMatrix* Ddg(const Vector &);
-   SparseMatrix * lDddg(const Vector &, const Vector &);
    FiniteElementSpace GetVh1();
    FiniteElementSpace GetVh2();
-   SparseMatrix *zeroMatdd;
+
 public:
    /** default constructor */
-   ExContactBlockTL(Mesh *, Mesh *, int);
+   ExContactBlockTL(int );
    
 
    /** default destructor */
    virtual ~ExContactBlockTL();
 
+   ///**@name Overloaded from TNLP */
+   ///** Method to return some info about the nlp */
+   //virtual bool get_nlp_info(
+   //   Index&          n,
+   //   Index&          m,
+   //   Index&          nnz_jac_g,
+   //   Index&          nnz_h_lag,
+   //   IndexStyleEnum& index_style
+   //);
+
+   ///** Method to return the bounds for my problem */
+   //virtual bool get_bounds_info(
+   //   Index   n,
+   //   Number* x_l,
+   //   Number* x_u,
+   //   Index   m,
+   //   Number* g_l,
+   //   Number* g_u
+   //);
+
+   ///** Method to return the starting point for the algorithm */
+   //virtual bool get_starting_point(
+   //   Index   n,
+   //   bool    init_x,
+   //   Number* x,
+   //   bool    init_z,
+   //   Number* z_L,
+   //   Number* z_U,
+   //   Index   m,
+   //   bool    init_lambda,
+   //   Number* lambda
+   //);
+
+   /* Method to return the objective value */
+   virtual bool eval_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number&       obj_value
+   ) const;
+
+   /* Method to return the gradient of the objective */
+   virtual bool eval_grad_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number*       grad_f
+   ) const;
+
+   /* Method to return the constraint residuals */
+   virtual bool eval_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Number*       cons
+   ) const;
+
+   /* Method to return:
+      1) The structure of the Jacobian (if "values" is NULL)
+      2) The values of the Jacobian (if "values" is not NULL)
+   */
+   virtual bool eval_jac_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Index         nele_jac,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   ) const;
+
+   /* Method to return:
+    *   1) The structure of the Hessian of the Lagrangian (if "values" is NULL)
+    *   2) The values of the Hessian of the Lagrangian (if "values" is not NULL)
+   */
+   virtual bool eval_h(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number        obj_factor,
+      Index         m,
+      const Number* lambda,
+      bool          new_lambda,
+      Index         nele_hess,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   );
+
+   ///** This method is called when the algorithm is complete so the TNLP can store/write the solution */
+   //virtual void finalize_solution(
+   //   SolverReturn               status,
+   //   Index                      n,
+   //   const Number*              x,
+   //   const Number*              z_L,
+   //   const Number*              z_U,
+   //   Index                      m,
+   //   const Number*              g,
+   //   const Number*              lambda,
+   //   Number                     obj_value,
+   //   const IpoptData*           ip_data,
+   //   IpoptCalculatedQuantities* ip_cq
+   //);
+
 private:
    void update_g() const;
+   void update_jac();
+   void update_hess();
 
 private:
    /**@name Methods to block default compiler methods.
@@ -240,6 +337,8 @@ private:
    GridFunction* nodes2;
    mutable GridFunction* x1;
    mutable GridFunction* x2;
+   LinearForm* b1;
+   LinearForm* b2;
    PWConstCoefficient* lambda1_func;
    PWConstCoefficient* lambda2_func;
    PWConstCoefficient* mu1_func;
@@ -271,7 +370,7 @@ private:
    mfem::Vector B1, X1;
    SparseMatrix A2;
    mfem::Vector B2, X2;
-   BlockVector *B;
+
    SparseMatrix* K;
    mutable mfem::Vector gapv;
    mutable mfem::Vector m_xi;
@@ -285,16 +384,13 @@ private:
 
    Array<int> Dirichlet_dof;
    Array<double> Dirichlet_val;
-   Array<int> block_offsets;
+
 public: 
    Mesh * GetMesh1() {return mesh1;}
    Mesh * GetMesh2() {return mesh2;}
-   GridFunction & GetMesh1GridFunction() {return *x1;}
-   GridFunction & GetMesh2GridFunction() {return *x2;}
-   Array<int> & GetMesh1DirichletDofs() {return ess_tdof_list1;}
-   Array<int> & GetMesh2DirichletDofs() {return ess_tdof_list2;}
+   Array<int> GetDirichletDofs() {return Dirichlet_dof;}
+   Array<double> GetDirichletVals() {return Dirichlet_val;}
+
 };
 
-
-
 #endif

examples/ex25.cpp

@@ -32,6 +32,7 @@
 //               We recommend viewing Example 22 before viewing this example.
 
 #include "mfem.hpp"
+#include <memory>
 #include <fstream>
 #include <iostream>
 
@@ -44,7 +45,7 @@ using namespace std;
 using namespace mfem;
 
 // Class for setting up a simple Cartesian PML region
-class CartesianPML
+class PML
 {
 private:
    Mesh *mesh;
@@ -69,7 +70,7 @@ private:
 
 public:
    // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<double> length_);
 
    // Return Computational Domain Boundary
    Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
@@ -91,12 +92,12 @@ public:
 class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   PML * pml = nullptr;
+   void (*Function)(const Vector &, PML *, Vector &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, PML *,
                                               Vector &),
-                            CartesianPML * pml_)
+                            PML * pml_)
       : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
 
@@ -125,13 +126,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D);
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D);
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -267,7 +268,7 @@ int main(int argc, char *argv[])
          length = 0.25;
          break;
    }
-   CartesianPML * pml = new CartesianPML(mesh,length);
+   PML * pml = new PML(mesh,length);
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
@@ -467,16 +468,14 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
       double s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
       if (pa)
       {
          // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
       else
       {
@@ -485,15 +484,13 @@ int main(int argc, char *argv[])
          prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
 
          // Gauss-Seidel Smoother
-         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
-         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
-         pc_r = gs00;
-         pc_i = gs11;
+         pc_r.reset(new GSSmoother(*PCOpAh.As<SparseMatrix>()));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
 
       BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());
 
       GMRESSolver gmres;
       gmres.SetPrintLevel(1);
@@ -807,7 +804,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -824,7 +821,7 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -841,7 +838,7 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -858,7 +855,7 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -883,7 +880,7 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -907,7 +904,7 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -931,14 +928,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<double> length_)
    : mesh(mesh_), length(length_)
 {
    dim = mesh->Dimension();
    SetBoundaries();
 }
 
-void CartesianPML::SetBoundaries()
+void PML::SetBoundaries()
 {
    comp_dom_bdr.SetSize(dim, 2);
    dom_bdr.SetSize(dim, 2);
@@ -953,7 +950,7 @@ void CartesianPML::SetBoundaries()
    }
 }
 
-void CartesianPML::SetAttributes(Mesh *mesh_)
+void PML::SetAttributes(Mesh *mesh_)
 {
    // Initialize bdr attributes
    for (int i = 0; i < mesh_->GetNBE(); ++i)
@@ -1002,8 +999,8 @@ void CartesianPML::SetAttributes(Mesh *mesh_)
    mesh_->SetAttributes();
 }
 
-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs)
+void PML::StretchFunction(const Vector &x,
+                          vector<complex<double>> &dxs)
 {
    complex<double> zi = complex<double>(0., 1.);
 

examples/ex25p.cpp

@@ -44,7 +44,7 @@ using namespace std;
 using namespace mfem;
 
 // Class for setting up a simple Cartesian PML region
-class CartesianPML
+class PML
 {
 private:
    Mesh *mesh;
@@ -69,7 +69,7 @@ private:
 
 public:
    // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<double> length_);
 
    // Return Computational Domain Boundary
    Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
@@ -91,12 +91,12 @@ public:
 class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   PML * pml = nullptr;
+   void (*Function)(const Vector &, PML *, Vector &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, PML *,
                                               Vector &),
-                            CartesianPML * pml_)
+                            PML * pml_)
       : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
 
@@ -125,13 +125,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D);
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D);
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D);
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D);
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -295,7 +295,7 @@ int main(int argc, char *argv[])
          length = 0.25;
          break;
    }
-   CartesianPML * pml = new CartesianPML(mesh,length);
+   PML * pml = new PML(mesh,length);
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
@@ -478,11 +478,11 @@ int main(int argc, char *argv[])
    if (!pa && mumps_solver)
    {
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
-      MUMPSSolver mumps;
+      MUMPSSolver mumps(A->GetComm());
       mumps.SetPrintLevel(0);
       mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
       mumps.SetOperator(*A);
-      mumps.Mult(B,X);
+      mumps.Mult(B, X);
       delete A;
    }
 #endif
@@ -524,16 +524,14 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
       int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
       if (pa)
       {
          // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
       else
       {
@@ -541,15 +539,13 @@ int main(int argc, char *argv[])
          prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
 
          // Hypre AMS
-         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
-         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
-         pc_r = ams00;
-         pc_i = ams11;
+         pc_r.reset(new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
 
       BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());
 
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetPrintLevel(1);
@@ -884,7 +880,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -901,7 +897,7 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -918,7 +914,7 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -935,7 +931,7 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -960,7 +956,7 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -984,7 +980,7 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -1008,14 +1004,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<double> length_)
    : mesh(mesh_), length(length_)
 {
    dim = mesh->Dimension();
    SetBoundaries();
 }
 
-void CartesianPML::SetBoundaries()
+void PML::SetBoundaries()
 {
    comp_dom_bdr.SetSize(dim, 2);
    dom_bdr.SetSize(dim, 2);
@@ -1030,7 +1026,7 @@ void CartesianPML::SetBoundaries()
    }
 }
 
-void CartesianPML::SetAttributes(ParMesh *pmesh)
+void PML::SetAttributes(ParMesh *pmesh)
 {
    // Initialize bdr attributes
    for (int i = 0; i < pmesh->GetNBE(); ++i)
@@ -1080,8 +1076,8 @@ void CartesianPML::SetAttributes(ParMesh *pmesh)
    pmesh->SetAttributes();
 }
 
-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs)
+void PML::StretchFunction(const Vector &x,
+                          vector<complex<double>> &dxs)
 {
    complex<double> zi = complex<double>(0., 1.);
 

examples/ex34.cpp

@@ -0,0 +1,622 @@
+//                                MFEM Example 34
+//
+// Compile with: make ex34
+//
+// Sample runs:  ex34 -o 2
+//               ex34 -o 2 -pa -hex
+//
+// Device sample runs:
+//               ex34 -o 2 -pa -hex -d cuda
+//               ex34 -o 2 -no-pa -d cuda
+//
+// Description:  This example code solves a simple magnetostatic problem
+//               curl curl A = J where the current density J is computed on a
+//               subset of the domain as J = -sigma grad phi. We discretize the
+//               vector potential with Nedelec finite elements, the scalar
+//               potential with Lagrange finite elements, and the current
+//               density with Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of a SubMesh to compute the
+//               scalar potential and its associated current density which is
+//               then transferred to the original mesh and used as a source
+//               function.
+//
+//               Note that this example takes certain liberties with the
+//               current density which is not necessarily divergence free
+//               as it should be. This was done to focus on the use of the
+//               SubMesh to transfer information between a full mesh and a
+//               sub-domain. A more rigorous implementation might employ an
+//               H(div) saddle point solver to obtain a divergence free J on
+//               the SubMesh. It would then also need to ensure that the r.h.s.
+//               of curl curl A = J does in fact lie in the range of the weak
+//               curl operator by performing a divergence cleaning procedure
+//               before the solve. After divergence cleaning the delta
+//               parameter would probably not be needed.
+//
+//               This example is designed to make use of a specific mesh which
+//               has a known configuration of elements and boundary attributes.
+//               Other meshes could be used but extra care would be required to
+//               properly define the SubMesh and the necessary boundaries.
+//
+//               We recommend viewing examples 1 and 3 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+static bool pa_ = false;
+static bool algebraic_ceed_ = false;
+
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    GridFunction &j_cond);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/fichera-mixed.mesh";
+   Array<int> cond_attr;
+   Array<int> submesh_elems;
+   Array<int> sym_plane_attr;
+   Array<int> phi0_attr;
+   Array<int> phi1_attr;
+   Array<int> jn_zero_attr;
+   int ref_levels = 1;
+   int order = 1;
+   double delta_const = 1e-6;
+   bool mixed = true;
+   bool static_cond = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_const, "-mc", "--magnetic-cond",
+                  "Magnetic Conductivity");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&mixed, "-mixed", "--mixed-mesh", "-hex",
+                  "--hex-mesh", "Mixed mesh of hexahedral mesh.");
+   args.AddOption(&pa_, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+#ifdef MFEM_USE_CEED
+   args.AddOption(&algebraic_ceed_, "-a", "--algebraic", "-no-a", "--no-algebraic",
+                  "Use algebraic Ceed solver");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (!mixed || pa_)
+   {
+      mesh_file = "../data/fichera.mesh";
+   }
+
+   if (submesh_elems.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0)
+      {
+         submesh_elems.SetSize(5);
+         submesh_elems[0] = 0;
+         submesh_elems[1] = 2;
+         submesh_elems[2] = 3;
+         submesh_elems[3] = 4;
+         submesh_elems[4] = 9;
+      }
+      else if (strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         submesh_elems.SetSize(7);
+         submesh_elems[0] = 10;
+         submesh_elems[1] = 14;
+         submesh_elems[2] = 34;
+         submesh_elems[3] = 36;
+         submesh_elems[4] = 37;
+         submesh_elems[5] = 38;
+         submesh_elems[6] = 39;
+      }
+   }
+   if (sym_plane_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         sym_plane_attr.SetSize(8);
+         sym_plane_attr[0] =  9;
+         sym_plane_attr[1] = 10;
+         sym_plane_attr[2] = 11;
+         sym_plane_attr[3] = 12;
+         sym_plane_attr[4] = 13;
+         sym_plane_attr[5] = 14;
+         sym_plane_attr[6] = 15;
+         sym_plane_attr[7] = 16;
+      }
+   }
+   if (phi0_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi0_attr.Append(2);
+      }
+   }
+   if (phi1_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi1_attr.Append(23);
+      }
+   }
+   if (jn_zero_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         jn_zero_attr.Append(25);
+      }
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         jn_zero_attr.Append(sym_plane_attr[i]);
+      }
+   }
+
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (!mixed || pa_)
+   {
+      mesh.UniformRefinement();
+
+      if (ref_levels > 0)
+      {
+         ref_levels--;
+      }
+   }
+
+   int submesh_attr = -1;
+   if (cond_attr.Size() == 0 && submesh_elems.Size() > 0)
+   {
+      int max_attr = mesh.attributes.Max();
+      submesh_attr = max_attr + 1;
+
+      for (int i=0; i<submesh_elems.Size(); i++)
+      {
+         mesh.SetAttribute(submesh_elems[i], submesh_attr);
+      }
+      mesh.SetAttributes();
+
+      if (cond_attr.Size() == 0)
+      {
+         cond_attr.Append(submesh_attr);
+      }
+   }
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement.
+   {
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   // 5b. Extract a submesh covering a portion of the domain
+   SubMesh mesh_cond(SubMesh::CreateFromDomain(mesh, cond_attr));
+
+   // 6. Define a suitable finite element space on the SubMesh and compute
+   //    the current density as an H(div) field.
+   RT_FECollection fec_cond_rt(order - 1, dim);
+   FiniteElementSpace fes_cond_rt(&mesh_cond, &fec_cond_rt);
+   GridFunction j_cond(&fes_cond_rt);
+
+   ComputeCurrentDensityOnSubMesh(order, phi0_attr, phi1_attr, jn_zero_attr,
+                                  j_cond);
+
+   // 6a. Save the SubMesh and associated current density in parallel. This
+   //     output can be viewed later using GLVis:
+   //        "glvis -np <np> -m cond_mesh -g cond_j"
+   {
+      ostringstream mesh_name, cond_name;
+      mesh_name << "cond.mesh";
+      cond_name << "cond_j.gf";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh_cond.Print(mesh_ofs);
+
+      ofstream cond_ofs(cond_name.str().c_str());
+      cond_ofs.precision(8);
+      j_cond.Save(cond_ofs);
+   }
+   // 6b. Send the current density, computed on the SubMesh, to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock.precision(8);
+      port_sock << "solution\n" << mesh_cond << j_cond
+                << "window_title 'Conductor J'"
+                << "window_geometry 400 0 400 350" << flush;
+   }
+
+   // 7. Define a parallel finite element space on the full mesh. Here we
+   //    use the H(curl) finite elements for the vector potential and H(div)
+   //    for the current density.
+   ND_FECollection fec_nd(order, dim);
+   RT_FECollection fec_rt(order - 1, dim);
+   FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+   FiniteElementSpace fespace_rt(&mesh, &fec_rt);
+
+   GridFunction j_full(&fespace_rt);
+   j_full = 0.0;
+   mesh_cond.Transfer(j_cond, j_full);
+
+   // 7a. Send the transferred current density to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << j_full
+               << "window_title 'J Full'"
+               << "window_geometry 400 430 400 350" << flush;
+   }
+
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes except for those on a symmetry
+   //    plane as essential (Dirichlet) and converting them to a list of
+   //    true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         ess_bdr[sym_plane_attr[i]-1] = 0;
+      }
+      fespace_nd.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   //  9. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system, which in this case is
+   //     (J,W_i) where J is given by the function H(div) field transferred
+   //     from the SubMesh and W_i are the basis functions in the finite
+   //     element fespace.
+   VectorGridFunctionCoefficient jCoef(&j_full);
+   LinearForm b(&fespace_nd);
+   b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(jCoef));
+   b.Assemble();
+
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x to zero.
+   GridFunction x(&fespace_nd);
+   x = 0.0;
+
+   // 11. Set up the parallel bilinear form corresponding to the EM
+   //     diffusion operator curl muinv curl + delta I, by adding the
+   //     curl-curl and the mass domain integrators. For standard
+   //     magnetostatics equations choose delta << 1. Larger values of
+   //     delta should make the linear system easier to solve at the
+   //     expense of resembling a diffusive quasistatic magnetic field.
+   //     A reasonable balance must be found whenever the mesh or problem
+   //     setup is altered.
+   ConstantCoefficient muinv(1.0);
+   ConstantCoefficient delta(delta_const);
+   BilinearForm a(&fespace_nd);
+   if (pa_) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.AddDomainIntegrator(new CurlCurlIntegrator(muinv));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(delta));
+
+   // 12. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // 13. Solve the system AX=B
+   if (pa_) // Jacobi preconditioning in partial assembly mode
+   {
+      cout << "\nSolving for magnetic vector potential "
+           << "using CG with a Jacobi preconditioner" << endl;
+
+      OperatorJacobiSmoother M(a, ess_tdof_list);
+      PCG(*A, M, B, X, 1, 1000, 1e-12, 0.0);
+   }
+   else
+   {
+#ifndef MFEM_USE_SUITESPARSE
+      cout << "\nSolving for magnetic vector potential "
+           << "using CG with a Gauss-Seidel preconditioner" << endl;
+
+      // 13a. Define a simple symmetric Gauss-Seidel preconditioner and use
+      //     it to solve the system Ax=b with PCG.
+      GSSmoother M((SparseMatrix&)(*A));
+      PCG(*A, M, B, X, 1, 500, 1e-12, 0.0);
+#else
+      cout << "\nSolving for magnetic vector potential "
+           << "using UMFPack" << endl;
+
+      // 13a. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
+      //     system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(*A);
+      umf_solver.Mult(B, X);
+#endif
+   }
+
+   // 14. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "refined.mesh";
+      sol_name << "sol.gf";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << x
+               << "window_title 'Vector Potential'"
+               << "window_geometry 800 0 400 350" << flush;
+   }
+
+   // 17. Compute the magnetic flux as the curl of the solution
+   DiscreteLinearOperator curl(&fespace_nd, &fespace_rt);
+   curl.AddDomainInterpolator(new CurlInterpolator);
+   curl.Assemble();
+   curl.Finalize();
+
+   GridFunction dx(&fespace_rt);
+   curl.Mult(x, dx);
+
+   // 18. Save the curl of the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g dsol".
+   {
+      ostringstream dsol_name;
+      dsol_name << "dsol.gf";
+
+      ofstream dsol_ofs(dsol_name.str().c_str());
+      dsol_ofs.precision(8);
+      dx.Save(dsol_ofs);
+   }
+
+   // 19. Send the curl of the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << dx
+               << "window_title 'Magnetic Flux'"
+               << "window_geometry 1200 0 400 350" << flush;
+   }
+
+   // 20. Clean exit
+   return 0;
+}
+
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    GridFunction &j_cond)
+{
+   // Exract the finite element space and mesh on which j_cond is defined
+   FiniteElementSpace &fes_cond_rt = *j_cond.FESpace();
+   Mesh &mesh_cond = *fes_cond_rt.GetMesh();
+   int dim  = mesh_cond.Dimension();
+
+   // Define a parallel finite element space on the SubMesh. Here we use the
+   // H1 finite elements for the electrostatic potential.
+   H1_FECollection fec_h1(order, dim);
+   FiniteElementSpace fes_cond_h1(&mesh_cond, &fec_h1);
+
+   // Define the conductivity coefficient and the boundaries associated with
+   // the fixed potentials phi0 and phi1 which will drive the current.
+   ConstantCoefficient sigmaCoef(1.0);
+   Array<int> ess_bdr_phi(mesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_j(mesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_tdof_phi;
+   ess_bdr_phi = 0;
+   ess_bdr_j   = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi0_attr[i]-1] = 1;
+   }
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi1_attr[i]-1] = 1;
+   }
+   for (int i=0; i<jn_zero_attr.Size(); i++)
+   {
+      ess_bdr_j[jn_zero_attr[i]-1] = 1;
+   }
+   fes_cond_h1.GetEssentialTrueDofs(ess_bdr_phi, ess_bdr_tdof_phi);
+
+   // Setup the bilinear form corresponding to -Div(sigma Grad phi)
+   BilinearForm a_h1(&fes_cond_h1);
+   a_h1.AddDomainIntegrator(new DiffusionIntegrator(sigmaCoef));
+   a_h1.Assemble();
+
+   // Set the r.h.s. to zero
+   LinearForm b_h1(&fes_cond_h1);
+   b_h1 = 0.0;
+
+   // Setup the boundary conditions on phi
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   GridFunction phi_h1(&fes_cond_h1);
+   phi_h1 = 0.0;
+
+   Array<int> bdr0(mesh_cond.bdr_attributes.Max()); bdr0 = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      bdr0[phi0_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(zero, bdr0);
+
+   Array<int> bdr1(mesh_cond.bdr_attributes.Max()); bdr1 = 0;
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      bdr1[phi1_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(one, bdr1);
+
+   {
+      OperatorPtr A;
+      Vector B, X;
+      a_h1.FormLinearSystem(ess_bdr_tdof_phi, phi_h1, b_h1, A, X, B);
+
+      // Solve the linear system
+      if (!pa_)
+      {
+#ifndef MFEM_USE_SUITESPARSE
+         cout << "\nSolving for electric potential using PCG "
+              << "with a Gauss-Seidel preconditioner" << endl;
+
+         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+         GSSmoother M((SparseMatrix&)(*A));
+         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+         cout << "\nSolving for electric potential using UMFPack" << endl;
+
+         // If MFEM was compiled with SuiteSparse,
+         // use UMFPACK to solve the system.
+         UMFPackSolver umf_solver;
+         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         umf_solver.SetOperator(*A);
+         umf_solver.Mult(B, X);
+#endif
+      }
+      else
+      {
+         cout << "\nSolving for electric potential using CG" << endl;
+
+         if (UsesTensorBasis(fes_cond_h1))
+         {
+            if (algebraic_ceed_)
+            {
+               ceed::AlgebraicSolver M(a_h1, ess_bdr_tdof_phi);
+               PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+            }
+            else
+            {
+               OperatorJacobiSmoother M(a_h1, ess_bdr_tdof_phi);
+               PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+            }
+         }
+         else
+         {
+            CG(*A, B, X, 1, 400, 1e-12, 0.0);
+         }
+      }
+      a_h1.RecoverFEMSolution(X, b_h1, phi_h1);
+   }
+
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock.precision(8);
+      port_sock << "solution\n" << mesh_cond << phi_h1
+                << "window_title 'Conductor Potential'"
+                << "window_geometry 0 0 400 350" << flush;
+   }
+
+   // Solve for the current density J = -sigma Grad phi with boundary
+   // conditions J.n = 0 on the walls of the conductor but not on the
+   // ports where phi=0 and phi=1.
+
+   // J will be computed in H(div) so we need an RT mass matrix
+   BilinearForm m_rt(&fes_cond_rt);
+   m_rt.AddDomainIntegrator(new VectorFEMassIntegrator);
+   m_rt.Assemble();
+
+   // Assemble the (sigma Grad phi) operator
+   MixedBilinearForm d_h1(&fes_cond_h1, &fes_cond_rt);
+   d_h1.AddDomainIntegrator(new MixedVectorGradientIntegrator(sigmaCoef));
+   d_h1.Assemble();
+
+   // Compute the r.h.s, b_rt = sigma E = -sigma Grad phi
+   LinearForm b_rt(&fes_cond_rt);
+   d_h1.Mult(phi_h1, b_rt);
+   b_rt *= -1.0;
+
+   // Apply the necessary boundary conditions and solve for J in H(div)
+   cout << "\nSolving for current density in H(Div) "
+        << "using diagonally scaled CG" << endl;
+   cout << "Size of linear system: "
+        << fes_cond_rt.GetTrueVSize() << endl;
+
+   Array<int> ess_bdr_tdof_rt;
+   OperatorPtr M;
+   Vector B, X;
+
+   fes_cond_rt.GetEssentialTrueDofs(ess_bdr_j, ess_bdr_tdof_rt);
+
+   j_cond = 0.0;
+   m_rt.FormLinearSystem(ess_bdr_tdof_rt, j_cond, b_rt, M, X, B);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*M);
+   cg.Mult(B, X);
+   m_rt.RecoverFEMSolution(X, b_rt, j_cond);
+}

examples/ex34p.cpp

@@ -0,0 +1,649 @@
+//                       MFEM Example 34 - Parallel Version
+//
+// Compile with: make ex34p
+//
+// Sample runs:  mpirun -np 4 ex34p -o 2
+//               mpirun -np 4 ex34p -o 2 -hex -pa
+//
+// Device sample runs:
+//               mpirun -np 4 ex34p -o 2 -hex -pa -d cuda
+//               mpirun -np 4 ex34p -o 2 -no-pa -d cuda
+//
+// Description:  This example code solves a simple magnetostatic problem
+//               curl curl A = J where the current density J is computed on a
+//               subset of the domain as J = -sigma grad phi. We discretize the
+//               vector potential with Nedelec finite elements, the scalar
+//               potential with Lagrange finite elements, and the current
+//               density with Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of a SubMesh to compute the
+//               scalar potential and its associated current density which is
+//               then transferred to the original mesh and used as a source
+//               function.
+//
+//               Note that this example takes certain liberties with the
+//               current density which is not necessarily divergence free
+//               as it should be. This was done to focus on the use of the
+//               SubMesh to transfer information between a full mesh and a
+//               sub-domain. A more rigorous implementation might employ an
+//               H(div) saddle point solver to obtain a divergence free J on
+//               the SubMesh. It would then also need to ensure that the r.h.s.
+//               of curl curl A = J does in fact lie in the range of the weak
+//               curl operator by performing a divergence cleaning procedure
+//               before the solve. After divergence cleaning the delta
+//               parameter would probably not be needed.
+//
+//               This example is designed to make use of a specific mesh which
+//               has a known configuration of elements and boundary attributes.
+//               Other meshes could be used but extra care would be required to
+//               properly define the SubMesh and the necessary boundaries.
+//
+//               We recommend viewing examples 1 and 3 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    ParGridFunction &j_cond);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/fichera-mixed.mesh";
+   Array<int> cond_attr;
+   Array<int> submesh_elems;
+   Array<int> sym_plane_attr;
+   Array<int> phi0_attr;
+   Array<int> phi1_attr;
+   Array<int> jn_zero_attr;
+   int ser_ref_levels = 1;
+   int par_ref_levels = 1;
+   int order = 1;
+   double delta_const = 1e-6;
+   bool mixed = true;
+   bool static_cond = false;
+   bool pa = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
+#ifdef MFEM_USE_AMGX
+   bool useAmgX = false;
+#endif
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_const, "-mc", "--magnetic-cond",
+                  "Magnetic Conductivity");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&mixed, "-mixed", "--mixed-mesh", "-hex",
+                  "--hex-mesh", "Mixed mesh of hexahedral mesh.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+#ifdef MFEM_USE_AMGX
+   args.AddOption(&useAmgX, "-amgx", "--useAmgX", "-no-amgx",
+                  "--no-useAmgX",
+                  "Enable or disable AmgX in MatrixFreeAMS.");
+#endif
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (!mixed || pa)
+   {
+      mesh_file = "../data/fichera.mesh";
+   }
+
+   if (submesh_elems.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0)
+      {
+         submesh_elems.SetSize(5);
+         submesh_elems[0] = 0;
+         submesh_elems[1] = 2;
+         submesh_elems[2] = 3;
+         submesh_elems[3] = 4;
+         submesh_elems[4] = 9;
+      }
+      else if (strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         submesh_elems.SetSize(7);
+         submesh_elems[0] = 10;
+         submesh_elems[1] = 14;
+         submesh_elems[2] = 34;
+         submesh_elems[3] = 36;
+         submesh_elems[4] = 37;
+         submesh_elems[5] = 38;
+         submesh_elems[6] = 39;
+      }
+   }
+   if (sym_plane_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         sym_plane_attr.SetSize(8);
+         sym_plane_attr[0] =  9;
+         sym_plane_attr[1] = 10;
+         sym_plane_attr[2] = 11;
+         sym_plane_attr[3] = 12;
+         sym_plane_attr[4] = 13;
+         sym_plane_attr[5] = 14;
+         sym_plane_attr[6] = 15;
+         sym_plane_attr[7] = 16;
+      }
+   }
+   if (phi0_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi0_attr.Append(2);
+      }
+   }
+   if (phi1_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi1_attr.Append(23);
+      }
+   }
+   if (jn_zero_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         jn_zero_attr.Append(25);
+      }
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         jn_zero_attr.Append(sym_plane_attr[i]);
+      }
+   }
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   if (!mixed || pa)
+   {
+      mesh->UniformRefinement();
+
+      if (ser_ref_levels > 0)
+      {
+         ser_ref_levels--;
+      }
+      else
+      {
+         par_ref_levels--;
+      }
+   }
+
+   int submesh_attr = -1;
+   if (cond_attr.Size() == 0 && submesh_elems.Size() > 0)
+   {
+      int max_attr = mesh->attributes.Max();
+      submesh_attr = max_attr + 1;
+
+      for (int i=0; i<submesh_elems.Size(); i++)
+      {
+         mesh->SetAttribute(submesh_elems[i], submesh_attr);
+      }
+      mesh->SetAttributes();
+
+      if (cond_attr.Size() == 0)
+      {
+         cond_attr.Append(submesh_attr);
+      }
+   }
+
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement.
+   {
+      int ref_levels = ser_ref_levels;
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh.UniformRefinement();
+      }
+   }
+
+   // 6b. Extract a submesh covering a portion of the domain
+   ParSubMesh pmesh_cond(ParSubMesh::CreateFromDomain(pmesh, cond_attr));
+
+   // 7. Define a suitable finite element space on the SubMesh and compute
+   //    the current density as an H(div) field.
+   RT_FECollection fec_cond_rt(order - 1, dim);
+   ParFiniteElementSpace fes_cond_rt(&pmesh_cond, &fec_cond_rt);
+   ParGridFunction j_cond(&fes_cond_rt);
+
+   ComputeCurrentDensityOnSubMesh(order, phi0_attr, phi1_attr, jn_zero_attr,
+                                  j_cond);
+
+   // 7a. Save the SubMesh and associated current density in parallel. This
+   //     output can be viewed later using GLVis:
+   //        "glvis -np <np> -m cond_mesh -g cond_j"
+   {
+      ostringstream mesh_name, cond_name;
+      mesh_name << "cond_mesh." << setfill('0') << setw(6) << myid;
+      cond_name << "cond_j." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh_cond.Print(mesh_ofs);
+
+      ofstream cond_ofs(cond_name.str().c_str());
+      cond_ofs.precision(8);
+      j_cond.Save(cond_ofs);
+   }
+   // 7b. Send the current density, computed on the SubMesh, to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock << "parallel " << num_procs << " " << myid << "\n";
+      port_sock.precision(8);
+      port_sock << "solution\n" << pmesh_cond << j_cond
+                << "window_title 'Conductor J'"
+                << "window_geometry 400 0 400 350" << flush;
+   }
+
+   // 8. Define a parallel finite element space on the full mesh. Here we
+   //    use the H(curl) finite elements for the vector potential and H(div)
+   //    for the current density.
+   ND_FECollection fec_nd(order, dim);
+   RT_FECollection fec_rt(order - 1, dim);
+   ParFiniteElementSpace fespace_nd(&pmesh, &fec_nd);
+   ParFiniteElementSpace fespace_rt(&pmesh, &fec_rt);
+
+   ParGridFunction j_full(&fespace_rt);
+   j_full = 0.0;
+   pmesh_cond.Transfer(j_cond, j_full);
+
+   // 8a. Send the transferred current density to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << j_full
+               << "window_title 'J Full'"
+               << "window_geometry 400 430 400 350" << flush;
+   }
+
+   // 9. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes except for those on a symmetry
+   //    plane as essential (Dirichlet) and converting them to a list of
+   //    true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         ess_bdr[sym_plane_attr[i]-1] = 0;
+      }
+      fespace_nd.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system, which in this case is
+   //     (J,W_i) where J is given by the function H(div) field transferred
+   //     from the SubMesh and W_i are the basis functions in the finite
+   //     element fespace.
+   VectorGridFunctionCoefficient jCoef(&j_full);
+   ParLinearForm b(&fespace_nd);
+   b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(jCoef));
+   b.Assemble();
+
+   // 11. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x to zero.
+   ParGridFunction x(&fespace_nd);
+   x = 0.0;
+
+   // 12. Set up the parallel bilinear form corresponding to the EM
+   //     diffusion operator curl muinv curl + delta I, by adding the
+   //     curl-curl and the mass domain integrators. For standard
+   //     magnetostatics equations choose delta << 1. Larger values of
+   //     delta should make the linear system easier to solve at the
+   //     expense of resembling a diffusive quasistatic magnetic field.
+   //     A reasonable balance must be found whenever the mesh or problem
+   //     setup is altered.
+   ConstantCoefficient muinv(1.0);
+   ConstantCoefficient delta(delta_const);
+   ParBilinearForm a(&fespace_nd);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.AddDomainIntegrator(new CurlCurlIntegrator(muinv));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(delta));
+
+   // 13. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "\nSolving for magnetic vector potential "
+           << "using CG with AMS" << endl;
+   }
+
+   // 14. Solve the system AX=B using PCG with an AMS preconditioner.
+   if (pa)
+   {
+#ifdef MFEM_USE_AMGX
+      MatrixFreeAMS ams(a, *A, fespace_nd, &muinv, &delta, NULL, ess_bdr,
+                        useAmgX);
+#else
+      MatrixFreeAMS ams(a, *A, fespace_nd, &muinv, &delta, NULL, ess_bdr);
+#endif
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(1000);
+      cg.SetPrintLevel(1);
+      cg.SetOperator(*A);
+      cg.SetPreconditioner(ams);
+      cg.Mult(B, X);
+   }
+   else
+   {
+      if (myid == 0)
+      {
+         cout << "Size of linear system: "
+              << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
+      }
+
+      ParFiniteElementSpace *prec_fespace =
+         (a.StaticCondensationIsEnabled() ? a.SCParFESpace() : &fespace_nd);
+      HypreAMS ams(*A.As<HypreParMatrix>(), prec_fespace);
+      HyprePCG pcg(*A.As<HypreParMatrix>());
+      pcg.SetTol(1e-12);
+      pcg.SetMaxIter(500);
+      pcg.SetPrintLevel(2);
+      pcg.SetPreconditioner(ams);
+      pcg.Mult(B, X);
+   }
+
+   // 15. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 16. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 17. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x
+               << "window_title 'Vector Potential'"
+               << "window_geometry 800 0 400 350" << flush;
+   }
+
+   // 18. Compute the magnetic flux as the curl of the solution
+   ParDiscreteLinearOperator curl(&fespace_nd, &fespace_rt);
+   curl.AddDomainInterpolator(new CurlInterpolator);
+   curl.Assemble();
+   curl.Finalize();
+
+   ParGridFunction dx(&fespace_rt);
+   curl.Mult(x, dx);
+
+   // 19. Save the curl of the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g dsol".
+   {
+      ostringstream dsol_name;
+      dsol_name << "dsol." << setfill('0') << setw(6) << myid;
+
+      ofstream dsol_ofs(dsol_name.str().c_str());
+      dsol_ofs.precision(8);
+      dx.Save(dsol_ofs);
+   }
+
+   // 20. Send the curl of the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << dx
+               << "window_title 'Magnetic Flux'"
+               << "window_geometry 1200 0 400 350" << flush;
+   }
+
+   // 21. Clean exit
+   return 0;
+}
+
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    ParGridFunction &j_cond)
+{
+   // Exract the finite element space and mesh on which j_cond is defined
+   ParFiniteElementSpace &fes_cond_rt = *j_cond.ParFESpace();
+   ParMesh &pmesh_cond = *fes_cond_rt.GetParMesh();
+   int myid = fes_cond_rt.GetMyRank();
+   int dim  = pmesh_cond.Dimension();
+
+   // Define a parallel finite element space on the SubMesh. Here we use the
+   // H1 finite elements for the electrostatic potential.
+   H1_FECollection fec_h1(order, dim);
+   ParFiniteElementSpace fes_cond_h1(&pmesh_cond, &fec_h1);
+
+   // Define the conductivity coefficient and the boundaries associated with
+   // the fixed potentials phi0 and phi1 which will drive the current.
+   ConstantCoefficient sigmaCoef(1.0);
+   Array<int> ess_bdr_phi(pmesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_j(pmesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_tdof_phi;
+   ess_bdr_phi = 0;
+   ess_bdr_j   = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi0_attr[i]-1] = 1;
+   }
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi1_attr[i]-1] = 1;
+   }
+   for (int i=0; i<jn_zero_attr.Size(); i++)
+   {
+      ess_bdr_j[jn_zero_attr[i]-1] = 1;
+   }
+   fes_cond_h1.GetEssentialTrueDofs(ess_bdr_phi, ess_bdr_tdof_phi);
+
+   // Setup the bilinear form corresponding to -Div(sigma Grad phi)
+   ParBilinearForm a_h1(&fes_cond_h1);
+   a_h1.AddDomainIntegrator(new DiffusionIntegrator(sigmaCoef));
+   a_h1.Assemble();
+
+   // Set the r.h.s. to zero
+   ParLinearForm b_h1(&fes_cond_h1);
+   b_h1 = 0.0;
+
+   // Setup the boundary conditions on phi
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   ParGridFunction phi_h1(&fes_cond_h1);
+   phi_h1 = 0.0;
+
+   Array<int> bdr0(pmesh_cond.bdr_attributes.Max()); bdr0 = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      bdr0[phi0_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(zero, bdr0);
+
+   Array<int> bdr1(pmesh_cond.bdr_attributes.Max()); bdr1 = 0;
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      bdr1[phi1_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(one, bdr1);
+
+   // Solve the linear system using algebraic multigrid
+   {
+      if (myid == 0)
+      {
+         cout << "\nSolving for electric potential "
+              << "using CG with AMG" << endl;
+      }
+      OperatorPtr A;
+      Vector B, X;
+      a_h1.FormLinearSystem(ess_bdr_tdof_phi, phi_h1, b_h1, A, X, B);
+
+      HypreBoomerAMG prec;
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(prec);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      a_h1.RecoverFEMSolution(X, b_h1, phi_h1);
+   }
+   {
+      int num_procs = fes_cond_h1.GetNRanks();
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock << "parallel " << num_procs << " " << myid << "\n";
+      port_sock.precision(8);
+      port_sock << "solution\n" << pmesh_cond << phi_h1
+                << "window_title 'Conductor Potential'"
+                << "window_geometry 0 0 400 350" << flush;
+   }
+
+   // Solve for the current density J = -sigma Grad phi with boundary
+   // conditions J.n = 0 on the walls of the conductor but not on the
+   // ports where phi=0 and phi=1.
+
+   // J will be computed in H(div) so we need an RT mass matrix
+   ParBilinearForm m_rt(&fes_cond_rt);
+   m_rt.AddDomainIntegrator(new VectorFEMassIntegrator);
+   m_rt.Assemble();
+
+   // Assemble the (sigma Grad phi) operator
+   ParMixedBilinearForm d_h1(&fes_cond_h1, &fes_cond_rt);
+   d_h1.AddDomainIntegrator(new MixedVectorGradientIntegrator(sigmaCoef));
+   d_h1.Assemble();
+
+   // Compute the r.h.s, b_rt = sigma E = -sigma Grad phi
+   ParLinearForm b_rt(&fes_cond_rt);
+   d_h1.Mult(phi_h1, b_rt);
+   b_rt *= -1.0;
+
+   // Apply the necessary boundary conditions and solve for J in H(div)
+   HYPRE_BigInt glb_size_rt = fes_cond_rt.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "\nSolving for current density in H(Div) "
+           << "using diagonally scaled CG" << endl;
+      cout << "Size of linear system: "
+           << glb_size_rt << endl;
+   }
+   Array<int> ess_bdr_tdof_rt;
+   OperatorPtr M;
+   Vector B, X;
+
+   fes_cond_rt.GetEssentialTrueDofs(ess_bdr_j, ess_bdr_tdof_rt);
+
+   j_cond = 0.0;
+   m_rt.FormLinearSystem(ess_bdr_tdof_rt, j_cond, b_rt, M, X, B);
+
+   HypreDiagScale prec;
+
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(prec);
+   cg.SetOperator(*M);
+   cg.Mult(B, X);
+   m_rt.RecoverFEMSolution(X, b_rt, j_cond);
+}

examples/ex35p.cpp

@@ -0,0 +1,818 @@
+//                       MFEM Example 35 - Parallel Version
+//
+// Compile with: make ex35p
+//
+// Sample runs:  mpirun -np 4 ex35p -p 0 -o 2
+//               mpirun -np 4 ex35p -p 0 -o 2 -pbc '22 23 24' -em 0
+//               mpirun -np 4 ex35p -p 1 -o 1 -rp 2
+//               mpirun -np 4 ex35p -p 1 -o 2
+//               mpirun -np 4 ex35p -p 2 -o 1 -rp 2 -c 15
+//
+// Device sample runs:
+//
+// Description:  This example code demonstrates the use of MFEM to define and
+//               solve simple complex-valued linear systems. It implements three
+//               variants of a damped harmonic oscillator:
+//
+//               1) A scalar H1 field
+//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
+//
+//               2) A vector H(Curl) field
+//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
+//
+//               3) A vector H(Div) field
+//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
+//
+//               In each case the field is driven by a forced oscillation, with
+//               angular frequency omega, imposed at the boundary or a portion
+//               of the boundary. The spatial variation of the boundary
+//               condition is computed as an eigenmode of an appropriate
+//               operator defined on a portion of the boundary i.e. a port
+//               boundary condition.
+//
+//               In electromagnetics the coefficients are typically named the
+//               permeability, mu = 1/a, permittivity, epsilon = b, and
+//               conductivity, sigma = c. The user can specify these constants
+//               using either set of names.
+//
+//               This example demonstrates how to transfer fields computed on
+//               a boundary generated SubMesh to the full mesh and apply them
+//               as boundary conditions. The default mesh and corresponding
+//               boundary attriburtes were chosen to verify proper behavior on
+//               both triangular and quadrilateral faces of tetrahedral,
+//               wedge-shaped, and hexahedral elements.
+//
+//               The example also demonstrates how to display a time-varying
+//               solution as a sequence of fields sent to a single GLVis socket.
+//
+//               We recommend viewing examples 11, 13, and 22 before viewing
+//               this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+static double mu_ = 1.0;
+static double epsilon_ = 1.0;
+static double sigma_ = 2.0;
+
+void SetPortBC(int prob, int dim, int mode, ParGridFunction &port_bc);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/fichera-mixed.mesh";
+   int ser_ref_levels = 1;
+   int par_ref_levels = 1;
+   int order = 1;
+   Array<int> port_bc_attr;
+   int prob = 0;
+   int mode = 1;
+   double freq = -1.0;
+   double omega = 2.0 * M_PI;
+   double a_coef = 0.0;
+   bool herm_conv = true;
+   bool slu_solver  = false;
+   bool visualization = 1;
+   bool mixed = true;
+   bool pa = false;
+   const char *device_config = "cpu";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&prob, "-p", "--problem-type",
+                  "Choose between 0: H_1, 1: H(Curl), or 2: H(Div) "
+                  "damped harmonic oscillator.");
+   args.AddOption(&mode, "-em", "--eigenmode",
+                  "Choose the index of the port eigenmode.");
+   args.AddOption(&a_coef, "-a", "--stiffness-coef",
+                  "Stiffness coefficient (spring constant or 1/mu).");
+   args.AddOption(&epsilon_, "-b", "--mass-coef",
+                  "Mass coefficient (or epsilon).");
+   args.AddOption(&sigma_, "-c", "--damping-coef",
+                  "Damping coefficient (or sigma).");
+   args.AddOption(&mu_, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon_, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma_, "-sigma", "--conductivity",
+                  "Conductivity (or damping constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&port_bc_attr, "-pbc", "--port-bc-attr",
+                  "Attributes of port boundary condition");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+#ifdef MFEM_USE_SUPERLU
+   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
+                  "--no-superlu", "Use the SuperLU Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&mixed, "-mixed", "--mixed-mesh", "-hex",
+                  "--hex-mesh", "Mixed mesh of hexahedral mesh.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   if (!mixed || pa)
+   {
+      mesh_file = "../data/fichera.mesh";
+   }
+
+   if ( a_coef != 0.0 )
+   {
+      mu_ = 1.0 / a_coef;
+   }
+   if ( freq > 0.0 )
+   {
+      omega = 2.0 * M_PI * freq;
+   }
+   if (port_bc_attr.Size() == 0 &&
+       (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+        strcmp(mesh_file, "../data/fichera.mesh") == 0))
+   {
+      port_bc_attr.SetSize(4);
+      port_bc_attr[0] =  7;
+      port_bc_attr[1] =  8;
+      port_bc_attr[2] = 11;
+      port_bc_attr[3] = 12;
+   }
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   MFEM_VERIFY(prob >= 0 && prob <=2,
+               "Unrecognized problem type: " << prob);
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution.
+   for (int l = 0; l < ser_ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 6a. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int l = 0; l < par_ref_levels; l++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   // 6b. Extract a submesh covering a portion of the boundary
+   ParSubMesh pmesh_port(ParSubMesh::CreateFromBoundary(pmesh, port_bc_attr));
+
+   // 7a. Define a parallel finite element space on the parallel mesh. Here we
+   //     use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements
+   //     of the specified order.
+   if (dim == 1 && prob != 0 )
+   {
+      if (myid == 0)
+      {
+         cout << "Switching to problem type 0, H1 basis functions, "
+              << "for 1 dimensional mesh." << endl;
+      }
+      prob = 0;
+   }
+
+   FiniteElementCollection *fec = NULL;
+   switch (prob)
+   {
+      case 0:  fec = new H1_FECollection(order, dim);      break;
+      case 1:  fec = new ND_FECollection(order, dim);      break;
+      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
+      default: break; // This should be unreachable
+   }
+   ParFiniteElementSpace fespace(&pmesh, fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7b. Define a parallel finite element space on the sub-mesh. Here we
+   //    use continuous Lagrange, Nedelec, or L2 finite elements of
+   //    the specified order.
+   FiniteElementCollection *fec_port = NULL;
+   switch (prob)
+   {
+      case 0:  fec_port = new H1_FECollection(order, dim-1);      break;
+      case 1:
+         if (dim == 3)
+         {
+            fec_port = new ND_FECollection(order, dim-1);
+         }
+         else
+         {
+            fec_port = new L2_FECollection(order - 1, dim-1,
+                                           BasisType::GaussLegendre,
+                                           FiniteElement::INTEGRAL);
+         }
+         break;
+      case 2:  fec_port = new L2_FECollection(order - 1, dim-1,
+                                                 BasisType::GaussLegendre,
+                                                 FiniteElement::INTEGRAL); break;
+      default: break; // This should be unreachable
+   }
+   ParFiniteElementSpace fespace_port(&pmesh_port, fec_port);
+   HYPRE_BigInt size_port = fespace_port.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element port BC unknowns: " << size_port
+           << endl;
+   }
+
+   // 8a. Define a parallel grid function on the SubMesh which will contain
+   //     the field to be applied as a port boundary condition.
+   ParGridFunction port_bc(&fespace_port);
+   port_bc = 0.0;
+
+   SetPortBC(prob, dim, mode, port_bc);
+
+   // 8b. Save the SubMesh and associated port boundary condition in parallel.
+   //     This output can be viewed later using GLVis:
+   //        "glvis -np <np> -m port_mesh -g port_mode"
+   {
+      ostringstream mesh_name, port_name;
+      mesh_name << "port_mesh." << setfill('0') << setw(6) << myid;
+      port_name << "port_mode." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh_port.Print(mesh_ofs);
+
+      ofstream port_ofs(port_name.str().c_str());
+      port_ofs.precision(8);
+      port_bc.Save(port_ofs);
+   }
+   // 8c. Send the port bc, computed on the SubMesh, to a GLVis server.
+   if (visualization && dim == 3)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock << "parallel " << num_procs << " " << myid << "\n";
+      port_sock.precision(8);
+      port_sock << "solution\n" << pmesh_port << port_bc
+                << "window_title 'Port BC'"
+                << "window_geometry 0 0 400 350" << flush;
+   }
+
+   // 9. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    using an eigenmode of the appropriate type computed on the SubMesh.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system.
+   ParComplexLinearForm b(&fespace, conv);
+   b.Vector::operator=(0.0);
+
+   // 11a. Define the solution vector u as a parallel complex finite element
+   //      grid function corresponding to fespace. Initialize u to equal zero.
+   ParComplexGridFunction u(&fespace);
+   u = 0.0;
+   pmesh_port.Transfer(port_bc, u.real());
+
+   // 11b. Send the transferred port bc field to a GLVis server.
+   {
+      ParGridFunction full_bc(&fespace);
+      ParTransferMap port_to_full(port_bc, full_bc);
+
+      full_bc = 0.0;
+      port_to_full.Transfer(port_bc, full_bc);
+
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream full_sock(vishost, visport);
+         full_sock << "parallel " << num_procs << " " << myid << "\n";
+         full_sock.precision(8);
+         full_sock << "solution\n" << pmesh << full_bc
+                   << "window_title 'Transferred BC'"
+                   << "window_geometry 400 0 400 350"<< flush;
+      }
+   }
+
+   // 12. Set up the parallel sesquilinear form a(.,.) on the finite element
+   //     space corresponding to the damped harmonic oscillator operator of the
+   //     appropriate type:
+   //
+   //     0) A scalar H1 field
+   //        -Div(a Grad) - omega^2 b + i omega c
+   //
+   //     1) A vector H(Curl) field
+   //        Curl(a Curl) - omega^2 b + i omega c
+   //
+   //     2) A vector H(Div) field
+   //        -Grad(a Div) - omega^2 b + i omega c
+   //
+   ConstantCoefficient stiffnessCoef(1.0/mu_);
+   ConstantCoefficient massCoef(-omega * omega * epsilon_);
+   ConstantCoefficient lossCoef(omega * sigma_);
+   ConstantCoefficient negMassCoef(omega * omega * epsilon_);
+
+   ParSesquilinearForm a(&fespace, conv);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   switch (prob)
+   {
+      case 0:
+         a.AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
+                               NULL);
+         a.AddDomainIntegrator(new MassIntegrator(massCoef),
+                               new MassIntegrator(lossCoef));
+         break;
+      case 1:
+         a.AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
+                               NULL);
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                               new VectorFEMassIntegrator(lossCoef));
+         break;
+      case 2:
+         a.AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
+                               NULL);
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                               new VectorFEMassIntegrator(lossCoef));
+         break;
+      default: break; // This should be unreachable
+   }
+
+   // 13. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, etc.
+   a.Assemble();
+
+   OperatorHandle A;
+   Vector B, U;
+
+   a.FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: "
+           << 2 * size << endl << endl;
+   }
+
+   if (!slu_solver)
+   {
+      // 14a. Set up the parallel bilinear form for the preconditioner
+      //      corresponding to the appropriate operator
+      //
+      //      0) A scalar H1 field
+      //         -Div(a Grad) - omega^2 b + i omega c
+      //
+      //      1) A vector H(Curl) field
+      //         Curl(a Curl) + omega^2 b + i omega c
+      //
+      //      2) A vector H(Div) field
+      //         -Grad(a Div) - omega^2 b + i omega c
+      //
+      ParBilinearForm pcOp(&fespace);
+      if (pa) { pcOp.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+      switch (prob)
+      {
+         case 0:
+            pcOp.AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef));
+            pcOp.AddDomainIntegrator(new MassIntegrator(massCoef));
+            pcOp.AddDomainIntegrator(new MassIntegrator(lossCoef));
+            break;
+         case 1:
+            pcOp.AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(negMassCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
+            break;
+         case 2:
+            pcOp.AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(massCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
+            break;
+         default: break; // This should be unreachable
+      }
+      pcOp.Assemble();
+
+      // 14b. Define and apply a parallel FGMRES solver for AU=B with a block
+      //     diagonal preconditioner based on the appropriate multigrid
+      //     preconditioner from hypre.
+      Array<int> blockTrueOffsets;
+      blockTrueOffsets.SetSize(3);
+      blockTrueOffsets[0] = 0;
+      blockTrueOffsets[1] = A->Height() / 2;
+      blockTrueOffsets[2] = A->Height() / 2;
+      blockTrueOffsets.PartialSum();
+
+      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
+
+      Operator * pc_r = NULL;
+      Operator * pc_i = NULL;
+
+      if (pa)
+      {
+         pc_r = new OperatorJacobiSmoother(pcOp, ess_tdof_list);
+      }
+      else
+      {
+         OperatorHandle PCOp;
+         pcOp.FormSystemMatrix(ess_tdof_list, PCOp);
+
+         switch (prob)
+         {
+            case 0:
+               pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
+               break;
+            case 1:
+               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), &fespace);
+               break;
+            case 2:
+               if (dim == 2 )
+               {
+                  pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), &fespace);
+               }
+               else
+               {
+                  pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), &fespace);
+               }
+               break;
+            default: break; // This should be unreachable
+         }
+      }
+      pc_i = new ScaledOperator(pc_r,
+                                (conv == ComplexOperator::HERMITIAN) ?
+                                -1.0:1.0);
+
+      BDP.SetDiagonalBlock(0, pc_r);
+      BDP.SetDiagonalBlock(1, pc_i);
+      BDP.owns_blocks = 1;
+
+      FGMRESSolver fgmres(MPI_COMM_WORLD);
+      fgmres.SetPreconditioner(BDP);
+      fgmres.SetOperator(*A.Ptr());
+      fgmres.SetRelTol(1e-6);
+      fgmres.SetMaxIter(1000);
+      fgmres.SetPrintLevel(1);
+      fgmres.Mult(B, U);
+   }
+#ifdef MFEM_USE_SUPERLU
+   else
+   {
+      // 14. Solve using a direct solver
+      // Transform to monolithic HypreParMatrix
+      HypreParMatrix *A_hyp = A.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      SuperLURowLocMatrix SA(*A_hyp);
+      SuperLUSolver superlu(MPI_COMM_WORLD);
+      superlu.SetPrintStatistics(true);
+      superlu.SetSymmetricPattern(false);
+      superlu.SetColumnPermutation(superlu::PARMETIS);
+      superlu.SetOperator(SA);
+      superlu.Mult(B, U);
+      delete A_hyp;
+   }
+#endif
+
+   // 15. Recover the parallel grid function corresponding to U. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(U, b, u);
+
+   // 16. Save the refined mesh and the solution in parallel. This output can be
+   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol_r" or
+   //     "glvis -np <np> -m mesh -g sol_i".
+   {
+      ostringstream mesh_name, sol_r_name, sol_i_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
+      sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_r_ofs(sol_r_name.str().c_str());
+      ofstream sol_i_ofs(sol_i_name.str().c_str());
+      sol_r_ofs.precision(8);
+      sol_i_ofs.precision(8);
+      u.real().Save(sol_r_ofs);
+      u.imag().Save(sol_i_ofs);
+   }
+
+   // 17. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_r << "solution\n" << pmesh << u.real()
+                 << "window_title 'Solution: Real Part'"
+                 << "window_geometry 800 0 400 350" << flush;
+
+      MPI_Barrier(MPI_COMM_WORLD);
+
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i.precision(8);
+      sol_sock_i << "solution\n" << pmesh << u.imag()
+                 << "window_title 'Solution: Imaginary Part'"
+                 << "window_geometry 1200 0 400 350" << flush;
+   }
+   if (visualization)
+   {
+      ParGridFunction u_t(&fespace);
+      u_t = u.real();
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << u_t
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "window_geometry 0 432 600 450"
+               << "pause\n" << flush;
+      if (myid == 0)
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 32;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos( 2.0 * M_PI * t), u.real(),
+             sin(-2.0 * M_PI * t), u.imag(), u_t);
+         sol_sock << "parallel " << num_procs << " " << myid << "\n";
+         sol_sock << "solution\n" << pmesh << u_t
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
+   }
+
+   // 18. Free the used memory.
+   delete fec_port;
+   delete fec;
+
+   return 0;
+}
+
+/**
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with
+   homogeneous Dirichlet boundary conditions on the boundary of the
+   domain. Returns mode number "mode" (counting from zero) in the
+   ParGridFunction "x".
+*/
+void ScalarWaveGuide(int mode, ParGridFunction &x)
+{
+   int nev = std::max(mode + 2, 5);
+   int seed = 75;
+
+   ParFiniteElementSpace &fespace = *x.ParFESpace();
+   ParMesh &pmesh = *fespace.GetParMesh();
+
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator);
+   a.Assemble();
+   a.EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a.Finalize();
+
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m.Finalize();
+
+   HypreParMatrix *A = a.ParallelAssemble();
+   HypreParMatrix *M = m.ParallelAssemble();
+
+   HypreBoomerAMG amg(*A);
+   amg.SetPrintLevel(0);
+
+   HypreLOBPCG lobpcg(MPI_COMM_WORLD);
+   lobpcg.SetNumModes(nev);
+   lobpcg.SetRandomSeed(seed);
+   lobpcg.SetPreconditioner(amg);
+   lobpcg.SetMaxIter(200);
+   lobpcg.SetTol(1e-8);
+   lobpcg.SetPrecondUsageMode(1);
+   lobpcg.SetPrintLevel(1);
+   lobpcg.SetMassMatrix(*M);
+   lobpcg.SetOperator(*A);
+   lobpcg.Solve();
+
+   x = lobpcg.GetEigenvector(mode);
+
+   delete A;
+   delete M;
+}
+
+/**
+   Solves the eigenvalue problem -Curl(Curl x) = lambda x with
+   homogeneous Dirichlet boundary conditions, on the tangential
+   component of x, on the boundary of the domain. Returns mode number
+   "mode" (counting from zero) in the ParGridFunction "x".
+*/
+void VectorWaveGuide(int mode, ParGridFunction &x)
+{
+   int nev = std::max(mode + 2, 5);
+
+   ParFiniteElementSpace &fespace = *x.ParFESpace();
+   ParMesh &pmesh = *fespace.GetParMesh();
+
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new CurlCurlIntegrator);
+   a.Assemble();
+   a.EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a.Finalize();
+
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new VectorFEMassIntegrator);
+   m.Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m.Finalize();
+
+   HypreParMatrix *A = a.ParallelAssemble();
+   HypreParMatrix *M = m.ParallelAssemble();
+
+   HypreAMS ams(*A,&fespace);
+   ams.SetPrintLevel(0);
+   ams.SetSingularProblem();
+
+   HypreAME ame(MPI_COMM_WORLD);
+   ame.SetNumModes(nev);
+   ame.SetPreconditioner(ams);
+   ame.SetMaxIter(100);
+   ame.SetTol(1e-8);
+   ame.SetPrintLevel(1);
+   ame.SetMassMatrix(*M);
+   ame.SetOperator(*A);
+   ame.Solve();
+
+   x = ame.GetEigenvector(mode);
+
+   delete A;
+   delete M;
+}
+
+/**
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with
+   homogeneous Neumann boundary conditions on the boundary of the
+   domain. Returns mode number "mode" (counting from zero) in the
+   ParGridFunction "x_l2". Note that mode 0 is a constant field so
+   higher mode numbers are often more interesting. The eigenmode is
+   solved using continuous H1 basis of the appropriate order and then
+   projected onto the L2 basis and returned.
+*/
+void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
+{
+   int nev = std::max(mode + 2, 5);
+   int seed = 75;
+
+   ParFiniteElementSpace &fespace_l2 = *x_l2.ParFESpace();
+   ParMesh &pmesh = *fespace_l2.GetParMesh();
+   int order_l2 = fespace_l2.FEColl()->GetOrder();
+
+   H1_FECollection fec(order_l2+1, pmesh.Dimension());
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   GridFunctionCoefficient xCoef(&x);
+
+   if (mode == 0)
+   {
+      x = 1.0;
+      x_l2.ProjectCoefficient(xCoef);
+      return;
+   }
+
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator);
+   a.AddDomainIntegrator(new MassIntegrator); // Shift eigenvalues by 1
+   a.Assemble();
+   a.Finalize();
+
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+   m.Finalize();
+
+   HypreParMatrix *A = a.ParallelAssemble();
+   HypreParMatrix *M = m.ParallelAssemble();
+
+   HypreBoomerAMG amg(*A);
+   amg.SetPrintLevel(0);
+
+   HypreLOBPCG lobpcg(MPI_COMM_WORLD);
+   lobpcg.SetNumModes(nev);
+   lobpcg.SetRandomSeed(seed);
+   lobpcg.SetPreconditioner(amg);
+   lobpcg.SetMaxIter(200);
+   lobpcg.SetTol(1e-8);
+   lobpcg.SetPrecondUsageMode(1);
+   lobpcg.SetPrintLevel(1);
+   lobpcg.SetMassMatrix(*M);
+   lobpcg.SetOperator(*A);
+   lobpcg.Solve();
+
+   x = lobpcg.GetEigenvector(mode);
+
+   x_l2.ProjectCoefficient(xCoef);
+
+   delete A;
+   delete M;
+}
+
+// Compute eigenmode "mode" of either a Dirichlet or Neumann Laplacian
+// or of a Dirichlet curl curl operator based on the problem type and
+// dimension of the domain.
+void SetPortBC(int prob, int dim, int mode, ParGridFunction &port_bc)
+{
+   switch (prob)
+   {
+      case 0:
+         ScalarWaveGuide(mode, port_bc);
+         break;
+      case 1:
+         if (dim == 3)
+         {
+            VectorWaveGuide(mode, port_bc);
+         }
+         else
+         {
+            PseudoScalarWaveGuide(mode, port_bc);
+         }
+         break;
+      case 2:
+         PseudoScalarWaveGuide(mode, port_bc);
+         break;
+   }
+}

examples/ex36.cpp

@@ -0,0 +1,463 @@
+//                                MFEM Example 36
+//
+//
+// Compile with: make ex36
+//
+// Sample runs: ex36 -o 2
+//              ex36 -o 2 -r 4
+//
+//
+// Description: This example code demonstrates the use of MFEM to solve the
+//              bound-constrained energy minimization problem
+//
+//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
+//
+//              This is known as the obstacle problem, and it is a simple
+//              mathematical model for contact mechanics.
+//
+//              In this example, the obstacle ϕ is a half-sphere centered
+//              at the origin of a circular domain Ω. After solving to a
+//              specified tolerance, the numerical solution is compared to
+//              a closed-form exact solution to assess accuracy.
+//
+//              The problem is discretized and solved using the proximal
+//              Galerkin finite element method, introduced by Keith and
+//              Surowiec [1].
+//
+//              This example highlights the ability of MFEM to deliver high-
+//              order solutions to variation inequality problems and
+//              showcases how to set up and solve nonlinear mixed methods.
+//
+//
+// [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//     preserving finite element method for pointwise bound constraints.
+//     arXiv:2307.12444 [math.NA]
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double spherical_obstacle(const Vector &pt);
+double exact_solution_obstacle(const Vector &pt);
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad);
+
+class LogarithmGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+
+public:
+   LogarithmGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                    double min_val_=-36)
+      : u(&u_), obstacle(&obst_), min_val(min_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+class ExponentialGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+   double max_val;
+
+public:
+   ExponentialGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                      double min_val_=0.0, double max_val_=1e6)
+      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int order = 1;
+   int max_it = 10;
+   int ref_levels = 3;
+   double alpha = 1.0;
+   double tol = 1e-5;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   const char *mesh_file = "../data/disc-nurbs.mesh";
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 3C. Rescale the domain to a unit circle (radius = 1).
+   GridFunction *nodes = mesh.GetNodes();
+   double scale = 2*sqrt(2);
+   *nodes /= scale;
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   H1_FECollection H1fec(order+1, dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   L2_FECollection L2fec(order-1, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec);
+
+   cout << "Number of H1 finite element unknowns: "
+        << H1fes.GetTrueVSize() << endl;
+   cout << "Number of L2 finite element unknowns: "
+        << L2fes.GetTrueVSize() << endl;
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = H1fes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   // 6. Define an initial guess for the solution.
+   auto IC_func = [](const Vector &x)
+   {
+      double r0 = 1.0;
+      double rr = 0.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         rr += x(i)*x(i);
+      }
+      return r0*r0 - rr;
+   };
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+
+   // 7. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_psi_gf;
+
+   u_gf.MakeRef(&H1fes,x,offsets[0]);
+   delta_psi_gf.MakeRef(&L2fes,x,offsets[1]);
+   delta_psi_gf = 0.0;
+
+   GridFunction u_old_gf(&H1fes);
+   GridFunction psi_old_gf(&L2fes);
+   GridFunction psi_gf(&L2fes);
+   u_old_gf = 0.0;
+   psi_old_gf = 0.0;
+
+   // 8. Define the function coefficients for the solution and use them to
+   //    initialize the initial guess
+   FunctionCoefficient exact_coef(exact_solution_obstacle);
+   VectorFunctionCoefficient exact_grad_coef(dim,exact_solution_gradient_obstacle);
+   FunctionCoefficient IC_coef(IC_func);
+   ConstantCoefficient f(0.0);
+   FunctionCoefficient obstacle(spherical_obstacle);
+   u_gf.ProjectCoefficient(IC_coef);
+   u_old_gf = u_gf;
+
+   // 9. Initialize the slack variable ψₕ = exp(uₕ)
+   LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
+   psi_gf.ProjectCoefficient(ln_u);
+   psi_old_gf = psi_gf;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 10. Iterate
+   int k;
+   int total_iterations = 0;
+   double increment_u = 0.1;
+   for (k = 0; k < max_it; k++)
+   {
+      GridFunction u_tmp(&H1fes);
+      u_tmp = u_old_gf;
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      int j;
+      for ( j = 0; j < 10; j++)
+      {
+         total_iterations++;
+
+         ConstantCoefficient alpha_cf(alpha);
+
+         LinearForm b0,b1;
+         b0.Update(&H1fes,rhs.GetBlock(0),0);
+         b1.Update(&L2fes,rhs.GetBlock(1),0);
+
+         ExponentialGridFunctionCoefficient exp_psi(psi_gf, zero);
+         ProductCoefficient neg_exp_psi(-1.0,exp_psi);
+         GradientGridFunctionCoefficient grad_u_old(&u_old_gf);
+         ProductCoefficient alpha_f(alpha, f);
+         GridFunctionCoefficient psi_cf(&psi_gf);
+         GridFunctionCoefficient psi_old_cf(&psi_old_gf);
+         SumCoefficient psi_old_minus_psi(psi_old_cf, psi_cf, 1.0, -1.0);
+
+         b0.AddDomainIntegrator(new DomainLFIntegrator(alpha_f));
+         b0.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+         b0.Assemble();
+
+         b1.AddDomainIntegrator(new DomainLFIntegrator(exp_psi));
+         b1.AddDomainIntegrator(new DomainLFIntegrator(obstacle));
+         b1.Assemble();
+
+         BilinearForm a00(&H1fes);
+         a00.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         a00.AddDomainIntegrator(new DiffusionIntegrator(alpha_cf));
+         a00.Assemble();
+         a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),
+                                  mfem::Operator::DIAG_ONE);
+         a00.Finalize();
+         SparseMatrix &A00 = a00.SpMat();
+
+         MixedBilinearForm a10(&H1fes,&L2fes);
+         a10.AddDomainIntegrator(new MixedScalarMassIntegrator());
+         a10.Assemble();
+         a10.EliminateTrialDofs(ess_bdr, x.GetBlock(0), rhs.GetBlock(1));
+         a10.Finalize();
+         SparseMatrix &A10 = a10.SpMat();
+
+         SparseMatrix *A01 = Transpose(A10);
+
+         BilinearForm a11(&L2fes);
+         a11.AddDomainIntegrator(new MassIntegrator(neg_exp_psi));
+         // NOTE: Shift the spectrum of the Hessian matrix for additional
+         //       stability (Quasi-Newton).
+         ConstantCoefficient eps_cf(-1e-6);
+         if (order == 1)
+         {
+            // NOTE: ∇ₕuₕ = 0 for constant functions.
+            //       Therefore, we use the mass matrix to shift the spectrum
+            a11.AddDomainIntegrator(new MassIntegrator(eps_cf));
+         }
+         else
+         {
+            a11.AddDomainIntegrator(new DiffusionIntegrator(eps_cf));
+         }
+         a11.Assemble();
+         a11.Finalize();
+         SparseMatrix &A11 = a11.SpMat();
+
+         BlockOperator A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         BlockDiagonalPreconditioner prec(offsets);
+         prec.SetDiagonalBlock(0,new GSSmoother(A00));
+         prec.SetDiagonalBlock(1,new GSSmoother(A11));
+         prec.owns_blocks = 1;
+
+         GMRES(A,prec,rhs,x,0,10000,500,1e-12,0.0);
+
+         u_gf.MakeRef(&H1fes, x.GetBlock(0), 0);
+         delta_psi_gf.MakeRef(&L2fes, x.GetBlock(1), 0);
+
+         u_tmp -= u_gf;
+         double Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         double gamma = 1.0;
+         delta_psi_gf *= gamma;
+         psi_gf += delta_psi_gf;
+
+         if (visualization)
+         {
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         delete A01;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      mfem::out << "H1-error  (|| u - uₕᵏ||)       = " << H1_error << endl;
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << H1fes.GetTrueVSize() + L2fes.GetTrueVSize()
+             << endl;
+
+   // 11. Exact solution.
+   if (visualization)
+   {
+      socketstream err_sock(vishost, visport);
+      err_sock.precision(8);
+
+      GridFunction error_gf(&H1fes);
+      error_gf.ProjectCoefficient(exact_coef);
+      error_gf -= u_gf;
+
+      err_sock << "solution\n" << mesh << error_gf << "window_title 'Error'"  <<
+               flush;
+   }
+
+   {
+      double L2_error = u_gf.ComputeL2Error(exact_coef);
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+
+      ExponentialGridFunctionCoefficient u_alt_cf(psi_gf,obstacle);
+      GridFunction u_alt_gf(&L2fes);
+      u_alt_gf.ProjectCoefficient(u_alt_cf);
+      double L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);
+
+      mfem::out << "\n Final L2-error (|| u - uₕ||)          = " << L2_error <<
+                endl;
+      mfem::out << " Final H1-error (|| u - uₕ||)          = " << H1_error << endl;
+      mfem::out << " Final L2-error (|| u - ϕ - exp(ψₕ)||) = " << L2_error_alt <<
+                endl;
+   }
+
+   return 0;
+}
+
+double LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
+   return max(min_val, log(val));
+}
+
+double ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                                const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip);
+   return min(max_val, max(min_val, exp(val) + obstacle->Eval(T, ip)));
+}
+
+double spherical_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double beta = 0.9;
+
+   double b = r0*beta;
+   double tmp = sqrt(r0*r0 - b*b);
+   double B = tmp + b*b/tmp;
+   double C = -b/tmp;
+
+   if (r > b)
+   {
+      return B + r * C;
+   }
+   else
+   {
+      return sqrt(r0*r0 - r*r);
+   }
+}
+
+double exact_solution_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      return A * log(r);
+   }
+   else
+   {
+      return sqrt(r0*r0-r*r);
+   }
+}
+
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      grad(0) =  A * x / (r*r);
+      grad(1) =  A * y / (r*r);
+   }
+   else
+   {
+      grad(0) = - x / sqrt( r0*r0 - r*r );
+      grad(1) = - y / sqrt( r0*r0 - r*r );
+   }
+}

examples/ex36p.cpp

@@ -0,0 +1,528 @@
+//                         MFEM Example 36 - Parallel Version
+//
+//
+// Compile with: make ex36p
+//
+// Sample runs: mpirun -np 4 ex36p -o 2
+//              mpirun -np 4 ex36p -o 2 -r 4
+//
+//
+// Description: This example code demonstrates the use of MFEM to solve the
+//              bound-constrained energy minimization problem
+//
+//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
+//
+//              This is known as the obstacle problem, and it is a simple
+//              mathematical model for contact mechanics.
+//
+//              In this example, the obstacle ϕ is a half-sphere centered
+//              at the origin of a circular domain Ω. After solving to a
+//              specified tolerance, the numerical solution is compared to
+//              a closed-form exact solution to assess accuracy.
+//
+//              The problem is discretized and solved using the proximal
+//              Galerkin finite element method, introduced by Keith and
+//              Surowiec [1].
+//
+//              This example highlights the ability of MFEM to deliver high-
+//              order solutions to variation inequality problems and
+//              showcases how to set up and solve nonlinear mixed methods.
+//
+//
+// [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//     preserving finite element method for pointwise bound constraints.
+//     arXiv:2307.12444 [math.NA]
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double spherical_obstacle(const Vector &pt);
+double exact_solution_obstacle(const Vector &pt);
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad);
+
+class LogarithmGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+
+public:
+   LogarithmGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                    double min_val_=-36)
+      : u(&u_), obstacle(&obst_), min_val(min_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+class ExponentialGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+   double max_val;
+
+public:
+   ExponentialGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                      double min_val_=0.0, double max_val_=1e6)
+      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   int order = 1;
+   int max_it = 10;
+   int ref_levels = 3;
+   double alpha = 1.0;
+   double tol = 1e-5;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Read the mesh from the mesh file.
+   const char *mesh_file = "../data/disc-nurbs.mesh";
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 3C. Rescale the domain to a unit circle (radius = 1).
+   GridFunction *nodes = mesh.GetNodes();
+   double scale = 2*sqrt(2);
+   *nodes /= scale;
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   H1_FECollection H1fec(order+1, dim);
+   ParFiniteElementSpace H1fes(&pmesh, &H1fec);
+
+   L2_FECollection L2fec(order-1, dim);
+   ParFiniteElementSpace L2fes(&pmesh, &L2fec);
+
+   int num_dofs_H1 = H1fes.GetTrueVSize();
+   MPI_Allreduce(MPI_IN_PLACE, &num_dofs_H1, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+   int num_dofs_L2 = L2fes.GetTrueVSize();
+   MPI_Allreduce(MPI_IN_PLACE, &num_dofs_L2, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+   if (myid == 0)
+   {
+      cout << "Number of H1 finite element unknowns: "
+           << num_dofs_H1 << endl;
+      cout << "Number of L2 finite element unknowns: "
+           << num_dofs_L2 << endl;
+   }
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = H1fes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   Array<int> toffsets(3);
+   toffsets[0] = 0;
+   toffsets[1] = H1fes.GetTrueVSize();
+   toffsets[2] = L2fes.GetTrueVSize();
+   toffsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   BlockVector tx(toffsets), trhs(toffsets);
+   tx = 0.0; trhs = 0.0;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   Array<int> empty;
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 6. Define an initial guess for the solution.
+   auto IC_func = [](const Vector &x)
+   {
+      double r0 = 1.0;
+      double rr = 0.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         rr += x(i)*x(i);
+      }
+      return r0*r0 - rr;
+   };
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+
+   // 7. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   ParGridFunction u_gf, delta_psi_gf;
+   u_gf.MakeRef(&H1fes,x,offsets[0]);
+   delta_psi_gf.MakeRef(&L2fes,x,offsets[1]);
+   delta_psi_gf = 0.0;
+
+   ParGridFunction u_old_gf(&H1fes);
+   ParGridFunction psi_old_gf(&L2fes);
+   ParGridFunction psi_gf(&L2fes);
+   u_old_gf = 0.0;
+   psi_old_gf = 0.0;
+
+
+   // 8. Define the function coefficients for the solution and use them to
+   //    initialize the initial guess
+   FunctionCoefficient exact_coef(exact_solution_obstacle);
+   VectorFunctionCoefficient exact_grad_coef(dim,exact_solution_gradient_obstacle);
+   FunctionCoefficient IC_coef(IC_func);
+   ConstantCoefficient f(0.0);
+   FunctionCoefficient obstacle(spherical_obstacle);
+   u_gf.ProjectCoefficient(IC_coef);
+   u_old_gf = u_gf;
+
+   // 9. Initialize the slack variable ψₕ = exp(uₕ)
+   LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
+   psi_gf.ProjectCoefficient(ln_u);
+   psi_old_gf = psi_gf;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 10. Iterate
+   int k;
+   int total_iterations = 0;
+   double increment_u = 0.1;
+   for (k = 0; k < max_it; k++)
+   {
+      ParGridFunction u_tmp(&H1fes);
+      u_tmp = u_old_gf;
+
+      if (myid == 0)
+      {
+         mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+      }
+
+      int j;
+      for ( j = 0; j < 10; j++)
+      {
+         total_iterations++;
+
+         ConstantCoefficient alpha_cf(alpha);
+
+         ParLinearForm b0,b1;
+         b0.Update(&H1fes,rhs.GetBlock(0),0);
+         b1.Update(&L2fes,rhs.GetBlock(1),0);
+
+         ExponentialGridFunctionCoefficient exp_psi(psi_gf, zero);
+         ProductCoefficient neg_exp_psi(-1.0,exp_psi);
+         GradientGridFunctionCoefficient grad_u_old(&u_old_gf);
+         ProductCoefficient alpha_f(alpha, f);
+         GridFunctionCoefficient psi_cf(&psi_gf);
+         GridFunctionCoefficient psi_old_cf(&psi_old_gf);
+         SumCoefficient psi_old_minus_psi(psi_old_cf, psi_cf, 1.0, -1.0);
+
+         b0.AddDomainIntegrator(new DomainLFIntegrator(alpha_f));
+         b0.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+         b0.Assemble();
+
+         b1.AddDomainIntegrator(new DomainLFIntegrator(exp_psi));
+         b1.AddDomainIntegrator(new DomainLFIntegrator(obstacle));
+         b1.Assemble();
+
+         ParBilinearForm a00(&H1fes);
+         a00.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         a00.AddDomainIntegrator(new DiffusionIntegrator(alpha_cf));
+         a00.Assemble();
+         HypreParMatrix A00;
+         a00.FormLinearSystem(ess_tdof_list, x.GetBlock(0), rhs.GetBlock(0),
+                              A00, tx.GetBlock(0), trhs.GetBlock(0));
+
+
+         ParMixedBilinearForm a10(&H1fes,&L2fes);
+         a10.AddDomainIntegrator(new MixedScalarMassIntegrator());
+         a10.Assemble();
+         HypreParMatrix A10;
+         a10.FormRectangularLinearSystem(ess_tdof_list, empty, x.GetBlock(0),
+                                         rhs.GetBlock(1),
+                                         A10, tx.GetBlock(0), trhs.GetBlock(1));
+
+         HypreParMatrix *A01 = A10.Transpose();
+
+         ParBilinearForm a11(&L2fes);
+         a11.AddDomainIntegrator(new MassIntegrator(neg_exp_psi));
+         // NOTE: Shift the spectrum of the Hessian matrix for additional
+         //       stability (Quasi-Newton).
+         ConstantCoefficient eps_cf(-1e-6);
+         if (order == 1)
+         {
+            // NOTE: ∇ₕuₕ = 0 for constant functions.
+            //       Therefore, we use the mass matrix to shift the spectrum
+            a11.AddDomainIntegrator(new MassIntegrator(eps_cf));
+         }
+         else
+         {
+            a11.AddDomainIntegrator(new DiffusionIntegrator(eps_cf));
+         }
+         a11.Assemble();
+         a11.Finalize();
+         HypreParMatrix A11;
+         a11.FormSystemMatrix(empty, A11);
+
+         BlockOperator A(toffsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         BlockDiagonalPreconditioner prec(toffsets);
+         HypreBoomerAMG P00(A00);
+         P00.SetPrintLevel(0);
+         HypreSmoother P11(A11);
+         prec.SetDiagonalBlock(0,&P00);
+         prec.SetDiagonalBlock(1,&P11);
+
+         GMRESSolver gmres(MPI_COMM_WORLD);
+         gmres.SetPrintLevel(-1);
+         gmres.SetRelTol(1e-8);
+         gmres.SetMaxIter(20000);
+         gmres.SetKDim(500);
+         gmres.SetOperator(A);
+         gmres.SetPreconditioner(prec);
+         gmres.Mult(trhs,tx);
+
+         u_gf.SetFromTrueDofs(tx.GetBlock(0));
+         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(1));
+
+         u_tmp -= u_gf;
+         double Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         double gamma = 1.0;
+         delta_psi_gf *= gamma;
+         psi_gf += delta_psi_gf;
+
+         if (visualization)
+         {
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << pmesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         if (myid == 0)
+         {
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         delete A01;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      if (myid == 0)
+      {
+         mfem::out << "Number of Newton iterations = " << j+1 << endl;
+         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+      }
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      if (myid == 0)
+      {
+         mfem::out << "H1-error  (|| u - uₕᵏ||)       = " << H1_error << endl;
+      }
+
+   }
+
+   if (myid == 0)
+   {
+      mfem::out << "\n Outer iterations: " << k+1
+                << "\n Total iterations: " << total_iterations
+                << "\n Total dofs:       " << num_dofs_H1 + num_dofs_L2
+                << endl;
+   }
+
+   // 11. Exact solution.
+   if (visualization)
+   {
+      socketstream err_sock(vishost, visport);
+      err_sock.precision(8);
+
+      ParGridFunction error_gf(&H1fes);
+      error_gf.ProjectCoefficient(exact_coef);
+      error_gf -= u_gf;
+
+      err_sock << "parallel " << num_procs << " " << myid << "\n";
+      err_sock << "solution\n" << pmesh << error_gf << "window_title 'Error'"  <<
+               flush;
+   }
+
+   {
+      double L2_error = u_gf.ComputeL2Error(exact_coef);
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+
+      ExponentialGridFunctionCoefficient u_alt_cf(psi_gf,obstacle);
+      ParGridFunction u_alt_gf(&L2fes);
+      u_alt_gf.ProjectCoefficient(u_alt_cf);
+      double L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);
+
+      if (myid == 0)
+      {
+         mfem::out << "\n Final L2-error (|| u - uₕ||)          = " << L2_error <<
+                   endl;
+         mfem::out << " Final H1-error (|| u - uₕ||)          = " << H1_error << endl;
+         mfem::out << " Final L2-error (|| u - ϕ - exp(ψₕ)||) = " << L2_error_alt <<
+                   endl;
+      }
+   }
+
+   return 0;
+}
+
+double LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
+   return max(min_val, log(val));
+}
+
+double ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                                const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip);
+   return min(max_val, max(min_val, exp(val) + obstacle->Eval(T, ip)));
+}
+
+double spherical_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double beta = 0.9;
+
+   double b = r0*beta;
+   double tmp = sqrt(r0*r0 - b*b);
+   double B = tmp + b*b/tmp;
+   double C = -b/tmp;
+
+   if (r > b)
+   {
+      return B + r * C;
+   }
+   else
+   {
+      return sqrt(r0*r0 - r*r);
+   }
+}
+
+double exact_solution_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      return A * log(r);
+   }
+   else
+   {
+      return sqrt(r0*r0-r*r);
+   }
+}
+
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      grad(0) =  A * x / (r*r);
+      grad(1) =  A * y / (r*r);
+   }
+   else
+   {
+      grad(0) = - x / sqrt( r0*r0 - r*r );
+      grad(1) = - y / sqrt( r0*r0 - r*r );
+   }
+}

examples/ex9p.cpp

@@ -536,8 +536,10 @@ int main(int argc, char *argv[])
       if (!sout)
       {
          if (Mpi::Root())
+         {
             cout << "Unable to connect to GLVis server at "
                  << vishost << ':' << visport << endl;
+         }
          visualization = false;
          if (Mpi::Root())
          {
@@ -552,8 +554,10 @@ int main(int argc, char *argv[])
          sout << "pause\n";
          sout << flush;
          if (Mpi::Root())
+         {
             cout << "GLVis visualization paused."
                  << " Press space (in the GLVis window) to resume it.\n";
+         }
       }
    }
 

examples/makefile

@@ -23,20 +23,20 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31 ex33
+  ex31 ex33 ex34 ex36
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-  ex24p ex25p ex26p
+  ex24p ex25p ex26p ex34p ex35p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
    EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
 endif
-SUBDIRS = contact 
+SUBDIRS =
 ifeq ($(MFEM_USE_AMGX),YES)
    SUBDIRS += amgx
 endif
@@ -183,3 +183,4 @@ clean-exec:
 	@rm -f ex23.mesh ex23-*.gf
 	@rm -f ex25.mesh ex25-*.gf ex25p-*.*
 	@rm -rf ex28_* ex28p_*
+	@rm -rf cond.* cond_mesh.* cond_j.* dsol.* port_mesh.* port_mode.*

examples/sundials/CMakeLists.txt

@@ -68,11 +68,43 @@ if (MFEM_ENABLE_TESTING)
       add_test(NAME ${TEST_NAME}_ser
         COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
     else()
-      add_test(NAME ${TEST_NAME}_np=4
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
         ${MPIEXEC_PREFLAGS}
         $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
         ${MPIEXEC_POSTFLAGS})
     endif()
   endforeach()
-endif()
+
+  # Add CUDA/HIP tests.
+  set(DEVICE_EXAMPLES
+      # serial examples with device support:
+      ex9
+      # parallel examples with device support:
+      ex9p)
+  set(MFEM_TEST_DEVICE)
+  if (MFEM_USE_CUDA)
+    set(MFEM_TEST_DEVICE "cuda")
+  elseif (MFEM_USE_HIP)
+    set(MFEM_TEST_DEVICE "hip")
+  endif()
+  if (MFEM_TEST_DEVICE)
+    foreach(TEST_NAME ${DEVICE_EXAMPLES})
+      string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+
+      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+
+      if (NOT (${TEST_NAME} MATCHES ".*p$"))
+        add_test(NAME ${PFX}${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+          COMMAND ${PFX}${TEST_NAME} ${THIS_TEST_OPTIONS})
+      else()
+        add_test(NAME ${PFX}${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+          COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+          ${MPIEXEC_PREFLAGS}
+          $<TARGET_FILE:${PFX}${TEST_NAME}> ${THIS_TEST_OPTIONS}
+          ${MPIEXEC_POSTFLAGS})
+      endif()
+    endforeach()
+  endif(MFEM_TEST_DEVICE)
+endif(MFEM_ENABLE_TESTING)

examples/sundials/README

@@ -12,8 +12,7 @@ use of MFEM features based on the SUNDIALS suite of time integration and
 non-linear solvers.
 
 To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_SUNDIALS = YES", see the top-level INSTALL file for details (version
-2.7 or higher of SUNDIALS is required).
+"MFEM_USE_SUNDIALS = YES", see the top-level INSTALL file for details.
 
 We recommend comparing the original example codes with the corresponding files
 in the current directory.

examples/sundials/ex9.cpp

@@ -280,15 +280,16 @@ int main(int argc, char *argv[])
       k.SetAssemblyLevel(AssemblyLevel::FULL);
    }
    m.AddDomainIntegrator(new MassIntegrator);
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
    k.AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
    k.AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
 
    LinearForm b(&fes);
    b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));
 
    m.Assemble();
    int skip_zeros = 0;

examples/sundials/ex9p.cpp

@@ -63,6 +63,66 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
+// Type of preconditioner for implicit time integrator
+enum class PrecType : int
+{
+   ILU = 0,
+   AIR = 1
+};
+
+#if MFEM_HYPRE_VERSION >= 21800
+// Algebraic multigrid preconditioner for advective problems based on
+// approximate ideal restriction (AIR). Most effective when matrix is
+// first scaled by DG block inverse, and AIR applied to scaled matrix.
+// See https://doi.org/10.1137/17M1144350.
+class AIR_prec : public Solver
+{
+private:
+   const HypreParMatrix *A;
+   // Copy of A scaled by block-diagonal inverse
+   HypreParMatrix A_s;
+
+   HypreBoomerAMG *AIR_solver;
+   int blocksize;
+
+public:
+   AIR_prec(int blocksize_) : AIR_solver(NULL), blocksize(blocksize_) { }
+
+   void SetOperator(const Operator &op)
+   {
+      width = op.Width();
+      height = op.Height();
+
+      A = dynamic_cast<const HypreParMatrix *>(&op);
+      MFEM_VERIFY(A != NULL, "AIR_prec requires a HypreParMatrix.")
+
+      // Scale A by block-diagonal inverse
+      BlockInverseScale(A, &A_s, NULL, NULL, blocksize,
+                        BlockInverseScaleJob::MATRIX_ONLY);
+      delete AIR_solver;
+      AIR_solver = new HypreBoomerAMG(A_s);
+      AIR_solver->SetAdvectiveOptions(1, "", "FA");
+      AIR_solver->SetPrintLevel(0);
+      AIR_solver->SetMaxLevels(50);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      // Scale the rhs by block inverse and solve system
+      HypreParVector z_s;
+      BlockInverseScale(A, NULL, &x, &z_s, blocksize,
+                        BlockInverseScaleJob::RHS_ONLY);
+      AIR_solver->Mult(z_s, y);
+   }
+
+   ~AIR_prec()
+   {
+      delete AIR_solver;
+   }
+};
+#endif
+
+
 class DG_Solver : public Solver
 {
 private:
@@ -70,24 +130,37 @@ private:
    SparseMatrix M_diag;
    HypreParMatrix *A;
    GMRESSolver linear_solver;
-   BlockILU prec;
+   Solver *prec;
    double dt;
 public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes,
+             PrecType prec_type)
       : M(M_),
         K(K_),
         A(NULL),
         linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
         dt(-1.0)
    {
+      int block_size = fes.GetFE(0)->GetDof();
+      if (prec_type == PrecType::ILU)
+      {
+         prec = new BlockILU(block_size,
+                             BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
+      }
+      else if (prec_type == PrecType::AIR)
+      {
+#if MFEM_HYPRE_VERSION >= 21800
+         prec = new AIR_prec(block_size);
+#else
+         MFEM_ABORT("Must have MFEM_HYPRE_VERSION >= 21800 to use AIR.\n");
+#endif
+      }
       linear_solver.iterative_mode = false;
       linear_solver.SetRelTol(1e-9);
       linear_solver.SetAbsTol(0.0);
       linear_solver.SetMaxIter(100);
       linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
+      linear_solver.SetPreconditioner(*prec);
 
       M.GetDiag(M_diag);
    }
@@ -120,10 +193,12 @@ public:
 
    ~DG_Solver()
    {
+      delete prec;
       delete A;
    }
 };
 
+
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
     and advection matrices, and b describes the flow on the boundary. This can
@@ -141,7 +216,8 @@ private:
    mutable Vector z;
 
 public:
-   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_);
+   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_,
+                PrecType prec_type);
 
    virtual void Mult(const Vector &x, Vector &y) const;
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
@@ -178,6 +254,11 @@ int main(int argc, char *argv[])
    bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
+#if MFEM_HYPRE_VERSION >= 21800
+   PrecType prec_type = PrecType::AIR;
+#else
+   PrecType prec_type = PrecType::ILU;
+#endif
 
    // Relative and absolute tolerances for CVODE and ARKODE.
    const double reltol = 1e-2, abstol = 1e-2;
@@ -218,6 +299,8 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
+                  "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -238,13 +321,13 @@ int main(int argc, char *argv[])
    args.Parse();
    if (!args.Good())
    {
-      if (myid == 0)
+      if (Mpi::Root())
       {
          args.PrintUsage(cout);
       }
       return 1;
    }
-   if (myid == 0)
+   if (Mpi::Root())
    {
       args.PrintOptions(cout);
    }
@@ -252,7 +335,7 @@ int main(int argc, char *argv[])
    // check for valid ODE solver option
    if (ode_solver_type < 1 || ode_solver_type > 9)
    {
-      if (myid == 0)
+      if (Mpi::Root())
       {
          cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
       }
@@ -260,7 +343,7 @@ int main(int argc, char *argv[])
    }
 
    Device device(device_config);
-   if (myid == 0) { device.Print(); }
+   if (Mpi::Root()) { device.Print(); }
 
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle geometrically periodic meshes in this code.
@@ -297,7 +380,7 @@ int main(int argc, char *argv[])
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
    HYPRE_BigInt global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
+   if (Mpi::Root())
    {
       cout << "Number of unknowns: " << global_vSize << endl;
    }
@@ -328,15 +411,16 @@ int main(int argc, char *argv[])
    }
 
    m->AddDomainIntegrator(new MassIntegrator);
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
    k->AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
    k->AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
 
    ParLinearForm *b = new ParLinearForm(fes);
    b->AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));
 
    int skip_zeros = 0;
    m->Assemble();
@@ -435,11 +519,13 @@ int main(int argc, char *argv[])
       sout.open(vishost, visport);
       if (!sout)
       {
-         if (myid == 0)
+         if (Mpi::Root())
+         {
             cout << "Unable to connect to GLVis server at "
                  << vishost << ':' << visport << endl;
+         }
          visualization = false;
-         if (myid == 0)
+         if (Mpi::Root())
          {
             cout << "GLVis visualization disabled.\n";
          }
@@ -451,15 +537,17 @@ int main(int argc, char *argv[])
          sout << "solution\n" << *pmesh << *u;
          sout << "pause\n";
          sout << flush;
-         if (myid == 0)
+         if (Mpi::Root())
+         {
             cout << "GLVis visualization paused."
                  << " Press space (in the GLVis window) to resume it.\n";
+         }
       }
    }
 
    // 9. Define the time-dependent evolution operator describing the ODE
    //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution adv(*m, *k, *B, prec_type);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -511,7 +599,7 @@ int main(int argc, char *argv[])
 
       if (done || ti % vis_steps == 0)
       {
-         if (myid == 0)
+         if (Mpi::Root())
          {
             cout << "time step: " << ti << ", time: " << t << endl;
             if (cvode) { cvode->PrintInfo(); }
@@ -590,7 +678,7 @@ int main(int argc, char *argv[])
 
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
-                           const Vector &b_)
+                           const Vector &b_, PrecType prec_type)
    : TimeDependentOperator(M_.Height()),
      b(b_),
      M_solver(M_.ParFESpace()->GetComm()),
@@ -617,7 +705,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
       HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
       M_prec = hypre_prec;
 
-      dg_solver = new DG_Solver(M_mat, K_mat, *M_.FESpace());
+      dg_solver = new DG_Solver(M_mat, K_mat, *M_.FESpace(), prec_type);
    }
    else
    {
@@ -633,6 +721,10 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
    M_solver.SetPrintLevel(0);
 }
 
+// Solve the equation:
+//    u_t = M^{-1}(Ku + b),
+// by solving associated linear system
+//    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
 {
    K->Mult(x, z);

examples/sundials/makefile

@@ -23,6 +23,8 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex9 ex10 ex16
 PAR_EXAMPLES = ex9p ex10p ex16p
+SEQ_DEVICE_EXAMPLES = ex9
+PAR_DEVICE_EXAMPLES = ex9p
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -54,10 +56,22 @@ include $(MFEM_TEST_MK)
 RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 SERIAL_NAME := Serial SUNDIALS example
 PARALLEL_NAME := Parallel SUNDIALS example
+SERIAL_CUDA_NAME := Serial SUNDIALS CUDA example
+PARALLEL_CUDA_NAME := Parallel SUNDIALS CUDA example
+SERIAL_HIP_NAME := Serial SUNDIALS HIP example
+PARALLEL_HIP_NAME := Parallel SUNDIALS HIP example
 %-test-par: %
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
 %-test-seq: %
 	@$(call mfem-test,$<,, $(SERIAL_NAME))
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_CUDA_NAME),-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, $(SERIAL_CUDA_NAME),-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_HIP_NAME),-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, $(SERIAL_HIP_NAME),-d hip)
 
 # Testing: Specific execution options:
 # Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
@@ -68,6 +82,16 @@ ex9-test-seq: ex9
 	@$(call mfem-test,$<,, $(SERIAL_NAME),$(EX9_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX9P_ARGS))
+ex9-test-seq-cuda: ex9
+	@$(call mfem-test,$<,, $(SERIAL_CUDA_NAME),-d cuda $(EX9_ARGS))
+ex9p-test-par-cuda: ex9p
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_CUDA_NAME),-d cuda \
+	   $(EX9P_ARGS))
+ex9-test-seq-hip: ex9
+	@$(call mfem-test,$<,, $(SERIAL_HIP_NAME),-d hip $(EX9_ARGS))
+ex9p-test-par-hip: ex9p
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_HIP_NAME),-d hip \
+	   $(EX9P_ARGS))
 # Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
 EX10_COMMON_ARGS := -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10
 EX10_ARGS  := $(EX10_COMMON_ARGS) -r  2

examples/superlu/ex1p.cpp

@@ -67,6 +67,7 @@ int main(int argc, char *argv[])
    int slu_colperm = 4;
    int slu_rowperm = 1;
    int slu_iterref = 2;
+   int slu_npdep = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -85,9 +86,11 @@ int main(int argc, char *argv[])
                   "6-ZOLTAN");
    args.AddOption(&slu_rowperm, "-rp", "--rowperm",
                   "SuperLU Row Permutation Method:  0-NOROWPERM, 1-LargeDiag");
-   args.AddOption(&slu_iterref, "-rp", "--rowperm",
+   args.AddOption(&slu_iterref, "-ir", "--iterref",
                   "SuperLU Iterative Refinement:  0-NOREFINE, 1-Single, "
                   "2-Double, 3-Extra");
+   args.AddOption(&slu_npdep, "-npdep", "--npdepth",
+                  "Depth of 3D parition for SuperLU (>= 7.2.0)");
 
    args.Parse();
    if (!args.Good())
@@ -214,7 +217,7 @@ int main(int argc, char *argv[])
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
    // 13. Solve the linear system A X = B utilizing SuperLU.
-   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD);
+   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD, slu_npdep);
    Operator *SLU_A = new SuperLURowLocMatrix(*A.As<HypreParMatrix>());
    superlu->SetPrintStatistics(true);
    superlu->SetSymmetricPattern(false);
@@ -281,10 +284,9 @@ int main(int argc, char *argv[])
    superlu->SetOperator(*SLU_A);
    superlu->SetPrintStatistics(true);
    superlu->Mult(B, X);
-   superlu->DismantleGrid();
 
-   delete SLU_A;
    delete superlu;
+   delete SLU_A;
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.

fem/CMakeLists.txt

@@ -13,28 +13,44 @@ set(SRCS
   bilinearform.cpp
   bilinearform_ext.cpp
   bilininteg.cpp
-  bilininteg_br2.cpp
-  bilininteg_convection_mf.cpp
-  bilininteg_convection_pa.cpp
-  bilininteg_convection_ea.cpp
-  bilininteg_dgtrace_pa.cpp
-  bilininteg_dgtrace_ea.cpp
-  bilininteg_diffusion_mf.cpp
-  bilininteg_diffusion_pa.cpp
-  bilininteg_diffusion_ea.cpp
-  bilininteg_divergence.cpp
-  bilininteg_hcurl.cpp
-  bilininteg_hdiv.cpp
-  bilininteg_vectorfe.cpp
-  bilininteg_gradient.cpp
-  bilininteg_mass_mf.cpp
-  bilininteg_mass_pa.cpp
-  bilininteg_mass_ea.cpp
-  bilininteg_transpose_ea.cpp
-  bilininteg_vecdiffusion.cpp
-  bilininteg_vecdiffusion_mf.cpp
-  bilininteg_vecmass.cpp
-  bilininteg_vecmass_mf.cpp
+  integ/bilininteg_br2.cpp
+  integ/bilininteg_convection_mf.cpp
+  integ/bilininteg_convection_pa.cpp
+  integ/bilininteg_convection_ea.cpp
+  integ/bilininteg_curlcurl_pa.cpp
+  integ/bilininteg_dgtrace_pa.cpp
+  integ/bilininteg_dgtrace_ea.cpp
+  integ/bilininteg_diffusion_mf.cpp
+  integ/bilininteg_diffusion_pa.cpp
+  integ/bilininteg_diffusion_ea.cpp
+  integ/bilininteg_divdiv_pa.cpp
+  integ/bilininteg_gradient_pa.cpp
+  integ/bilininteg_interp_pa.cpp
+  integ/bilininteg_mass_mf.cpp
+  integ/bilininteg_mass_pa.cpp
+  integ/bilininteg_mass_ea.cpp
+  integ/bilininteg_mixedcurl_pa.cpp
+  integ/bilininteg_mixedvecgrad_pa.cpp
+  integ/bilininteg_transpose_ea.cpp
+  integ/bilininteg_vecdiffusion_mf.cpp
+  integ/bilininteg_vecdiffusion_pa.cpp
+  integ/bilininteg_vecdiv_pa.cpp
+  integ/bilininteg_vecmass_mf.cpp
+  integ/bilininteg_vecmass_pa.cpp
+  integ/bilininteg_vectorfediv_pa.cpp
+  integ/bilininteg_vectorfemass_pa.cpp
+  integ/bilininteg_diffusion_kernels.cpp
+  integ/bilininteg_hcurl_kernels.cpp
+  integ/bilininteg_hdiv_kernels.cpp
+  integ/bilininteg_hcurlhdiv_kernels.cpp
+  integ/bilininteg_mass_kernels.cpp
+  integ/lininteg_boundary.cpp
+  integ/lininteg_boundary_flux.cpp
+  integ/lininteg_domain.cpp
+  integ/lininteg_domain_grad.cpp
+  integ/lininteg_domain_vectorfe.cpp
+  integ/nonlininteg_vecconvection_pa.cpp
+  integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
   convergence.cpp
@@ -74,11 +90,6 @@ set(SRCS
   linearform.cpp
   linearform_ext.cpp
   lininteg.cpp
-  lininteg_boundary.cpp
-  lininteg_boundary_flux.cpp
-  lininteg_domain.cpp
-  lininteg_domain_grad.cpp
-  lininteg_vectorfe_domain.cpp
   lor/lor.cpp
   lor/lor_ads.cpp
   lor/lor_ams.cpp
@@ -91,8 +102,6 @@ set(SRCS
   nonlinearform_ext.cpp
   nonlininteg.cpp
   fespacehierarchy.cpp
-  nonlininteg_vectorconvection.cpp
-  nonlininteg_vectorconvection_mf.cpp
   qfunction.cpp
   qinterp/det.cpp
   qinterp/eval_by_nodes.cpp
@@ -143,7 +152,11 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
-  bilininteg_mass_pa.hpp
+  integ/bilininteg_diffusion_kernels.hpp
+  integ/bilininteg_hcurl_kernels.hpp
+  integ/bilininteg_hdiv_kernels.hpp
+  integ/bilininteg_hcurlhdiv_kernels.hpp
+  integ/bilininteg_mass_kernels.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp

fem/bilinearform_ext.cpp

@@ -56,6 +56,9 @@ void MFBilinearFormExtension::Assemble()
    {
       integrators[i]->AssembleMF(*a->FESpace());
    }
+
+   MFEM_VERIFY(a->GetBBFI()->Size() == 0, "AddBoundaryIntegrator is not "
+               "currently supported in MFBilinearFormExtension");
 }
 
 void MFBilinearFormExtension::AssembleDiagonal(Vector &y) const
@@ -275,7 +278,9 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
       int_face_Y.UseDevice(true); // ensure 'int_face_Y = 0.0' is done on device
    }
 
-   if (bdr_face_restrict_lex == NULL && a->GetBFBFI()->Size() > 0)
+   const bool has_bdr_integs = (a->GetBFBFI()->Size() > 0 ||
+                                a->GetBBFI()->Size() > 0);
+   if (bdr_face_restrict_lex == NULL && has_bdr_integs)
    {
       bdr_face_restrict_lex = trial_fes->GetFaceRestriction(
                                  ElementDofOrdering::LEXICOGRAPHIC,
@@ -292,27 +297,27 @@ void PABilinearFormExtension::Assemble()
    SetupRestrictionOperators(L2FaceValues::DoubleValued);
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
+   for (BilinearFormIntegrator *integ : integrators)
    {
-      integrators[i]->AssemblePA(*a->FESpace());
+      integ->AssemblePA(*a->FESpace());
    }
 
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
+   Array<BilinearFormIntegrator*> &bdr_integrators = *a->GetBBFI();
+   for (BilinearFormIntegrator *integ : bdr_integrators)
+   {
+      integ->AssemblePABoundary(*a->FESpace());
+   }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int intFaceIntegratorCount = intFaceIntegrators.Size();
-   for (int i = 0; i < intFaceIntegratorCount; ++i)
+   for (BilinearFormIntegrator *integ : intFaceIntegrators)
    {
-      intFaceIntegrators[i]->AssemblePAInteriorFaces(*a->FESpace());
+      integ->AssemblePAInteriorFaces(*a->FESpace());
    }
 
    Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int boundFaceIntegratorCount = bdrFaceIntegrators.Size();
-   for (int i = 0; i < boundFaceIntegratorCount; ++i)
+   for (BilinearFormIntegrator *integ : bdrFaceIntegrators)
    {
-      bdrFaceIntegrators[i]->AssemblePABoundaryFaces(*a->FESpace());
+      integ->AssemblePABoundaryFaces(*a->FESpace());
    }
 }
 
@@ -323,20 +328,27 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
    const int iSz = integrators.Size();
    if (elem_restrict && !DeviceCanUseCeed())
    {
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      if (iSz > 0)
       {
-         integrators[i]->AssembleDiagonalPA(localY);
-      }
-      const ElementRestriction* H1elem_restrict =
-         dynamic_cast<const ElementRestriction*>(elem_restrict);
-      if (H1elem_restrict)
-      {
-         H1elem_restrict->MultTransposeUnsigned(localY, y);
+         localY = 0.0;
+         for (int i = 0; i < iSz; ++i)
+         {
+            integrators[i]->AssembleDiagonalPA(localY);
+         }
+         const ElementRestriction* H1elem_restrict =
+            dynamic_cast<const ElementRestriction*>(elem_restrict);
+         if (H1elem_restrict)
+         {
+            H1elem_restrict->MultTransposeUnsigned(localY, y);
+         }
+         else
+         {
+            elem_restrict->MultTranspose(localY, y);
+         }
       }
       else
       {
-         elem_restrict->MultTranspose(localY, y);
+         y = 0.0;
       }
    }
    else
@@ -348,6 +360,18 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
          integrators[i]->AssembleDiagonalPA(y);
       }
    }
+
+   Array<BilinearFormIntegrator*> &bdr_integs = *a->GetBBFI();
+   const int n_bdr_integs = bdr_integs.Size();
+   if (bdr_face_restrict_lex && n_bdr_integs > 0)
+   {
+      bdr_face_Y = 0.0;
+      for (int i = 0; i < n_bdr_integs; ++i)
+      {
+         bdr_integs[i]->AssembleDiagonalPA(bdr_face_Y);
+      }
+      bdr_face_restrict_lex->AddMultTransposeUnsigned(bdr_face_Y, y);
+   }
 }
 
 void PABilinearFormExtension::Update()
@@ -397,13 +421,20 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    }
    else
    {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      if (iSz)
       {
-         integrators[i]->AddMultPA(localX, localY);
+         elem_restrict->Mult(x, localX);
+         localY = 0.0;
+         for (int i = 0; i < iSz; ++i)
+         {
+            integrators[i]->AddMultPA(localX, localY);
+         }
+         elem_restrict->MultTranspose(localY, y);
+      }
+      else
+      {
+         y = 0.0;
       }
-      elem_restrict->MultTranspose(localY, y);
    }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
@@ -422,17 +453,24 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       }
    }
 
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
+   Array<BilinearFormIntegrator*> &bdr_integs = *a->GetBBFI();
+   Array<BilinearFormIntegrator*> &bdr_face_integs = *a->GetBFBFI();
+   const int n_bdr_integs = bdr_integs.Size();
+   const int n_bdr_face_integs = bdr_face_integs.Size();
+   const bool has_bdr_integs = (n_bdr_face_integs > 0 || n_bdr_integs > 0);
+   if (bdr_face_restrict_lex && has_bdr_integs)
    {
       bdr_face_restrict_lex->Mult(x, bdr_face_X);
       if (bdr_face_X.Size()>0)
       {
          bdr_face_Y = 0.0;
-         for (int i = 0; i < bFISz; ++i)
+         for (int i = 0; i < n_bdr_integs; ++i)
          {
-            bdrFaceIntegrators[i]->AddMultPA(bdr_face_X, bdr_face_Y);
+            bdr_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
+         }
+         for (int i = 0; i < n_bdr_face_integs; ++i)
+         {
+            bdr_face_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
@@ -596,7 +634,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
       auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
       auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-      MFEM_FORALL(glob_j, ne*NDOFS,
+      mfem::forall(ne*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
       {
          const int e = glob_j/NDOFS;
          const int j = glob_j%NDOFS;
@@ -631,7 +669,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          if (!factorize_face_terms)
          {
             auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
             {
                const int f = glob_j/NDOFS;
                const int j = glob_j%NDOFS;
@@ -650,7 +688,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             });
          }
          auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
+         mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -687,7 +725,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
          auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
          auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
+         mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -724,7 +762,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
       auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
       auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-      MFEM_FORALL(glob_j, ne*NDOFS,
+      mfem::forall(ne*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
       {
          const int e = glob_j/NDOFS;
          const int j = glob_j%NDOFS;
@@ -759,7 +797,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          if (!factorize_face_terms)
          {
             auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
             {
                const int f = glob_j/NDOFS;
                const int j = glob_j%NDOFS;
@@ -778,7 +816,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             });
          }
          auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
+         mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -815,7 +853,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
          auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
          auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
+         mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -1030,13 +1068,13 @@ void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const
       const int local_size = a->FESpace()->GetVSize();
       auto dg_x_ptr = dg_x.Write();
       auto x_ptr = x.Read();
-      MFEM_FORALL(i,local_size,
+      mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[i] = x_ptr[i];
       });
       const int shared_size = shared_x.Size();
       auto shared_x_ptr = shared_x.Read();
-      MFEM_FORALL(i,shared_size,
+      mfem::forall(shared_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[local_size+i] = shared_x_ptr[i];
       });
@@ -1047,7 +1085,7 @@ void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const
          // DG Restriction
          auto dg_y_ptr = dg_y.Read();
          auto y_ptr = y.ReadWrite();
-         MFEM_FORALL(i,local_size,
+         mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
          {
             y_ptr[i] += dg_y_ptr[i];
          });
@@ -1091,13 +1129,13 @@ void FABilinearFormExtension::DGMultTranspose(const Vector &x, Vector &y) const
       const int local_size = a->FESpace()->GetVSize();
       auto dg_x_ptr = dg_x.Write();
       auto x_ptr = x.Read();
-      MFEM_FORALL(i,local_size,
+      mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[i] = x_ptr[i];
       });
       const int shared_size = shared_x.Size();
       auto shared_x_ptr = shared_x.Read();
-      MFEM_FORALL(i,shared_size,
+      mfem::forall(shared_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[local_size+i] = shared_x_ptr[i];
       });
@@ -1108,7 +1146,7 @@ void FABilinearFormExtension::DGMultTranspose(const Vector &x, Vector &y) const
          // DG Restriction
          auto dg_y_ptr = dg_y.Read();
          auto y_ptr = y.ReadWrite();
-         MFEM_FORALL(i,local_size,
+         mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
          {
             y_ptr[i] += dg_y_ptr[i];
          });
@@ -1446,7 +1484,7 @@ void PADiscreteLinearOperatorExtension::Assemble()
    }
 
    auto tm = test_multiplicity.ReadWrite();
-   MFEM_FORALL(i, test_multiplicity.Size(),
+   mfem::forall(test_multiplicity.Size(), [=] MFEM_HOST_DEVICE (int i)
    {
       tm[i] = 1.0 / tm[i];
    });
@@ -1498,7 +1536,7 @@ void PADiscreteLinearOperatorExtension::AddMultTranspose(
    MFEM_VERIFY(x.Size() == test_multiplicity.Size(), "Input vector of wrong size");
    auto xs = xscaled.ReadWrite();
    auto tm = test_multiplicity.Read();
-   MFEM_FORALL(i, x.Size(),
+   mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
    {
       xs[i] *= tm[i];
    });

fem/bilininteg.cpp

@@ -22,41 +22,47 @@ namespace mfem
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePA(fes)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                         const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
+              "   is not implemented for this class.");
+}
+
+void BilinearFormIntegrator::AssemblePABoundary(const FiniteElementSpace&)
+{
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePABoundary(fes)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePAInteriorFaces(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePAInteriorFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePAInteriorFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePABoundaryFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePABoundaryFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
                                         Vector &emat,
                                         const bool add)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEA(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
@@ -65,8 +71,8 @@ void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
                                                      Vector &ea_data_ext,
                                                      const bool add)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
@@ -74,8 +80,8 @@ void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
                                                      Vector &ea_data_bdr,
                                                      const bool add)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
@@ -86,62 +92,62 @@ void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
 
 void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::MultAssembled(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::MultAssembled(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultTransposePA(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AddMultTransposePA(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleMF(const FiniteElementSpace &fes)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AddMultMF(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AddMultMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AddMultTransposeMF(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleElementMatrix (
    const FiniteElement &el, ElementTransformation &Trans,
    DenseMatrix &elmat )
 {
-   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleElementMatrix2 (
    const FiniteElement &el1, const FiniteElement &el2,
    ElementTransformation &Trans, DenseMatrix &elmat )
 {
-   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix2(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix2(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleFaceMatrix (
    const FiniteElement &el1, const FiniteElement &el2,
    FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleFaceMatrix(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleFaceMatrix(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleFaceMatrix(
@@ -153,6 +159,16 @@ void BilinearFormIntegrator::AssembleFaceMatrix(
               " Integrator class.");
 }
 
+void BilinearFormIntegrator::AssembleTraceFaceMatrix (int elem,
+                                                      const FiniteElement &trial_face_fe,
+                                                      const FiniteElement &test_fe1,
+                                                      FaceElementTransformations &Trans,
+                                                      DenseMatrix &elmat)
+{
+   MFEM_ABORT("AssembleTraceFaceMatrix (DPG form) is not implemented for this"
+              " Integrator class.");
+}
+
 void BilinearFormIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
    Vector &elvect)
@@ -2633,7 +2649,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
    }
    else
    {
-      mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+      MFEM_ABORT("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
                  "   is not implemented for given trial and test bases.");
    }
 }
@@ -3997,6 +4013,216 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
    }
 }
 
+void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                              const FiniteElement &trial_face_fe,
+                                              const FiniteElement &test_fe,
+                                              FaceElementTransformations & Trans,
+                                              DenseMatrix &elmat)
+{
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::VALUE,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Test space should be H1");
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::INTEGRAL,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Trial space should be RT trace");
+
+   int i, j, face_ndof, ndof;
+   int order;
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   face_shape.SetSize(face_ndof);
+   shape.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      trial_face_fe.CalcPhysShape(Trans,face_shape);
+
+      // Finite element shape function
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcPhysShape(*eltrans, shape);
+
+      face_shape *= Trans.Weight()*ip.weight*scale;
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += shape(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                    const FiniteElement &trial_face_fe,
+                                                    const FiniteElement &test_fe,
+                                                    FaceElementTransformations &Trans,
+                                                    DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof, dim;
+   int order;
+
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::H_DIV,
+               "NormalTraceIntegrator::AssembleTraceFaceMatrix: Test space should be RT");
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE,
+               "NormalTraceIntegrator::AssembleTraceFaceMatrix: Trial space should be H1 (trace)");
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+   dim = test_fe.GetDim();
+
+   face_shape.SetSize(face_ndof);
+   normal.SetSize(dim);
+   shape.SetSize(ndof,dim);
+   shape_n.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      Trans.SetAllIntPoints(&ip);
+      trial_face_fe.CalcPhysShape(Trans, face_shape);
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * etrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*etrans, shape);
+      shape.Mult(normal, shape_n);
+      face_shape *= ip.weight*scale;
+
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += shape_n(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void TangentTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                     const FiniteElement &trial_face_fe,
+                                                     const FiniteElement &test_fe,
+                                                     FaceElementTransformations & Trans,
+                                                     DenseMatrix &elmat)
+{
+
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::H_CURL,
+               "TangentTraceIntegrator::AssembleTraceFaceMatrix: Test space should be ND");
+
+   int face_ndof, ndof, dim;
+   int order;
+   dim = test_fe.GetDim();
+   if (dim == 3)
+   {
+      std::string msg =
+         "Trial space should be ND face trace and test space should be a ND vector field in 3D ";
+      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::H_CURL &&
+                  trial_face_fe.GetDim() == 2 && test_fe.GetDim() == 3, msg);
+   }
+   else
+   {
+      std::string msg =
+         "Trial space should be H1 edge trace and test space should be a ND vector field in 2D";
+      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE &&
+                  trial_face_fe.GetDim() == 1 && test_fe.GetDim() == 2, msg);
+   }
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   int dimc = (dim == 3) ? 3 : 1;
+
+   face_shape.SetSize(face_ndof,dimc);
+   shape_n.SetSize(ndof,dimc);
+   shape.SetSize(ndof,dim);
+   normal.SetSize(dim);
+   DenseMatrix face_shape_n(face_ndof,dimc);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      if (dim == 3)
+      {
+         trial_face_fe.CalcVShape(Trans,face_shape);
+      }
+      else
+      {
+         face_shape.GetColumnReference(0,temp);
+         trial_face_fe.CalcPhysShape(Trans,temp);
+      }
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*eltrans, shape);
+
+      // rotate
+      cross_product(normal, shape, shape_n);
+
+      const double w = scale*ip.weight;
+      AddMult_a_ABt(w,shape_n, face_shape, elmat);
+   }
+}
 
 void NormalInterpolator::AssembleElementMatrix2(
    const FiniteElement &dom_fe, const FiniteElement &ran_fe,

fem/bilininteg.hpp

@@ -61,6 +61,8 @@ public:
    virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                            const FiniteElementSpace &test_fes);
 
+   virtual void AssemblePABoundary(const FiniteElementSpace &fes);
+
    virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes);
 
    virtual void AssemblePABoundaryFaces(const FiniteElementSpace &fes);
@@ -159,6 +161,15 @@ public:
                                    FaceElementTransformations &Trans,
                                    DenseMatrix &elmat);
 
+   /** Abstract method used for assembling TraceFaceIntegrators for
+       DPG weak formulations. */
+   virtual void AssembleTraceFaceMatrix(int elem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+
+
    /// @brief Perform the local action of the BilinearFormIntegrator.
    /// Note that the default implementation in the base class is general but not
    /// efficient.
@@ -292,6 +303,12 @@ public:
       bfi->AssemblePA(fes);
    }
 
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes)
+   {
+      bfi->AssemblePA(test_fes, trial_fes); // Reverse test and trial
+   }
+
    virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes)
    {
       bfi->AssemblePAInteriorFaces(fes);
@@ -2183,8 +2200,9 @@ protected:
    // PA extension
    const FiniteElementSpace *fespace;
    Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
+   const DofToQuad *maps;                 ///< Not owned
+   const GeometricFactors *geom;          ///< Not owned
+   const FaceGeometricFactors *face_geom; ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
 
 public:
@@ -2211,6 +2229,8 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
+   virtual void AssemblePABoundary(const FiniteElementSpace &fes);
+
    virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
                            const bool add);
 
@@ -3301,6 +3321,87 @@ public:
                                    DenseMatrix &elmat);
 };
 
+/** Integrator for the DPG form: < v, w > over a face (the interface) where
+    the trial variable v is defined on the interface
+    (H^-1/2 i.e., v:=u⋅n normal trace of H(div))
+    and the test variable w is in an H1-conforming space. */
+class TraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, shape;
+public:
+   TraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
+/** Integrator for the form: < v, w.n > over a face (the interface) where
+    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
+    and the test variable w is in an H(div)-conforming space. */
+class NormalTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, normal, shape_n;
+   DenseMatrix shape;
+
+public:
+   NormalTraceIntegrator() { }
+   virtual void AssembleTraceFaceMatrix(int ielem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+};
+
+
+/** Integrator for the form: < v, w × n > over a face (the interface)
+ *  In 3D the trial variable v is defined on the interface (H^-1/2(curl), trace of H(curl))
+ *  In 2D it's defined on the interface (H^1/2, trace of H1)
+ *  The test variable w is in an H(curl)-conforming space. */
+class TangentTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   DenseMatrix face_shape, shape, shape_n;
+   Vector normal;
+   Vector temp;
+
+   void cross_product(const Vector & x, const DenseMatrix & Y, DenseMatrix & Z)
+   {
+      int dim = x.Size();
+      MFEM_VERIFY(Y.Width() == dim, "Size missmatch");
+      int dimc = dim == 3 ? dim : 1;
+      int h = Y.Height();
+      Z.SetSize(h,dimc);
+      if (dim == 3)
+      {
+         for (int i = 0; i<h; i++)
+         {
+            Z(i,0) = x(2) * Y(i,1) - x(1) * Y(i,2);
+            Z(i,1) = x(0) * Y(i,2) - x(2) * Y(i,0);
+            Z(i,2) = x(1) * Y(i,0) - x(0) * Y(i,1);
+         }
+      }
+      else
+      {
+         for (int i = 0; i<h; i++)
+         {
+            Z(i,0) = x(1) * Y(i,0) - x(0) * Y(i,1);
+         }
+      }
+   }
+
+public:
+   TangentTraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
 /** Abstract class to serve as a base for local interpolators to be used in the
     DiscreteLinearOperator class. */
 class DiscreteInterpolator : public BilinearFormIntegrator { };
@@ -3336,7 +3437,7 @@ public:
 
 private:
    /// 1D finite element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * dofquad_fe;
+   FiniteElement *dofquad_fe;
 
    bool B_id; // is the B basis operator (maps_C_C) the identity?
    const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
@@ -3351,6 +3452,8 @@ private:
 class IdentityInterpolator : public DiscreteInterpolator
 {
 public:
+   IdentityInterpolator(): dofquad_fe(NULL) { }
+
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
                                        ElementTransformation &Trans,
@@ -3365,9 +3468,11 @@ public:
    virtual void AddMultPA(const Vector &x, Vector &y) const;
    virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
 
+   virtual ~IdentityInterpolator() { delete dofquad_fe; }
+
 private:
    /// 1D finite element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * dofquad_fe;
+   FiniteElement *dofquad_fe;
 
    const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
    const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
@@ -3522,17 +3627,5 @@ protected:
    VectorCoefficient *VQ;
 };
 
-
-
-// PA Diffusion Assemble 2D kernel
-template<const int T_SDIM>
-void PADiffusionSetup2D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        const Vector &c,
-                        Vector &d);
-
 }
 #endif

fem/bilininteg_mass_pa.cpp

@@ -1,736 +0,0 @@
-// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "qfunction.hpp"
-#include "ceed/integrators/mass/mass.hpp"
-#include "bilininteg_mass_pa.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Mass Integrator
-
-// PA Mass Assemble kernel
-
-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
-
-   // Assuming the same element type
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T0 = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
-      }
-      return;
-   }
-   int map_type = el.GetMapType();
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
-   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, mt);
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
-   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
-   if (dim==2)
-   {
-      const int NE = ne;
-      const int Q1D = quad1D;
-      const bool const_c = coeff.Size() == 1;
-      const bool by_val = map_type == FiniteElement::VALUE;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
-      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
-      MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               const double detJ = J(qx,qy,e);
-               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
-            }
-         }
-      });
-   }
-   if (dim==3)
-   {
-      const int NE = ne;
-      const int Q1D = quad1D;
-      const bool const_c = coeff.Size() == 1;
-      const bool by_val = map_type == FiniteElement::VALUE;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
-      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,Q1D,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
-      MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qz,z,Q1D)
-               {
-                  const double detJ = J(qx,qy,qz,e);
-                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
-               }
-            }
-         }
-      });
-   }
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal2D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QD[MQ1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassAssembleDiagonal2D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
-      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               // might need absolute values on next line
-               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal3D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QQD[MQ1][MQ1][MD1];
-      double QDD[MQ1][MD1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassAssembleDiagonal3D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
-      MFEM_SHARED double QDD[MQ1][MD1][MD1];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassAssembleDiagonal(const int dim, const int D1D,
-                                   const int Q1D, const int NE,
-                                   const Array<double> &B,
-                                   const Vector &D,
-                                   Vector &Y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
-         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
-         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
-         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
-         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
-         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
-         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
-         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
-         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
-         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
-         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
-         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
-         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
-         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
-         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
-         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
-         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
-         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
-}
-
-
-#ifdef MFEM_USE_OCCA
-// OCCA PA Mass Apply 2D kernel
-static void OccaPAMassApply2D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &D,
-                              const Vector &X,
-                              Vector &Y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
-   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
-   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply2D_cpu;
-      if (OccaMassApply2D_cpu.find(id) == OccaMassApply2D_cpu.end())
-      {
-         const occa::kernel MassApply2D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_CPU", props);
-         OccaMassApply2D_cpu.emplace(id, MassApply2D_CPU);
-      }
-      OccaMassApply2D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply2D_gpu;
-      if (OccaMassApply2D_gpu.find(id) == OccaMassApply2D_gpu.end())
-      {
-         const occa::kernel MassApply2D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_GPU", props);
-         OccaMassApply2D_gpu.emplace(id, MassApply2D_GPU);
-      }
-      OccaMassApply2D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-}
-
-// OCCA PA Mass Apply 3D kernel
-static void OccaPAMassApply3D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &D,
-                              const Vector &X,
-                              Vector &Y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
-   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
-   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply3D_cpu;
-      if (OccaMassApply3D_cpu.find(id) == OccaMassApply3D_cpu.end())
-      {
-         const occa::kernel MassApply3D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_CPU", props);
-         OccaMassApply3D_cpu.emplace(id, MassApply3D_CPU);
-      }
-      OccaMassApply3D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply3D_gpu;
-      if (OccaMassApply3D_gpu.find(id) == OccaMassApply3D_gpu.end())
-      {
-         const occa::kernel MassApply3D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_GPU", props);
-         OccaMassApply3D_gpu.emplace(id, MassApply3D_GPU);
-      }
-      OccaMassApply3D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-}
-#endif // MFEM_USE_OCCA
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassApply2D(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
-   MFEM_FORALL(e, NE,
-   {
-      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassApply2D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(bt_);
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   const auto b = b_.Read();
-   const auto D = d_.Read();
-   const auto x = x_.Read();
-   auto Y = y_.ReadWrite();
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassApply3D(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
-   MFEM_FORALL(e, NE,
-   {
-      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassApply3D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(bt_);
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= M1D, "");
-   MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = b_.Read();
-   auto d = d_.Read();
-   auto x = x_.Read();
-   auto y = y_.ReadWrite();
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
-   });
-}
-
-static void PAMassApply(const int dim,
-                        const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const Array<double> &B,
-                        const Array<double> &Bt,
-                        const Vector &D,
-                        const Vector &X,
-                        Vector &Y)
-{
-#ifdef MFEM_USE_OCCA
-   if (DeviceCanUseOcca())
-   {
-      if (dim == 2)
-      {
-         return OccaPAMassApply2D(D1D,Q1D,NE,B,Bt,D,X,Y);
-      }
-      if (dim == 3)
-      {
-         return OccaPAMassApply3D(D1D,Q1D,NE,B,Bt,D,X,Y);
-      }
-      MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
-   }
-#endif // MFEM_USE_OCCA
-   const int id = (D1D << 4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
-         case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply2D<3,5,16>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
-         case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply2D<4,6,8>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
-         case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
-         case 0x57: return SmemPAMassApply2D<5,7,8>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
-         case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
-         case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
-         case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
-         case 0x99: return SmemPAMassApply2D<9,9,2>(NE,B,Bt,D,X,Y);
-         default:   return PAMassApply2D(NE,B,Bt,D,X,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
-         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
-         case 0x26: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
-         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
-         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
-         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
-         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
-         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
-         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
-         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->AddMult(x, y);
-   }
-   else
-   {
-      PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
-   }
-}
-
-void MassIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
-{
-   // Mass integrator is symmetric
-   AddMultPA(x, y);
-}
-
-} // namespace mfem

fem/ceed/interface/coefficient.hpp

@@ -288,7 +288,7 @@ void InitCoefficientWithIndices(mfem::Coefficient *Q, mfem::Mesh &mesh,
       auto in = Reshape(qFun.Read(), nq, ne);
       auto d_indices = Read(m_indices, nelem);
       auto out = Reshape(ceedCoeff->coeff.Write(), nq, nelem);
-      MFEM_FORALL(i, nelem * nq,
+      mfem::forall(nelem * nq, [=] MFEM_HOST_DEVICE (int i)
       {
          const int q = i%nq;
          const int sub_e = i/nq;
@@ -378,7 +378,7 @@ void InitCoefficientWithIndices(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
       auto in = Reshape(qFun.Read(), dim, nq, ne);
       auto d_indices = Read(m_indices, nelem);
       auto out = Reshape(ceedCoeff->coeff.Write(), dim, nq, nelem);
-      MFEM_FORALL(i, nelem * nq,
+      mfem::forall(nelem * nq, [=] MFEM_HOST_DEVICE (int i)
       {
          const int q = i%nq;
          const int sub_e = i/nq;

fem/ceed/interface/util.hpp

@@ -13,13 +13,13 @@
 #define MFEM_LIBCEED_UTIL
 
 #include "../../../config/config.hpp"
+#include <functional>
+#include <string>
 #include <tuple>
 #include <unordered_map>
-#include <string>
 
 #include "ceed.hpp"
 #ifdef MFEM_USE_CEED
-#include <ceed/hash.h>
 #include <ceed/backend.h>  // for CeedOperatorField
 #endif
 
@@ -105,6 +105,21 @@ const IntegrationRule & GetRule(
 /// Return the path to the libCEED q-function headers.
 const std::string &GetCeedPath();
 
+/// Wrapper for std::hash.
+template <typename T>
+inline std::size_t CeedHash(const T key)
+{
+   return std::hash<T> {}(key);
+}
+
+/// Effective way to combine hashes (from libCEED).
+inline std::size_t CeedHashCombine(std::size_t seed, std::size_t hash)
+{
+   // See https://doi.org/10.1002/asi.10170, or
+   //     https://dl.acm.org/citation.cfm?id=759509.
+   return seed ^ (hash + (seed << 6) + (seed >> 2));
+}
+
 // Hash table for CeedBasis
 using BasisKey = std::tuple<const mfem::FiniteElementSpace*,
       const mfem::IntegrationRule*,
@@ -115,12 +130,12 @@ struct BasisHash
    {
       return CeedHashCombine(
                 CeedHashCombine(
-                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
+                   CeedHash(std::get<0>(k)),
+                   CeedHash(std::get<1>(k))),
                 CeedHashCombine(
-                   CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                   CeedHashInt(std::get<3>(k))),
-                   CeedHashInt(std::get<4>(k))));
+                   CeedHashCombine(CeedHash(std::get<2>(k)),
+                                   CeedHash(std::get<3>(k))),
+                   CeedHash(std::get<4>(k))));
    }
 };
 using BasisMap = std::unordered_map<const BasisKey, CeedBasis, BasisHash>;
@@ -137,11 +152,11 @@ struct RestrHash
       return CeedHashCombine(
                 CeedHashCombine(
                    CeedHashCombine(
-                      CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                      CeedHashInt(std::get<1>(k))),
-                   CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                   CeedHashInt(std::get<3>(k)))),
-                CeedHashInt(std::get<4>(k)));
+                      CeedHash(std::get<0>(k)),
+                      CeedHash(std::get<1>(k))),
+                   CeedHashCombine(CeedHash(std::get<2>(k)),
+                                   CeedHash(std::get<3>(k)))),
+                CeedHash(std::get<4>(k)));
    }
 };
 using RestrMap =

fem/ceed/solvers/algebraic.cpp

@@ -519,7 +519,7 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
    ierr = CeedVectorGetArray(a, mem, &a_data); CeedChk(ierr);
    ierr = CeedVectorGetArrayRead(b, mem, &b_data); CeedChk(ierr);
    MFEM_VERIFY(int(length) == length, "length overflow");
-   MFEM_FORALL(i, length,
+   mfem::forall(length, [=] MFEM_HOST_DEVICE (int i)
    {a_data[i] *= b_data[i];});
 
    ierr = CeedVectorRestoreArray(a, &a_data); CeedChk(ierr);
@@ -593,7 +593,7 @@ void AlgebraicInterpolation::MultTranspose(const mfem::Vector& x,
                                  &multiplicitydata); PCeedChk(ierr);
    ierr = CeedVectorGetArrayWrite(fine_work, mem, &workdata); PCeedChk(ierr);
    MFEM_VERIFY((int)length == length, "length overflow");
-   MFEM_FORALL(i, length,
+   mfem::forall(length, [=] MFEM_HOST_DEVICE (int i)
    {workdata[i] = in_ptr[i] * multiplicitydata[i];});
    ierr = CeedVectorRestoreArrayRead(fine_multiplicity_r,
                                      &multiplicitydata);

fem/coefficient.cpp

@@ -144,11 +144,54 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
    }
 }
 
+double CartesianCoefficient::Eval(ElementTransformation & T,
+                                  const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return transip[comp];
+}
+
+double CylindricalRadialCoefficient::Eval(ElementTransformation & T,
+                                          const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return sqrt(transip[0] * transip[0] + transip[1] * transip[1]);
+}
+
+double CylindricalAzimuthalCoefficient::Eval(ElementTransformation & T,
+                                             const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return atan2(transip[1], transip[0]);
+}
+
+double SphericalRadialCoefficient::Eval(ElementTransformation & T,
+                                        const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return sqrt(transip * transip);
+}
+
+double SphericalAzimuthalCoefficient::Eval(ElementTransformation & T,
+                                           const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return atan2(transip[1], transip[0]);
+}
+
+double SphericalPolarCoefficient::Eval(ElementTransformation & T,
+                                       const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return atan2(sqrt(transip[0] * transip[0] + transip[1] * transip[1]),
+                transip[2]);
+}
+
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridF->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       return GridF->GetValue(T, ip, Component);
    }
@@ -313,6 +356,13 @@ void PWVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
    V = 0.0;
 }
 
+void PositionVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
+                                     const IntegrationPoint &ip)
+{
+   V.SetSize(vdim);
+   T.Transform(ip, V);
+}
+
 void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
@@ -396,7 +446,7 @@ void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       GridFunc->GetVectorValue(T, ip, V);
    }
@@ -444,7 +494,7 @@ void GradientGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                            const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       GridFunc->GetGradient(T, V);
    }
@@ -485,7 +535,7 @@ void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       GridFunc->GetCurl(T, V);
    }
@@ -507,7 +557,7 @@ double DivergenceGridFunctionCoefficient::Eval(ElementTransformation &T,
                                                const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       return GridFunc->GetDivergence(T);
    }

fem/coefficient.hpp

@@ -258,6 +258,124 @@ public:
                        const IntegrationPoint &ip);
 };
 
+/// A common base class for returning individual components of the domain's
+/// Cartesian coordinates.
+class CartesianCoefficient : public Coefficient
+{
+protected:
+   int comp;
+   mutable Vector transip;
+
+   /// @a comp_ index of the desired component (0 -> x, 1 -> y, 2 -> z)
+   CartesianCoefficient(int comp_) : comp(comp_), transip(3) {}
+
+public:
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the x-component of the evaluation point
+class CartesianXCoefficient : public CartesianCoefficient
+{
+public:
+   CartesianXCoefficient() : CartesianCoefficient(0) {}
+};
+
+/// Scalar coefficient which returns the y-component of the evaluation point
+class CartesianYCoefficient : public CartesianCoefficient
+{
+public:
+   CartesianYCoefficient() : CartesianCoefficient(1) {}
+};
+
+/// Scalar coefficient which returns the z-component of the evaluation point
+class CartesianZCoefficient : public CartesianCoefficient
+{
+public:
+   CartesianZCoefficient() : CartesianCoefficient(2) {}
+};
+
+/// Scalar coefficient which returns the radial distance from the axis of
+/// the evaluation point in the cylindrical coordinate system
+class CylindricalRadialCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   CylindricalRadialCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the angular position or azimuth (often
+/// denoted by theta) of the evaluation point in the cylindrical coordinate
+/// system
+class CylindricalAzimuthalCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   CylindricalAzimuthalCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the height or altitude of
+/// the evaluation point in the cylindrical coordinate system
+typedef CartesianZCoefficient CylindricalZCoefficient;
+
+/// Scalar coefficient which returns the radial distance from the origin of
+/// the evaluation point in the spherical coordinate system
+class SphericalRadialCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   SphericalRadialCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the azimuthal angle (often denoted by phi)
+/// of the evaluation point in the spherical coordinate system
+class SphericalAzimuthalCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   SphericalAzimuthalCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the polar angle (often denoted by theta)
+/// of the evaluation point in the spherical coordinate system
+class SphericalPolarCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   SphericalPolarCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
 class GridFunction;
 
 /// Coefficient defined by a GridFunction. This coefficient is mesh dependent.
@@ -600,6 +718,22 @@ public:
    using VectorCoefficient::Eval;
 };
 
+/// A vector coefficient which returns the physical location of the
+/// evaluation point in the Cartesian coordinate system.
+class PositionVectorCoefficient : public VectorCoefficient
+{
+public:
+
+   PositionVectorCoefficient(int dim) : VectorCoefficient(dim) {}
+
+   using VectorCoefficient::Eval;
+   /// Evaluate the vector coefficient at @a ip.
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+
+   virtual ~PositionVectorCoefficient() { }
+};
+
 /// A general vector function coefficient
 class VectorFunctionCoefficient : public VectorCoefficient
 {

fem/complex_fem.cpp

@@ -497,7 +497,7 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
          auto d_X_r = X_r.Read();
          auto d_X_i = X_i.Read();
          auto d_idx = ess_tdof_list.Read();
-         MFEM_FORALL(i, n,
+         mfem::forall(n, [=] MFEM_HOST_DEVICE (int i)
          {
             const int j = d_idx[i];
             d_B_r[j] = d_X_r[j];
@@ -1230,7 +1230,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       auto d_X_r = X_r.Read();
       auto d_X_i = X_i.Read();
       auto d_idx = ess_tdof_list.Read();
-      MFEM_FORALL(i, n,
+      mfem::forall(n, [=] MFEM_HOST_DEVICE (int i)
       {
          const int j = d_idx[i];
          d_B_r[j] = d_X_r[j];

fem/dgmassinv.cpp

@@ -107,7 +107,7 @@ void DGMassInverse::Update()
 {
    M->Assemble();
    M->AssembleDiagonal(diag_inv);
-   internal::MakeReciprocal(diag_inv.Size(), diag_inv.ReadWrite());
+   diag_inv.Reciprocal();
 }
 
 DGMassInverse::~DGMassInverse()
@@ -168,7 +168,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
 
    constexpr int NB = Q1D ? Q1D : 1; // block size
 
-   MFEM_FORALL_2D(e, NE, NB, NB, 1,
+   mfem::forall_2D(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
    {
       constexpr int NB = Q1D ? Q1D : 1; // redefine here for some compilers
 

fem/dgmassinv.hpp

@@ -87,6 +87,8 @@ public:
    ///
    /// If @ref iterative_mode is @a true, @a u is used as an initial guess.
    void Mult(const Vector &b, Vector &u) const;
+   /// Same as Mult() since the mass matrix is symmetric.
+   void MultTranspose(const Vector &b, Vector &u) const { Mult(b, u); }
    /// Not implemented. Aborts.
    void SetOperator(const Operator &op);
    /// Set the relative tolerance.
@@ -101,8 +103,8 @@ public:
    ~DGMassInverse();
 
    /// @brief Solve the system M b = u. <b>Not part of the public interface.</b>
-   /// @note This member function must be public because it contains an
-   /// MFEM_FORALL kernel (nvcc limitation)
+   /// @note This member function must be public because it defines an
+   /// extended lambda used in an mfem::forall kernel (nvcc limitation)
    template<int DIM, int D1D = 0, int Q1D = 0>
    void DGMassCGIteration(const Vector &b_, Vector &u_) const;
 };

fem/dgmassinv_kernels.hpp

@@ -12,9 +12,9 @@
 #ifndef MFEM_DGMASSINV_KERNELS_HPP
 #define MFEM_DGMASSINV_KERNELS_HPP
 
-#include "bilininteg_mass_pa.hpp"
 #include "../linalg/kernels.hpp"
 #include "kernels.hpp"
+#include "integ/bilininteg_mass_kernels.hpp"
 
 namespace mfem
 {
@@ -22,11 +22,6 @@ namespace mfem
 namespace internal
 {
 
-void MakeReciprocal(int n, double *x)
-{
-   MFEM_FORALL(i, n, x[i] = 1.0/x[i]; );
-}
-
 template <int DIM, int D1D, int Q1D>
 MFEM_HOST_DEVICE inline
 void DGMassApply(const int e,

fem/doftrans.cpp

@@ -14,54 +14,6 @@
 namespace mfem
 {
 
-void DofTransformation::TransformPrimal(Vector &v) const
-{
-   TransformPrimal(v.GetData());
-}
-
-void DofTransformation::TransformPrimalCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformPrimal(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::TransformDual(Vector &v) const
-{
-   TransformDual(v.GetData());
-}
-
-void DofTransformation::TransformDual(DenseMatrix &V) const
-{
-   TransformDualCols(V);
-   TransformDualRows(V);
-}
-
-void DofTransformation::TransformDualRows(DenseMatrix &V) const
-{
-   Vector row;
-   for (int r=0; r<V.Height(); r++)
-   {
-      V.GetRow(r, row);
-      TransformDual(row);
-      V.SetRow(r, row);
-   }
-}
-
-void DofTransformation::TransformDualCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformDual(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::InvTransformPrimal(Vector &v) const
-{
-   InvTransformPrimal(v.GetData());
-}
-
 void TransformPrimal(const DofTransformation *ran_dof_trans,
                      const DofTransformation *dom_dof_trans,
                      DenseMatrix &elmat)
@@ -85,11 +37,6 @@ void TransformPrimal(const DofTransformation *ran_dof_trans,
    }
 }
 
-void DofTransformation::InvTransformDual(Vector &v) const
-{
-   InvTransformDual(v.GetData());
-}
-
 void TransformDual(const DofTransformation *ran_dof_trans,
                    const DofTransformation *dom_dof_trans,
                    DenseMatrix &elmat)
@@ -113,15 +60,16 @@ void TransformDual(const DofTransformation *ran_dof_trans,
    }
 }
 
-void VDofTransformation::TransformPrimal(double *v) const
+void StatelessVDofTransformation::TransformPrimal(const Array<int> & face_ori,
+                                                  double *v) const
 {
-   int size = doftrans_->Size();
+   int size = sdoftrans_->Size();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->TransformPrimal(&v[i*size]);
+         sdoftrans_->TransformPrimal(face_ori, &v[i*size]);
       }
    }
    else
@@ -133,7 +81,7 @@ void VDofTransformation::TransformPrimal(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->TransformPrimal(vec);
+         sdoftrans_->TransformPrimal(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -142,15 +90,17 @@ void VDofTransformation::TransformPrimal(double *v) const
    }
 }
 
-void VDofTransformation::InvTransformPrimal(double *v) const
+void StatelessVDofTransformation::InvTransformPrimal(
+   const Array<int> & face_ori,
+   double *v) const
 {
-   int size = doftrans_->Height();
+   int size = sdoftrans_->Height();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->InvTransformPrimal(&v[i*size]);
+         sdoftrans_->InvTransformPrimal(face_ori, &v[i*size]);
       }
    }
    else
@@ -162,7 +112,7 @@ void VDofTransformation::InvTransformPrimal(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->InvTransformPrimal(vec);
+         sdoftrans_->InvTransformPrimal(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -171,15 +121,16 @@ void VDofTransformation::InvTransformPrimal(double *v) const
    }
 }
 
-void VDofTransformation::TransformDual(double *v) const
+void StatelessVDofTransformation::TransformDual(const Array<int> & face_ori,
+                                                double *v) const
 {
-   int size = doftrans_->Size();
+   int size = sdoftrans_->Size();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->TransformDual(&v[i*size]);
+         sdoftrans_->TransformDual(face_ori, &v[i*size]);
       }
    }
    else
@@ -191,7 +142,7 @@ void VDofTransformation::TransformDual(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->TransformDual(vec);
+         sdoftrans_->TransformDual(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -200,15 +151,16 @@ void VDofTransformation::TransformDual(double *v) const
    }
 }
 
-void VDofTransformation::InvTransformDual(double *v) const
+void StatelessVDofTransformation::InvTransformDual(const Array<int> & face_ori,
+                                                   double *v) const
 {
-   int size = doftrans_->Size();
+   int size = sdoftrans_->Size();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->InvTransformDual(&v[i*size]);
+         sdoftrans_->InvTransformDual(face_ori, &v[i*size]);
       }
    }
    else
@@ -220,7 +172,7 @@ void VDofTransformation::InvTransformDual(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->InvTransformDual(vec);
+         sdoftrans_->InvTransformDual(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -229,7 +181,8 @@ void VDofTransformation::InvTransformDual(double *v) const
    }
 }
 
-const double ND_DofTransformation::T_data[24] =
+// ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
+const double ND_StatelessDofTransformation::T_data[24] =
 {
    1.0,  0.0,  0.0,  1.0,
    -1.0, -1.0,  0.0,  1.0,
@@ -239,10 +192,11 @@ const double ND_DofTransformation::T_data[24] =
    0.0,  1.0,  1.0,  0.0
 };
 
-const DenseTensor ND_DofTransformation
-::T(const_cast<double*>(ND_DofTransformation::T_data), 2, 2, 6);
+const DenseTensor ND_StatelessDofTransformation
+::T(const_cast<double*>(ND_StatelessDofTransformation::T_data), 2, 2, 6);
 
-const double ND_DofTransformation::TInv_data[24] =
+// ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
+const double ND_StatelessDofTransformation::TInv_data[24] =
 {
    1.0,  0.0,  0.0,  1.0,
    -1.0, -1.0,  0.0,  1.0,
@@ -252,301 +206,113 @@ const double ND_DofTransformation::TInv_data[24] =
    0.0,  1.0,  1.0,  0.0
 };
 
-const DenseTensor ND_DofTransformation
+const DenseTensor ND_StatelessDofTransformation
 ::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
 
-ND_DofTransformation::ND_DofTransformation(int size, int p)
-   : DofTransformation(size)
+ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
+                                                             int num_edges,
+                                                             int num_tri_faces)
+   : StatelessDofTransformation(size)
    , order(p)
    , nedofs(p)
    , nfdofs(p*(p-1))
+   , nedges(num_edges)
+   , nfaces(num_tri_faces)
 {
 }
 
-ND_TriDofTransformation::ND_TriDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2), p)
-{
-}
-
-void ND_TriDofTransformation::TransformPrimal(double *v) const
+void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
+                                                    double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }
 
-void
-ND_TriDofTransformation::InvTransformPrimal(double *v) const
+void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
+                                                       double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }
 
-void
-ND_TriDofTransformation::TransformDual(double *v) const
+void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
+                                                  double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }
 
-void
-ND_TriDofTransformation::InvTransformDual(double *v) const
+void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
+                                                     double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-ND_TetDofTransformation::ND_TetDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
-{
-}
-
-void ND_TetDofTransformation::TransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::InvTransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::TransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::InvTransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-ND_WedgeDofTransformation::ND_WedgeDofTransformation(int p)
-   : ND_DofTransformation(3 * p * ((p + 1) * (p + 2))/2, p)
-{
-}
-
-void ND_WedgeDofTransformation::TransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[9*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_WedgeDofTransformation::InvTransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[9*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_WedgeDofTransformation::TransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[9*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_WedgeDofTransformation::InvTransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[9*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }

fem/doftrans.hpp

@@ -15,19 +15,31 @@
 #include "../config/config.hpp"
 #include "../linalg/linalg.hpp"
 #include "intrules.hpp"
-#include "fe.hpp"
 
 namespace mfem
 {
 
-/** The DofTransformation class is an abstract base class for a family of
-    transformations that map local degrees of freedom (DoFs), contained within
-    individual elements, to global degrees of freedom, stored within
-    GridFunction objects. These transformations are necessary to ensure that
-    basis functions in neighboring elements align correctly. Closely related but
+/** The StatelessDofTransformation class is an abstract base class for a family
+    of transformations that map local degrees of freedom (DoFs), contained
+    within individual elements, to global degrees of freedom, stored within
+    GridFunction objects.
+
+    In this context "stateless" means that the concrete classes derived from
+    StatelessDofTransformation do not store information about the relative
+    orientations of the faces with respect to their neighboring elements. In
+    other words there is no information specific to a particular element (aside
+    from the element type e.g. tetrahedron, wedge, or pyramid). The
+    StatelessDofTransformation provides access to the transformation operators
+    for specific relative face orientations. These are useful, for example, when
+    relating DoFs associated with distinct overlapping meshes such as parent and
+    sub-meshes.
+
+    These transformations are necessary to ensure that basis functions in
+    neighboring (or overlapping) elements align correctly. Closely related but
     complementary transformations are required for the entries stored in
-    LinearForm and BilinearForm objects. The DofTransformation class is designed
-    to apply the action of both of these types of DoF transformations.
+    LinearForm and BilinearForm objects. The StatelessDofTransformation class
+    is designed to apply the action of both of these types of DoF
+    transformations.
 
     Let the "primal transformation" be given by the operator T. This means that
     given a local element vector v the data that must be placed into a
@@ -53,24 +65,84 @@ namespace mfem
     D_t = T * D * T^{-1}. This can be accomplished by using a primal
     transformation on the columns of D and a dual transformation on its rows.
 */
-class DofTransformation
+class StatelessDofTransformation
 {
 protected:
    int size_;
 
-   Array<int> Fo;
-
-   DofTransformation(int size)
+   StatelessDofTransformation(int size)
       : size_(size) {}
 
 public:
-
    inline int Size() const { return size_; }
    inline int Height() const { return size_; }
    inline int NumRows() const { return size_; }
    inline int Width() const { return size_; }
    inline int NumCols() const { return size_; }
 
+   /** Transform local DoFs to align with the global DoFs. For example, this
+       transformation can be used to map the local vector computed by
+       FiniteElement::Project() to the transformed vector stored within a
+       GridFunction object. */
+   virtual void TransformPrimal(const Array<int> & face_orientation,
+                                double *v) const = 0;
+   inline void TransformPrimal(const Array<int> & face_orientation,
+                               Vector &v) const
+   { TransformPrimal(face_orientation, v.GetData()); }
+
+   /** Inverse transform local DoFs. Used to transform DoFs from a global vector
+       back to their element-local form. For example, this must be used to
+       transform the vector obtained using GridFunction::GetSubVector before it
+       can be used to compute a local interpolation.
+   */
+   virtual void InvTransformPrimal(const Array<int> & face_orientation,
+                                   double *v) const = 0;
+   inline void InvTransformPrimal(const Array<int> & face_orientation,
+                                  Vector &v) const
+   { InvTransformPrimal(face_orientation, v.GetData()); }
+
+   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
+       into a LinearForm object. */
+   virtual void TransformDual(const Array<int> & face_orientation,
+                              double *v) const = 0;
+   inline void TransformDual(const Array<int> & face_orientation,
+                             Vector &v) const
+   { TransformDual(face_orientation, v.GetData()); }
+
+   /** Inverse Transform dual DoFs */
+   virtual void InvTransformDual(const Array<int> & face_orientation,
+                                 double *v) const = 0;
+   inline void InvTransformDual(const Array<int> & face_orientation,
+                                Vector &v) const
+   { InvTransformDual(face_orientation, v.GetData()); }
+};
+
+/** The DofTransformation class is an extension of the
+    StatelessDofTransformation which stores the face orientations used to
+    select the necessary transformations which allows it to offer a collection
+    of convenience methods.
+
+    DofTransformation objects are provided by the FiniteElementSpace which has
+    access to the mesh and can therefore provide the face orientations. This is
+    convenient when working with GridFunction, LinearForm, or BilinearForm
+    obejcts or their parallel counterparts.
+
+    StatelessDofTransformation objects are provided by FiniteElement or
+    FiniteElementCollection objects which do not have access to face
+    orientation information. This can be useful in non-standard contexts such as
+    transferring finite element degrees of freedom between different meshes.
+    For examples of its use see the TransferMap used by the SubMesh class.
+   */
+class DofTransformation : virtual public StatelessDofTransformation
+{
+protected:
+   Array<int> Fo;
+
+   DofTransformation(int size)
+      : StatelessDofTransformation(size) {}
+
+public:
+
    /** @brief Configure the transformation using face orientations for the
        current element. */
    /// The face_orientation array can be obtained from Mesh::GetElementFaces.
@@ -79,42 +151,82 @@ public:
 
    inline const Array<int> & GetFaceOrientations() const { return Fo; }
 
+   using StatelessDofTransformation::TransformPrimal;
+   using StatelessDofTransformation::InvTransformPrimal;
+   using StatelessDofTransformation::TransformDual;
+   using StatelessDofTransformation::InvTransformDual;
+
    /** Transform local DoFs to align with the global DoFs. For example, this
        transformation can be used to map the local vector computed by
        FiniteElement::Project() to the transformed vector stored within a
        GridFunction object. */
-   virtual void TransformPrimal(double *v) const = 0;
-   virtual void TransformPrimal(Vector &v) const;
+   inline void TransformPrimal(double *v) const
+   { TransformPrimal(Fo, v); }
+   inline void TransformPrimal(Vector &v) const
+   { TransformPrimal(v.GetData()); }
 
    /// Transform groups of DoFs stored as dense matrices
-   virtual void TransformPrimalCols(DenseMatrix &V) const;
+   inline void TransformPrimalCols(DenseMatrix &V) const
+   {
+      for (int c=0; c<V.Width(); c++)
+      {
+         TransformPrimal(V.GetColumn(c));
+      }
+   }
 
    /** Inverse transform local DoFs. Used to transform DoFs from a global vector
        back to their element-local form. For example, this must be used to
        transform the vector obtained using GridFunction::GetSubVector before it
        can be used to compute a local interpolation.
    */
-   virtual void InvTransformPrimal(double *v) const = 0;
-   virtual void InvTransformPrimal(Vector &v) const;
+   inline void InvTransformPrimal(double *v) const
+   { InvTransformPrimal(Fo, v); }
+   inline void InvTransformPrimal(Vector &v) const
+   { InvTransformPrimal(v.GetData()); }
 
    /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
        into a LinearForm object. */
-   virtual void TransformDual(double *v) const = 0;
-   virtual void TransformDual(Vector &v) const;
+   inline void TransformDual(double *v) const
+   { TransformDual(Fo, v); }
+   inline void TransformDual(Vector &v) const
+   { TransformDual(v.GetData()); }
 
    /** Inverse Transform dual DoFs */
-   virtual void InvTransformDual(double *v) const = 0;
-   virtual void InvTransformDual(Vector &v) const;
+   inline void InvTransformDual(double *v) const
+   { InvTransformDual(Fo, v); }
+   inline void InvTransformDual(Vector &v) const
+   { InvTransformDual(v.GetData()); }
 
    /** Transform a matrix of dual DoFs entries as computed by a
        BilinearFormIntegrator before summing into a BilinearForm object. */
-   virtual void TransformDual(DenseMatrix &V) const;
+   inline void TransformDual(DenseMatrix &V) const
+   {
+      TransformDualCols(V);
+      TransformDualRows(V);
+   }
 
-   /// Transform groups of dual DoFs stored as dense matrices
-   virtual void TransformDualRows(DenseMatrix &V) const;
-   virtual void TransformDualCols(DenseMatrix &V) const;
+   /// Transform rows of a dense matrix containing dual DoFs
+   inline void TransformDualRows(DenseMatrix &V) const
+   {
+      Vector row;
+      for (int r=0; r<V.Height(); r++)
+      {
+         V.GetRow(r, row);
+         TransformDual(row);
+         V.SetRow(r, row);
+      }
+   }
 
-   virtual ~DofTransformation() {}
+   /// Transform columns of a dense matrix containing dual DoFs
+   inline void TransformDualCols(DenseMatrix &V) const
+   {
+      for (int c=0; c<V.Width(); c++)
+      {
+         TransformDual(V.GetColumn(c));
+      }
+   }
+
+   virtual ~DofTransformation() = default;
 };
 
 /** Transform a matrix of DoFs entries from different finite element spaces as
@@ -133,66 +245,143 @@ void TransformDual(const DofTransformation *ran_dof_trans,
                    const DofTransformation *dom_dof_trans,
                    DenseMatrix &elmat);
 
-/** The VDofTransformation class implements a nested transformation where an
-    arbitrary DofTransformation is replicated with a vdim >= 1.
+/** The StatelessVDofTransformation class implements a nested transformation
+    where an arbitrary StatelessDofTransformation is replicated with a
+    vdim >= 1.
 */
-class VDofTransformation : public DofTransformation
+class StatelessVDofTransformation : virtual public StatelessDofTransformation
 {
-private:
+protected:
    int vdim_;
    int ordering_;
-   DofTransformation * doftrans_;
+   StatelessDofTransformation * sdoftrans_;
 
 public:
    /** @brief Default constructor which requires that SetDofTransformation be
        called before use. */
-   VDofTransformation(int vdim = 1, int ordering = 0)
-      : DofTransformation(0),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(NULL) {}
+   StatelessVDofTransformation(int vdim = 1, int ordering = 0)
+      : StatelessDofTransformation(0)
+      , vdim_(vdim)
+      , ordering_(ordering)
+      , sdoftrans_(NULL)
+   {}
 
-   /// Constructor with a known DofTransformation
-   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
-                      int ordering = 0)
-      : DofTransformation(vdim * doftrans.Size()),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(&doftrans) {}
+   /// Constructor with a known StatelessDofTransformation
+   StatelessVDofTransformation(StatelessDofTransformation & doftrans,
+                               int vdim = 1,
+                               int ordering = 0)
+      : StatelessDofTransformation(vdim * doftrans.Size())
+      , vdim_(vdim)
+      , ordering_(ordering)
+      , sdoftrans_(&doftrans)
+   {}
 
    /// Set or change the vdim parameter
    inline void SetVDim(int vdim)
    {
       vdim_ = vdim;
-      if (doftrans_)
+      if (sdoftrans_)
       {
-         size_ = vdim_ * doftrans_->Size();
+         size_ = vdim_ * sdoftrans_->Size();
       }
    }
 
    /// Return the current vdim value
    inline int GetVDim() const { return vdim_; }
 
-   /// Set or change the nested DofTransformation object
-   inline void SetDofTransformation(DofTransformation & doftrans)
+   /// Set or change the nested StatelessDofTransformation object
+   inline void SetDofTransformation(StatelessDofTransformation & doftrans)
    {
       size_ = vdim_ * doftrans.Size();
+      sdoftrans_ = &doftrans;
+   }
+
+   /// Return the nested StatelessDofTransformation object
+   inline StatelessDofTransformation * GetDofTransformation() const
+   { return sdoftrans_; }
+
+   using StatelessDofTransformation::TransformPrimal;
+   using StatelessDofTransformation::InvTransformPrimal;
+   using StatelessDofTransformation::TransformDual;
+   using StatelessDofTransformation::InvTransformDual;
+
+   /** Specializations of these base class methods which account for the vdim
+       and ordering of the full set of DoFs.
+   */
+   void TransformPrimal(const Array<int> & face_ori, double *v) const;
+   void InvTransformPrimal(const Array<int> & face_ori, double *v) const;
+   void TransformDual(const Array<int> & face_ori, double *v) const;
+   void InvTransformDual(const Array<int> & face_ori, double *v) const;
+};
+
+/** The VDofTransformation class implements a nested transformation where an
+    arbitrary DofTransformation is replicated with a vdim >= 1.
+*/
+class VDofTransformation : public StatelessVDofTransformation,
+   public DofTransformation
+{
+protected:
+   DofTransformation * doftrans_;
+
+public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   VDofTransformation(int vdim = 1, int ordering = 0)
+      : StatelessDofTransformation(0)
+      , StatelessVDofTransformation(vdim, ordering)
+      , DofTransformation(0)
+      , doftrans_(NULL)
+   {}
+
+   /// Constructor with a known DofTransformation
+   /// @note The face orientations in @a doftrans will be copied into the
+   /// new VDofTransformation object.
+   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
+                      int ordering = 0)
+      : StatelessDofTransformation(vdim * doftrans.Size())
+      , StatelessVDofTransformation(doftrans, vdim, ordering)
+      , DofTransformation(vdim * doftrans.Size())
+      , doftrans_(&doftrans)
+   {
+      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
+   }
+
+   using StatelessVDofTransformation::SetDofTransformation;
+
+   /// Set or change the nested DofTransformation object
+   /// @note The face orientations in @a doftrans will be copied into the
+   /// VDofTransformation object.
+   void SetDofTransformation(DofTransformation & doftrans)
+   {
       doftrans_ = &doftrans;
+      StatelessVDofTransformation::SetDofTransformation(doftrans);
+      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
    }
 
    /// Return the nested DofTransformation object
    inline DofTransformation * GetDofTransformation() const { return doftrans_; }
 
-   inline void SetFaceOrientation(const Array<int> & face_orientation)
-   { Fo = face_orientation; doftrans_->SetFaceOrientations(face_orientation); }
+   /// Set new face orientations in both the VDofTransformation and the
+   /// DofTransformation contained within (if there is one).
+   inline void SetFaceOrientations(const Array<int> & face_orientation)
+   {
+      DofTransformation::SetFaceOrientations(face_orientation);
+      if (doftrans_) { doftrans_->SetFaceOrientations(face_orientation); }
+   }
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
    using DofTransformation::InvTransformDual;
 
-   void TransformPrimal(double *v) const;
-   void InvTransformPrimal(double *v) const;
-   void TransformDual(double *v) const;
-   void InvTransformDual(double *v) const;
+   inline void TransformPrimal(double *v) const
+   { TransformPrimal(Fo, v); }
+   inline void InvTransformPrimal(double *v) const
+   { InvTransformPrimal(Fo, v); }
+   inline void TransformDual(double *v) const
+   { TransformDual(Fo, v); }
+   inline void InvTransformDual(double *v) const
+   { InvTransformDual(Fo, v); }
 };
 
 /** Abstract base class for high-order Nedelec spaces on elements with
@@ -207,17 +396,22 @@ public:
     be accessed as DenseMatrices using the GetFaceTransform() and
     GetFaceInverseTransform() methods.
 */
-class ND_DofTransformation : public DofTransformation
+class ND_StatelessDofTransformation : virtual public StatelessDofTransformation
 {
-protected:
+private:
    static const double T_data[24];
    static const double TInv_data[24];
    static const DenseTensor T, TInv;
-   int order;
-   int nedofs; // number of DoFs per edge
-   int nfdofs; // number of DoFs per face
 
-   ND_DofTransformation(int size, int order);
+protected:
+   const int order;  // basis function order
+   const int nedofs; // number of DoFs per edge
+   const int nfdofs; // number of DoFs per face
+   const int nedges; // number of edges per element
+   const int nfaces; // number of triangular faces per element
+
+   ND_StatelessDofTransformation(int size, int order,
+                                 int num_edges, int num_tri_faces);
 
 public:
    // Return the 2x2 transformation operator for the given face orientation
@@ -226,67 +420,117 @@ public:
    // Return the 2x2 inverse transformation operator
    static const DenseMatrix & GetFaceInverseTransform(int ori)
    { return TInv(ori); }
+
+   void TransformPrimal(const Array<int> & face_orientation,
+                        double *v) const;
+
+   void InvTransformPrimal(const Array<int> & face_orientation,
+                           double *v) const;
+
+   void TransformDual(const Array<int> & face_orientation,
+                      double *v) const;
+
+   void InvTransformDual(const Array<int> & face_orientation,
+                         double *v) const;
+};
+
+/// Stateless DoF transformation implementation for the Nedelec basis on
+/// triangles
+class ND_TriStatelessDofTransformation : public ND_StatelessDofTransformation
+{
+public:
+   ND_TriStatelessDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2))
+      , ND_StatelessDofTransformation(order*(order + 2), order, 3, 1)
+   {}
 };
 
 /// DoF transformation implementation for the Nedelec basis on triangles
-class ND_TriDofTransformation : public ND_DofTransformation
+class ND_TriDofTransformation : public DofTransformation,
+   public ND_TriStatelessDofTransformation
 {
 public:
-   ND_TriDofTransformation(int order);
+   ND_TriDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2))
+      , DofTransformation(order*(order + 2))
+      , ND_TriStatelessDofTransformation(order)
+   {}
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-
-   void InvTransformDual(double *v) const;
    using DofTransformation::InvTransformDual;
+
+   using ND_TriStatelessDofTransformation::TransformPrimal;
+   using ND_TriStatelessDofTransformation::InvTransformPrimal;
+   using ND_TriStatelessDofTransformation::TransformDual;
+   using ND_TriStatelessDofTransformation::InvTransformDual;
 };
 
 /// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public ND_DofTransformation
+class ND_TetStatelessDofTransformation : public ND_StatelessDofTransformation
 {
 public:
-   ND_TetDofTransformation(int order);
+   ND_TetStatelessDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
+      , ND_StatelessDofTransformation(order*(order + 2)*(order + 3)/2, order,
+                                      6, 4)
+   {}
+};
+
+/// DoF transformation implementation for the Nedelec basis on tetrahedra
+class ND_TetDofTransformation : public DofTransformation,
+   public ND_TetStatelessDofTransformation
+{
+public:
+   ND_TetDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
+      , DofTransformation(order*(order + 2)*(order + 3)/2)
+      , ND_TetStatelessDofTransformation(order)
+   {}
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
    using DofTransformation::InvTransformDual;
 
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-
-   void InvTransformDual(double *v) const;
+   using ND_TetStatelessDofTransformation::TransformPrimal;
+   using ND_TetStatelessDofTransformation::InvTransformPrimal;
+   using ND_TetStatelessDofTransformation::TransformDual;
+   using ND_TetStatelessDofTransformation::InvTransformDual;
 };
 
 /// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeDofTransformation : public ND_DofTransformation
+class ND_WedgeStatelessDofTransformation : public ND_StatelessDofTransformation
 {
 public:
-   ND_WedgeDofTransformation(int order);
+   ND_WedgeStatelessDofTransformation(int order)
+      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , ND_StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2,
+                                      order, 9, 2)
+   {}
+};
+
+/// DoF transformation implementation for the Nedelec basis on wedge elements
+class ND_WedgeDofTransformation : public DofTransformation,
+   public ND_WedgeStatelessDofTransformation
+{
+public:
+   ND_WedgeDofTransformation(int order)
+      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , DofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , ND_WedgeStatelessDofTransformation(order)
+   {}
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
    using DofTransformation::InvTransformDual;
 
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-
-   void InvTransformDual(double *v) const;
-
+   using ND_WedgeStatelessDofTransformation::TransformPrimal;
+   using ND_WedgeStatelessDofTransformation::InvTransformPrimal;
+   using ND_WedgeStatelessDofTransformation::TransformDual;
+   using ND_WedgeStatelessDofTransformation::InvTransformDual;
 };
 
 } // namespace mfem

fem/fe/fe_base.cpp

@@ -401,7 +401,7 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
          }
       }
    }
-   else
+   else if (range_type == VECTOR)
    {
       d2q->B.SetSize(nqpt*dim*dof);
       d2q->Bt.SetSize(dof*nqpt*dim);
@@ -419,6 +419,10 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
          }
       }
    }
+   else
+   {
+      // Skip B and Bt for unknown range type
+   }
    switch (deriv_type)
    {
       case GRAD:
@@ -472,7 +476,7 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
             {
                for (int j = 0; j < dof; j++)
                {
-                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                  d2q->G[i+nqpt*(d+cdim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
                }
             }
          }
@@ -480,7 +484,8 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
       }
       case NONE:
       default:
-         MFEM_ABORT("invalid finite element derivative type");
+         // Skip G and Gt for unknown derivative type
+         break;
    }
    dof2quad_array.Append(d2q);
    return *d2q;

fem/fe/fe_base.hpp

@@ -14,6 +14,7 @@
 
 #include "../intrules.hpp"
 #include "../geom.hpp"
+#include "../doftrans.hpp"
 
 #include <map>
 
@@ -576,6 +577,7 @@ public:
    virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                          DofToQuad::Mode mode) const;
 
+
    /** @brief Return the mapping from lexicographic face DOFs to lexicographic
        element DOFs for the given local face @a face_id. */
    /** Given the @a ith DOF (lexicographically ordered) on the face referenced
@@ -590,6 +592,12 @@ public:
        when simplex elements are supported in the future. */
    virtual void GetFaceMap(const int face_id, Array<int> &face_map) const;
 
+   /** @brief Return a DoF transformation object for this particular type of
+       basis.
+   */
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return NULL; }
+
    /// Deconstruct the FiniteElement
    virtual ~FiniteElement();
 
@@ -1288,9 +1296,9 @@ public:
    const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                  DofToQuad::Mode mode) const override
    {
-      MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
-      return GetTensorDofToQuad(*this, ir, mode, basis1d, true,
-                                dof2quad_array);
+      return (mode == DofToQuad::FULL) ?
+             FiniteElement::GetDofToQuad(ir, mode) :
+             GetTensorDofToQuad(*this, ir, mode, basis1d, true, dof2quad_array);
    }
 
    const DofToQuad &GetDofToQuadOpen(const IntegrationRule &ir,

fem/fe/fe_nd.cpp

@@ -845,7 +845,7 @@ const double ND_TetrahedronElement::c = 1./4.;
 
 ND_TetrahedronElement::ND_TetrahedronElement(const int p)
    : VectorFiniteElement(3, Geometry::TETRAHEDRON, p*(p + 2)*(p + 3)/2, p,
-                         H_CURL, FunctionSpace::Pk), dof2tk(dof)
+                         H_CURL, FunctionSpace::Pk), dof2tk(dof), doftrans(p)
 {
    const double *eop = poly1d.OpenPoints(p - 1);
    const double *fop = (p > 1) ? poly1d.OpenPoints(p - 2) : NULL;
@@ -1108,7 +1108,7 @@ const double ND_TriangleElement::c = 1./3.;
 ND_TriangleElement::ND_TriangleElement(const int p)
    : VectorFiniteElement(2, Geometry::TRIANGLE, p*(p + 2), p,
                          H_CURL, FunctionSpace::Pk),
-     dof2tk(dof)
+     dof2tk(dof), doftrans(p)
 {
    const double *eop = poly1d.OpenPoints(p - 1);
    const double *iop = (p > 1) ? poly1d.OpenPoints(p - 2) : NULL;
@@ -1302,6 +1302,7 @@ ND_WedgeElement::ND_WedgeElement(const int p,
      dof2tk(dof),
      t_dof(dof),
      s_dof(dof),
+     doftrans(p),
      H1TriangleFE(p, cb_type),
      NDTriangleFE(p),
      H1SegmentFE(p, cb_type),

fem/fe/fe_nd.hpp

@@ -179,6 +179,8 @@ class ND_TetrahedronElement : public VectorFiniteElement
    Array<int> dof2tk;
    DenseMatrixInverse Ti;
 
+   mutable ND_TetStatelessDofTransformation doftrans;
+
 public:
    /// Construct the ND_TetrahedronElement of order @a p
    ND_TetrahedronElement(const int p);
@@ -199,6 +201,8 @@ public:
                                   ElementTransformation &Trans,
                                   DenseMatrix &I) const
    { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return &doftrans; }
    using FiniteElement::Project;
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
@@ -238,6 +242,8 @@ class ND_TriangleElement : public VectorFiniteElement
    Array<int> dof2tk;
    DenseMatrixInverse Ti;
 
+   mutable ND_TriStatelessDofTransformation doftrans;
+
 public:
    /// Construct the ND_TriangleElement of order @a p
    ND_TriangleElement(const int p);
@@ -258,6 +264,8 @@ public:
                                   ElementTransformation &Trans,
                                   DenseMatrix &I) const
    { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return &doftrans; }
    using FiniteElement::Project;
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
@@ -338,6 +346,8 @@ private:
 #endif
    Array<int> dof2tk, t_dof, s_dof;
 
+   mutable ND_WedgeStatelessDofTransformation doftrans;
+
    H1_TriangleElement H1TriangleFE;
    ND_TriangleElement NDTriangleFE;
    H1_SegmentElement  H1SegmentFE;
@@ -369,6 +379,9 @@ public:
                                   DenseMatrix &I) const
    { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
 
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return &doftrans; }
+
    using FiniteElement::Project;
 
    virtual void Project(VectorCoefficient &vc,

fem/fe_coll.cpp

@@ -2886,6 +2886,19 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
    }
 }
 
+StatelessDofTransformation *
+ND_FECollection::DofTransformationForGeometry(Geometry::Type GeomType) const
+{
+   if (!Geometry::IsTensorProduct(GeomType) && this->GetOrder() > 1)
+   {
+      return FiniteElementForGeometry(GeomType)->GetDofTransformation();
+   }
+   else
+   {
+      return NULL;
+   }
+}
+
 const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {

fem/fe_coll.hpp

@@ -61,6 +61,13 @@ public:
 
    virtual int DofForGeometry(Geometry::Type GeomType) const = 0;
 
+   /** @brief Returns a DoF transformation object compatible with this basis
+       and geometry type.
+   */
+   virtual StatelessDofTransformation *
+   DofTransformationForGeometry(Geometry::Type GeomType) const
+   { return NULL; }
+
    /** @brief Returns an array, say p, that maps a local permuted index i to a
        local base index: base_i = p[i].
 
@@ -466,8 +473,12 @@ public:
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return ND_dof[GeomType]; }
 
+   virtual StatelessDofTransformation *
+   DofTransformationForGeometry(Geometry::Type GeomType) const;
+
    virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
                                              int Or) const;
+
    virtual const char *Name() const { return nd_name; }
    virtual int GetContType() const { return TANGENTIAL; }
    FiniteElementCollection *GetTraceCollection() const;

fem/fespace.cpp

@@ -385,21 +385,38 @@ void FiniteElementSpace::BuildBdrElementToDofTable() const
    if (bdr_elem_dof) { return; }
 
    Table *bel_dof = new Table;
+   Table *bel_fos = (mesh->Dimension() == 3) ? (new Table) : NULL;
    Array<int> dofs;
+   int F, Fo;
    bel_dof->MakeI(mesh->GetNBE());
+   if (bel_fos) { bel_fos->MakeI(mesh->GetNBE()); }
    for (int i = 0; i < mesh->GetNBE(); i++)
    {
       GetBdrElementDofs(i, dofs);
       bel_dof->AddColumnsInRow(i, dofs.Size());
+
+      if (bel_fos)
+      {
+         bel_fos->AddAColumnInRow(i);
+      }
    }
    bel_dof->MakeJ();
+   if (bel_fos) { bel_fos->MakeJ(); }
    for (int i = 0; i < mesh->GetNBE(); i++)
    {
       GetBdrElementDofs(i, dofs);
       bel_dof->AddConnections(i, (int *)dofs, dofs.Size());
+
+      if (bel_fos)
+      {
+         mesh->GetBdrElementFace(i, &F, &Fo);
+         bel_fos->AddConnection(i, Fo);
+      }
    }
    bel_dof->ShiftUpI();
+   if (bel_fos) { bel_fos->ShiftUpI(); }
    bdr_elem_dof = bel_dof;
+   bdr_elem_fos = bel_fos;
 }
 
 void FiniteElementSpace::BuildFaceToDofTable() const
@@ -1542,6 +1559,10 @@ FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
    delete old_elem_dof;
    delete old_elem_fos;
+   for (int i=0; i<old_DoFTrans.Size(); i++)
+   {
+      delete old_DoFTrans[i];
+   }
 }
 
 void FiniteElementSpace::RefinementOperator

fem/fespace.hpp

@@ -377,17 +377,6 @@ protected:
    /// Return number of possible DOF variants for edge/face (var. order spaces).
    int GetNVariants(int entity, int index) const;
 
-   /// Helper to encode a sign flip into a DOF index (for Hcurl/Hdiv shapes).
-   static inline int EncodeDof(int entity_base, int idx)
-   { return (idx >= 0) ? (entity_base + idx) : (-1-(entity_base + (-1-idx))); }
-
-   /// Helpers to remove encoded sign from a DOF
-   static inline int DecodeDof(int dof)
-   { return (dof >= 0) ? dof : (-1 - dof); }
-
-   static inline int DecodeDof(int dof, double& sign)
-   { return (dof >= 0) ? (sign = 1, dof) : (sign = -1, (-1 - dof)); }
-
    /// Helper to get vertex, edge or face DOFs (entity=0,1,2 resp.).
    int GetEntityDofs(int entity, int index, Array<int> &dofs,
                      Geometry::Type master_geom = Geometry::INVALID,
@@ -985,6 +974,18 @@ public:
    /// well on sets of @ref ldof "Local Dofs".
    static void AdjustVDofs(Array<int> &vdofs);
 
+   /// Helper to encode a sign flip into a DOF index (for Hcurl/Hdiv shapes).
+   static inline int EncodeDof(int entity_base, int idx)
+   { return (idx >= 0) ? (entity_base + idx) : (-1-(entity_base + (-1-idx))); }
+
+   /// Helper to return the DOF associated with a sign encoded DOF
+   static inline int DecodeDof(int dof)
+   { return (dof >= 0) ? dof : (-1 - dof); }
+
+   /// Helper to determine the DOF and sign of a sign encoded DOF
+   static inline int DecodeDof(int dof, double& sign)
+   { return (dof >= 0) ? (sign = 1, dof) : (sign = -1, (-1 - dof)); }
+
    /// @anchor getvdof @name Local Vector DoF Access Members
    /// These member functions produce arrays of local vector degree of freedom
    /// indices, see @ref ldof and @ref vdof. These indices can be used to
@@ -994,7 +995,9 @@ public:
 
    /// @brief Returns indices of degrees of freedom for the @a i'th element.
    /// The returned indices are offsets into an @ref ldof vector with @b vdim
-   /// not necessarily equal to 1. See also GetElementDofs().
+   /// not necessarily equal to 1. The returned indexes are always ordered
+   /// byNODES, irrespective of whether the space is byNODES or byVDIM.
+   /// See also GetElementDofs().
    ///
    /// @note In many cases the returned DofTransformation object will be NULL.
    /// In other cases see the documentation of the DofTransformation class for

fem/gridfunc.cpp

@@ -397,8 +397,6 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
 {
    Array<int> vdofs;
 
-   int k;
-
    DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
    const FiniteElement *FElem = fes->GetFE(i);
    const IntegrationRule *ElemVert =
@@ -419,7 +417,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
       Vector shape(dof);
       if (FElem->GetMapType() == FiniteElement::VALUE)
       {
-         for (k = 0; k < n; k++)
+         for (int k = 0; k < n; k++)
          {
             FElem->CalcShape(ElemVert->IntPoint(k), shape);
             nval[k] = shape * (&loc_data[dof * vdim]);
@@ -428,7 +426,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
       else
       {
          ElementTransformation *Tr = fes->GetElementTransformation(i);
-         for (k = 0; k < n; k++)
+         for (int k = 0; k < n; k++)
          {
             Tr->SetIntPoint(&ElemVert->IntPoint(k));
             FElem->CalcPhysShape(*Tr, shape);
@@ -440,7 +438,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
    {
       ElementTransformation *Tr = fes->GetElementTransformation(i);
       DenseMatrix vshape(dof, FElem->GetDim());
-      for (k = 0; k < n; k++)
+      for (int k = 0; k < n; k++)
       {
          Tr->SetIntPoint(&ElemVert->IntPoint(k));
          FElem->CalcVShape(*Tr, vshape);
@@ -2401,7 +2399,11 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
                                      loc_mass);
             vals.SetSize(fe->GetDof());
             fe->ProjectDelta(j, vals);
-            fes->GetElementVDofs(i, vdofs);
+            const DofTransformation* const doftrans = fes->GetElementVDofs(i, vdofs);
+            if (doftrans)
+            {
+               doftrans->TransformPrimal(vals);
+            }
             SetSubVector(vdofs, vals);
             loc_mass_vals.SetSize(vals.Size());
             loc_mass.Mult(vals, loc_mass_vals);
@@ -2755,7 +2757,11 @@ void GridFunction::ProjectBdrCoefficientNormal(
          CalcOrtho(T->Jacobian(), nor);
          lvec(j) = (vc * nor);
       }
-      fes->GetBdrElementDofs(i, dofs);
+      const DofTransformation* const doftrans = fes->GetBdrElementDofs(i, dofs);
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(lvec);
+      }
       SetSubVector(dofs, lvec);
    }
 #endif
@@ -4031,11 +4037,19 @@ double ZZErrorEstimator(BilinearFormIntegrator &blfi,
       {
          if (with_subdomains && ufes->GetAttribute(i) != s) { continue; }
 
-         ufes->GetElementVDofs(i, udofs);
-         ffes->GetElementVDofs(i, fdofs);
+         const DofTransformation* const utrans = ufes->GetElementVDofs(i, udofs);
+         const DofTransformation* const ftrans = ffes->GetElementVDofs(i, fdofs);
 
          u.GetSubVector(udofs, ul);
          flux.GetSubVector(fdofs, fla);
+         if (utrans)
+         {
+            utrans->InvTransformPrimal(ul);
+         }
+         if (ftrans)
+         {
+            ftrans->InvTransformPrimal(fla);
+         }
 
          Transf = ufes->GetElementTransformation(i);
          blfi.ComputeElementFlux(*ufes->GetFE(i), *Transf, ul,
@@ -4330,8 +4344,12 @@ double LSZZErrorEstimator(BilinearFormIntegrator &blfi,  // input
                                                     flux_order));
          int num_integration_pts = ir->GetNPoints();
 
-         ufes->GetElementVDofs(ielem, udofs);
+         const DofTransformation* const utrans = ufes->GetElementVDofs(ielem, udofs);
          u.GetSubVector(udofs, ul);
+         if (utrans)
+         {
+            utrans->InvTransformPrimal(ul);
+         }
          Transf = ufes->GetElementTransformation(ielem);
          FiniteElement *dummy = nullptr;
          blfi.ComputeElementFlux(*ufes->GetFE(ielem), *Transf, ul,

fem/gridfunc.hpp

@@ -684,6 +684,10 @@ public:
    /// Transform by the Space UpdateMatrix (e.g., on Mesh change).
    virtual void Update();
 
+   /** Return update counter, similar to Mesh::GetSequence(). Used to
+       check if it is up to date with the space. */
+   long GetSequence() const { return fes_sequence; }
+
    FiniteElementSpace *FESpace() { return fes; }
    const FiniteElementSpace *FESpace() const { return fes; }
 

fem/gslib.cpp

@@ -108,8 +108,7 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
                             const int npt_max)
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
-   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
-               "Variable order mesh is not currently supported.");
+   const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
 
    // call FreeData if FindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -117,30 +116,36 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    crystal_init(cr, gsl_comm);
    mesh = &m;
    dim  = mesh->Dimension();
-   const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
-   unsigned dof1D = fe->GetOrder() + 1;
+   unsigned dof1D = meshOrder + 1;
 
    SetupSplitMeshes();
    if (dim == 2)
    {
       if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
       ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
    }
    else if (dim == 3)
    {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
 
       if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
       ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
 
       if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
       ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
 
       if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
       ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
    }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
@@ -179,7 +184,7 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
    gsl_ref.SetSize(points_cnt * dim);
    gsl_dist.SetSize(points_cnt);
 
-   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
+   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[])
    {
       for (int d = 0; d < dim; d++)
       {
@@ -199,7 +204,7 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[2];
       unsigned xv_stride[2];
-      xvFill(xv_base, xv_stride, dim);
+      xvFill(xv_base, xv_stride);
       findpts_2(gsl_code.GetData(), sizeof(unsigned int),
                 gsl_proc.GetData(), sizeof(unsigned int),
                 gsl_elem.GetData(), sizeof(unsigned int),
@@ -211,7 +216,7 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[3];
       unsigned xv_stride[3];
-      xvFill(xv_base, xv_stride, dim);
+      xvFill(xv_base, xv_stride);
       findpts_3(gsl_code.GetData(), sizeof(unsigned int),
                 gsl_proc.GetData(), sizeof(unsigned int),
                 gsl_elem.GetData(), sizeof(unsigned int),
@@ -333,9 +338,14 @@ void FindPointsGSLIB::SetupSplitMeshes()
             (*gf_rst_map[0])(j+k*npt) = quad_v[j][k];
          }
       }
+
+      mesh_split[1] = new Mesh(Mesh::MakeCartesian2D(1, 1,
+                                                     Element::QUADRILATERAL));
    }
    else if (mesh->Dimension() == 3)
    {
+      mesh_split[0] = new Mesh(Mesh::MakeCartesian3D(1, 1, 1,
+                                                     Element::HEXAHEDRON));
       // Tetrahedron
       {
          int Nvert = 15;
@@ -565,11 +575,12 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
    const GridFunction *nodes     = gf_in;
    const FiniteElementSpace *fes = nodes->FESpace();
    const int NE                  = mesh->GetNE();
-   const int vdim                = gf_in->FESpace()->GetVDim();
+   const int vdim                = fes->GetVDim();
 
    IntegrationRule *ir_split_temp = NULL;
 
-   const int dof_1D =  nodes->FESpace()->GetFE(0)->GetOrder()+1;
+   const int maxOrder = fes->GetMaxElementOrder();
+   const int dof_1D =  maxOrder+1;
    const int pts_el = std::pow(dof_1D, dim);
    const int pts_cnt = NE_split_total * pts_el;
    node_vals.SetSize(vdim * pts_cnt);
@@ -579,7 +590,7 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
 
    for (int e = 0; e < NE; e++)
    {
-      const FiniteElement *fe   = nodes->FESpace()->GetFE(e);
+      const FiniteElement *fe   = fes->GetFE(e);
       const Geometry::Type gt   = fe->GetGeomType();
       bool el_to_split = true;
       if (gt == Geometry::TRIANGLE)
@@ -598,16 +609,22 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
       {
          ir_split_temp = ir_split[3];
       }
-      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
+      else if (gt == Geometry::SQUARE)
       {
-         el_to_split = false;
+         ir_split_temp = ir_split[1];
+         el_to_split = gf_in->FESpace()->IsVariableOrder();
+      }
+      else if (gt == Geometry::CUBE)
+      {
+         ir_split_temp = ir_split[0];
+         el_to_split = gf_in->FESpace()->IsVariableOrder();
       }
       else
       {
          MFEM_ABORT("Unsupported geometry type.");
       }
 
-      if (el_to_split) // Triangle/Tet/Prism
+      if (el_to_split) // Triangle/Tet/Prism or Quads/Hex but variable order
       {
          // Fill gsl_mesh with location of split points.
          Vector locval(vdim);
@@ -622,7 +639,7 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
             gsl_mesh_pt_index++;
          }
       }
-      else // Quad/Hex
+      else // Quad/Hex and constant polynomial order
       {
          const int dof_cnt_split = fe->GetDof();
 
@@ -803,8 +820,8 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
 void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
                                   Vector &field_out)
 {
-   const int  gf_order   = field_in.FESpace()->GetFE(0)->GetOrder(),
-              mesh_order = mesh->GetNodalFESpace()->GetFE(0)->GetOrder();
+   const int  gf_order   = field_in.FESpace()->GetMaxElementOrder(),
+              mesh_order = mesh->GetNodalFESpace()->GetMaxElementOrder();
 
    const FiniteElementCollection *fec_in =  field_in.FESpace()->FEColl();
    const H1_FECollection *fec_h1 = dynamic_cast<const H1_FECollection *>(fec_in);
@@ -812,7 +829,8 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
 
    if (fec_h1 && gf_order == mesh_order &&
        fec_h1->GetBasisType() == BasisType::GaussLobatto &&
-       !field_in.FESpace()->IsVariableOrder())
+       field_in.FESpace()->IsVariableOrder() ==
+       mesh->GetNodalFESpace()->IsVariableOrder())
    {
       InterpolateH1(field_in, field_out);
       return;
@@ -886,12 +904,21 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
                                     Vector &field_out)
 {
    FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
+   if (field_in.FESpace()->IsVariableOrder())
+   {
+      for (int e = 0; e < ind_fes.GetMesh()->GetNE(); e++)
+      {
+         ind_fes.SetElementOrder(e, field_in.FESpace()->GetElementOrder(e));
+      }
+      ind_fes.Update(false);
+   }
    GridFunction field_in_scalar(&ind_fes);
    Vector node_vals;
 
    const int ncomp      = field_in.FESpace()->GetVDim(),
-             points_fld = field_in.Size() / ncomp,
-             points_cnt = gsl_code.Size();
+             points_fld = field_in.Size() / ncomp;
+   MFEM_VERIFY(points_cnt == gsl_code.Size(),
+               "FindPointsGSLIB::InterpolateH1: Inconsistent size of gsl_code");
 
    field_out.SetSize(points_cnt*ncomp);
    field_out = default_interp_value;
@@ -1111,8 +1138,7 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
                                    const int npt_max)
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
-   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
-               "Variable order mesh is not currently supported.");
+   const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
 
    // FreeData if OversetFindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -1128,21 +1154,29 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
    {
       if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
       ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
    }
    else if (dim == 3)
    {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
       if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
       ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
 
       if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
       ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
 
       if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
       ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
    }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
@@ -1274,6 +1308,298 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
    Interpolate(field_in, field_out);
 }
 
+#ifdef MFEM_USE_MPI
+
+GSLIBCommunicator::GSLIBCommunicator(MPI_Comm comm_)
+   : cr(NULL), gsl_comm(NULL)
+{
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
+   comm_init(gsl_comm, comm_);
+   crystal_init(cr, gsl_comm);
+}
+
+void GSLIBCommunicator::SendData(int dim, const Array<unsigned int> & gsl_proc,
+                                 const Array<unsigned int> & elem_send,
+                                 const Vector &ref_send,
+                                 const Vector &coords_send,
+                                 const Array<int> &s_conn_send,
+                                 Array<unsigned int> & proc_recv,
+                                 Array<unsigned int> & index_recv,
+                                 Array<unsigned int> & elem_recv,
+                                 Vector &ref_recv,
+                                 Vector &coords_recv,
+                                 Array<int> &s_conn_recv)
+{
+   int nptsend = gsl_proc.Size();
+   int nptElem   = elem_send.Size();
+   int nptRST    = ref_send.Size();
+
+   MFEM_VERIFY(nptElem == nptsend,
+               "Incompatible Elem size.");
+   MFEM_VERIFY(nptsend*dim == nptRST,
+               "Incompatible nptRST size.");
+   MFEM_VERIFY(dim <= 3,
+               "Incompatible dimension.");
+
+   // Pack data to send via crystal router
+   struct gslib::array *outpt = new gslib::array;
+
+   struct out_pt { double rst[3], coords[3]; int s_conn; uint index, elem, proc; };
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->index = index;
+      pt->elem = elem_send[index];
+      pt->proc  = gsl_proc[index];
+      pt->s_conn = s_conn_send[index];
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->rst[d]= ref_send(index*dim + d);
+         pt->coords[d]= coords_send(index + d*nptsend);
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
+   // unpack
+   int npt = outpt->n;
+   proc_recv.SetSize(npt);
+   elem_recv.SetSize(npt);
+   index_recv.SetSize(npt);
+   ref_recv.SetSize(npt*dim);
+   coords_recv.SetSize(npt*dim);
+   s_conn_recv.SetSize(npt);
+
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      index_recv[index] = pt->index;
+      elem_recv[index] = pt->elem;
+      proc_recv[index] = pt->proc;
+      s_conn_recv[index] = pt->s_conn;
+      for (int d = 0; d < dim; ++d)
+      {
+         ref_recv(index*dim + d)= pt->rst[d]; // by VDIM
+         coords_recv(index + d*npt)= pt->coords[d]; // by NODES
+      }
+      ++pt;
+   }
+
+   array_free(outpt);
+   delete outpt;
+}
+
+void GSLIBCommunicator::SendData2(int dim,
+                                  const Array<unsigned int> & gsl_proc,
+                                  const Vector &xyz_send,
+                                  const Vector &xi_send,
+                                  const Array<int> &s_conn_send,
+                                  const Array<int> &conn_send,
+                                  const DenseMatrix &coords_send,
+                                  Vector &xyz_recv,
+                                  Vector &xi_recv,
+                                  Array<int> &s_conn_recv,
+                                  Array<int> &conn_recv,
+                                  DenseMatrix &coords_recv)
+{
+   int nptsend = gsl_proc.Size();
+
+   struct gslib::array *outpt = new gslib::array;
+   struct out_pt {double xyz[3], xi[2], coords[12]; int s_conn; int conn[4]; uint proc;};
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->proc  = gsl_proc[index];
+      pt->s_conn  = s_conn_send[index];
+      for (int d = 0; d < dim-1; ++d)
+      {
+         pt->xi[d]= xi_send(index*(dim-1) + d);
+      }
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->xyz[d]= xyz_send(index + d*nptsend);
+      }
+      for (int j = 0; j<4; j++)
+      {
+         pt->conn[j] = conn_send[index*4+j];
+         for (int d = 0; d < dim; ++d)
+         {
+            pt->coords[j*dim+d]= coords_send(index*4+j,d);
+         }
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+   // unpack
+   int npt = outpt->n;
+   xi_recv.SetSize(npt*(dim-1));
+   xyz_recv.SetSize(npt*dim);
+   s_conn_recv.SetSize(npt);
+   conn_recv.SetSize(npt*4);
+   coords_recv.SetSize(npt*4,dim);
+
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      s_conn_recv[index] = pt->s_conn;
+      for (int d = 0; d < dim-1; ++d)
+      {
+         xi_recv(index*(dim-1) + d) = pt->xi[d];
+      }
+      for (int d = 0; d < dim; ++d)
+      {
+         xyz_recv(index + d*npt)= pt->xyz[d]; // by NODES
+      }
+      for (int j = 0; j<4; j++)
+      {
+         conn_recv[index*4+j] = pt->conn[j];
+         for (int d = 0; d < dim; ++d)
+         {
+            coords_recv(index*4+j,d) = pt->coords[j*dim+d];
+         }
+      }
+      ++pt;
+   }
+   array_free(outpt);
+   delete outpt;
+}
+
+
+void GSLIBCommunicator::ExchangeNormal(Mesh & mesh,
+                                       const Array<unsigned int> &gsl_proc,
+                                       const Array<unsigned int> &gsl_mfem_elem,
+                                       const Vector &gsl_mfem_ref,
+                                       Vector &recv_normals)
+{
+   int dim = mesh.Dimension();
+   int nptsend = gsl_proc.Size();
+   int nptElem   = gsl_mfem_elem.Size();
+   int nptRST    = gsl_mfem_ref.Size();
+
+   recv_normals.SetSize(nptRST);
+   int nptNormal    = recv_normals.Size();
+
+   MFEM_VERIFY(nptElem == nptsend,
+               "Incompatible Elem size.");
+   MFEM_VERIFY(nptsend*dim == nptRST,
+               "Incompatible nptRST size.");
+   MFEM_VERIFY(dim <= 3,
+               "Incompatible dimension.");
+
+   // Pack data to send via crystal router
+   struct gslib::array *outpt = new gslib::array;
+
+   struct out_pt { double rst[3]; uint index, elem, proc; };
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->index = index;
+      pt->elem = gsl_mfem_elem[index];
+      pt->proc  = gsl_proc[index];
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->rst[d]= gsl_mfem_ref(index*dim + d);
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
+   // Get normal vector
+   int npt = outpt->n;
+   pt = (struct out_pt *)outpt->ptr;
+   Vector normal(npt*dim);
+   for (int index = 0; index < npt; index++)
+   {
+      IntegrationPoint ip;
+      ip.Set3(&pt->rst[0]);
+      Vector localval(normal.GetData()+index*dim, dim);
+      // get the normal at this integration point here
+      // for now I just put back this proc's rank + the input rst coordinates
+      for (int d = 0; d < dim; d++)
+      {
+         localval(d) = gsl_comm->id + pt->rst[d];
+      }
+      ++pt;
+   }
+
+   // Save index and proc data in a struct
+   struct gslib::array *savpt = new gslib::array;
+   struct sav_pt { uint index, proc; };
+   struct sav_pt *spt;
+   array_init(struct sav_pt, savpt, npt);
+   savpt->n=npt;
+   spt = (struct sav_pt *)savpt->ptr;
+   pt  = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      spt->index = pt->index;
+      spt->proc  = pt->proc;
+      ++pt; ++spt;
+   }
+
+   array_free(outpt);
+   delete outpt;
+
+   // Copy data from save struct to send struct and send component wise
+   struct gslib::array *sendpt = new gslib::array;
+   struct send_pt { double ival; uint index, proc; };
+   struct send_pt *sdpt;
+   for (int j = 0; j < dim; j++)
+   {
+      array_init(struct send_pt, sendpt, npt);
+      sendpt->n=npt;
+      spt  = (struct sav_pt *)savpt->ptr;
+      sdpt = (struct send_pt *)sendpt->ptr;
+      for (int index = 0; index < npt; index++)
+      {
+         sdpt->index = spt->index;
+         sdpt->proc  = spt->proc;
+         sdpt->ival  = normal(j + index*dim);
+         ++sdpt; ++spt;
+      }
+
+      sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
+      sdpt = (struct send_pt *)sendpt->ptr;
+      for (int index = 0; index < static_cast<int>(sendpt->n); index++)
+      {
+         int idx = sdpt->index*dim + j;
+         recv_normals(idx) = sdpt->ival;
+         ++sdpt;
+      }
+      array_free(sendpt);
+   }
+   array_free(savpt);
+   delete sendpt;
+   delete savpt;
+}
+
+void GSLIBCommunicator::FreeData()
+{
+   crystal_free(cr);
+}
+
+GSLIBCommunicator::~GSLIBCommunicator()
+{
+   delete gsl_comm;
+   delete cr;
+}
+#endif
 
 } // namespace mfem
 

fem/gslib.hpp

@@ -57,7 +57,9 @@ public:
 protected:
    Mesh *mesh;
    Array<Mesh *> mesh_split;  // Meshes used to split simplices.
-   Array<IntegrationRule *> ir_split; // IntegrationRules for simplex->Quad/Hex
+   // IntegrationRules for simplex->Quad/Hex and to project to highest polynomial
+   // order in-case of p-refinement.
+   Array<IntegrationRule *> ir_split;
    Array<FiniteElementSpace *>
    fes_rst_map; // FESpaces to map info Quad/Hex->Simplex
    Array<GridFunction *> gf_rst_map; // GridFunctions to map info Quad/Hex->Simplex
@@ -288,6 +290,55 @@ public:
    using FindPointsGSLIB::Interpolate;
 };
 
+#ifdef MFEM_USE_MPI
+// Use to send info to certain processes
+class GSLIBCommunicator
+{
+protected:
+   struct gslib::crystal *cr;             // gslib's internal data
+   struct gslib::comm *gsl_comm;          // gslib's internal data
+
+public:
+   GSLIBCommunicator(MPI_Comm comm_);
+
+   virtual ~GSLIBCommunicator();
+
+   void ExchangeNormal(Mesh& mesh,
+                       const Array<unsigned int> &gsl_proc,
+                       const Array<unsigned int> &gsl_mfem_elem,
+                       const Vector &gsl_mfem_ref,
+                       Vector &recv_normals); //npt*dim
+
+   void SendData(int dim,
+                 const Array<unsigned int> & gsl_proc,
+                 const Array<unsigned int> & elem_send,
+                 const Vector &ref_send,
+                 const Vector &coords_send,
+                 const Array<int> &s_conn_send,
+                 Array<unsigned int> & proc_recv,
+                 Array<unsigned int> & index_recv,
+                 Array<unsigned int> & elem_recv,
+                 Vector &ref_recv,
+                 Vector &coords_recv,
+                 Array<int> & s_conn_recv);
+
+   void SendData2(int dim,
+                  const Array<unsigned int> & gsl_proc,
+                  const Vector &xyz_send,
+                  const Vector &xi_send,
+                  const Array<int> &s_conn_send,
+                  const Array<int> &conn_send,
+                  const DenseMatrix &coords_send,
+                  Vector &xyz_recv,
+                  Vector &ref_recv,
+                  Array<int> &s_conn_recv,
+                  Array<int> &conn_recv,
+                  DenseMatrix &coords_recv);
+
+   virtual void FreeData();
+};
+#endif
+
 } // namespace mfem
 
 #endif // MFEM_USE_GSLIB

fem/integ/bilininteg_br2.cpp

@@ -9,8 +9,8 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "bilininteg.hpp"
-#include "pfespace.hpp"
+#include "../bilininteg.hpp"
+#include "../pfespace.hpp"
 #include <algorithm>
 
 namespace mfem

fem/integ/bilininteg_convection_ea.cpp

@@ -9,9 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 
 namespace mfem
 {
@@ -34,7 +34,7 @@ static void EAConvectionAssemble1D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -86,7 +86,7 @@ static void EAConvectionAssemble2D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -163,7 +163,7 @@ static void EAConvectionAssemble3D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 3, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/integ/bilininteg_convection_mf.cpp

@@ -9,12 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/integrators/convection/convection.hpp"
-
-using namespace std;
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../ceed/integrators/convection/convection.hpp"
 
 namespace mfem
 {