Merge branch 'fusion-ldrd-dev' into epic-dev

Update version numbers to 4.3.1 -- a new development version

.github/workflows/builds-and-tests.yml

@@ -47,26 +47,36 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04, macos-10.15]
-        target: [debug, optim]
-        mpi: [sequential, parallel]
+        target: [dbg, opt]
+        mpi: [seq, par]
         build-system: [make]
-        # 'include' allows us to
-        # - add a variable without creating a new matrix dimension.
-        # - add a new combination ('build-system: cmake' case here)
+        hypre-target: [int32]
+        # 'include' allows us to:
+        # - Add a variable to all jobs without creating a new matrix dimension.
+        #   Codecov is defined that way.
+        # - Add a new combination.
+        #   'build-system: cmake' and 'hypre-target: int64'
         #
         # note: we will gather coverage info for any non-debug run except the
         # CMake build.
         include:
-          - target: debug
+          - target: dbg
             codecov: NO
-          - target: optim
+          - target: opt
             codecov: YES
           - os: ubuntu-18.04
-            target: optim
+            target: opt
             codecov: NO
-            mpi: parallel
+            mpi: par
             build-system: cmake
-    name: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.build-system }}
+            hypre-target: int32
+          - os: ubuntu-18.04
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int64
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
 
     runs-on: ${{ matrix.os }}
 
@@ -92,7 +102,7 @@ jobs:
     # TODO: It would be nice to have only one step, e.g. with a dedicated
     # action, but I (@adrienbernede) don't see how at the moment.
     - name: get MPI (Linux)
-      if: matrix.mpi == 'parallel' && matrix.os == 'ubuntu-18.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
       run: |
         sudo apt-get install mpich libmpich-dev
         export MAKE_CXX_FLAG="MPICXX=mpic++"
@@ -103,11 +113,11 @@ jobs:
         sudo apt-get install lcov
 
     - name: Set up Homebrew
-      if: ( matrix.mpi == 'parallel' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
       uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
 
     - name: get MPI (MacOS)
-      if: matrix.mpi == 'parallel' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
       run: |
         export HOMEBREW_NO_INSTALL_CLEANUP=1
         brew install openmpi
@@ -123,39 +133,40 @@ jobs:
     # Install will only run on cache miss.
     - name: cache hypre
       id: hypre-cache
-      if: matrix.mpi == 'parallel'
+      if: matrix.mpi == 'par'
       uses: actions/cache@v2
       with:
         path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0
 
     - name: get hypre
-      if: matrix.mpi == 'parallel' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v1.0
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v2.0
       with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}
 
     # Get Metis through cache, or build it.
     # Install will only run on cache miss.
     - name: cache metis
       id: metis-cache
-      if: matrix.mpi == 'parallel'
+      if: matrix.mpi == 'par'
       uses: actions/cache@v2
       with:
         path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
 
     - name: install metis
-      if: matrix.mpi == 'parallel' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v1.0
+      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.0
       with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}
 
     # MFEM build and test
     - name: build
-      uses: mfem/github-actions/build-mfem@v1.0
+      uses: mfem/github-actions/build-mfem@v2.0
       with:
         os: ${{ matrix.os }}
         target: ${{ matrix.target }}
@@ -168,17 +179,17 @@ jobs:
 
     # Run checks (and only checks) on debug targets
     - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'debug'
+      if: matrix.build-system == 'make' && matrix.target == 'dbg'
       run: |
         cd ${{ env.MFEM_TOP_DIR }}  && make check
 
     - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
       run: |
         cd ${{ env.MFEM_TOP_DIR }} && make unittest
 
     - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
       run: |
         cd ${{ env.MFEM_TOP_DIR }} && make test
 
@@ -190,8 +201,8 @@ jobs:
     # Code coverage (process and upload reports)
     - name: codecov
       if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v1.0
+      uses: mfem/github-actions/upload-coverage@v2.0
       with:
-        name: ${{ matrix.os }}-${{ matrix.mpi }}
+        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
         project_dir: ${{ env.MFEM_TOP_DIR }}
         directories: "fem general linalg mesh"

.github/workflows/mfem-analysis.yml

@@ -53,32 +53,33 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0
 
     - name: Get Hypre
       if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@master
+      uses: mfem/github-actions/build-hypre@v2.0
       with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: int32
 
     - name: Cache Metis Install
       id: metis-cache
       uses: actions/cache@v2
       with:
         path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
 
     - name: Install Metis
       if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@master
+      uses: mfem/github-actions/build-metis@v2.0
       with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}
 
     # MFEM build and test
     - name: build-mfem
-      uses: mfem/github-actions/build-mfem@master
+      uses: mfem/github-actions/build-mfem@v2.0
       with:
         os: ${{ runner.os }}
         target: optim

.github/workflows/repo-check.yml

@@ -28,49 +28,24 @@ jobs:
         access_token: ${{ github.token }}
     - name: checkout mfem
       uses: actions/checkout@v2
-      with:
-        path: mfem
 
     - name: copyright check
       id: copyright
       run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt
-        then
-          echo "Please update the following files to Copyright (c) 2010-2021:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated copyright found."
-        fi
+        ./config/githooks/pre-push --copyright
+
       continue-on-error: true
 
     - name: license check
       id: license
       run: |
-        cd mfem
-        if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt
-        then
-          echo "Please update the following files to the BSD-3 license:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No GNU GPL license found."
-        fi
+        ./config/githooks/pre-push --license
       continue-on-error: true
 
     - name: release check
       id: release
       run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-        then
-          echo "Please update the following files to LLNL-CODE-806117:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated release number found."
-        fi
+        ./config/githooks/pre-push --release
       continue-on-error: true
 
     - name: wrap-up
@@ -100,8 +75,7 @@ jobs:
 
     - name: style check
       run: |
-        cd tests/scripts
-        ./runtest code-style
+        ./config/githooks/pre-push --style
 
   documentation:
     runs-on: ubuntu-18.04
@@ -133,5 +107,4 @@ jobs:
       run: |
         git fetch origin master:master
         git checkout -b gh-actions-branch-history
-        cd tests/scripts
-        ./runtest branch-history
+        ./config/githooks/pre-push --history

.gitignore

@@ -26,6 +26,7 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
+config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
@@ -251,6 +252,7 @@ miniapps/shifted/ParaViewDistance
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
+miniapps/shifted/ParaViewDiffusion
 
 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -295,6 +297,7 @@ tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
+tests/unit/debug_device_tests
 
 # Test script output
 tests/scripts/*.err

.gitlab-ci.yml

@@ -48,39 +48,58 @@ variables:
   TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
   TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
   AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
   ARTIFACTS_DIR: artifacts
 
-# The pipeline is divided into stages. Usually, these are also synchronization
-# points, however, we use "needs" keyword to express the DAG of jobs for more
-# efficiency.
-# - We use setup phase to download content outside of mfem directory.
+# The pipeline is divided into stages. Usually, jobs in a given stage wait for
+# the preceding stages to complete before to start. However, we sometimes use
+# the "needs" keyword and express the DAG of jobs for more efficiency.
+# - We use setup and setup_baseline phases to download content outside of mfem
+#   directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
+  - setup
   - q_allocate_resources
   - q_build_and_test
   - q_release_resources
   - l_build_and_test
   - c_build_and_test
-  - setup
+  - setup_baseline
   - baseline_check
   - baseline_to_autotest
   - baseline_publish
 
-# The setup job in setup stage don't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO: updating tests and tpls is not necessary anymore since pipelines are
-# now using unique directories so repo are never shared with another pipeline.
-# This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
+# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
+# then symlinks the repo to the parent directory of the MFEM source directory.
+# Unit tests that depend on the mfem/data repo will then detect that this
+# directory is present and be enabled.
 setup:
   tags:
     - shell
     - quartz
   stage: setup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+
+# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
+# It prepares a pipeline-wide working directory downloading/updating external
+# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
+# are now using unique directories so repo are never shared with another
+# pipeline. This is not memory efficient (we keep a lot of data), hence this
+# reminder.
+# Note: This job can start immediately.
+setup_baseline:
+  tags:
+    - shell
+    - quartz
+  stage: setup_baseline
   variables:
     GIT_STRATEGY: none
   script:
@@ -106,10 +125,10 @@ setup:
   script:
     - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
 
-# Lassen uses a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-# We use pdebug queue on lassen to speed-up the allocation.
-# However this would not be scalable to multiple builds.
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
+# speed-up the allocation.  However this would not be scalable to multiple
+# builds.
 .build_blueos_3_ppc64le_ib_script:
   script:
     - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test

.gitlab/lassen.yml

@@ -22,10 +22,11 @@
 
 # Spack helped builds
 # Generic lassen build job, extending build script
+# Note: Lassen jobs can start as soon as the setup job is complete.
 .build_and_test_on_lassen:
   extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
   stage: l_build_and_test
-  needs: []
+  needs: [setup]
 
 opt_mpi_cuda_xl_16_1_1_8:
   variables:

.gitlab/quartz.yml

@@ -16,13 +16,13 @@
     - shell
     - quartz
   rules:
-    # Don’t run quartz jobs if...
+    # Don't run quartz jobs if...
     - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
       when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
       when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status
@@ -37,6 +37,18 @@
     # Default is to run if previous stage succeeded
     - when: on_success
 
+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
 # Allocate
 q_allocate_resources:
   variables:
@@ -65,10 +77,11 @@ q_report_success:
   stage: q_release_resources
   script:
     - echo "Can only run if all the quartz jobs passed"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
     - git push origin master
@@ -80,10 +93,11 @@ q_report_failure:
   stage: q_release_resources
   script:
     - echo "Runs if there was at least one failure on quartz"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
@@ -136,24 +150,29 @@ opt_par_gcc_6_1_0_pumi:
     SPEC: "%gcc@6.1.0 +pumi"
   extends: .build_and_test_on_quartz
 
-# Baseline
+# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
+# that "setup-baseline" can start immediately. Then, we have to use needs for
+# each one of the baseline jobs, otherwise they will wait for the rest of the
+# pipeline.
+
 baselinecheck_mfem_intel_quartz:
   extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup]
+  needs: [setup_baseline]
 
 update_autotest:
   extends: [.on_quartz]
   needs: [baselinecheck_mfem_intel_quartz]
   stage: baseline_to_autotest
   script:
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
     # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
     - |
       if [[ -f ${rundir}/*.err ]]
       then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
         cp ${rundir}/*.err ${rundir}/autotest-email.html
       fi
     - git add ${rundir}

.travis.yml

@@ -1,469 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-language: cpp
-
-os: linux
-dist: bionic
-
-stages:
-  - checks
-  - tests
-  - optional
-
-env:
-  global:
-    - HYPRE_ARCHIVE=v2.19.0.tar.gz
-      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
-      HYPRE_TOP_DIR=hypre-2.19.0
-
-jobs:
-  include:
-
-    # ========================
-    #         Checks
-    # ========================
-    # - code-style
-    # - documentation
-    # - gitignore
-
-    - stage: checks
-      os: linux
-      dist: xenial
-      name: "code-style"
-      addons:
-        apt:
-          packages:
-            - astyle=2.05.1-0ubuntu1
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest code-style
-
-    - stage: checks
-      os: linux
-      name: "documentation"
-      addons:
-        apt:
-          packages:
-            - doxygen
-            - graphviz
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest documentation
-
-    - stage: checks
-      os: linux
-      name: "gitignore"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-      before_script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - mpicxx -v
-        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
-        - make all -j3
-        - make test-noclean
-      script:
-        - cd tests/scripts
-        - ./runtest gitignore
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    # ========================
-    #  Optional Checks/Tests
-    # ========================
-    # - branch-history
-
-    - stage: optional
-      name: "branch-history"
-      if: branch != next
-      # need full git history for the binary/big files check
-      git:
-        depth: false
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        # update master
-        - git fetch origin master:master
-        # checkout a branch (otherwise Travis works in detached head)
-        - git checkout -b travis_tests
-        - cd tests/scripts
-        - ./runtest branch-history
-
-    # ========================
-    #       Linux tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - stage: tests
-      os: linux
-      compiler: gcc
-      name: "Linux: Serial + Debug"
-      env: DEBUG=YES
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Serial"
-      env: DEBUG=NO
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel + Debug"
-      addons:
-        apt:
-          # sources:
-            # - ubuntu-toolchain-r-test
-          packages:
-            # GCC 4.9
-            # - g++-4.9
-            # MPICH
-            - mpich
-            - libmpich-dev
-            # OpenMPI
-            # - openmpi-bin
-            # - libopenmpi-dev
-      env: DEBUG=YES
-           MPI=YES
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-           NPROCS=2
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel"
-      addons:
-        apt:
-          # sources:
-            # - ubuntu-toolchain-r-test
-          packages:
-            # GCC 4.9
-            # - g++-4.9
-            # MPICH
-            - mpich
-            - libmpich-dev
-            # OpenMPI
-            # - openmpi-bin
-            # - libopenmpi-dev
-      env: DEBUG=NO
-           MPI=YES
-           CODECOV=YES
-           MFEM_TEST_TARGET=test
-           NPROCS=2
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel (cmake)"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-           NPROCS=2
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - mkdir ${TRAVIS_BUILD_DIR}/build
-        - cd ${TRAVIS_BUILD_DIR}/build
-        - cmake ..
-             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
-             -DMFEM_MPI_NP=$NPROCS
-        - make -j3 mfem examples
-        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
-        - make -j3
-        - ctest --output-on-failure
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    # ========================
-    #      Mac OS X tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Serial + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=YES
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Serial"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=NO
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Parallel + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=YES
-           MPI=YES
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-           NPROCS=4
-           TMPDIR=/tmp
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-          - $HOME/local-cached
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Parallel"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=NO
-           MPI=YES
-           CODECOV=YES
-           MFEM_TEST_TARGET=test
-           NPROCS=4
-           TMPDIR=/tmp
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-          - $HOME/local-cached
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-before_install:
-   # No addon for brew yet, have to install OSX packages this way.
-   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
-   #      brew install open-mpi;
-   #   fi
-
-   # Disable ccache while building dependencies that are cached:
-   - echo "before \$PATH = $PATH";
-     export PATH=${PATH//\/usr\/lib\/ccache:/};
-     echo "after  \$PATH = $PATH"
-
-   # On Mac OS X, build and cache OpenMPI 2.1.6:
-   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
-        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
-           mkdir -p $HOME/builds && cd $HOME/builds &&
-           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
-           tar jxf openmpi-2.1.6.tar.bz2 &&
-           mkdir openmpi-build && cd openmpi-build &&
-           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
-           make -j3 all && make install;
-        fi;
-        PATH=$HOME/local-cached/bin:$PATH;
-        cd $TRAVIS_BUILD_DIR;
-     fi
-
-   # Update environment to find g++ 4.9 installation first.
-   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
-   #      mkdir -p latest-gcc-symlinks;
-   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
-   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
-   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
-   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
-   #   fi
-
-   # Install tool to upload code coverage reports to coveralls.io
-   - if [ "$CODECOV" == "YES" ]; then
-        export PYTHONUSERBASE=$HOME/local;
-        pip install --user cpp-coveralls;
-        pip install --user pyyaml;
-        PATH=$HOME/local/bin:$PATH;
-     fi
-
-install:
-   # Set MPI compilers, print compiler version
-   - if [ $MPI == "YES" ]; then
-        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-           export MPICH_CC="$CC";
-           export MPICH_CXX="$CXX";
-        else
-           export OMPI_CC="$CC";
-           export OMPI_CXX="$CXX";
-           mpic++ --showme:version;
-        fi;
-        mpic++ -v;
-     else
-        $CXX -v;
-     fi
-
-   # Back out of the mfem directory to install the libraries
-   - cd ..
-
-   # hypre
-   - if [ $MPI == "YES" ]; then
-        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
-           wget $HYPRE_URL;
-           rm -rf $HYPRE_TOP_DIR;
-           tar xvzf $HYPRE_ARCHIVE;
-           cd $HYPRE_TOP_DIR/src;
-           ./configure --disable-fortran CC=mpicc CXX=mpic++;
-           make -j3;
-           cd ../..;
-        else
-            echo "Reusing cached $HYPRE_TOP_DIR/";
-        fi;
-        ln -s $HYPRE_TOP_DIR hypre;
-     else
-        echo "Serial build, not using hypre";
-     fi
-
-   # METIS, use a mirror because the original source server is not always up.
-   # Original url:
-   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
-   - if [ $MPI == "YES" ]; then
-        if [ ! -e metis-4.0/libmetis.a ]; then
-           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
-           tar xvzf metis-4.0.3.tar.gz;
-           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
-           rm -rf metis-4.0;
-           mv metis-4.0.3 metis-4.0;
-        else
-           echo "Reusing cached metis-4.0/";
-        fi;
-     fi
-
-   # Re-enable ccache on linux; enable ccache on mac os:
-   - if [ $TRAVIS_OS_NAME == "linux" ]; then
-        export PATH="/usr/lib/ccache:$PATH";
-     else
-        if [ $TRAVIS_OS_NAME == "osx" ]; then
-           export PATH="/usr/local/opt/ccache/libexec:$PATH";
-        fi;
-     fi
-
-   - printf "which \$CC  = "; which $CC;
-     printf "which \$CXX = "; which $CXX
-
-script:
-   # Compiler
-   - if [ $MPI == "YES" ]; then
-        export MYCXX=mpic++;
-        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
-     else
-        export MYCXX="$CXX";
-        export MAKE_CXX_FLAG=CXX=$MYCXX;
-     fi
-
-   # Print the compiler version
-   - $MYCXX -v
-
-   # Set some variables
-   - cd $TRAVIS_BUILD_DIR;
-     CPPFLAGS="";
-     SKIP_TEST_DIRS="";
-     if [ "$CODECOV" == "YES" ]; then
-        CPPFLAGS="--coverage -g";
-     fi;
-     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
-        CPPFLAGS+=" -pedantic -Wall -Werror";
-     fi
-
-   # Configure the library
-   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
-        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
-   # Show the configuration
-   - make info
-   # Build the library
-   - make -j3
-   # Build the examples and the miniapps
-   - make -j3 all
-   # Run tests
-   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
-
-after_success:
-   - if [ "$CODECOV" == "YES" ]; then
-        coveralls --include fem --include general --include linalg --include
-           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
-     fi

CHANGELOG

@@ -8,40 +8,71 @@
                                https://mfem.org
 
 
-Version 4.2.1 (development)
+Version 4.3.1 (development)
 ===========================
-- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
 
-- Memory management:
-  * Added method Device::SetMemoryTypes that can be used to change the default
-    host and device MemoryTypes before Device setup.
-  * In class MemoryManager, added methods GetDualMemoryType and
-    SetDualMemoryType; dual MemoryTypes are used to determine the second
-    MemoryType (host or device) when only one MemoryType is specified in methods
-    of class Memory.
-  * Added Memory constructor for setting both the host and device MemoryTypes.
-  * Switched the default behavior of device memory allocations so that they
-    are deferred until the device pointer is needed.
-  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
-    corresponding allocator that can be set with the method
-    MemoryManager::SetUmpireDevice2AllocatorName.
-  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
 
-- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
+Version 4.3, released on July 29, 2021
+======================================
 
-- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
+Discretization improvements
+---------------------------
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.
 
-- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Extended the support for field transfer between high-order and low-order
+  refined finite element spaces to include: dual fields and H1 fields (both
+  primary and dual). These are illustrated in the lor-transfer miniapp.
 
-- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of
-  the domain decomposition.
+- Improved libCEED integration, including support for VectorCoefficient,
+  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Changed the interface for the error estimator and implemented the Kelly error
+  indicator for scalar-valued problems, supported in serial and parallel builds.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+Linear and nonlinear solvers
+----------------------------
+- Added support for AMG preconditioners on GPUs based on the hypre library
+  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
+  MFEM examples that use hypre have been ported to support this functionality.
+  The GPU preconditioners require that both hypre and MFEM are built with CUDA
+  support. Hypre builds with CUDA and unified memory are also supported and
+  can be used with `-d cuda:uvm` as a command-line option.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+
+- Generalized the Multigrid class to support non-geometric multigrid. Previous
+  functionality, based on FiniteElementSpaceHierarchy, is now available in the
+  derived class GeometricMultigrid.
 
 - Introduced solver interface for linear problems with constraints, a few
   concrete solvers that implement the interface, and a demonstration of their
@@ -52,19 +83,18 @@ Version 4.2.1 (development)
   as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
   Example 3p and linalg/auxiliary.?pp.
 
-- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
+- Improved interface for using the Ginkgo library, including: support for matrix-
+  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
+  support, and reduction of unnecessary data copies.
 
-- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p, ex3p, and ex4p, and may not work in more general settings.
 
-- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.
 
 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
   See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -72,6 +102,17 @@ Version 4.2.1 (development)
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
   ESDIRK-32, and A-stable ESDIRK-33.
 
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+Meshing improvements
+--------------------
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
   of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
   parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -80,109 +121,26 @@ Version 4.2.1 (development)
   NC data files are compatible with serial code, e.g., can be viewed with serial
   GLVis. Loading of legacy NC mesh files is still supported.
 
-- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
-- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
-- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
-- Generalized the Multigrid class to support non-geometric multigrid. The
-  previous functionality, based on FiniteElementSpaceHierarchy, is now available
-  in the derived class GeometricMultigrid.
-
-- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
-     and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
-- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
-- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
-- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
-- Added HIP support to the CMake build system.
-
-- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
-- Added support for reading VTK meshes in XML format.
-
-- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
-- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
+  unstructured high-order meshes with general high-order finite element fields
+  on them. When enabled, mfem can convert data collections to/from FMS data
+  collections in memory. In addition, an FMS data collection class was added so
+  the convert-dc miniapp can read and generate data files in FMS format.
 
 - Added new mesh quality metrics and improved the untangling capabilities of the
   TMOP-based mesh optimization algorithms.
 
-- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
-- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
-- Changed the interface for the error estimator.
-
-- Implemented the Kelly error indicator for scalar-valued problems, supported
-  in serial and parallel builds.
-
-- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
-- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
-- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
-- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
+- The TMOP mesh optimization algorithms were extended to GPU:
+  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
+  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+  * Kernels for normalization and limiting have been added
+  * The AdvectorCG now also supports AssemblyLevel::PARTIAL
 
 - Added support for creating refined meshes for all element types (e.g. by
   splitting high-order elements into low-order refined elements), including
   mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
   supports meshes with any element geometry.
 
-- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
-
-- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
-- Added a new, very simple example (ex0 and parallel version ex0p). This
-  example solves a simple Poisson problem using H1 elements (the same problem as
-  ex1), but is intended to be extremely simple and approachable for new users.
-
 - Meshes consisting of any type of elements (including mixed meshes) can be
   converted to all-simplex meshes using Mesh::MakeSimplicial.
 
@@ -195,56 +153,147 @@ Version 4.2.1 (development)
   requisite periodic vertex mappings can be created with
   Mesh::CreatePeriodicVertexMapping.
 
-- Added support for transferring dual fields between high-order and low-order
-  refined finite element spaces using the transposed versions of the
-  L2ProjectionGridTransfer operators. This functionality is illustrated in the
-  lor-transfer miniapp.
-
-- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
-- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p and may not work in more general settings.
-
-- Added support for transferring fields (primary and dual) between high-order
-  and low-order refined H1 finite element spaces using the
-  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
-  through the lor-transfer miniapp when run with the -h1 option.
-
-- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).
 
 - Added sample meshes in the `data` subdirectory showing the reference elements
   of the six currently supported element types; ref-segment.mesh,
   ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
   ref-prism.mesh.
 
+High-performance computing
+--------------------------
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Added method Device::SetMemoryTypes that can be used to change the default
+  host and device MemoryTypes before Device setup.
+
+- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
+  dual MemoryTypes are used to determine the second MemoryType (host or device)
+  when only one MemoryType is specified in methods of class Memory.
+
+- Added Memory constructor for setting both the host and device MemoryTypes.
+
+- Switched the default behavior of device memory allocations so that they are
+  deferred until the device pointer is needed.
+
+- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
+  allocator that can be set with the method SetUmpireDevice2AllocatorName.
+
+- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
+
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new, very simple example (ex0 and parallel version ex0p). This example
+  solves a simple Poisson problem using H1 elements (the same problem as ex1),
+  but is intended to be extremely simple and approachable for new users.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.
+
 - Added a high-order extension of the shifted boundary method to solve PDEs on
   non body-fitted meshes. This is illustrated in the new Shifted Diffusion
   miniapp, see miniapps/shifted/diffusion.cpp.
 
+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.
+
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of the
+  domain decomposition.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+Improved testing
+----------------
+- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
+
+- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
+
+- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
+  support.
+
+- Added a set of suggested git hooks for developers in config/githooks.
+
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+Miscellaneous
+-------------
+- The following integrations have updated minimum version requirements:
+  * CUDA >= 10.1.168
+  * Ginkgo >= 1.4.0
+  * GSLIB >= 1.0.7
+  * HIOP >= 0.4
+  * HYPRE >= 2.20.0 for mixedint support
+  * HYPRE >= 2.22.0 for CUDA support
+  * libCEED >= 0.8
+  * PETSc >= 3.15.0 for CUDA support
+  * RAJA >= 0.13.0
+  see INSTALL for more details.
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for reading VTK meshes in XML format.
+
 - Added makefile rule to generate TAGS table for vi or Emacs users.
 
-libCEED integration improvements
---------------------------------
-- Refactor the libCEED integration
+- Added HIP support to the CMake build system.
 
-- Add support for VectorCoefficient with libCEED backends.
+- Various other simplifications, extensions, and bugfixes in the code.
 
-- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
-  libCEED backends.
+API changes
+-----------
+- Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
+  and `L2FaceRestriction`.
+  In order to conform with the semantic of `MultTranspose` in `mfem::Operator`,
+  `mfem::FaceRestriction::MultTranspose` now sets instead of adding values, and
+  `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
+  `mfem::FaceRestriction::MultTranspose`.
 
 
 Version 4.2, released on October 30, 2020
 =========================================
 
-High-Performance Computing
+High-performance computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
   code, which can now take advantage of specific classes on the following
@@ -326,9 +375,6 @@ Linear and nonlinear solvers
   matrix with the function HypreParMatrixFromBlocks. This could be useful for
   solving block systems with parallel direct solvers such as STRUMPACK.
 
-- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
   preconditioner. The latter requires hypre version 2.19.0 or later.
 
@@ -439,7 +485,7 @@ New and updated examples and miniapps
     L2, with partial assembly support in Example 24/24p.
   * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
 
-Data management and Visualization
+Data management and visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
   Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection

CMakeLists.txt

@@ -16,7 +16,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO) 
+if (MFEM_USE_GINKGO)
   set(CMAKE_CXX_STANDARD 14)
 endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -54,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.2.1)
+set(${PROJECT_NAME}_VERSION 4.3.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,7 +102,7 @@ if (MFEM_USE_CUDA)
    endif()
    enable_language(CUDA)
    set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO) 
+   if (MFEM_USE_GINKGO)
      set(CMAKE_CUDA_STANDARD 14)
    endif()
    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
@@ -246,6 +246,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
     message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
   endif()
   find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()
 
 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -266,6 +267,15 @@ if (MFEM_USE_SUNDIALS)
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
+# EPIC
+if (MFEM_USE_EPIC)
+  if (NOT (MFEM_USE_MPI AND MFEM_USE_SUNDIALS AND MFEM_USE_LAPACK) )
+    message(FATAL_ERROR " *** EPIC requires that MPI, SUNDIALS and LAPACK be enabled.")
+  else()
+    find_package(EPIC REQUIRED SUNDIALS NVector_Serial NVector_Parallel BLAS LAPACK)
+  endif()
+endif()
+
 # Mesquite
 if (MFEM_USE_MESQUITE)
   find_package(Mesquite REQUIRED)
@@ -330,6 +340,10 @@ if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
 
+if (MFEM_USE_FMS)
+    find_package(FMS REQUIRED fms )
+endif()
+
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Axom)
@@ -423,10 +437,11 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS EPIC PETSC
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
     CUSPARSE MKL_CPARDISO AMGX CALIPER)
+
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -445,6 +460,9 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
   message(STATUS "MFEM: using package OpenMP")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
 endif()
 
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")

CONTRIBUTING.md

@@ -4,7 +4,9 @@
 
 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
@@ -63,6 +65,8 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
   development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
   with regards to documentation and code styling.
+- Please do not commit large/binary files to the central repository (use a fork
+  instead).
 - Pull requests  should be issued toward `mfem:master`. Make sure
   to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
@@ -71,6 +75,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
   reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
   fix issues and implement improvements.
+- During review there should be no force pushes/rewriting history in the branch.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -91,8 +96,8 @@ The MFEM source code has the following structure:
 ```
   .
   ├── config
-  │   └── cmake
-  │       └── ...
+  │   ├── cmake
+  │   └── githooks
   ├── data
   ├── doc
   ├── examples
@@ -129,10 +134,10 @@ The MFEM source code has the following structure:
   └── tests
       ├── convergence
       ├── gitlab
+      ├── mem_manager
       ├── par-mesh-format
       ├── scripts
       └── unit
-          └── ...
 ```
 
 #### Main directories and classes
@@ -363,6 +368,10 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
   the PR and work with the author to implement improvements and fix issues.
 
+- Once the `ready-for-review` label has been applied and reviewers have been
+  assigned, the PR is considered under review. To help with the review process
+  there should be no force pushes/rewriting history in the branch.
+
 - After approval, the PR is [tested](#masternext-workflow) for a week with
   other approved PRs in the `mfem:next` branch.
 
@@ -370,16 +379,20 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   `mfem:next`, see the [README](tests/scripts/README) file in that directory
   for more details.
 
-- Track the Travis CI, Github Actions and Appveyor [continuous integration](#automated-testing)
+- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
   builds at the end of the PR. These should generally run clean, so address any
   errors as soon as possible. Please ask if you are unsure how to do that.
 
-- Note that some tests, such as the `branch-history` check in Travis and Github
-  Actions are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in GitHub Actions
+  are safeguards that are allowed to fail in certain cases.
 
 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in Travis and Github Actions enforce MFEM-specific rules which are
-  explained in the error messages and the `tests/scripts` directory.
+  checks in GitHub Actions enforce MFEM-specific rules which are explained in
+  the error messages and the `tests/scripts` directory.
+
+- Also note that the tests `branch-history` and `repos-checks` found in GitHub
+  Actions can be triggered automatically before each push using git hooks. See
+  the [git hooks README](config/githooks/README.md) for a detailed explanation.
 
 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
   one of the _LLNL developers_ for details.
@@ -399,7 +412,7 @@ Before a PR can be merged, it should satisfy the following:
     - [ ] Does `make` or `cmake` have a new target?
     - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.travis.yml`
+    - [ ] `.github`
     - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
     - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
@@ -516,7 +529,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.travis.yml`
+    - [ ] `.github`
     - [ ] `.appveyor.yml`
 - [ ] Update the `CHANGELOG` to organize all release contributions
 - [ ] Review the whole source code once over
@@ -578,14 +591,17 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.
 
-### Linux and Mac smoke tests
-We use Travis CI and Github Actions to drive the default tests on the `master`
-and `next` branches. See the `.travis` file and the logs at
-[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).
+In addition, developers can set local git hooks to run some quick checks on
+commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
+directory.
 
-Testing using Travis CI and Github Actions should be kept lightweight, as there
-is a time constraint on jobs. Two virtual machines are configured - Mac (OS X)
-and Linux.
+### Linux and Mac smoke tests
+We use GitHub Actions to drive the default tests on the `master` and `next`
+branches. See the `.github/workflows` files and the logs at
+[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+
+Testing using GitHub Actions should be kept lightweight, as there is a time
+constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
 
 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

INSTALL

@@ -474,7 +474,7 @@ MFEM_USE_HIP = YES/NO
    Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
    interface for portability developed by AMD that can target both AMD and
    NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
    option uses the HIP_* build options, see below.
 
 MFEM_USE_RAJA = YES/NO
@@ -516,6 +516,13 @@ MFEM_USE_CALIPER = YES/NO
    profiling at runtime with Caliper's configuration API. Alternatively, one
    can configure Caliper through environment variables or config files.
 
+MFEM_USE_FMS = YES/NO
+   Enables support for the FMS library which consists of the DataCollection
+   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
+   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   DataCollection class, see the header file fem/fmsconvert.hpp.
+
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -540,8 +547,9 @@ The specific libraries and their options are:
   See also the "Specific options for hypre" section at the end of this file.
   URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b,
-            HYPRE >= 2.20.0 for '--enable-mixedint' support.
+  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
+            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
+            HYPRE >= 2.22.0   (HYPRE built with CUDA)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -615,7 +623,7 @@ The specific libraries and their options are:
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0. 
+  Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX
@@ -753,6 +761,11 @@ The specific libraries and their options are:
   URL: https://zlib.net
   Options: ZLIB_OPT, ZLIB_LIB.
 
+- FMS (optional), used when MFEM_USE_FMS = YES.
+  URL: https://github.com/CEED/FMS
+  Options: FMS_OPT, FMS_LIB.
+  Versions: FMS >= 0.2.
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -884,6 +897,7 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
+MFEM_USE_FMS
 
 The following options are CMake specific:
 
@@ -938,6 +952,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - UMPIRE
  - AXOM - Used when MFEM_USE_SIDRE is enabled
  - CALIPER
+ - FMS
 
 The following built-in CMake packages are also used:
 
@@ -955,7 +970,7 @@ config/config.hpp.in:
 
    cp config/config.hpp.in config/_config.hpp
 
-The file config/_config.hpp can then be edited to enable desired options.  The
+The file config/_config.hpp can then be edited to enable desired options. The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.
 
@@ -963,7 +978,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.
 
 MFEM will expect the launcher command, plus the command line option to allow it

config/XSDKDefaults.cmake

@@ -256,6 +256,10 @@ IF (DEFINED TPL_ENABLE_SIDRE)
   SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_FMS)
+  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_CONDUIT)
   SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -29,6 +29,7 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_EPIC @MFEM_USE_EPIC@)
 set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
@@ -44,6 +45,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
+set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)

config/cmake/config.hpp.in

@@ -119,6 +119,9 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD
 
+// Enable MFEM functionality based on the FMS library
+#cmakedefine MFEM_USE_FMS
+
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 
@@ -162,6 +165,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 #cmakedefine MFEM_USE_SUNDIALS
 
+// Enable MFEM functionality based on the EPIC libraries.
+#cmakedefine MFEM_USE_EPIC
+
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 

config/cmake/modules/FindEPIC.cmake

@@ -0,0 +1,21 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - EPIC_FOUND
+#   - EPIC_LIBRARIES
+#   - EPIC_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(EPIC EPIC EPIC_DIR
+       "include" Epic.h "lib" epic1.0.0
+  "Paths to headers required by EPIC." "Libraries required by EPIC.")
+

config/cmake/modules/FindFMS.cmake

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - FMS_FOUND
+#   - FMS_LIBRARIES
+#   - FMS_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(FMS FMS FMS_DIR
+  "include" fms.h "lib" fms
+  "Paths to headers required by FMS." "Libraries required by FMS.")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -759,7 +759,7 @@ function(mfem_export_mk_files)
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_EPIC MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
       MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
       MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI

config/config.hpp.in

@@ -85,6 +85,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS
 
+// Enable MFEM functionality based on the EPIC libraries.
+// #define MFEM_USE_EPIC
+
 // Enable MFEM functionality based on the Mesquite library.
 // #define MFEM_USE_MESQUITE
 
@@ -117,6 +120,9 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD
 
+// Enable FMS support
+// #define MFEM_USE_FMS
+
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT
 

config/config.mk.in

@@ -29,6 +29,7 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_EPIC          = @MFEM_USE_EPIC@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
@@ -43,6 +44,7 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
+MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@

config/defaults.cmake

@@ -30,6 +30,7 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_EPIC "Enable EPIC usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
@@ -45,6 +46,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
+option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -96,6 +98,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -109,6 +116,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")
 
+set(EPIC_DIR "${MFEM_DIR}/../epic-cpp/instdir" CACHE PATH
+    "Path to the EPIC library.")
+
 set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
     "Path to the Mesquite library.")
 
@@ -132,10 +142,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
     "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")    
+    "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")    
+#     "Additional libraries required by MUMPS.")
 
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
     "Path to the STRUMPACK library.")
@@ -187,6 +197,12 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
 
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")
 
+set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
+    "Path to the FMS library.")
+# If FMS is built with Conduit:
+# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
+#     "Additional packages required by FMS.")
+
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
     "Path to the Conduit library.")
 

config/defaults.mk

@@ -122,6 +122,7 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_EPIC          = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
@@ -136,6 +137,7 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
+MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -174,6 +176,10 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
+ifeq (YES,$(MFEM_USE_CUDA))
+   # This is only necessary when hypre is built with cuda:
+   HYPRE_LIB += -lcusparse -lcurand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -226,6 +232,11 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
+# EPIC library configuration
+MESQUITE_DIR = @MFEM_DIR@/../epic-cpp/instdir
+MESQUITE_OPT = -I$(EPIC_DIR)/include
+MESQUITE_LIB = -L$(EPIC_DIR)/lib -lepic1.0.0
+
 # MESQUITE library configuration
 MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
 MESQUITE_OPT = -I$(MESQUITE_DIR)/include
@@ -357,6 +368,11 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr
 
+# FMS and required libraries configuration
+FMS_DIR = $(MFEM_DIR)/../fms
+FMS_OPT = -I$(FMS_DIR)/include
+FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
+
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit

config/githooks/README.md

@@ -0,0 +1,41 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+
+This directory contains recommended git hooks, which are scripts that can be
+used to improve your development experience with MFEM:
+
+### The hooks
+
+* `pre-commit` is a hook that will be applied before each commit and run
+`astyle` on the code. This will ensure that your changes comply with the MFEM
+code styling guidelines.
+
+* `pre-push` is a hook that will be applied before each push to run a quick set
+of tests that verify that your files headers are in compliance, and that you did
+not add any large files to the repo.
+
+### Setup
+
+To setup the git hooks, run `make hooks`, which creates symlinks to the hooks in
+the `.git/hooks` directory. Individual hooks can be enabled by manually creating
+symlinks.
+
+(You may also copy the scripts directly and customize them further, but this way
+you may miss additional updates in the future.)
+
+### Failures
+
+The `branch-history` check can fail in some cases when the history is OK. For
+example, when a large number of files were modified for a legitimate reason, or
+when a picture was added for documentation.
+
+If that is the case, make sure the failure is indeed justified, and rerun the
+push command with the `--no-verify` option. This will skip the hooks, allowing
+you to push those changes.

config/githooks/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Apply automated code formatting
+make -C $(git rev-parse --show-toplevel) style

config/githooks/pre-push

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+option=${1:-""}
+
+if [[ "${option}" == "--help" ]]; then
+  echo "This script runs checks on the repository."
+  echo "It has 2 modes: with and without an option."
+  echo ""
+  echo "Options are used in GitHub Actions and can be:"
+  echo "  --copyright"
+  echo "  --license"
+  echo "  --release"
+  echo "  --style"
+  echo "  --history"
+  echo ""
+  echo "As a githook, the script is used without options."
+  echo "In that case, it will run all the checks except style."
+  echo ""
+  echo "Use --help to print this help message."
+fi
+
+cd $(git rev-parse --show-toplevel)
+
+# copyright check
+copyright=true
+if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
+  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
+    echo "Please update the following files to Copyright (c) 2010-2021:"
+    cat matches.txt
+    copyright=false
+  fi
+fi
+
+# license check
+license=true
+if [[ "${option}" == "--license" || "${option}" == "" ]]; then
+  if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt; then
+    echo "Please update the following files to the BSD-3 license:"
+    cat matches.txt
+    license=false
+  fi
+fi
+
+# release check
+release=true
+if [[ "${option}" == "--release" || "${option}" == "" ]]; then
+  if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
+  then
+    echo "Please update the following files to LLNL-CODE-806117:"
+    cat matches.txt
+    release=false
+  fi
+fi
+
+# wrap-up
+code=0
+if ! $copyright ; then
+  echo "copyright check failed, unroll log for details"
+  code=1
+fi
+if ! $license ; then
+  echo "license check failed, unroll log for details"
+  code=1
+fi
+if ! $release ; then
+  echo "release check failed, unroll log for details"
+  code=1
+fi
+
+# `code-style` is not just a check, it will actually reformat the code if
+# necessary. This means that if one pushes while the repo is in dirty state
+# (changes not staged), those changes may be mixed with format changes.
+# To activate this, you will need to hard-copy this hook script in the hook
+# directory and uncomment only then. (See README.md)
+#
+## style check
+#if [[ "${option}" == "--style" || "${option}" == "" ]]; then
+if [[ "${option}" == "--style" ]]; then
+  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
+    cd tests/scripts
+    if ! ./runtest code-style; then code=1; fi
+    cd -
+  else
+    echo "Warning: astyle not found or version is not 2.05.1"
+  fi
+fi
+
+# branch-history
+if [[ "${option}" == "--history" || "${option}" == "" ]]; then
+  git fetch origin master:master
+  cd tests/scripts
+  if ! ./runtest branch-history; then code=1; fi
+  cd -
+fi
+
+exit $code

config/test.mk

@@ -57,22 +57,27 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code
+# Test runs of the examples/miniapps with parameters - check exit code:
+# 0 means success, 255 means the test was skipped, anything else means error
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
-   if [ "$$3" = 0 ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
-   rm -f $(1).stderr; exit $$3
+   err="$$3"; \
+   if [ "$$3" = 0 ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
+   rm -f $(1).stderr; exit $$err
 
 # Test runs of the examples/miniapps - check exit code and if a file exists
+# See mfem-test for the interpretation of the error code
 mfem-test-file = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
    rm -f $(1).stderr; exit $$err
 
 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -80,6 +85,26 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
+ifeq ($(MFEM_USE_CUDA),YES)
+.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
+test-par-YES: test-par-YES-cuda
+test-par-NO:  test-par-NO-cuda
+test-par-YES-cuda: test-par-cuda test-ser-cuda
+test-par-NO-cuda:  test-ser-cuda
+test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
+test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
+test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
+endif
+ifeq ($(MFEM_USE_HIP),YES)
+.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
+test-par-YES: test-par-YES-hip
+test-par-NO:  test-par-NO-hip
+test-par-YES-hip: test-par-hip test-ser-hip
+test-par-NO-hip:  test-ser-hip
+test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
+test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
+test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
+endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec

data/star-q3.fms

@@ -0,0 +1,246 @@
+FMS: 100
+DataCollection/Name: star
+DataCollection/NumberOfFieldDescriptors: 1
+DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
+DataCollection/FieldDescriptors/0/ComponentName: volume
+DataCollection/FieldDescriptors/0/Type: 0
+DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
+DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
+DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
+DataCollection/FieldDescriptors/0/NumDofs: 211
+DataCollection/NumberOfFields: 1
+DataCollection/Fields/0/Name: Coords
+DataCollection/Fields/0/LayoutType: 0
+DataCollection/Fields/0/NumberOfVectorComponents: 2
+DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
+DataCollection/Fields/0/Data/Size: 422
+DataCollection/Fields/0/Data/Type: FMS_DOUBLE
+DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
+1.309020, -0.809017, -0.500000, 
+-0.809017, -1.618030, 0.309017, 
+-0.500000, 1.309020, 0.519420, 
+1.154510, 0.809019, 0.147680, 
+-0.095492, -0.654508, -0.415586, 
+-1.213520, -1.213520, -0.392210, 
+-0.654508, -0.095492, 0.139949, 
+0.809019, 1.154510, 0.660184, 
+-0.264063, -0.800064, -0.231060, 
+0.663691, 0.183114, 0.317639, 
+0.543082, 0.598483, 0.345112, 
+0.478298, 0.027703, 0.095229, 
+0.012368, -0.092534, -0.334412, 
+-0.313767, -0.140526, -0.293881, 
+-0.534056, -0.660290, -0.537646, 
+-0.655590, -0.121396, -0.274504, 
+-0.346497, -0.296570, 0.004737, 
+-0.098835, 0.069287, 0.082675, 
+0.318799, 0.467183, 0.564505, 
+0.595190, 0.846237, 0.671735, 
+1.051500, 1.103010, 0.964008, 
+0.821603, 1.257520, 1.206010, 
+1.142350, 0.975686, 0.781273, 
+0.717257, 0.475684, 0.642352, 
+0.268930, 0.211049, 0.174181, 
+0.039345, -0.147746, -0.177481, 
+-0.365164, -0.230328, -0.551503, 
+-0.603005, -0.497587, -0.389864, 
+-0.757514, -0.706011, -0.675487, 
+-0.528946, -0.943851, -1.078690, 
+-1.087600, -0.955467, -1.483190, 
+-1.348360, -1.483190, -1.348360, 
+-1.085930, -0.938010, -0.943851, 
+-1.078690, -0.681476, -0.540944, 
+-0.757514, -0.706011, -0.540614, 
+-0.367058, -0.551503, -0.603005, 
+-0.365164, -0.230328, -0.138552, 
+-0.206896, 0.174181, 0.039345, 
+0.268468, 0.222269, 0.475684, 
+0.642352, 0.759791, 0.719381, 
+1.142350, 0.975686, 1.257520, 
+1.206010, 0.972837, 0.836119, 
+1.051500, 1.103010, 0.214572, 
+0.407449, 0.288323, 0.449827, 
+-0.086700, -0.027358, -0.200560, 
+-0.166595, -0.271802, -0.418426, 
+-0.426131, -0.551441, -0.096117, 
+-0.206969, -0.027946, -0.184969, 
+0.211136, 0.260131, 0.407172, 
+0.430781, 0.718277, 0.885068, 
+0.753103, 0.957692, 0.866273, 
+1.024530, 0.934099, 1.093820, 
+0.348422, 0.524463, 0.404903, 
+0.587376, 0.054525, 0.146431, 
+-0.078026, -0.007795, -0.329488, 
+-0.302967, -0.488115, -0.439332, 
+-0.498515, -0.453527, -0.633059, 
+-0.570251, -0.655787, -0.791132, 
+-0.802013, -0.956872, -1.094720, 
+-1.208560, -1.207720, -1.339910, 
+-0.693371, -0.795404, -0.803594, 
+-0.945068, -0.464668, -0.631721, 
+-0.458968, -0.547876, -0.332878, 
+-0.485696, -0.286385, -0.408481, 
+0.053021, -0.058200, 0.110846, 
+-0.002086, 0.381384, 0.416784, 
+0.551133, 0.613261, 0.872474, 
+0.901208, 1.038300, 1.084660, 
+0.737459, 0.751250, 0.890002, 
+0.915210, 0.010915, 0.000000, 
+0.951057, 0.951057, 0.587785, 
+1.538840, -0.587785, 0.000000, 
+-0.951057, -1.538840, -0.951057, 
+-0.015847, 0.475529, 0.951057, 
+0.492248, 1.244950, 1.063310, 
+0.274399, 0.293893, -0.293892, 
+-0.296404, -1.063310, -1.244950, 
+-0.453865, -0.951057, -0.475529, 
+0.466620, 0.792932, -0.013913, 
+-0.748783, -0.497528, 0.021382, 
+-0.017158, 0.172591, 0.330125, 
+0.458568, 0.457971, 0.137740, 
+0.299049, 0.588394, 0.667324, 
+0.432341, 0.634346, 0.117322, 
+0.193603, 0.211702, 0.098278, 
+-0.199438, -0.077304, -0.082243, 
+-0.216296, -0.458634, -0.592374, 
+-0.563926, -0.680404, -0.135751, 
+-0.302942, -0.469005, -0.453640, 
+-0.182727, -0.314240, 0.024270, 
+0.021546, 0.158510, 0.317019, 
+0.485799, 0.492951, 0.792548, 
+0.634038, 0.951057, 0.951057, 
+0.777915, 0.613430, 0.951057, 
+0.951057, 0.793994, 0.635800, 
+1.049020, 1.146990, 1.084480, 
+0.924310, 1.440880, 1.342910, 
+1.380330, 1.221820, 0.948209, 
+0.856297, 0.746293, 0.904802, 
+0.476242, 0.393234, 0.489821, 
+0.391857, 0.194471, 0.075751, 
+0.097964, 0.195929, -0.097964, 
+-0.195928, -0.173234, -0.078922, 
+-0.489821, -0.391856, -0.467007, 
+-0.397859, -0.746293, -0.904802, 
+-0.945206, -0.849559, -1.380330, 
+-1.221820, -1.440880, -1.342910, 
+-1.100830, -0.923191, -1.049020, 
+-1.146990, -0.774515, -0.621542, 
+-0.951057, -0.951057, -0.803055, 
+-0.635255, -0.951057, -0.951057, 
+-0.792548, -0.634038, -0.454301, 
+-0.479369, -0.158510, -0.317019, 
+0.149331, 0.178643, 0.295860, 
+0.306275, 0.246225, 0.406610, 
+0.367954, 0.498458, -0.014929, 
+0.084917, -0.094272, -0.000726, 
+-0.245374, -0.340755, -0.435351, 
+-0.490564, -0.176355, -0.294974, 
+-0.148010, -0.328246, 0.136248, 
+0.182883, 0.328957, 0.309903, 
+0.646446, 0.622546, 0.800859, 
+0.801803, 0.643087, 0.654473, 
+0.796963, 0.816799, 0.748189, 
+0.898148, 0.819092, 0.970033, 
+1.049540, 1.162660, 1.115310, 
+1.266700, 0.539960, 0.728396, 
+0.656318, 0.783152, 0.302716, 
+0.403858, 0.219006, 0.309252, 
+0.009667, 0.107161, -0.118211, 
+0.011786, -0.284704, -0.179858, 
+-0.411567, -0.296676, -0.570168, 
+-0.635348, -0.726784, -0.793211, 
+-1.016860, -1.112970, -1.170910, 
+-1.281530, -0.737860, -0.851723, 
+-0.883732, -0.995070, -0.653153, 
+-0.769939, -0.631918, -0.797688, 
+-0.616289, -0.806819, -0.638485, 
+-0.790356, -0.136399, -0.322769, 
+-0.165339, -0.309622]
+DataCollection/Mesh/PartitionInfo/Size: 2
+DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
+DataCollection/Mesh/PartitionInfo/Values: [0, 1]
+DataCollection/Mesh/NumDomainNames: 1
+DataCollection/Mesh/NumComponents: 1
+DataCollection/Mesh/NumTags: 0
+DataCollection/Mesh/DomainNames/0/Name: Domain
+DataCollection/Mesh/DomainNames/0/NumDomains: 1
+DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
+DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
+11, 26, 14, 
+14, 0, 27, 
+14, 27, 17, 
+17, 0, 28, 
+17, 28, 20, 
+20, 0, 29, 
+20, 29, 23, 
+23, 0, 30, 
+23, 30, 11, 
+11, 1, 12, 
+1, 26, 12, 
+12, 3, 13, 
+3, 26, 13, 
+13, 2, 14, 
+2, 15, 2, 
+27, 15, 15, 
+5, 16, 5, 
+27, 16, 16, 
+4, 17, 4, 
+18, 4, 28, 
+18, 18, 7, 
+19, 7, 28, 
+19, 19, 6, 
+20, 6, 21, 
+6, 29, 21, 
+21, 9, 22, 
+9, 29, 22, 
+22, 8, 23, 
+8, 24, 8, 
+30, 24, 24, 
+10, 25, 10, 
+30, 25, 25, 1]
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
+3, 3, 4, 
+5, 6, 6, 
+7, 8, 9, 
+9, 10, 11, 
+12, 12, 13, 
+14, 0, 15, 
+16, 17, 1, 
+17, 18, 19, 
+20, 2, 20, 
+21, 22, 22, 
+23, 24, 4, 
+24, 25, 26, 
+27, 5, 27, 
+28, 29, 29, 
+30, 31, 7, 
+31, 32, 33, 
+34, 8, 34, 
+35, 36, 36, 
+37, 38, 10, 
+38, 39, 40, 
+41, 11, 41, 
+42, 43, 43, 
+44, 45, 13, 
+45, 46, 47, 
+48, 14, 48, 
+49, 15]
+DataCollection/Mesh/Components/0/Name: volume
+DataCollection/Mesh/Components/0/Dimension: 2
+DataCollection/Mesh/Components/0/NumEntities: 20
+DataCollection/Mesh/Components/0/Coordinates: Coords
+DataCollection/Mesh/Components/0/NumParts: 1
+DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
+DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
+DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
+DataCollection/Mesh/Components/0/Relations/Size: 0
+DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.2.1
+PROJECT_NUMBER         = v4.3.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

examples/CMakeLists.txt

@@ -84,8 +84,9 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
-    set(THIS_TEST_OPTIONS "-no-vis")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  if (${TEST_NAME} MATCHES "ex0p?")
+    set(THIS_TEST_OPTIONS)
   endif()
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -107,6 +108,34 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   endif()
 endforeach()
 
+# Add CUDA/HIP tests.
+set(DEVICE_EXAMPLES
+    # serial examples with device support:
+    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+    # parallel examples with device support:
+    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+set(MFEM_TEST_DEVICE)
+if (MFEM_USE_CUDA)
+  set(MFEM_TEST_DEVICE "cuda")
+elseif (MFEM_USE_HIP)
+  set(MFEM_TEST_DEVICE "hip")
+endif()
+if (MFEM_TEST_DEVICE)
+  foreach(TEST_NAME ${DEVICE_EXAMPLES})
+    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    elseif (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
+
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
   add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
@@ -130,6 +159,11 @@ if (MFEM_USE_AMGX)
   add_subdirectory(amgx)
 endif()
 
+# Include the examples/epic directory if EPIC is enabled.
+if (MFEM_USE_EPIC)
+  add_subdirectory(epic)
+endif()
+
 # Include the examples/ginkgo directory if GINKGO is enabled.
 if (MFEM_USE_GINKGO)
   add_subdirectory(ginkgo)

examples/caliper/README

@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the Caliper performance profiling library.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
+2.5.0 of Caliper is recommended, though older versions may work too).
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.

examples/epic/CMakeLists.txt

@@ -0,0 +1,64 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(EPIC_EXAMPLES_SRCS)
+list(APPEND EPIC_EXAMPLES_SRCS
+  ex16.cpp
+  )
+
+if (MFEM_USE_MPI)
+list(APPEND EPIC_EXAMPLES_SRCS
+    ex16p.cpp
+    )
+endif()
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_epic" target, see below.
+add_custom_target(test_epic
+  ${CMAKE_CTEST_COMMAND} -R epic USES_TERMINAL)
+
+# Add one executable per cpp file, adding "epic_" as prefix. Sets
+# "test_epic" as a target that depends on the given examples.
+set(PFX epic_)
+add_mfem_examples(EPIC_EXAMPLES_SRCS ${PFX} "" test_epic)
+
+# Testing.
+# The EPIC tests can be run separately using the target "test_epic"
+# which builds the examples and runs:
+#   ctest -R epic
+
+# Example 16: use the default options
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${EPIC_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
+

examples/epic/README

@@ -0,0 +1,17 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               http://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the EPIC suite of time integration.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_EPIC = YES".
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.

examples/epic/ex16.cpp

@@ -0,0 +1,610 @@
+//                                MFEM Example 16
+//                               EPIC Modification
+//
+// Compile with: make ex16
+//
+// Sample runs:  ex16
+//               ex16 -m ../../data/inline-tri.mesh
+//               ex16 -m ../../data/disc-nurbs.mesh -tf 2
+//               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
+//               ex16 -m ../../data/fichera-q2.mesh
+//               ex16 -m ../../data/escher.mesh
+//               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
+//               ex16 -m ../../data/amr-quad.mesh -o 4 -r 0
+//               ex16 -m ../../data/amr-hex.mesh -o 2 -r 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ImplicitSolveOperator;
+class JacobianOperator;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-K(u) u)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   FiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   BilinearForm *M;
+   mutable BilinearForm *K;
+   mutable BilinearForm *dK;
+   mutable BilinearForm *J_K;
+
+   SparseMatrix Mmat;
+   mutable SparseMatrix J_K_mat;
+
+   mutable CGSolver M_solver; // Krylov solver for inverting the mass matrix M
+   DSmoother M_prec;  // Preconditioner for the mass matrix M
+
+   CGSolver Jg_solver; // Krylov solver for inverting the Jacobian in the nonlinear solve
+   DSmoother Jg_prec;  // Preconditioner for the Jacobian Jg
+
+   NewtonSolver newton_solver;
+   mutable JacobianOperator *jac;
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+   mutable int nRhsMult, nSetJac, nJacMult, nImpSolve, nImpIter, nImpMult, nImpSet;
+
+public:
+   Vector u0;
+
+   ConductionOperator(FiniteElementSpace &f, double alpha, double kappa, const Vector &u);
+
+   void UpdateStats();
+   void PrintStats(ostream& out);
+
+   void ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2);
+
+   BilinearForm& GetKLambda(const Vector& u) const;
+   BilinearForm& GetdKLambda(const Vector& u) const;
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+   virtual Operator& GetGradient(const Vector &k) const;
+
+   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
+
+   virtual ~ConductionOperator();
+};
+
+class ImplicitSolveOperator : public Operator
+{
+private:
+   double dt;
+   const Vector* x;
+   ConductionOperator* oper;
+
+   const SparseMatrix* M;
+   mutable SparseMatrix* Jg;
+
+   mutable Vector u, z;
+   mutable int nMult, nSet;
+
+public:
+   ImplicitSolveOperator(ConductionOperator* oper, const SparseMatrix* M, double dt, const Vector* x);
+
+   int GetnMult() { return nMult; }
+   int GetnSet() { return nSet; }
+   virtual void Mult(const Vector &k, Vector &gk) const;
+   virtual Operator &GetGradient(const Vector &k) const;
+};
+
+class JacobianOperator : public Operator
+{
+private:
+   Operator* J;
+   Operator* M_solver;
+
+   mutable int nMult;
+   mutable Vector z;
+public:
+   JacobianOperator(Operator* J, Operator* M_solver);
+
+   int GetnMult() { return nMult; }
+
+   void ExtractJacobian(const Vector& x, std::ostream &out);
+   virtual void Mult(const Vector &k, Vector &gk) const;
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ref_levels = 2;
+   int order = 2;
+   int ode_solver_type = 8; // Exponential Euler
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver:\n\t"
+                  "1  - Forward Euler,\n\t"
+                  "2  - RK2,\n\t"
+                  "3  - RK3 SSP,\n\t"
+                  "4  - RK4,\n\t"
+                  "5  - Backward Euler,\n\t"
+                  "6  - SDIRK 2,\n\t"
+                  "7  - SDIRK 3,\n\t"
+                  "8  - EPIC (exponential euler)\n\t");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (ode_solver_type < 1 || ode_solver_type > 9)
+   {
+      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      return 3;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
+   //    command-line parameter.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   FiniteElementSpace fespace(mesh, &fe_coll);
+
+   int fe_size = fespace.GetTrueVSize();
+   cout << "Number of temperature unknowns: " << fe_size << endl;
+
+   GridFunction u_gf(&fespace);
+
+   // 5. Set the initial conditions for u. All boundaries are considered
+   //    natural.
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 6. Initialize the conduction operator and the visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ofstream omesh("ex16.mesh");
+      omesh.precision(precision);
+      mesh->Print(omesh);
+      ofstream osol("ex16-init.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16", mesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+         visualization = false;
+         cout << "GLVis visualization disabled.\n";
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *mesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
+   }
+
+   // 7. Define the ODE solver used for time integration.
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // MFEM explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
+      // EPIC 
+      case 8: ode_solver = new EPI2();break;
+      case 9: ode_solver = new EPIRK4(); break;
+   }
+
+   // Initialize integrators
+   ode_solver->Init(oper);
+
+   // 8. Perform time-integration (looping over the time iterations, ti, with a
+   //    time-step dt).
+   cout << "Integrating the ODE ..." << endl;
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   /*ofstream out_jac_an("jacobian_an.txt");
+   ofstream out_jac_fd("jacobian_fd.txt");
+   oper.ExtractJacobians(u, out_jac_fd, out_jac_an);*/
+
+   bool last_step = false;
+   int ti;
+   for (ti = 1; !last_step; ti++)
+   {
+      double dt_real = min(dt, t_final - t);
+
+      // Note that since we are using the "one-step" mode of the SUNDIALS
+      // solvers, they will, generally, step over the final time and will not
+      // explicitly perform the interpolation to t_final as they do in the
+      // "normal" step mode.
+      ode_solver->Step(u, t, dt_real);
+
+      oper.UpdateStats();
+
+      last_step = (t >= t_final - 1e-8*dt);
+
+      if (last_step || (ti % vis_steps) == 0) {
+         cout << "step " << ti << ", t = " << t << endl;
+
+         u_gf.SetFromTrueDofs(u);
+         if (visualization) {
+            sout << "solution\n" << *mesh << u_gf << flush;
+         }
+
+         if (visit) {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+   }
+   tic_toc.Stop();
+   double comp_time = tic_toc.RealTime();
+   cout << "Done, " << comp_time << "s." << endl;
+
+   // 9. Save the final solution. This output can be viewed later using GLVis:
+   //    "glvis -m ex16.mesh -g ex16-final.gf".
+   {
+      ofstream osol("ex16-final.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+
+      ofstream ostats("ex16-stats.txt");
+      ostats << "time " << comp_time << endl;
+      oper.PrintStats(ostats);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete mesh;
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al, double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL), dK(NULL), J_K(NULL), jac(NULL), z(height), u0(height),
+     nRhsMult(0), nSetJac(0), nJacMult(0), nImpSolve(0), nImpIter(0), nImpMult(0), nImpSet(0)
+{
+   const double rel_tol = 1e-8;
+
+   M = new BilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble();
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(50);
+   M_solver.SetPrintLevel(0);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(Mmat);
+
+   Jg_solver.SetRelTol(rel_tol);
+   Jg_solver.SetAbsTol(0.0);
+   Jg_solver.SetMaxIter(50);
+   Jg_solver.SetPrintLevel(0);
+   Jg_solver.SetPreconditioner(Jg_prec);
+
+   newton_solver.SetMaxIter(10);
+   newton_solver.SetRelTol(rel_tol);
+   newton_solver.SetPrintLevel(-1);
+   newton_solver.SetSolver(Jg_solver);
+   newton_solver.SetMaxIter(100);
+   newton_solver.iterative_mode = false;
+
+   alpha = al;
+   kappa = kap;
+}
+
+void ConductionOperator::UpdateStats()
+{
+   if (jac)
+   {
+      nJacMult += jac->GetnMult();
+   }
+}
+
+void ConductionOperator::PrintStats(ostream &out)
+{
+   out  << "nRhsMult " << nRhsMult << endl
+        << "nSetJac " << nSetJac << endl
+        << "nJacMult " << nJacMult  << endl
+        << "nImplicitSolve " << nImpSolve  << endl
+        << "nImplicitIter " << nImpIter << endl
+        << "nImplicitMult " << nImpMult << endl
+        << "nImplicitSet " << nImpSet << endl;
+}
+
+BilinearForm& ConductionOperator::GetKLambda(const Vector &u) const
+{
+   GridFunction conductivity_gf(&fespace);
+   conductivity_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < conductivity_gf.Size(); i++)
+   {
+      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
+   }
+
+   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
+
+   delete K;
+   K = new BilinearForm(&fespace);
+   K->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
+   K->Assemble();
+
+   return *K;
+}
+
+BilinearForm& ConductionOperator::GetdKLambda(const Vector &u) const
+{
+   GridFunction conductivity_gf(&fespace);
+   conductivity_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < conductivity_gf.Size(); i++)
+   {
+      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
+   }
+
+   // Define diffusion form with conductivity = kappa(u0)
+   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
+
+   // Define advection form with velocity = grad kappa(u0)
+   GridFunction neg_cond_gf(conductivity_gf);
+   neg_cond_gf.Neg();
+   GradientGridFunctionCoefficient velocity_coeff(&neg_cond_gf);
+
+   delete dK;
+   dK = new BilinearForm(&fespace);
+
+   dK->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
+   dK->AddDomainIntegrator(new MixedScalarWeakDivergenceIntegrator(velocity_coeff));
+   dK->Assemble();
+
+   return *dK;
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   GetKLambda(u);
+   K->Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+   nRhsMult++;
+}
+
+void ConductionOperator::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+{
+   ImplicitSolveOperator imp_oper(this, &this->Mmat, dt, &x);
+   newton_solver.SetOperator(imp_oper);
+
+   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
+   newton_solver.Mult(zero, k);
+   MFEM_VERIFY(newton_solver.GetConverged(), "Newton solver did not converge.");
+
+   nImpSolve++;
+   nImpMult += imp_oper.GetnMult();
+   nImpSet  += imp_oper.GetnSet();
+   nImpIter += newton_solver.GetNumIterations();
+}
+
+Operator &ConductionOperator::GetGradient(const Vector &u) const
+{
+   delete jac;
+   GetdKLambda(u);
+   jac = new JacobianOperator(dK, &M_solver);
+
+   nSetJac++;
+
+   return *jac;
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete M;
+   delete K;
+   delete dK;
+   delete J_K;
+   delete jac;
+}
+
+ImplicitSolveOperator::ImplicitSolveOperator(ConductionOperator *oper_, const SparseMatrix* M_, double dt_, const Vector* x_):
+   Operator(oper_->Height()), oper(oper_), M(M_), dt(dt_), x(x_), u(height), z(height), Jg(NULL), nMult(0), nSet(0)
+{ }
+
+
+void ImplicitSolveOperator::Mult(const Vector& y, Vector& gy) const
+{
+   // Compute gy = g(y) = My + dt K(lambda(u)) u
+   // with u = x + dt y
+   add(*x, dt, y, u);
+   BilinearForm& K = oper->GetKLambda(u);
+   K.Mult(u, gy);
+
+   M->AddMult(y, gy);
+
+   nMult++;
+}
+
+Operator& ImplicitSolveOperator::GetGradient(const Vector &k) const
+{
+   add(*x, dt, k, u);
+
+   BilinearForm& dK = oper->GetdKLambda(u);
+   Array<int> ess_tdof_list;
+   SparseMatrix dK_mat;
+   dK.FormSystemMatrix(ess_tdof_list, dK_mat);
+
+   delete Jg;
+   Jg = Add(1.0, *M, dt, dK_mat);
+
+   nSet++;
+   return *Jg;
+}
+
+JacobianOperator::JacobianOperator(Operator* J_, Operator* M_solver_):
+   Operator(M_solver_->Height()), J(J_), M_solver(M_solver_), z(height), nMult(0)
+{ }
+
+void JacobianOperator::Mult(const Vector &v, Vector &Jv) const
+{
+   Vector temp(v);
+   J->Mult(v, z);
+   z.Neg(); // z = -z
+   M_solver->Mult(z, Jv);
+   nMult++;
+}
+
+
+void ConductionOperator::ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2)
+{
+   int n = x.Size();
+
+   Vector e(n);
+   e = 0.0;
+
+   double eps = 1e-8;
+   Vector fx(n), fx_eps(n), x_eps(n);
+   Mult(x, fx);
+
+   DenseMatrix J(n);
+
+   for (int i = 0; i < n; i++)
+   {
+      e[i] = 1.0;
+      add(x, eps, e, x_eps);
+      Mult(x_eps, fx_eps);
+      fx_eps -= fx;
+      fx_eps /= eps;
+      J.SetCol(i, fx_eps);
+      e[i] = 0.0;
+   }
+
+   J.PrintMatlab(out);
+   GetGradient(x);
+   jac->ExtractJacobian(x, out2);
+}
+
+void JacobianOperator::ExtractJacobian(const Vector& x, std::ostream &out)
+{
+   int n = z.Size();
+
+   Vector e(n);
+   e= 0.0;
+
+   Vector J_i(n);
+   DenseMatrix J(n);
+
+   for (int i = 0; i < n; i++)
+   {
+      e[i] = 1.0;
+      Mult(e, J_i);
+      J.SetCol(i, J_i);
+      e[i] = 0.0;
+   }
+
+   J.PrintMatlab(out);
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5) { return 2.0; }
+   else                  { return 1.0; }
+}

examples/epic/ex16p.cpp

@@ -0,0 +1,494 @@
+//                         MFEM Example 16 - Parallel Version
+//                             SUNDIALS Modification
+//
+// Compile with: make ex16p
+//
+// Sample runs:
+//     mpirun -np 4 ex16p
+//     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
+//     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
+//     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
+//     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
+//     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
+//     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
+//     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
+//     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
+//     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
+//     mpirun -np 4 ex16p -m ../../data/amr-quad.mesh -o 4 -rs 0 -rp 0
+//     mpirun -np 4 ex16p -m ../../data/amr-hex.mesh -o 2 -rs 0 -rp 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-Ku)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   ParFiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   ParBilinearForm *M;
+   ParBilinearForm *K;
+
+   HypreParMatrix Mmat;
+   HypreParMatrix Kmat;
+   HypreParMatrix *T; // T = M + dt K
+   double current_dt;
+
+   CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
+   HypreSmoother M_prec; // Preconditioner for the mass matrix M
+
+   CGSolver T_solver;    // Implicit solver for T = M + dt K
+   HypreSmoother T_prec; // Preconditioner for the implicit solver
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+public:
+   ConductionOperator(ParFiniteElementSpace &f, double alpha, double kappa,
+                      const Vector &u);
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
+
+   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+                                int jok, int *jcur, double gamma);
+
+   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
+
+   virtual ~ConductionOperator();
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 2;
+   int par_ref_levels = 1;
+   int order = 2;
+   int ode_solver_type = 8; // Exponential Euler 
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver:\n\t"
+                  "1  - Forward Euler,\n\t"
+                  "2  - RK2,\n\t"
+                  "3  - RK3 SSP,\n\t"
+                  "4  - RK4,\n\t"
+                  "5  - Backward Euler,\n\t"
+                  "6  - SDIRK 2,\n\t"
+                  "7  - SDIRK 3,\n\t"
+                  "8  - Exponential Euler,\n\t");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      MPI_Finalize();
+      return 1;
+   }
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // check for vaild ODE solver option
+   if (ode_solver_type < 1 || ode_solver_type > 8)
+   {
+      if (myid == 0)
+      {
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      }
+      MPI_Finalize();
+      return 1;
+   }
+
+   // 3. Read the serial mesh from the given mesh file on all processors. We can
+   //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
+   //    with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the mesh in serial to increase the resolution. In this example
+   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
+   //    a command-line parameter.
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   ParFiniteElementSpace fespace(pmesh, &fe_coll);
+
+   int fe_size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of temperature unknowns: " << fe_size << endl;
+   }
+
+   ParGridFunction u_gf(&fespace);
+
+   // 7. Set the initial conditions for u. All boundaries are considered
+   //    natural.
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 8. Initialize the conduction operator and the VisIt visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "ex16-mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "ex16-init." << setfill('0') << setw(6) << myid;
+      ofstream omesh(mesh_name.str().c_str());
+      omesh.precision(precision);
+      pmesh->Print(omesh);
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16-Parallel", pmesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      sout << "parallel " << num_procs << " " << myid << endl;
+      int good = sout.good(), all_good;
+      MPI_Allreduce(&good, &all_good, 1, MPI_INT, MPI_MIN, pmesh->GetComm());
+      if (!all_good)
+      {
+         sout.close();
+         visualization = false;
+         if (myid == 0)
+         {
+            cout << "Unable to connect to GLVis server at "
+                 << vishost << ':' << visport << endl;
+            cout << "GLVis visualization disabled.\n";
+         }
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *pmesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         if (myid == 0)
+         {
+            cout << "GLVis visualization paused."
+                 << " Press space (in the GLVis window) to resume it.\n";
+         }
+      }
+   }
+
+   // 9. Define the ODE solver used for time integration.
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   EPICSolver *epic_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // MFEM explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
+      // EPIC
+      case 8:
+         epic_solver = new EPICSolver();
+	 epic_solver->Init(oper);
+	 ode_solver = epic_solver;
+	 break;
+   }
+
+   // Initialize MFEM integrators
+   ode_solver->Init(oper);
+
+   // 10. Perform time-integration (looping over the time iterations, ti, with a
+   //     time-step dt).
+   if (myid == 0)
+   {
+      cout << "Integrating the ODE ..." << endl;
+   }
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   bool last_step = false;
+   for (int ti = 1; !last_step; ti++)
+   {
+      double dt_real = min(dt, t_final - t);
+
+      // Note that since we are using the "one-step" mode of the SUNDIALS
+      // solvers, they will, generally, step over the final time and will not
+      // explicitly perform the interpolation to t_final as they do in the
+      // "normal" step mode.
+
+      ode_solver->Step(u, t, dt_real);
+
+      last_step = (t >= t_final - 1e-8*dt);
+
+      if (last_step || (ti % vis_steps) == 0)
+      {
+         if (myid == 0)
+         {
+            cout << "step " << ti << ", t = " << t << endl;
+         }
+
+         u_gf.SetFromTrueDofs(u);
+         if (visualization)
+         {
+            sout << "parallel " << num_procs << " " << myid << "\n";
+            sout << "solution\n" << *pmesh << u_gf << flush;
+         }
+
+         if (visit)
+         {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+      oper.SetParameters(u);
+   }
+   tic_toc.Stop();
+   if (myid == 0)
+   {
+      cout << "Done, " << tic_toc.RealTime() << "s." << endl;
+   }
+
+   // 11. Save the final solution in parallel. This output can be viewed later
+   //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
+   {
+      ostringstream sol_name;
+      sol_name << "ex16-final." << setfill('0') << setw(6) << myid;
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   // 12. Free the used memory.
+   delete ode_solver;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
+     T(NULL),
+     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
+{
+   const double rel_tol = 1e-8;
+
+   M = new ParBilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble(0); // keep sparsity pattern of M and K the same
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(100);
+   M_solver.SetPrintLevel(0);
+   M_prec.SetType(HypreSmoother::Jacobi);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(Mmat);
+
+   alpha = al;
+   kappa = kap;
+
+   T_solver.iterative_mode = false;
+   T_solver.SetRelTol(rel_tol);
+   T_solver.SetAbsTol(0.0);
+   T_solver.SetMaxIter(100);
+   T_solver.SetPrintLevel(0);
+   T_solver.SetPreconditioner(T_prec);
+
+   SetParameters(u);
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Kmat.Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, dt, Kmat);
+   T_solver.SetOperator(*T);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
+}
+
+int ConductionOperator::SUNImplicitSetup(const Vector &x,
+                                         const Vector &fx, int jok, int *jcur,
+                                         double gamma)
+{
+   // Setup the ODE Jacobian T = M + gamma K.
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, gamma, Kmat);
+   T_solver.SetOperator(*T);
+   *jcur = 1;
+   return (0);
+}
+
+int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
+{
+   // Solve the system A x = z => (M - gamma K) x = M b.
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, x);
+   return (0);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
+{
+   ParGridFunction u_alpha_gf(&fespace);
+   u_alpha_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < u_alpha_gf.Size(); i++)
+   {
+      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
+   }
+
+   delete K;
+   K = new ParBilinearForm(&fespace);
+
+   GridFunctionCoefficient u_coeff(&u_alpha_gf);
+
+   K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
+   K->Assemble(0); // keep sparsity pattern of M and K the same
+   K->FormSystemMatrix(ess_tdof_list, Kmat);
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete T;
+   delete M;
+   delete K;
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5)
+   {
+      return 2.0;
+   }
+   else
+   {
+      return 1.0;
+   }
+}

examples/epic/makefile

@@ -0,0 +1,76 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/epic/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ex16
+PAR_EXAMPLES = ex16p
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_EPIC),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with EPIC)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+SERIAL_NAME := Serial EPIC example
+PARALLEL_NAME := Parallel EPIC example
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
+%-test-seq: %
+	@$(call mfem-test,$<,, $(SERIAL_NAME))
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f deformed.* velocity.* elastic_energy.*
+	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.* Example16*

examples/ex13p.cpp

@@ -55,6 +55,7 @@ int main(int argc, char *argv[])
    int order = 1;
    int nev = 5;
    bool visualization = 1;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -71,6 +72,8 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -86,13 +89,18 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement (2 by default, or
    //    specified on the command line with -rs).
    for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -100,7 +108,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution (1 time by
    //    default, or specified on the command line with -rp). Once the parallel
    //    mesh is defined, the serial mesh can be deleted.
@@ -112,7 +120,7 @@ int main(int argc, char *argv[])
    }
    pmesh->ReorientTetMesh();
 
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -122,7 +130,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
    //    element space. The first corresponds to the curl curl, while the second
    //    is a simple mass matrix needed on the right hand side of the
    //    generalized eigenvalue problem below. The boundary conditions are
@@ -164,7 +172,7 @@ int main(int argc, char *argv[])
    delete a;
    delete m;
 
-   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
    //    A to be used within the solver. Set the matrices which define the
    //    generalized eigenproblem A x = lambda M x.
    HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -180,15 +188,15 @@ int main(int argc, char *argv[])
    ame->SetMassMatrix(*M);
    ame->SetOperator(*A);
 
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
+   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //     parallel grid function to represent each of the eigenmodes returned by
+   //     the solver.
    Array<double> eigenvalues;
    ame->Solve();
    ame->GetEigenvalues(eigenvalues);
    ParGridFunction x(fespace);
 
-   // 10. Save the refined mesh and the modes in parallel. This output can be
+   // 11. Save the refined mesh and the modes in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
    {
       ostringstream mesh_name, mode_name;
@@ -213,7 +221,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 11. Send the solution by socket to a GLVis server.
+   // 12. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -253,7 +261,7 @@ int main(int argc, char *argv[])
       mode_sock.close();
    }
 
-   // 12. Free the used memory.
+   // 13. Free the used memory.
    delete ame;
    delete ams;
    delete M;

examples/ex19p.cpp

@@ -196,6 +196,12 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
    // 1. Initialize MPI
    MPI_Session mpi;
    const int myid = mpi.WorldRank();
@@ -438,16 +444,20 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
    // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
-
-   Vector disp_out(y.GetData() + block_trueOffsets[0],
+   Vector disp_in;
+   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
                    block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
+   Vector pres_in;
+   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
                    block_trueOffsets[2]-block_trueOffsets[1]);
 
+   Vector disp_out;
+   disp_out.MakeRef(y, block_trueOffsets[0],
+                    block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out;
+   pres_out.MakeRef(y, block_trueOffsets[1],
+                    block_trueOffsets[2]-block_trueOffsets[1]);
+
    Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
    Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
 
@@ -459,6 +469,9 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
    subtract(disp_in, temp, temp2);
 
    stiff_pcg->Mult(temp2, disp_out);
+
+   disp_out.SyncAliasMemory(y);
+   pres_out.SyncAliasMemory(y);
 }
 
 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -473,7 +486,10 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
 
       if (!spaces[0]->GetParMesh()->Nonconforming())
       {
+#ifndef HYPRE_USING_CUDA
+         // Not available yet when hypre is built with CUDA
          stiff_prec_amg->SetElasticityOptions(spaces[0]);
+#endif
       }
 
       stiff_prec = stiff_prec_amg;

examples/ex1p.cpp

@@ -89,7 +89,8 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
+                  "-no-a", "--no-algebraic",
                   "Use algebraic Ceed solver");
 #endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -197,15 +198,15 @@ int main(int argc, char *argv[])
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
    b.Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
    ParGridFunction x(&fespace);
    x = 0.0;
 
    // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));

examples/ex26.cpp

@@ -105,7 +105,7 @@ private:
       Vector diag(fespace.GetTrueVSize());
       bfs.Last()->AssembleDiagonal(diag);
 
-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
+      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
                                                        *essentialTrueDofs.Last(), 2);
       AddLevel(opr.Ptr(), smoother, true, true);
    }

examples/ex26p.cpp

@@ -115,7 +115,7 @@ private:
       Vector diag(fespace.GetTrueVSize());
       bfs.Last()->AssembleDiagonal(diag);
 
-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
+      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
                                                        *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());
 
       AddLevel(opr.Ptr(), smoother, true, true);

examples/ex28p.cpp

@@ -81,6 +81,12 @@ Mesh * build_trapezoid_mesh(double offset)
 
 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
    // 1. Initialize MPI.
    int num_procs, myid;
    MPI_Init(&argc, &argv);
@@ -360,6 +366,7 @@ int main(int argc, char *argv[])
    }
    delete pmesh;
 
+   // HYPRE_Finalize();
    MPI_Finalize();
 
    return 0;

examples/ex2p.cpp

@@ -61,6 +61,7 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool amg_elast = 0;
    bool reorder_space = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -78,6 +79,8 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                   "Use byNODES ordering of vector space instead of byVDIM");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -93,7 +96,12 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -109,14 +117,14 @@ int main(int argc, char *argv[])
       return 3;
    }
 
-   // 4. Select the order of the finite element discretization space. For NURBS
+   // 5. Select the order of the finite element discretization space. For NURBS
    //    meshes, we increase the order by degree elevation.
    if (mesh->NURBSext)
    {
       mesh->DegreeElevate(order, order);
    }
 
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 6. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
@@ -129,7 +137,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -142,7 +150,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
    //    use vector finite elements, i.e. dim copies of a scalar finite element
    //    space. We use the ordering by vector dimension (the last argument of
    //    the FiniteElementSpace constructor) which is expected in the systems
@@ -175,7 +183,7 @@ int main(int argc, char *argv[])
            << "Assembling: " << flush;
    }
 
-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 9. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined by
    //    marking only boundary attribute 1 from the mesh as essential and
    //    converting it to a list of true dofs.
@@ -184,14 +192,14 @@ int main(int argc, char *argv[])
    ess_bdr[0] = 1;
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system. In this case, b_i equals the
-   //    boundary integral of f*phi_i where f represents a "pull down" force on
-   //    the Neumann part of the boundary and phi_i are the basis functions in
-   //    the finite element fespace. The force is defined by the object f, which
-   //    is a vector of Coefficient objects. The fact that f is non-zero on
-   //    boundary attribute 2 is indicated by the use of piece-wise constants
-   //    coefficient for its last component.
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system. In this case, b_i equals the
+   //     boundary integral of f*phi_i where f represents a "pull down" force on
+   //     the Neumann part of the boundary and phi_i are the basis functions in
+   //     the finite element fespace. The force is defined by the object f, which
+   //     is a vector of Coefficient objects. The fact that f is non-zero on
+   //     boundary attribute 2 is indicated by the use of piece-wise constants
+   //     coefficient for its last component.
    VectorArrayCoefficient f(dim);
    for (int i = 0; i < dim-1; i++)
    {
@@ -212,13 +220,13 @@ int main(int argc, char *argv[])
    }
    b->Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid
+   // 11. Define the solution vector x as a parallel finite element grid
    //     function corresponding to fespace. Initialize x with initial guess of
    //     zero, which satisfies the boundary conditions.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
    //     corresponding to the linear elasticity integrator with piece-wise
    //     constants coefficient lambda and mu.
    Vector lambda(pmesh->attributes.Max());
@@ -233,7 +241,7 @@ int main(int argc, char *argv[])
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
 
-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 13. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, static condensation, etc.
@@ -250,7 +258,7 @@ int main(int argc, char *argv[])
       cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
    }
 
-   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
    //     preconditioner from hypre.
    HypreBoomerAMG *amg = new HypreBoomerAMG(A);
    if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -268,11 +276,11 @@ int main(int argc, char *argv[])
    pcg->SetPreconditioner(*amg);
    pcg->Mult(B, X);
 
-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 15. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
    //     space. This means that we define the mesh elements through a fespace
    //     based transformation of the reference element.  This allows us to save
    //     the displaced mesh as a curved mesh when using high-order finite
@@ -284,7 +292,7 @@ int main(int argc, char *argv[])
       pmesh->SetNodalFESpace(fespace);
    }
 
-   // 16. Save in parallel the displaced mesh and the inverted solution (which
+   // 17. Save in parallel the displaced mesh and the inverted solution (which
    //     gives the backward displacements to the original grid). This output
    //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
@@ -305,7 +313,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
    //     keys in GLVis to visualize the displacements.
    if (visualization)
    {
@@ -317,7 +325,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 18. Free the used memory.
+   // 19. Free the used memory.
    delete pcg;
    delete amg;
    delete a;

examples/ex3p.cpp

@@ -103,6 +103,7 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
+      // HYPRE_Finalize();
       MPI_Finalize();
       return 1;
    }

examples/ex5.cpp

@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
+      if (Device::IsEnabled()) { B.BuildTranspose(); }
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);
@@ -240,6 +241,7 @@ int main(int argc, char *argv[])
    {
       SparseMatrix &M(mVarf->SpMat());
       M.GetDiag(Md);
+      Md.HostReadWrite();
 
       SparseMatrix &B(bVarf->SpMat());
       MinvBt = Transpose(B);
@@ -287,12 +289,18 @@ int main(int argc, char *argv[])
    chrono.Stop();
 
    if (solver.GetConverged())
+   {
       std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
+                << " iterations with a residual norm of "
+                << solver.GetFinalNorm() << ".\n";
+   }
    else
+   {
       std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm()
+                << ".\n";
+   }
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
 
    // 12. Create the grid functions u and p. Compute the L2 error norms.
    GridFunction u, p;

examples/ex7p.cpp

@@ -47,6 +47,7 @@ int main(int argc, char *argv[])
    int order = 2;
    bool always_snap = false;
    bool visualization = 1;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&elem_type, "-e", "--elem",
@@ -65,6 +66,8 @@ int main(int argc, char *argv[])
                   "--snap-at-the-end",
                   "If true, snap nodes to the sphere initially and after each refinement "
                   "otherwise, snap only after the last refinement");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -80,7 +83,12 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
    //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
    //    the elements and the vertices of the mesh, and then make it high-order
    //    by specifying a finite element space for its nodes.
@@ -146,7 +154,7 @@ int main(int argc, char *argv[])
    FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
    mesh->SetNodalFESpace(&nodal_fes);
 
-   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
    //    refinements is fixed to 2.
    for (int l = 0; l <= ref_levels; l++)
    {
@@ -218,7 +226,7 @@ int main(int argc, char *argv[])
       SnapNodes(*pmesh);
    }
 
-   // 5. Define a finite element space on the mesh. Here we use isoparametric
+   // 6. Define a finite element space on the mesh. Here we use isoparametric
    //    finite elements -- the same as the mesh nodes.
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
    HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -227,7 +235,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
    //    the basis functions in the finite element fespace.
    ParLinearForm *b = new ParLinearForm(fespace);
@@ -237,27 +245,27 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
    b->Assemble();
 
-   // 7. Define the solution vector x as a finite element grid function
+   // 8. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 8. Set up the bilinear form a(.,.) on the finite element space
+   // 9. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    and Mass domain integrators.
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
    a->AddDomainIntegrator(new MassIntegrator(one));
 
-   // 9. Assemble the parallel linear system, applying any transformations
-   //    such as: parallel assembly, applying conforming constraints, etc.
+   // 10. Assemble the parallel linear system, applying any transformations
+   //     such as: parallel assembly, applying conforming constraints, etc.
    a->Assemble();
    HypreParMatrix A;
    Vector B, X;
    Array<int> empty_tdof_list;
    a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);
 
-   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
    //     preconditioner from hypre. Extract the parallel grid function x
    //     corresponding to the finite element approximation X. This is the local
    //     solution on each processor.
@@ -273,14 +281,14 @@ int main(int argc, char *argv[])
    delete a;
    delete b;
 
-   // 11. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
    double err = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
       cout << "\nL2 norm of error: " << err << endl;
    }
 
-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
    {
       ostringstream mesh_name, sol_name;
@@ -296,7 +304,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -307,7 +315,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 14. Free the used memory.
+   // 15. Free the used memory.
    delete pcg;
    delete amg;
    delete fespace;

examples/makefile

@@ -26,6 +26,9 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
+  ex24p ex25p ex26p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -42,6 +45,9 @@ endif
 ifeq ($(MFEM_USE_HIOP),YES)
    SUBDIRS += hiop
 endif
+ifeq ($(MFEM_USE_EPIC),YES)
+   SUBDIRS += epic
+endif
 ifeq ($(MFEM_USE_PETSC),YES)
    SUBDIRS += petsc
 endif
@@ -99,6 +105,14 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, Serial CUDA example,-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, Serial HIP example,-d hip)
 
 # Testing: Specific execution options
 ex0-test-seq: ex0

examples/superlu/ex1p.cpp

@@ -282,6 +282,10 @@ int main(int argc, char *argv[])
    superlu->SetOperator(*SLU_A);
    superlu->SetPrintStatistics(true);
    superlu->Mult(B, X);
+   superlu->DismantleGrid();
+
+   delete SLU_A;
+   delete superlu;
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.

fem/CMakeLists.txt

@@ -184,6 +184,11 @@ if (MFEM_USE_ADIOS2)
   list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
+if (MFEM_USE_FMS)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+endif()
+
 if (MFEM_USE_MPI)
   list(APPEND SRCS
     pbilinearform.cpp

fem/bilinearform.cpp

@@ -31,7 +31,7 @@ void BilinearForm::AllocMat()
    const Table &elem_dof = fes->GetElementToDofTable();
    Table dof_dof;
 
-   if (fbfi.Size() > 0)
+   if (interior_face_integs.Size() > 0)
    {
       // the sparsity pattern is defined from the map: face->element->dof
       Table face_dof, dof_face;
@@ -99,15 +99,15 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
    ext = NULL;
 
    // Copy the pointers to the integrators
-   dbfi = bf->dbfi;
+   domain_integs = bf->domain_integs;
 
-   bbfi = bf->bbfi;
-   bbfi_marker = bf->bbfi_marker;
+   boundary_integs = bf->boundary_integs;
+   boundary_integs_marker = bf->boundary_integs_marker;
 
-   fbfi = bf->fbfi;
+   interior_face_integs = bf->interior_face_integs;
 
-   bfbfi = bf->bfbfi;
-   bfbfi_marker = bf->bfbfi_marker;
+   boundary_face_integs = bf->boundary_face_integs;
+   boundary_face_integs_marker = bf->boundary_face_integs_marker;
 
    AllocMat();
 }
@@ -234,46 +234,47 @@ void BilinearForm::Finalize (int skip_zeros)
 
 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
 {
-   dbfi.Append(bfi);
-   dbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   domain_integs.Append(bfi);
+   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }
 
 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi,
                                        Array<int> &elem_marker)
 {
-   dbfi.Append(bfi);
-   dbfi_marker.Append(&elem_marker);
+   domain_integs.Append(bfi);
+   domain_integs_marker.Append(&elem_marker);
 }
 
 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }
 
 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                           Array<int> &bdr_marker)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(&bdr_marker);
 }
 
 void BilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi)
 {
-   fbfi.Append (bfi);
+   interior_face_integs.Append (bfi);
 }
 
 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   bfbfi.Append(bfi);
-   bfbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_face_integs.Append(bfi);
+   // NULL marker means apply everywhere
+   boundary_face_integs_marker.Append(NULL);
 }
 
 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                                         Array<int> &bdr_marker)
 {
-   bfbfi.Append(bfi);
-   bfbfi_marker.Append(&bdr_marker);
+   boundary_face_integs.Append(bfi);
+   boundary_face_integs_marker.Append(&bdr_marker);
 }
 
 void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
@@ -285,14 +286,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
       return;
    }
 
-   if (dbfi.Size())
+   if (domain_integs.Size())
    {
       const FiniteElement &fe = *fes->GetFE(i);
       ElementTransformation *eltrans = fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix(fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
       {
-         dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
          elmat += elemmat;
       }
    }
@@ -306,14 +307,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 
 void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (bbfi.Size())
+   if (boundary_integs.Size())
    {
       const FiniteElement &be = *fes->GetBE(i);
       ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
+      boundary_integs[0]->AssembleElementMatrix(be, *eltrans, elmat);
+      for (int k = 1; k < boundary_integs.Size(); k++)
       {
-         bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+         boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
          elmat += elemmat;
       }
    }
@@ -407,13 +408,14 @@ void BilinearForm::Assemble(int skip_zeros)
    }
 #endif
 
-   if (dbfi.Size())
+   if (domain_integs.Size())
    {
-      for (int k = 0; k < dbfi.Size(); k++)
+      for (int k = 0; k < domain_integs.Size(); k++)
       {
-         if (dbfi_marker[k] != NULL)
+         if (domain_integs_marker[k] != NULL)
          {
-            MFEM_VERIFY(mesh->attributes.Size() == dbfi_marker[k]->Size(),
+            MFEM_VERIFY(mesh->attributes.Size() ==
+                        domain_integs_marker[k]->Size(),
                         "invalid element marker for domain integrator #"
                         << k << ", counting from zero");
          }
@@ -430,14 +432,14 @@ void BilinearForm::Assemble(int skip_zeros)
          else
          {
             elmat.SetSize(0);
-            for (int k = 0; k < dbfi.Size(); k++)
+            for (int k = 0; k < domain_integs.Size(); k++)
             {
-               if ( dbfi_marker[k] == NULL ||
-                    (*(dbfi_marker[k]))[elem_attr-1] == 1)
+               if ( domain_integs_marker[k] == NULL ||
+                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
                {
                   const FiniteElement &fe = *fes->GetFE(i);
                   eltrans = fes->GetElementTransformation(i);
-                  dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
                   if (elmat.Size() == 0)
                   {
                      elmat = elemmat;
@@ -472,20 +474,20 @@ void BilinearForm::Assemble(int skip_zeros)
       }
    }
 
-   if (bbfi.Size())
+   if (boundary_integs.Size())
    {
       // Which boundary attributes need to be processed?
       Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                  mesh->bdr_attributes.Max() : 0);
       bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
       {
-         if (bbfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
          {
             bdr_attr_marker = 1;
             break;
          }
-         Array<int> &bdr_marker = *bbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
          MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                      "invalid boundary marker for boundary integrator #"
                      << k << ", counting from zero");
@@ -504,21 +506,21 @@ void BilinearForm::Assemble(int skip_zeros)
          fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
          int k = 0;
-         for (; k < bbfi.Size(); k++)
+         for (; k < boundary_integs.Size(); k++)
          {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
 
-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elmat);
+            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elmat);
             k++;
             break;
          }
-         for (; k < bbfi.Size(); k++)
+         for (; k < boundary_integs.Size(); k++)
          {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
 
-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
             elmat += elemmat;
          }
          if (!static_cond)
@@ -536,7 +538,7 @@ void BilinearForm::Assemble(int skip_zeros)
       }
    }
 
-   if (fbfi.Size())
+   if (interior_face_integs.Size())
    {
       FaceElementTransformations *tr;
       Array<int> vdofs2;
@@ -550,18 +552,19 @@ void BilinearForm::Assemble(int skip_zeros)
             fes -> GetElementVDofs (tr -> Elem1No, vdofs);
             fes -> GetElementVDofs (tr -> Elem2No, vdofs2);
             vdofs.Append (vdofs2);
-            for (int k = 0; k < fbfi.Size(); k++)
+            for (int k = 0; k < interior_face_integs.Size(); k++)
             {
-               fbfi[k] -> AssembleFaceMatrix (*fes -> GetFE (tr -> Elem1No),
-                                              *fes -> GetFE (tr -> Elem2No),
-                                              *tr, elemmat);
+               interior_face_integs[k]->
+               AssembleFaceMatrix(*fes->GetFE(tr->Elem1No),
+                                  *fes->GetFE(tr->Elem2No),
+                                  *tr, elemmat);
                mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
             }
          }
       }
    }
 
-   if (bfbfi.Size())
+   if (boundary_face_integs.Size())
    {
       FaceElementTransformations *tr;
       const FiniteElement *fe1, *fe2;
@@ -570,14 +573,14 @@ void BilinearForm::Assemble(int skip_zeros)
       Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                  mesh->bdr_attributes.Max() : 0);
       bdr_attr_marker = 0;
-      for (int k = 0; k < bfbfi.Size(); k++)
+      for (int k = 0; k < boundary_face_integs.Size(); k++)
       {
-         if (bfbfi_marker[k] == NULL)
+         if (boundary_face_integs_marker[k] == NULL)
          {
             bdr_attr_marker = 1;
             break;
          }
-         Array<int> &bdr_marker = *bfbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_face_integs_marker[k];
          MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                      "invalid boundary marker for boundary face integrator #"
                      << k << ", counting from zero");
@@ -601,12 +604,14 @@ void BilinearForm::Assemble(int skip_zeros)
             // but we can't dereference a NULL pointer, and we don't want to
             // actually make a fake element.
             fe2 = fe1;
-            for (int k = 0; k < bfbfi.Size(); k++)
+            for (int k = 0; k < boundary_face_integs.Size(); k++)
             {
-               if (bfbfi_marker[k] &&
-                   (*bfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_face_integs_marker[k] &&
+                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }
 
-               bfbfi[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elemmat);
+               boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
+                                                              elemmat);
                mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
             }
          }
@@ -720,8 +725,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       {
          // A, X and B point to the same data as mat, x and b
          EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.MakeRef(x, 0, x.Size());
+         B.MakeRef(b, 0, b.Size());
          if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
       }
    }
@@ -857,7 +862,7 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,
 
 void BilinearForm::ComputeElementMatrices()
 {
-   if (element_matrices || dbfi.Size() == 0 || fes->GetNE() == 0)
+   if (element_matrices || domain_integs.Size() == 0 || fes->GetNE() == 0)
    {
       return;
    }
@@ -886,11 +891,11 @@ void BilinearForm::ComputeElementMatrices()
 #endif
       fes->GetElementTransformation(i, &eltrans);
 
-      dbfi[0]->AssembleElementMatrix(fe, eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix(fe, eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
       {
          // note: some integrators may not be thread-safe
-         dbfi[k]->AssembleElementMatrix(fe, eltrans, tmp);
+         domain_integs[k]->AssembleElementMatrix(fe, eltrans, tmp);
          elmat += tmp;
       }
       elmat.ClearExternalData();
@@ -1105,10 +1110,12 @@ BilinearForm::~BilinearForm()
    if (!extern_bfs)
    {
       int k;
-      for (k=0; k < dbfi.Size(); k++) { delete dbfi[k]; }
-      for (k=0; k < bbfi.Size(); k++) { delete bbfi[k]; }
-      for (k=0; k < fbfi.Size(); k++) { delete fbfi[k]; }
-      for (k=0; k < bfbfi.Size(); k++) { delete bfbfi[k]; }
+      for (k=0; k < domain_integs.Size(); k++) { delete domain_integs[k]; }
+      for (k=0; k < boundary_integs.Size(); k++) { delete boundary_integs[k]; }
+      for (k=0; k < interior_face_integs.Size(); k++)
+      { delete interior_face_integs[k]; }
+      for (k=0; k < boundary_face_integs.Size(); k++)
+      { delete boundary_face_integs[k]; }
    }
 
    delete ext;
@@ -1141,13 +1148,13 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    ext = NULL;
 
    // Copy the pointers to the integrators
-   dbfi = mbf->dbfi;
-   bbfi = mbf->bbfi;
-   tfbfi = mbf->tfbfi;
-   btfbfi = mbf->btfbfi;
+   domain_integs = mbf->domain_integs;
+   boundary_integs = mbf->boundary_integs;
+   trace_face_integs = mbf->trace_face_integs;
+   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
 
-   bbfi_marker = mbf->bbfi_marker;
-   btfbfi_marker = mbf->btfbfi_marker;
+   boundary_integs_marker = mbf->boundary_integs_marker;
+   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;
 
    assembly = AssemblyLevel::LEGACY;
    ext = NULL;
@@ -1236,7 +1243,8 @@ MatrixInverse * MixedBilinearForm::Inverse() const
 {
    if (assembly != AssemblyLevel::LEGACY)
    {
-      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
+      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this "
+                   "assembly level!");
       return NULL;
    }
    else
@@ -1267,38 +1275,39 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
 
 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   dbfi.Append (bfi);
+   domain_integs.Append (bfi);
 }
 
 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }
 
 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                                Array<int> &bdr_marker)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(&bdr_marker);
 }
 
 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   tfbfi.Append (bfi);
+   trace_face_integs.Append (bfi);
 }
 
 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_trace_face_integs.Append(bfi);
+   // NULL marker means apply everywhere
+   boundary_trace_face_integs_marker.Append(NULL);
 }
 
 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
                                                   Array<int> &bdr_marker)
 {
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(&bdr_marker);
+   boundary_trace_face_integs.Append(bfi);
+   boundary_trace_face_integs_marker.Append(&bdr_marker);
 }
 
 void MixedBilinearForm::Assemble (int skip_zeros)
@@ -1320,37 +1329,37 @@ void MixedBilinearForm::Assemble (int skip_zeros)
       mat = new SparseMatrix(height, width);
    }
 
-   if (dbfi.Size())
+   if (domain_integs.Size())
    {
       for (int i = 0; i < test_fes -> GetNE(); i++)
       {
          trial_fes -> GetElementVDofs (i, tr_vdofs);
          test_fes  -> GetElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetElementTransformation (i);
-         for (int k = 0; k < dbfi.Size(); k++)
+         for (int k = 0; k < domain_integs.Size(); k++)
          {
-            dbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                               *test_fes  -> GetFE(i),
-                                               *eltrans, elemmat);
+            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                                        *test_fes  -> GetFE(i),
+                                                        *eltrans, elemmat);
             mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
 
-   if (bbfi.Size())
+   if (boundary_integs.Size())
    {
       // Which boundary attributes need to be processed?
       Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                  mesh->bdr_attributes.Max() : 0);
       bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
       {
-         if (bbfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
          {
             bdr_attr_marker = 1;
             break;
          }
-         Array<int> &bdr_marker = *bbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
          MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                      "invalid boundary marker for boundary integrator #"
                      << k << ", counting from zero");
@@ -1368,20 +1377,20 @@ void MixedBilinearForm::Assemble (int skip_zeros)
          trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
          test_fes  -> GetBdrElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetBdrElementTransformation (i);
-         for (int k = 0; k < bbfi.Size(); k++)
+         for (int k = 0; k < boundary_integs.Size(); k++)
          {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
 
-            bbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
-                                               *test_fes  -> GetBE(i),
-                                               *eltrans, elemmat);
+            boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                                        *test_fes  -> GetBE(i),
+                                                        *eltrans, elemmat);
             mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
 
-   if (tfbfi.Size())
+   if (trace_face_integs.Size())
    {
       FaceElementTransformations *ftr;
       Array<int> te_vdofs2;
@@ -1408,16 +1417,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
             // want to actually make a fake element.
             test_fe2 = test_fe1;
          }
-         for (int k = 0; k < tfbfi.Size(); k++)
+         for (int k = 0; k < trace_face_integs.Size(); k++)
          {
-            tfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                         *ftr, elemmat);
+            trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
+                                                     *test_fe2, *ftr, elemmat);
             mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
 
-   if (btfbfi.Size())
+   if (boundary_trace_face_integs.Size())
    {
       FaceElementTransformations *ftr;
       Array<int> te_vdofs2;
@@ -1427,17 +1436,17 @@ void MixedBilinearForm::Assemble (int skip_zeros)
       Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                  mesh->bdr_attributes.Max() : 0);
       bdr_attr_marker = 0;
-      for (int k = 0; k < btfbfi.Size(); k++)
+      for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
       {
-         if (btfbfi_marker[k] == NULL)
+         if (boundary_trace_face_integs_marker[k] == NULL)
          {
             bdr_attr_marker = 1;
             break;
          }
-         Array<int> &bdr_marker = *btfbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_trace_face_integs_marker[k];
          MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary trace face integrator #"
-                     << k << ", counting from zero");
+                     "invalid boundary marker for boundary trace face"
+                     "integrator #" << k << ", counting from zero");
          for (int i = 0; i < bdr_attr_marker.Size(); i++)
          {
             bdr_attr_marker[i] |= bdr_marker[i];
@@ -1460,13 +1469,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
             // boundaries, but we can't dereference a NULL pointer, and we don't
             // want to actually make a fake element.
             test_fe2 = test_fe1;
-            for (int k = 0; k < btfbfi.Size(); k++)
+            for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
             {
-               if (btfbfi_marker[k] &&
-                   (*btfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_trace_face_integs_marker[k] &&
+                   (*boundary_trace_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }
 
-               btfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                             *ftr, elemmat);
+               boundary_trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe,
+                                                                 *test_fe1,
+                                                                 *test_fe2,
+                                                                 *ftr, elemmat);
                mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
             }
          }
@@ -1557,15 +1569,17 @@ void MixedBilinearForm::ConformingAssemble()
 
 void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (dbfi.Size())
+   if (domain_integs.Size())
    {
       const FiniteElement &trial_fe = *trial_fes->GetFE(i);
       const FiniteElement &test_fe = *test_fes->GetFE(i);
       ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
+                                               elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
       {
-         dbfi[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elemmat);
+         domain_integs[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
+                                                  elemmat);
          elmat += elemmat;
       }
    }
@@ -1580,15 +1594,17 @@ void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 
 void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (bbfi.Size())
+   if (boundary_integs.Size())
    {
       const FiniteElement &trial_be = *trial_fes->GetBE(i);
       const FiniteElement &test_be = *test_fes->GetBE(i);
       ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
+      boundary_integs[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
+                                                 elmat);
+      for (int k = 1; k < boundary_integs.Size(); k++)
       {
-         bbfi[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elemmat);
+         boundary_integs[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
+                                                    elemmat);
          elmat += elemmat;
       }
    }
@@ -1688,10 +1704,10 @@ void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
       }
 }
 
-void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    OperatorHandle &A)
+void MixedBilinearForm::FormRectangularSystemMatrix(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   OperatorHandle &A)
 
 {
    if (ext)
@@ -1729,17 +1745,17 @@ void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
    A.Reset(mat, false);
 }
 
-void MixedBilinearForm::FormRectangularLinearSystem(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    Vector &x, Vector &b,
-                                                    OperatorHandle &A,
-                                                    Vector &X, Vector &B)
+void MixedBilinearForm::FormRectangularLinearSystem(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   Vector &x, Vector &b,
+   OperatorHandle &A,
+   Vector &X, Vector &B)
 {
    if (ext)
    {
-      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X,
-                                       B);
+      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                       x, b, A, X, B);
       return;
    }
 
@@ -1777,10 +1793,13 @@ MixedBilinearForm::~MixedBilinearForm()
    if (!extern_bfs)
    {
       int i;
-      for (i = 0; i < dbfi.Size(); i++) { delete dbfi[i]; }
-      for (i = 0; i < bbfi.Size(); i++) { delete bbfi[i]; }
-      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
-      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
+      for (i = 0; i < domain_integs.Size(); i++) { delete domain_integs[i]; }
+      for (i = 0; i < boundary_integs.Size(); i++)
+      { delete boundary_integs[i]; }
+      for (i = 0; i < trace_face_integs.Size(); i++)
+      { delete trace_face_integs[i]; }
+      for (i = 0; i < boundary_trace_face_integs.Size(); i++)
+      { delete boundary_trace_face_integs[i]; }
    }
    delete ext;
 }
@@ -1830,7 +1849,7 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
       mat = new SparseMatrix(height, width);
    }
 
-   if (dbfi.Size() > 0)
+   if (domain_integs.Size() > 0)
    {
       for (int i = 0; i < test_fes->GetNE(); i++)
       {
@@ -1840,17 +1859,19 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
          dom_fe = trial_fes->GetFE(i);
          ran_fe = test_fes->GetFE(i);
 
-         dbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < dbfi.Size(); j++)
+         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                  totelmat);
+         for (int j = 1; j < domain_integs.Size(); j++)
          {
-            dbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                     elmat);
             totelmat += elmat;
          }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
       }
    }
 
-   if (tfbfi.Size())
+   if (trace_face_integs.Size())
    {
       const int nfaces = test_fes->GetMesh()->GetNumFaces();
       for (int i = 0; i < nfaces; i++)
@@ -1861,10 +1882,12 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
          dom_fe = trial_fes->GetFaceElement(i);
          ran_fe = test_fes->GetFaceElement(i);
 
-         tfbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < tfbfi.Size(); j++)
+         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                      totelmat);
+         for (int j = 1; j < trace_face_integs.Size(); j++)
          {
-            tfbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                         elmat);
             totelmat += elmat;
          }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);

fem/bilinearform.hpp

@@ -84,28 +84,29 @@ protected:
        the BilinearForm. */
    long sequence;
 
-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #fbfi, and #bfbfi are owned by another BilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
+       #boundary_integs, #interior_face_integs, and #boundary_face_integs are
+       owned by another BilinearForm. */
    int extern_bfs;
 
    /// Set of Domain Integrators to be applied.
-   Array<BilinearFormIntegrator*> dbfi;
+   Array<BilinearFormIntegrator*> domain_integs;
    /// Element attribute marker (should be of length mesh->attributes)
    /// Includes all by default.
    /// 0 - ignore attribute
    /// 1 - include attribute
-   Array<Array<int>*>             dbfi_marker;
+   Array<Array<int>*>             domain_integs_marker;
 
    /// Set of Boundary Integrators to be applied.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_integs;
+   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.
 
    /// Set of interior face Integrators to be applied.
-   Array<BilinearFormIntegrator*> fbfi;
+   Array<BilinearFormIntegrator*> interior_face_integs;
 
    /// Set of boundary face Integrators to be applied.
-   Array<BilinearFormIntegrator*> bfbfi;
-   Array<Array<int>*>             bfbfi_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_face_integs;
+   Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.
 
    DenseMatrix elemmat;
    Array<int>  vdofs;
@@ -231,24 +232,25 @@ public:
    void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
 
    /// Access all the integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
 
    /// Access all the integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
    /** @brief Access all boundary markers added with AddBoundaryIntegrator().
        If no marker was specified when the integrator was added, the
        corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }
 
    /// Access all integrators added with AddInteriorFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetFBFI() { return &fbfi; }
+   Array<BilinearFormIntegrator*> *GetFBFI() { return &interior_face_integs; }
 
    /// Access all integrators added with AddBdrFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBFBFI() { return &bfbfi; }
+   Array<BilinearFormIntegrator*> *GetBFBFI() { return &boundary_face_integs; }
    /** @brief Access all boundary markers added with AddBdrFaceIntegrator().
        If no marker was specified when the integrator was added, the
        corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBFBFI_Marker() { return &bfbfi_marker; }
+   Array<Array<int>*> *GetBFBFI_Marker()
+   { return &boundary_face_integs_marker; }
 
    /// Returns a reference to: \f$ M_{ij} \f$
    const double &operator()(int i, int j) { return (*mat)(i,j); }
@@ -652,23 +654,25 @@ protected:
        Partial Assembly (PA), or Matrix Free assembly (MF). */
    MixedBilinearFormExtension *ext;
 
-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
+       #boundary_integs, #trace_face_integs and #boundary_trace_face_integs
+       are owned by another MixedBilinearForm. */
    int extern_bfs;
 
    /// Domain integrators.
-   Array<BilinearFormIntegrator*> dbfi;
+   Array<BilinearFormIntegrator*> domain_integs;
 
    /// Boundary integrators.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker;///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_integs;
+   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.
 
    /// Trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> tfbfi;
+   Array<BilinearFormIntegrator*> trace_face_integs;
 
    /// Boundary trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> btfbfi;
-   Array<Array<int>*>             btfbfi_marker;///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_trace_face_integs;
+   /// Entries are not owned.
+   Array<Array<int>*> boundary_trace_face_integs_marker;
 
    DenseMatrix elemmat;
    Array<int>  trial_vdofs, test_vdofs;
@@ -762,24 +766,26 @@ public:
                                    Array<int> &bdr_marker);
 
    /// Access all integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
 
    /// Access all integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
    /** @brief Access all boundary markers added with AddBoundaryIntegrator().
        If no marker was specified when the integrator was added, the
        corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }
 
    /// Access all integrators added with AddTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetTFBFI() { return &tfbfi; }
+   Array<BilinearFormIntegrator*> *GetTFBFI() { return &trace_face_integs; }
 
    /// Access all integrators added with AddBdrTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBTFBFI() { return &btfbfi; }
+   Array<BilinearFormIntegrator*> *GetBTFBFI()
+   { return &boundary_trace_face_integs; }
    /** @brief Access all boundary markers added with AddBdrTraceFaceIntegrator().
        If no marker was specified when the integrator was added, the
        corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }
+   Array<Array<int>*> *GetBTFBFI_Marker()
+   { return &boundary_trace_face_integs_marker; }
 
    /// Sets all sparse values of \f$ M \f$ to @a a.
    void operator=(const double a) { *mat = a; }
@@ -1004,7 +1010,7 @@ public:
    { AddTraceFaceIntegrator(di); }
 
    /// Access all interpolators added with AddDomainInterpolator().
-   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }
 
    /// Set the desired assembly level. The default is AssemblyLevel::FULL.
    /** This method must be called before assembly. */

fem/bilinearform_ext.cpp

@@ -160,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
          }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
       }
    }
 
@@ -176,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
          }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
       }
    }
 }
@@ -217,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
          }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
       }
    }
 
@@ -233,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
          }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
       }
    }
 }
@@ -417,7 +417,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultPA(faceIntX, faceIntY);
          }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
       }
    }
 
@@ -433,7 +433,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultPA(faceBdrX, faceBdrY);
          }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
       }
    }
 }
@@ -474,7 +474,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultTransposePA(faceIntX, faceIntY);
          }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
       }
    }
 
@@ -490,7 +490,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultTransposePA(faceBdrX, faceBdrY);
          }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
       }
    }
 }
@@ -657,7 +657,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             Y(j, 0, f) += res;
          });
          // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
       }
    }
 
@@ -688,7 +688,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             Y(j, f) += res;
          });
          // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
       }
    }
 }
@@ -783,7 +783,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             Y(j, 0, f) += res;
          });
          // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
       }
    }
 
@@ -814,7 +814,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             Y(j, f) += res;
          });
          // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
       }
    }
 }

fem/bilinearform_ext.hpp

@@ -72,8 +72,8 @@ protected:
    mutable Vector faceIntX, faceIntY;
    mutable Vector faceBdrX, faceBdrY;
    const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
+   const FaceRestriction *int_face_restrict_lex; // Not owned
+   const FaceRestriction *bdr_face_restrict_lex; // Not owned
 
 public:
    PABilinearFormExtension(BilinearForm*);
@@ -143,8 +143,8 @@ protected:
    mutable Vector faceIntX, faceIntY;
    mutable Vector faceBdrX, faceBdrY;
    const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
+   const FaceRestriction *int_face_restrict_lex; // Not owned
+   const FaceRestriction *bdr_face_restrict_lex; // Not owned
 
 public:
    MFBilinearFormExtension(BilinearForm *form);

fem/bilininteg.cpp

@@ -175,6 +175,11 @@ void BilinearFormIntegrator::AssembleFaceVector(
    elmat.Mult(elfun, elvect);
 }
 
+void TransposeIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   bfi->SetIntRule(ir);
+}
 
 void TransposeIntegrator::AssembleElementMatrix (
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
@@ -202,6 +207,12 @@ void TransposeIntegrator::AssembleFaceMatrix (
    elmat.Transpose (bfi_elmat);
 }
 
+void LumpedIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   bfi->SetIntRule(ir);
+}
+
 void LumpedIntegrator::AssembleElementMatrix (
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -209,6 +220,12 @@ void LumpedIntegrator::AssembleElementMatrix (
    elmat.Lump();
 }
 
+void InverseIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   integrator->SetIntRule(ir);
+}
+
 void InverseIntegrator::AssembleElementMatrix(
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -216,6 +233,15 @@ void InverseIntegrator::AssembleElementMatrix(
    elmat.Invert();
 }
 
+void SumIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->SetIntRule(ir);
+   }
+}
+
 void SumIntegrator::AssembleElementMatrix(
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -1751,15 +1777,16 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
    int dim = trial_fe.GetDim();
    int trial_nd = trial_fe.GetDof();
    int test_nd = test_fe.GetDof();
+   int spaceDim = Trans.GetSpaceDim();
 
    int i, l;
    double det;
 
    elmat.SetSize (test_nd,trial_nd);
    dshape.SetSize (trial_nd,dim);
-   dshapedxt.SetSize(trial_nd,dim);
+   dshapedxt.SetSize(trial_nd, spaceDim);
    dshapedxi.SetSize(trial_nd);
-   invdfdx.SetSize(dim);
+   invdfdx.SetSize(dim, spaceDim);
    shape.SetSize (test_nd);
 
    const IntegrationRule *ir = IntRule;

fem/bilininteg.hpp

@@ -261,6 +261,8 @@ public:
    TransposeIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
    { bfi = bfi_; own_bfi = own_bfi_; }
 
+   virtual void SetIntRule(const IntegrationRule *ir);
+
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
@@ -328,6 +330,8 @@ public:
    LumpedIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
    { bfi = bfi_; own_bfi = own_bfi_; }
 
+   virtual void SetIntRule(const IntegrationRule *ir);
+
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
@@ -346,6 +350,8 @@ public:
    InverseIntegrator(BilinearFormIntegrator *integ, int own_integ = 1)
    { integrator = integ; own_integrator = own_integ; }
 
+   virtual void SetIntRule(const IntegrationRule *ir);
+
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
@@ -364,6 +370,8 @@ private:
 public:
    SumIntegrator(int own_integs = 1) { own_integrators = own_integs; }
 
+   virtual void SetIntRule(const IntegrationRule *ir);
+
    void AddIntegrator(BilinearFormIntegrator *integ)
    { integrators.Append(integ); }
 
@@ -703,7 +711,7 @@ protected:
    {
       return "MixedScalarDerivativeIntegrator:  "
              "Trial and test spaces must both be scalar fields in 1D "
-             "and the trial space must implement CaldDShape.";
+             "and the trial space must implement CalcDShape.";
    }
 
    inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
@@ -2928,11 +2936,11 @@ public:
 
     - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
       compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
-      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77-88. Springer
       Berlin Heidelberg, 2000.
     - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
       of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
-      Numerical Analysis, 39(5):1749–1779, 2002.
+      Numerical Analysis, 39(5):1749-1779, 2002.
 */
 class DGDiffusionBR2Integrator : public BilinearFormIntegrator
 {

fem/ceed/algebraic.cpp

@@ -143,7 +143,7 @@ Solver *BuildSmootherFromCeed(ConstrainedOperator &op, bool chebyshev)
    if (chebyshev)
    {
       const int cheb_order = 3;
-      out = new OperatorChebyshevSmoother(&op, t_diag, ess_tdofs, cheb_order);
+      out = new OperatorChebyshevSmoother(op, t_diag, ess_tdofs, cheb_order);
    }
    else
    {

fem/complex_fem.cpp

@@ -1204,17 +1204,30 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       });
       // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      ess_tdof_list.HostRead();
       if (A_i.Type() == Operator::Hypre_ParCSR)
       {
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
+#ifndef HYPRE_USING_CUDA
+         ess_tdof_list.HostRead();
          for (int k = 0; k < n; k++)
          {
             const int j = ess_tdof_list[k];
             Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
+#else
+         Ah->HypreReadWrite();
+         const int *d_ess_tdof_list =
+            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+         const int *d_diag_i = Aih->diag->i;
+         double *d_diag_data = Aih->diag->data;
+         CuWrap1D(n, [=] MFEM_DEVICE (int k)
+         {
+            const int j = d_ess_tdof_list[k];
+            d_diag_data[d_diag_i[j]] = 0.0;
+         });
+#endif
       }
       else
       {

fem/datacollection.cpp

@@ -17,6 +17,7 @@
 
 #include <cerrno>      // errno
 #include <sstream>
+#include <regex>
 
 #ifndef _WIN32
 #include <sys/stat.h>  // mkdir
@@ -764,7 +765,8 @@ ParaViewDataCollection::ParaViewDataCollection(const std::string&
    : DataCollection(collection_name, mesh_),
      levels_of_detail(1),
      pv_data_format(VTKFormat::BINARY),
-     high_order_output(false)
+     high_order_output(false),
+     restart_mode(false)
 {
 #ifdef MFEM_USE_ZLIB
    compression = -1; // default zlib compression level, equivalent to 6
@@ -842,17 +844,60 @@ void ParaViewDataCollection::Save()
    }
    // the directory is created
 
-   // create pvd file if needed
+   // create pvd file if needed. If we are not in restart mode, a new pvd file
+   // is always created. In restart mode, we keep any previously defined
+   // timestep values as long as they are less than the currently defined time.
+
    if (myid == 0 && !pvd_stream.is_open())
    {
       std::string dpath=GenerateCollectionPath();
       std::string pvdname=dpath+"/"+GeneratePVDFileName();
-      pvd_stream.open(pvdname.c_str(),std::ios::out);
-      // initialize the file
-      pvd_stream << "<?xml version=\"1.0\"?>\n";
-      pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
-      pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
-      pvd_stream << "<Collection>" << std::endl;
+
+      std::ifstream pvd_in;
+      if (restart_mode && (pvd_in.open(pvdname,std::ios::binary),pvd_in.good()))
+      {
+         // PVD file exists and restart mode enabled: preserve existing time
+         // steps less than the current time.
+         std::fstream::pos_type pos_begin = pvd_in.tellg();
+         std::fstream::pos_type pos_end = pos_begin;
+
+         std::regex regexp("timestep=\"([^[:space:]]+)\".*file=\"Cycle(\\d+)");
+         std::smatch match;
+
+         std::string line;
+         while (getline(pvd_in,line))
+         {
+            if (regex_search(line,match,regexp))
+            {
+               MFEM_ASSERT(match.size() == 3, "Unable to parse DataSet");
+               double tvalue = std::stod(match[1]);
+               if (tvalue >= GetTime()) { break; }
+               int cvalue = std::stoi(match[2]);
+               MFEM_VERIFY(cvalue < GetCycle(), "Cycle " << GetCycle() <<
+                           " is too small for restart mode: trying to overwrite"
+                           " existing data.");
+               pos_end = pvd_in.tellg();
+            }
+         }
+         size_t count = pos_end - pos_begin;
+         std::vector<char> buf(count);
+         pvd_in.clear();
+         pvd_in.seekg(pos_begin);
+         pvd_in.read(buf.data(), count);
+         pvd_in.close();
+         pvd_stream.open(pvdname.c_str(),std::ios::out);
+         pvd_stream.write(buf.data(), count);
+      }
+      else
+      {
+         // initialize new pvd file
+         pvd_stream.open(pvdname.c_str(),std::ios::out);
+         // initialize the file
+         pvd_stream << "<?xml version=\"1.0\"?>\n";
+         pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
+         pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
+         pvd_stream << "<Collection>" << std::endl;
+      }
    }
 
    // define the vtu file
@@ -1091,6 +1136,11 @@ void ParaViewDataCollection::SetCompression(bool compression_)
    }
 }
 
+void ParaViewDataCollection::UseRestartMode(bool restart_mode_)
+{
+   restart_mode = restart_mode_;
+}
+
 const char *ParaViewDataCollection::GetDataFormatString() const
 {
    if (pv_data_format == VTKFormat::ASCII)

fem/datacollection.hpp

@@ -488,6 +488,7 @@ private:
    std::fstream pvd_stream;
    VTKFormat pv_data_format;
    bool high_order_output;
+   bool restart_mode;
 
 protected:
    void SaveDataVTU(std::ostream &out, int ref);
@@ -545,6 +546,11 @@ public:
    /// by default). Reading high-order data requires ParaView 5.5 or later.
    void SetHighOrderOutput(bool high_order_output_);
 
+   /// Enable or disable restart mode. If restart is enabled, new writes will
+   /// preserve timestep metadata for any solutions prior to the currently
+   /// defined time.
+   void UseRestartMode(bool restart_mode_);
+
    /// Load the collection - not implemented in the ParaView writer
    virtual void Load(int cycle_ = 0) override;
 };

fem/eltrans.hpp

@@ -316,7 +316,7 @@ public:
 
    /// Set the desired print level, useful for debugging.
    /** The valid options are: -1 - never print (default); 0 - print only errors;
-       1 - print the first and last last iterations; 2 - print every iteration;
+       1 - print the first and last iterations; 2 - print every iteration;
        and 3 - print every iteration including point coordinates. */
    void SetPrintLevel(int pr_level) { print_level = pr_level; }
 

fem/fe.cpp

@@ -495,6 +495,7 @@ void ScalarFiniteElement::ScalarLocalRestriction(
       R *= 1.0 / Trans.Weight();
    }
 }
+
 const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                                    DofToQuad::Mode mode) const
 {
@@ -7948,7 +7949,27 @@ VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                          p, M, FunctionSpace::Qk),
      TensorBasisElement(dims, p, VerifyNodal(cbtype), dmtype),
      cbasis1d(poly1d.GetBasis(p, VerifyClosed(cbtype))),
-     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(obtype))) { }
+     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(obtype)))
+{
+   MFEM_VERIFY(dims > 1, "Constructor for VectorTensorFiniteElement with both "
+               "open and closed bases is not valid for 1D elements.");
+}
+
+VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
+                                                     const int d,
+                                                     const int p,
+                                                     const int obtype,
+                                                     const int M,
+                                                     const DofMapType dmtype)
+   : VectorFiniteElement(dims, GetTensorProductGeometry(dims), d,
+                         p, M, FunctionSpace::Pk),
+     TensorBasisElement(dims, p, obtype, dmtype),
+     cbasis1d(poly1d.GetBasis(p, VerifyOpen(obtype))),
+     obasis1d(poly1d.GetBasis(p, VerifyOpen(obtype)))
+{
+   MFEM_VERIFY(dims == 1, "Constructor for VectorTensorFiniteElement without "
+               "closed basis is only valid for 1D elements.");
+}
 
 H1_SegmentElement::H1_SegmentElement(const int p, const int btype)
    : NodalTensorFiniteElement(1, p, VerifyClosed(btype), H1_DOF_MAP)
@@ -13055,9 +13076,8 @@ void ND_TriangleElement::CalcCurlShape(const IntegrationPoint &ip,
 const double ND_SegmentElement::tk[1] = { 1. };
 
 ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
-   : VectorFiniteElement(1, Geometry::SEGMENT, p, p - 1,
-                         H_CURL, FunctionSpace::Pk),
-     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(ob_type))),
+   : VectorTensorFiniteElement(1, p, p - 1, ob_type, H_CURL,
+                               DofMapType::L2_DOF_MAP),
      dof2tk(dof)
 {
    if (obasis1d.IsIntegratedType()) { is_nodal = false; }

fem/fe.hpp

@@ -97,7 +97,7 @@ public:
       {
          "Gauss-Legendre", "Gauss-Lobatto", "Positive (Bernstein)",
          "Open uniform", "Closed uniform", "Open half uniform",
-         "Seredipity", "Closed Gauss-Legendre",
+         "Serendipity", "Closed Gauss-Legendre",
          "Integrated Gauss-Lobatto indicator"
       };
       return name[Check(b_type)];
@@ -1126,7 +1126,7 @@ public:
    { dofs = 1.0; }
 };
 
-/// A 1D quadractic finite element with uniformly spaced nodes
+/// A 1D quadratic finite element with uniformly spaced nodes
 class Quad1DFiniteElement : public NodalFiniteElement
 {
 public:
@@ -2239,6 +2239,11 @@ public:
                              const int cbtype, const int obtype,
                              const int M, const DofMapType dmtype);
 
+   // For 1D elements: there is only an "open basis", no "closed basis"
+   VectorTensorFiniteElement(const int dims, const int d, const int p,
+                             const int obtype, const int M,
+                             const DofMapType dmtype);
+
    const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                  DofToQuad::Mode mode) const;
 
@@ -3311,11 +3316,9 @@ public:
 
 
 /// Arbitrary order Nedelec elements in 1D on a segment
-class ND_SegmentElement : public VectorFiniteElement
+class ND_SegmentElement : public VectorTensorFiniteElement
 {
    static const double tk[1];
-
-   Poly_1D::Basis &obasis1d;
    Array<int> dof2tk;
 
 public:

fem/fem.hpp

@@ -64,4 +64,9 @@
 #include "adios2datacollection.hpp"
 #endif
 
+#ifdef MFEM_USE_FMS
+#include "fmsconvert.hpp"
+#include "fmsdatacollection.hpp"
+#endif
+
 #endif

fem/fespace.cpp

@@ -1225,7 +1225,7 @@ const Operator *FiniteElementSpace::GetElementRestriction(
    return L2E_nat.Ptr();
 }
 
-const Operator *FiniteElementSpace::GetFaceRestriction(
+const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
    ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
 {
    const bool is_dg_space = IsDGSpace();
@@ -1239,7 +1239,7 @@ const Operator *FiniteElementSpace::GetFaceRestriction(
    }
    else
    {
-      Operator* res;
+      FaceRestriction *res;
       if (is_dg_space)
       {
          res = new L2FaceRestriction(*this, e_ordering, type, m);

fem/fespace.hpp

@@ -164,7 +164,7 @@ protected:
                 + 8 * (int)std::get<3>(k);
       }
    };
-   using map_L2F = std::unordered_map<const key_face,Operator*,key_hash>;
+   using map_L2F = std::unordered_map<const key_face,FaceRestriction*,key_hash>;
    mutable map_L2F L2F;
 
    mutable Array<QuadratureInterpolator*> E2Q_array;
@@ -488,7 +488,7 @@ public:
    const Operator *GetElementRestriction(ElementDofOrdering e_ordering) const;
 
    /// Return an Operator that converts L-vectors to E-vectors on each face.
-   virtual const Operator *GetFaceRestriction(
+   virtual const FaceRestriction *GetFaceRestriction(
       ElementDofOrdering e_ordering, FaceType,
       L2FaceValues mul = L2FaceValues::DoubleValued) const;
 

fem/fmsconvert.hpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef FMS_CONVERT
+#define FMS_CONVERT
+
+#include "../config/config.hpp"
+#include "datacollection.hpp"
+
+#ifdef MFEM_USE_FMS
+#include <fms.h>
+
+namespace mfem
+{
+
+/** In-memory conversion of FMS data collection to an MFEM data collection.
+    @param dc The FMS data collection to convert.
+    @param[out] mfem_dc A pointer to a new MFEM DataCollection containing the
+    FMS data.
+    @return 0 on success; non-zero on failure.
+*/
+int FmsDataCollectionToDataCollection(FmsDataCollection dc,
+                                      DataCollection **mfem_dc);
+
+/** In-memory conversion of MFEM data collection to an FMS data collection.
+    @param mfem_dc The MFEM data collection to convert.
+    @param[out] dc A pointer to a new FmsDataCollection containing the MFEM
+    data.
+    @return 0 on success; non-zero on failure.
+*/
+int DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
+                                      FmsDataCollection *dc);
+
+} // namespace mfem
+
+#endif
+
+#endif

fem/fmsdatacollection.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "fem.hpp"
+#include "../general/text.hpp"
+
+#include <fmsio.h>
+
+#include <string>
+#include <sstream>
+
+namespace mfem
+{
+
+// class FMSDataCollection implementation
+
+FMSDataCollection::FMSDataCollection(const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   appendRankToFileName = false; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#ifdef MFEM_USE_MPI
+
+FMSDataCollection::FMSDataCollection(MPI_Comm comm,
+                                     const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   m_comm = comm;
+   MPI_Comm_rank(comm, &myid);
+   MPI_Comm_size(comm, &num_procs);
+   appendRankToFileName = true; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#endif
+
+FMSDataCollection::~FMSDataCollection()
+{
+   // empty
+}
+
+void FMSDataCollection::Save()
+{
+   // Convert this to FmsDataCollection.
+
+   FmsDataCollection dc;
+   if (DataCollectionToFmsDataCollection(this, &dc) == 0)
+   {
+      std::string root(RootFileName());
+      int err = FmsIOWrite(root.c_str(), fms_protocol.c_str(), dc);
+      FmsDataCollectionDestroy(&dc);
+      if (err)
+      {
+         MFEM_ABORT("Error creating FMS file: " << root);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Error converting data collection");
+   }
+}
+
+void FMSDataCollection::Load(int cycle)
+{
+   DeleteAll();
+   this->cycle = cycle;
+
+   FmsDataCollection dc;
+   std::string root(RootFileName());
+   int err = FmsIORead(root.c_str(), fms_protocol.c_str(), &dc);
+
+   if (err == 0)
+   {
+      DataCollection *mdc = nullptr;
+      if (FmsDataCollectionToDataCollection(dc,&mdc) == 0)
+      {
+         // Tell the data collection we read that it does not own data.
+         // We will steal its data.
+         mdc->SetOwnData(false);
+
+         SetCycle(mdc->GetCycle());
+         SetTime(mdc->GetTime());
+         SetTimeStep(mdc->GetTimeStep());
+         name = mdc->GetCollectionName();
+
+         // Set mdc's mesh as our mesh.
+         SetMesh(mdc->GetMesh());
+
+         // Set mdc's fields/qfields as ours.
+         std::vector<std::string> names;
+         for (const auto &pair : mdc->GetFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         names.clear();
+         for (const auto &pair : mdc->GetQFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterQField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         // Indicate that we own the data.
+         SetOwnData(true);
+
+         // Delete mdc. We stole its contents.
+         delete mdc;
+      }
+      FmsDataCollectionDestroy(&dc);
+   }
+   else
+   {
+      MFEM_ABORT("Error reading data collection: " << root);
+   }
+}
+
+void FMSDataCollection::SetProtocol(const std::string &protocol)
+{
+   fms_protocol = protocol;
+}
+
+std::string FMSDataCollection::RootFileName()
+{
+   std::string res;
+   if (pad_digits_cycle)
+   {
+      res = prefix_path + name + "_" +
+            to_padded_string(cycle, pad_digits_cycle) +
+            ".fms";
+   }
+   else
+   {
+      res = prefix_path + name + ".fms";
+   }
+   return res;
+}
+
+} // namespace mfem
+
+#endif

fem/fmsdatacollection.hpp

@@ -0,0 +1,74 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FMSDATACOLLECTION
+#define MFEM_FMSDATACOLLECTION
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "datacollection.hpp"
+#include <fms.h>
+
+namespace mfem
+{
+
+/** @brief Data collection that uses FMS. */
+/** FMSDataCollection lets MFEM read/write data using FMS.
+
+    For more information, see:
+    - FMS project, https://ceed.exascaleproject.org/fms/
+*/
+
+/// Data collection with FMS I/O routines
+class FMSDataCollection : public DataCollection
+{
+protected:
+   // file name helpers
+
+   /// Returns file name for the current cycle
+   std::string RootFileName();
+
+   // holds currently active i/o protocol
+   std::string fms_protocol;
+
+public:
+   /// Constructor. The collection name is used when saving the data.
+   /** If @a mesh is NULL, then the mesh can be set later by calling either
+       SetMesh() or Load(). The latter works only in serial. */
+   FMSDataCollection(const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#ifdef MFEM_USE_MPI
+   /// Construct a parallel FMSDataCollection.
+   FMSDataCollection(MPI_Comm comm, const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#endif
+
+   /// We will delete the mesh and fields if we own them
+   virtual ~FMSDataCollection();
+
+   /// Set the FMS relay i/o protocol to use
+   /** Supported options: ascii (default), json, yaml, hdf5 */
+   void SetProtocol(const std::string &protocol);
+
+   /// Save the collection and a FMS blueprint root file
+   virtual void Save();
+
+   /// Load the collection based blueprint data
+   virtual void Load(int cycle = 0);
+};
+
+} // namespace mfem
+
+#endif
+
+#endif

fem/gridfunc.cpp

@@ -218,7 +218,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 {
-   if (!f->GetProlongationMatrix())
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
    {
       MakeRef(f, tv);
       t_vec.NewDataAndSize(tv, size);
@@ -232,7 +232,8 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
 {
-   if (!f->GetProlongationMatrix())
+   tv.UseDevice(true);
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
    {
       MakeRef(f, tv, tv_offset);
       t_vec.NewMemoryAndSize(data, size, false);
@@ -241,10 +242,7 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
    {
       MFEM_ASSERT(tv.Size() >= tv_offset + f->GetTrueVSize(), "");
       SetSpace(f); // works in parallel
-      tv.UseDevice(true);
-      const int tv_size = f->GetTrueVSize();
-      t_vec.NewMemoryAndSize(Memory<double>(tv.GetMemory(), tv_offset, tv_size),
-                             tv_size, true);
+      t_vec.MakeRef(tv, tv_offset, f->GetTrueVSize());
    }
 }
 
@@ -334,10 +332,10 @@ int GridFunction::VectorDim() const
 void GridFunction::GetTrueDofs(Vector &tv) const
 {
    const SparseMatrix *R = fes->GetRestrictionMatrix();
-   if (!R)
+   if (!R || IsIdentityProlongation(fes->GetProlongationMatrix()))
    {
-      // R is identity -> make tv a reference to *this
-      tv.MakeRef(const_cast<GridFunction &>(*this), 0, size);
+      // R is identity
+      tv = *this; // no real copy if 'tv' and '*this' use the same data
    }
    else
    {

fem/gridfunc.hpp

@@ -130,9 +130,7 @@ public:
        or set. */
    Vector &GetTrueVector() { return t_vec; }
 
-   /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
-   /// then `tv` will be set to point to the data of `*this`.
-   /** @warning This method breaks const-ness when all dofs are true. */
+   /// Extract the true-dofs from the GridFunction.
    void GetTrueDofs(Vector &tv) const;
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.

fem/gslib.cpp

@@ -19,7 +19,11 @@
 #pragma GCC diagnostic ignored "-Wunused-function"
 #endif
 
+// External GSLIB header (the MFEM header is gslib.hpp)
+namespace gslib
+{
 #include "gslib.h"
+}
 
 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
 #pragma GCC diagnostic pop
@@ -34,13 +38,13 @@ FindPointsGSLIB::FindPointsGSLIB()
      dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr       = new crystal;
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
 #ifdef MFEM_USE_MPI
    int initialized;
    MPI_Initialized(&initialized);
    if (!initialized) { MPI_Init(NULL, NULL); }
-   MPI_Comm comm = MPI_COMM_WORLD;;
+   MPI_Comm comm = MPI_COMM_WORLD;
    comm_init(gsl_comm, comm);
 #else
    comm_init(gsl_comm, 0);
@@ -62,8 +66,8 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
      dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr      = new crystal;
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
    comm_init(gsl_comm, comm_);
 }
 #endif
@@ -728,7 +732,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
       }
 
       // Pack data to send via crystal router
-      struct array *outpt = new array;
+      struct gslib::array *outpt = new gslib::array;
       struct out_pt { double r[3], ival; uint index, el, proc; };
       struct out_pt *pt;
       array_init(struct out_pt, outpt, nptsend);
@@ -788,7 +792,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          }
 
          // Save index and proc data in a struct
-         struct array *savpt = new array;
+         struct gslib::array *savpt = new gslib::array;
          struct sav_pt { uint index, proc; };
          struct sav_pt *spt;
          array_init(struct sav_pt, savpt, npt);
@@ -806,7 +810,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          delete outpt;
 
          // Copy data from save struct to send struct and send component wise
-         struct array *sendpt = new array;
+         struct gslib::array *sendpt = new gslib::array;
          struct send_pt { double ival; uint index, proc; };
          struct send_pt *sdpt;
          for (int j = 0; j < ncomp; j++)

fem/gslib.hpp

@@ -17,11 +17,13 @@
 
 #ifdef MFEM_USE_GSLIB
 
+namespace gslib
+{
 struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
-struct array;
 struct crystal;
+}
 
 namespace mfem
 {
@@ -50,10 +52,10 @@ public:
 protected:
    Mesh *mesh, *meshsplit;
    IntegrationRule *ir_simplex;    // IntegrationRule to split quads/hex -> simplex
-   struct findpts_data_2 *fdata2D; // gslib's internal data
-   struct findpts_data_3 *fdata3D; // gslib's internal data
-   struct crystal *cr;             // gslib's internal data
-   struct comm *gsl_comm;          // gslib's internal data
+   struct gslib::findpts_data_2 *fdata2D; // gslib's internal data
+   struct gslib::findpts_data_3 *fdata3D; // gslib's internal data
+   struct gslib::crystal *cr;             // gslib's internal data
+   struct gslib::comm *gsl_comm;          // gslib's internal data
    int dim, points_cnt;
    Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
    Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;

fem/intrules.cpp

@@ -34,6 +34,7 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry)
    nx = irx.GetNPoints();
    ny = iry.GetNPoints();
    SetSize(nx * ny);
+   SetPointIndices();
 
    for (j = 0; j < ny; j++)
    {
@@ -48,8 +49,6 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry)
          ip.weight = ipx.weight * ipy.weight;
       }
    }
-
-   SetPointIndices();
 }
 
 IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
@@ -59,6 +58,7 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
    const int ny = iry.GetNPoints();
    const int nz = irz.GetNPoints();
    SetSize(nx*ny*nz);
+   SetPointIndices();
 
    for (int iz = 0; iz < nz; ++iz)
    {
@@ -78,8 +78,6 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
          }
       }
    }
-
-   SetPointIndices();
 }
 
 const Array<double> &IntegrationRule::GetWeights() const
@@ -125,6 +123,7 @@ void IntegrationRule::GrundmannMollerSimplexRule(int s, int n)
    }
    np /= f;
    SetSize(np);
+   SetPointIndices();
 
    int pt = 0;
    for (int i = 0; i <= s; i++)
@@ -375,6 +374,7 @@ public:
 void QuadratureFunctions1D::GaussLegendre(const int np, IntegrationRule* ir)
 {
    ir->SetSize(np);
+   ir->SetPointIndices();
 
    switch (np)
    {
@@ -477,6 +477,7 @@ void QuadratureFunctions1D::GaussLobatto(const int np, IntegrationRule* ir)
    */
 
    ir->SetSize(np);
+   ir->SetPointIndices();
    if ( np == 1 )
    {
       ir->IntPoint(0).Set1w(0.5, 1.0);
@@ -576,6 +577,7 @@ void QuadratureFunctions1D::GaussLobatto(const int np, IntegrationRule* ir)
 void QuadratureFunctions1D::OpenUniform(const int np, IntegrationRule* ir)
 {
    ir->SetSize(np);
+   ir->SetPointIndices();
 
    // The Newton-Cotes quadrature is based on weights that integrate exactly the
    // interpolatory polynomial through the equally spaced quadrature points.
@@ -591,6 +593,7 @@ void QuadratureFunctions1D::ClosedUniform(const int np,
                                           IntegrationRule* ir)
 {
    ir->SetSize(np);
+   ir->SetPointIndices();
    if ( np == 1 ) // allow this case as "closed"
    {
       ir->IntPoint(0).Set1w(0.5, 1.0);
@@ -608,6 +611,7 @@ void QuadratureFunctions1D::ClosedUniform(const int np,
 void QuadratureFunctions1D::OpenHalfUniform(const int np, IntegrationRule* ir)
 {
    ir->SetSize(np);
+   ir->SetPointIndices();
 
    // Open half points: the centers of np uniform intervals
    for (int i = 0; i < np ; ++i)
@@ -621,6 +625,7 @@ void QuadratureFunctions1D::OpenHalfUniform(const int np, IntegrationRule* ir)
 void QuadratureFunctions1D::ClosedGL(const int np, IntegrationRule* ir)
 {
    ir->SetSize(np);
+   ir->SetPointIndices();
    ir->IntPoint(0).x = 0.0;
    ir->IntPoint(np-1).x = 1.0;
 

fem/intrules.hpp

@@ -96,9 +96,6 @@ private:
        by request with the method GetWeights(). */
    mutable Array<double> weights;
 
-   /// Sets the indices of each quadrature point on initialization.
-   void SetPointIndices();
-
    /// Define n-simplex rule (triangle/tetrahedron for n=2/3) of order (2s+1)
    void GrundmannMollerSimplexRule(int s, int n = 3);
 
@@ -227,6 +224,11 @@ public:
       }
    }
 
+   /// Sets the indices of each quadrature point on initialization.
+   /** Note that most calls to IntegrationRule::SetSize should be paired with a
+       call to SetPointIndices in order for the indices to be set correctly. */
+   void SetPointIndices();
+
    /// Tensor product of two 1D integration rules
    IntegrationRule(IntegrationRule &irx, IntegrationRule &iry);
 

fem/linearform.cpp

@@ -26,14 +26,14 @@ LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
    extern_lfs = 1;
 
    // Copy the pointers to the integrators
-   dlfi = lf->dlfi;
+   domain_integs = lf->domain_integs;
 
-   dlfi_delta = lf->dlfi_delta;
+   domain_delta_integs = lf->domain_delta_integs;
 
-   blfi = lf->blfi;
+   boundary_integs = lf->boundary_integs;
 
-   flfi = lf->flfi;
-   flfi_marker = lf->flfi_marker;
+   boundary_face_integs = lf->boundary_face_integs;
+   boundary_face_integs_marker = lf->boundary_face_integs_marker;
 }
 
 void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi)
@@ -42,13 +42,13 @@ void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi)
       dynamic_cast<DeltaLFIntegrator *>(lfi);
    if (!maybe_delta || !maybe_delta->IsDelta())
    {
-      dlfi.Append(lfi);
+      domain_integs.Append(lfi);
    }
    else
    {
-      dlfi_delta.Append(maybe_delta);
+      domain_delta_integs.Append(maybe_delta);
    }
-   dlfi_marker.Append(NULL);
+   domain_integs_marker.Append(NULL);
 }
 
 void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi,
@@ -58,44 +58,45 @@ void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi,
       dynamic_cast<DeltaLFIntegrator *>(lfi);
    if (!maybe_delta || !maybe_delta->IsDelta())
    {
-      dlfi.Append(lfi);
+      domain_integs.Append(lfi);
    }
    else
    {
-      dlfi_delta.Append(maybe_delta);
+      domain_delta_integs.Append(maybe_delta);
    }
-   dlfi_marker.Append(&elem_marker);
+   domain_integs_marker.Append(&elem_marker);
 }
 
 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi)
 {
-   blfi.Append (lfi);
-   blfi_marker.Append(NULL); // NULL -> all attributes are active
+   boundary_integs.Append (lfi);
+   boundary_integs_marker.Append(NULL); // NULL -> all attributes are active
 }
 
 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi,
                                         Array<int> &bdr_attr_marker)
 {
-   blfi.Append (lfi);
-   blfi_marker.Append(&bdr_attr_marker);
+   boundary_integs.Append (lfi);
+   boundary_integs_marker.Append(&bdr_attr_marker);
 }
 
 void LinearForm::AddBdrFaceIntegrator (LinearFormIntegrator * lfi)
 {
-   flfi.Append(lfi);
-   flfi_marker.Append(NULL); // NULL -> all attributes are active
+   boundary_face_integs.Append(lfi);
+   // NULL -> all attributes are active
+   boundary_face_integs_marker.Append(NULL);
 }
 
 void LinearForm::AddBdrFaceIntegrator(LinearFormIntegrator *lfi,
                                       Array<int> &bdr_attr_marker)
 {
-   flfi.Append(lfi);
-   flfi_marker.Append(&bdr_attr_marker);
+   boundary_face_integs.Append(lfi);
+   boundary_face_integs_marker.Append(&bdr_attr_marker);
 }
 
 void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
 {
-   iflfi.Append(lfi);
+   interior_face_integs.Append(lfi);
 }
 
 void LinearForm::Assemble()
@@ -112,14 +113,14 @@ void LinearForm::Assemble()
    // The first use of AddElementVector() below will move it back to host
    // because both 'vdofs' and 'elemvect' are on host.
 
-   if (dlfi.Size())
+   if (domain_integs.Size())
    {
-      for (int k = 0; k < dlfi.Size(); k++)
+      for (int k = 0; k < domain_integs.Size(); k++)
       {
-         if (dlfi_marker[k] != NULL)
+         if (domain_integs_marker[k] != NULL)
          {
             MFEM_VERIFY(fes->GetMesh()->attributes.Size() ==
-                        dlfi_marker[k]->Size(),
+                        domain_integs_marker[k]->Size(),
                         "invalid element marker for domain linear form "
                         "integrator #" << k << ", counting from zero");
          }
@@ -128,14 +129,15 @@ void LinearForm::Assemble()
       for (i = 0; i < fes -> GetNE(); i++)
       {
          int elem_attr = fes->GetMesh()->GetAttribute(i);
-         for (int k = 0; k < dlfi.Size(); k++)
+         for (int k = 0; k < domain_integs.Size(); k++)
          {
-            if ( dlfi_marker[k] == NULL ||
-                 (*(dlfi_marker[k]))[elem_attr-1] == 1 )
+            if ( domain_integs_marker[k] == NULL ||
+                 (*(domain_integs_marker[k]))[elem_attr-1] == 1 )
             {
                fes -> GetElementVDofs (i, vdofs);
                eltrans = fes -> GetElementTransformation (i);
-               dlfi[k]->AssembleRHSElementVect(*fes->GetFE(i), *eltrans, elemvect);
+               domain_integs[k]->AssembleRHSElementVect(*fes->GetFE(i),
+                                                        *eltrans, elemvect);
                AddElementVector (vdofs, elemvect);
             }
          }
@@ -143,7 +145,7 @@ void LinearForm::Assemble()
    }
    AssembleDelta();
 
-   if (blfi.Size())
+   if (boundary_integs.Size())
    {
       Mesh *mesh = fes->GetMesh();
 
@@ -151,14 +153,14 @@ void LinearForm::Assemble()
       Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                  mesh->bdr_attributes.Max() : 0);
       bdr_attr_marker = 0;
-      for (int k = 0; k < blfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
       {
-         if (blfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
          {
             bdr_attr_marker = 1;
             break;
          }
-         Array<int> &bdr_marker = *blfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
          MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                      "invalid boundary marker for boundary integrator #"
                      << k << ", counting from zero");
@@ -174,18 +176,19 @@ void LinearForm::Assemble()
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
          fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
-         for (int k=0; k < blfi.Size(); k++)
+         for (int k=0; k < boundary_integs.Size(); k++)
          {
-            if (blfi_marker[k] &&
-                (*blfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
 
-            blfi[k]->AssembleRHSElementVect(*fes->GetBE(i), *eltrans, elemvect);
+            boundary_integs[k]->AssembleRHSElementVect(*fes->GetBE(i),
+                                                       *eltrans, elemvect);
 
             AddElementVector (vdofs, elemvect);
          }
       }
    }
-   if (flfi.Size())
+   if (boundary_face_integs.Size())
    {
       FaceElementTransformations *tr;
       Mesh *mesh = fes->GetMesh();
@@ -194,14 +197,14 @@ void LinearForm::Assemble()
       Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                  mesh->bdr_attributes.Max() : 0);
       bdr_attr_marker = 0;
-      for (int k = 0; k < flfi.Size(); k++)
+      for (int k = 0; k < boundary_face_integs.Size(); k++)
       {
-         if (flfi_marker[k] == NULL)
+         if (boundary_face_integs_marker[k] == NULL)
          {
             bdr_attr_marker = 1;
             break;
          }
-         Array<int> &bdr_marker = *flfi_marker[k];
+         Array<int> &bdr_marker = *boundary_face_integs_marker[k];
          MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                      "invalid boundary marker for boundary face integrator #"
                      << k << ", counting from zero");
@@ -220,24 +223,26 @@ void LinearForm::Assemble()
          if (tr != NULL)
          {
             fes -> GetElementVDofs (tr -> Elem1No, vdofs);
-            for (int k = 0; k < flfi.Size(); k++)
+            for (int k = 0; k < boundary_face_integs.Size(); k++)
             {
-               if (flfi_marker[k] &&
-                   (*flfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_face_integs_marker[k] &&
+                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }
 
-               flfi[k] -> AssembleRHSElementVect (*fes->GetFE(tr -> Elem1No),
-                                                  *tr, elemvect);
+               boundary_face_integs[k]->
+               AssembleRHSElementVect(*fes->GetFE(tr->Elem1No),
+                                      *tr, elemvect);
                AddElementVector (vdofs, elemvect);
             }
          }
       }
    }
 
-   if (iflfi.Size())
+   if (interior_face_integs.Size())
    {
       Mesh *mesh = fes->GetMesh();
 
-      for (int k = 0; k < iflfi.Size(); k++)
+      for (int k = 0; k < interior_face_integs.Size(); k++)
       {
          for (i = 0; i < mesh->GetNumFaces(); i++)
          {
@@ -249,9 +254,10 @@ void LinearForm::Assemble()
                Array<int> vdofs2;
                fes -> GetElementVDofs (tr -> Elem2No, vdofs2);
                vdofs.Append(vdofs2);
-               iflfi[k] -> AssembleRHSElementVect (*fes->GetFE(tr -> Elem1No),
-                                                   *fes->GetFE(tr -> Elem2No),
-                                                   *tr, elemvect);
+               interior_face_integs[k]->
+               AssembleRHSElementVect(*fes->GetFE(tr->Elem1No),
+                                      *fes->GetFE(tr->Elem2No),
+                                      *tr, elemvect);
                AddElementVector (vdofs, elemvect);
             }
          }
@@ -261,56 +267,55 @@ void LinearForm::Assemble()
 
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
+   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
    fes = f;
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, f->GetVSize()),
-                    f->GetVSize(), false);
+   v.UseDevice(true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    ResetDeltaLocations();
 }
 
 void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 {
-   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
-   fes = f;
-   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+   Update(f, v, v_offset);
 }
 
 void LinearForm::AssembleDelta()
 {
-   if (dlfi_delta.Size() == 0) { return; }
+   if (domain_delta_integs.Size() == 0) { return; }
 
    if (!HaveDeltaLocations())
    {
       int sdim = fes->GetMesh()->SpaceDimension();
       Vector center;
-      DenseMatrix centers(sdim, dlfi_delta.Size());
+      DenseMatrix centers(sdim, domain_delta_integs.Size());
       for (int i = 0; i < centers.Width(); i++)
       {
          centers.GetColumnReference(i, center);
-         dlfi_delta[i]->GetDeltaCenter(center);
+         domain_delta_integs[i]->GetDeltaCenter(center);
          MFEM_VERIFY(center.Size() == sdim,
                      "Point dim " << center.Size() <<
                      " does not match space dim " << sdim);
       }
-      fes->GetMesh()->FindPoints(centers, dlfi_delta_elem_id, dlfi_delta_ip);
+      fes->GetMesh()->FindPoints(centers, domain_delta_integs_elem_id,
+                                 domain_delta_integs_ip);
    }
 
    Array<int> vdofs;
    Vector elemvect;
-   for (int i = 0; i < dlfi_delta.Size(); i++)
+   for (int i = 0; i < domain_delta_integs.Size(); i++)
    {
-      int elem_id = dlfi_delta_elem_id[i];
+      int elem_id = domain_delta_integs_elem_id[i];
       // The delta center may be outside of this sub-domain, or
       // (Par)Mesh::FindPoints() failed to find this point:
       if (elem_id < 0) { continue; }
 
-      const IntegrationPoint &ip = dlfi_delta_ip[i];
+      const IntegrationPoint &ip = domain_delta_integs_ip[i];
       ElementTransformation &Trans = *fes->GetElementTransformation(elem_id);
       Trans.SetIntPoint(&ip);
 
       fes->GetElementVDofs(elem_id, vdofs);
-      dlfi_delta[i]->AssembleDeltaElementVect(*fes->GetFE(elem_id), Trans,
-                                              elemvect);
+      domain_delta_integs[i]->AssembleDeltaElementVect(*fes->GetFE(elem_id),
+                                                       Trans, elemvect);
       AddElementVector(vdofs, elemvect);
    }
 }
@@ -333,11 +338,14 @@ LinearForm::~LinearForm()
    if (!extern_lfs)
    {
       int k;
-      for (k=0; k < dlfi_delta.Size(); k++) { delete dlfi_delta[k]; }
-      for (k=0; k < dlfi.Size(); k++) { delete dlfi[k]; }
-      for (k=0; k < blfi.Size(); k++) { delete blfi[k]; }
-      for (k=0; k < flfi.Size(); k++) { delete flfi[k]; }
-      for (k=0; k < iflfi.Size(); k++) { delete iflfi[k]; }
+      for (k=0; k < domain_delta_integs.Size(); k++)
+      { delete domain_delta_integs[k]; }
+      for (k=0; k < domain_integs.Size(); k++) { delete domain_integs[k]; }
+      for (k=0; k < boundary_integs.Size(); k++) { delete boundary_integs[k]; }
+      for (k=0; k < boundary_face_integs.Size(); k++)
+      { delete boundary_face_integs[k]; }
+      for (k=0; k < interior_face_integs.Size(); k++)
+      { delete interior_face_integs[k]; }
    }
 }
 

fem/linearform.hpp

@@ -26,43 +26,46 @@ protected:
    /// FE space on which the LinearForm lives. Not owned.
    FiniteElementSpace *fes;
 
-   /** @brief Indicates the LinearFormIntegrator%s stored in #dlfi, #dlfi_delta,
-       #blfi, and #flfi are owned by another LinearForm. */
+   /** @brief Indicates the LinearFormIntegrator%s stored in #domain_integs,
+       #domain_delta_integs, #boundary_integs, and #boundary_face_integs are
+       owned by another LinearForm. */
    int extern_lfs;
 
    /// Set of Domain Integrators to be applied.
-   Array<LinearFormIntegrator*> dlfi;
+   Array<LinearFormIntegrator*> domain_integs;
    /// Element attribute marker (should be of length mesh->attributes)
    /// Includes all by default.
    /// 0 - ignore attribute
    /// 1 - include attribute
-   Array<Array<int>*>           dlfi_marker;
+   Array<Array<int>*>           domain_integs_marker;
 
    /// Separate array for integrators with delta function coefficients.
-   Array<DeltaLFIntegrator*> dlfi_delta;
+   Array<DeltaLFIntegrator*>    domain_delta_integs;
 
    /// Set of Boundary Integrators to be applied.
-   Array<LinearFormIntegrator*> blfi;
-   Array<Array<int>*>           blfi_marker; ///< Entries are not owned.
+   Array<LinearFormIntegrator*> boundary_integs;
+   /// Entries are not owned.
+   Array<Array<int>*>           boundary_integs_marker;
 
    /// Set of Boundary Face Integrators to be applied.
-   Array<LinearFormIntegrator*> flfi;
-   Array<Array<int>*>           flfi_marker; ///< Entries are not owned.
+   Array<LinearFormIntegrator*> boundary_face_integs;
+   Array<Array<int>*> boundary_face_integs_marker; ///< Entries not owned.
 
    /// Set of Internal Face Integrators to be applied.
-   Array<LinearFormIntegrator*> iflfi;
+   Array<LinearFormIntegrator*> interior_face_integs;
 
    /// The element ids where the centers of the delta functions lie
-   Array<int> dlfi_delta_elem_id;
+   Array<int> domain_delta_integs_elem_id;
 
    /// The reference coordinates where the centers of the delta functions lie
-   Array<IntegrationPoint> dlfi_delta_ip;
+   Array<IntegrationPoint> domain_delta_integs_ip;
 
    /// If true, the delta locations are not (re)computed during assembly.
-   bool HaveDeltaLocations() { return (dlfi_delta_elem_id.Size() != 0); }
+   bool HaveDeltaLocations()
+   { return (domain_delta_integs_elem_id.Size() != 0); }
 
    /// Force (re)computation of delta locations.
-   void ResetDeltaLocations() { dlfi_delta_elem_id.SetSize(0); }
+   void ResetDeltaLocations() { domain_delta_integs_elem_id.SetSize(0); }
 
 private:
    /// Copy construction is not supported; body is undefined.
@@ -150,22 +153,25 @@ public:
    /** @brief Access all integrators added with AddDomainIntegrator() which are
        not DeltaLFIntegrator%s or they are DeltaLFIntegrator%s with non-delta
        coefficients. */
-   Array<LinearFormIntegrator*> *GetDLFI() { return &dlfi; }
+   Array<LinearFormIntegrator*> *GetDLFI() { return &domain_integs; }
 
    /** @brief Access all integrators added with AddDomainIntegrator() which are
        DeltaLFIntegrator%s with delta coefficients. */
-   Array<DeltaLFIntegrator*> *GetDLFI_Delta() { return &dlfi_delta; }
+   Array<DeltaLFIntegrator*> *GetDLFI_Delta() { return &domain_delta_integs; }
 
    /// Access all integrators added with AddBoundaryIntegrator().
-   Array<LinearFormIntegrator*> *GetBLFI() { return &blfi; }
+   Array<LinearFormIntegrator*> *GetBLFI() { return &boundary_integs; }
 
    /// Access all integrators added with AddBdrFaceIntegrator().
-   Array<LinearFormIntegrator*> *GetFLFI() { return &flfi; }
+   Array<LinearFormIntegrator*> *GetFLFI() { return &boundary_face_integs; }
+
+   /// Access all integrators added with AddInteriorFaceIntegrator().
+   Array<LinearFormIntegrator*> *GetIFLFI() { return &interior_face_integs; }
 
    /** @brief Access all boundary markers added with AddBdrFaceIntegrator().
        If no marker was specified when the integrator was added, the
        corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetFLFI_Marker() { return &flfi_marker; }
+   Array<Array<int>*> *GetFLFI_Marker() { return &boundary_face_integs_marker; }
 
    /// Assembles the linear form i.e. sums over all domain/bdr integrators.
    void Assemble();

fem/nonlinearform.cpp

@@ -630,7 +630,7 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
 
 double BlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs.Update(x.GetData(), block_offsets);
+   xs.Update(const_cast<Vector&>(x), block_offsets);
    return GetEnergyBlocked(xs);
 }
 
@@ -646,7 +646,9 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
    Array<const FiniteElement *> fe2(fes.Size());
    ElementTransformation *T;
 
+   by.UseDevice(true);
    by = 0.0;
+   by.SyncToBlocks();
    for (int s=0; s<fes.Size(); ++s)
    {
       el_x_const[s] = el_x[s] = new Vector();
@@ -785,6 +787,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
       delete el_y[s];
       delete el_x[s];
    }
+
+   by.SyncFromBlocks();
 }
 
 const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
@@ -805,8 +809,8 @@ const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
 
 void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
-   BlockVector by(y.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
+   BlockVector by(y, block_trueOffsets);
 
    const BlockVector &pbx = Prolongate(bx);
    if (needs_prolongation)
@@ -815,8 +819,8 @@ void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
    }
    BlockVector &pby = needs_prolongation ? aux2 : by;
 
-   xs.Update(pbx.GetData(), block_offsets);
-   ys.Update(pby.GetData(), block_offsets);
+   xs.Update(const_cast<BlockVector&>(pbx), block_offsets);
+   ys.Update(pby, block_offsets);
    MultBlocked(xs, ys);
 
    for (int s = 0; s < fes.Size(); s++)
@@ -979,6 +983,8 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
 
             for (int k = 0; k < bfnfi.Size(); ++k)
             {
+               if (bfnfi_marker[k] &&
+                   (*bfnfi_marker[k])[bdr_attr-1] == 0) { continue; }
                bfnfi[k]->AssembleFaceGrad(fe, fe2, *tr, el_x_const, elmats);
                for (int l=0; l<fes.Size(); ++l)
                {
@@ -1019,7 +1025,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
 
 Operator &BlockNonlinearForm::GetGradient(const Vector &x) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
    const BlockVector &pbx = Prolongate(bx);
 
    ComputeGradientBlocked(pbx);

fem/nonlininteg.hpp

@@ -40,10 +40,10 @@ protected:
 public:
    /** @brief Prescribe a fixed IntegrationRule to use (when @a ir != NULL) or
        let the integrator choose (when @a ir == NULL). */
-   void SetIntRule(const IntegrationRule *ir) { IntRule = ir; }
+   virtual void SetIntRule(const IntegrationRule *ir) { IntRule = ir; }
 
    /// Prescribe a fixed IntegrationRule to use.
-   void SetIntegrationRule(const IntegrationRule &irule) { IntRule = &irule; }
+   void SetIntegrationRule(const IntegrationRule &ir) { SetIntRule(&ir); }
 
    /// Set the memory type used for GeometricFactors and other large allocations
    /// in PA extensions.
@@ -121,7 +121,7 @@ public:
        @param[in,out] y  The result Vector: @f$ y += G x @f$. */
    virtual void AddMultGradPA(const Vector &x, Vector &y) const;
 
-   /// Method for computing the diagonal of the gradient with partial assmebly.
+   /// Method for computing the diagonal of the gradient with partial assembly.
    /** The result Vector @a diag is an E-Vector. This method can be called only
        after the method AssembleGradPA() has been called.
 

fem/pbilinearform.cpp

@@ -130,7 +130,7 @@ void ParBilinearForm::ParallelAssemble(OperatorHandle &A, SparseMatrix *A_local)
 
    OperatorHandle dA(A.Type()), Ph(A.Type()), hdA;
 
-   if (fbfi.Size() == 0)
+   if (interior_face_integs.Size() == 0)
    {
       // construct a parallel block-diagonal matrix 'A' based on 'a'
       dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
@@ -214,11 +214,12 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
          }
       }
       vdofs_all.Append(vdofs2);
-      for (int k = 0; k < fbfi.Size(); k++)
+      for (int k = 0; k < interior_face_integs.Size(); k++)
       {
-         fbfi[k]->AssembleFaceMatrix(*pfes->GetFE(T->Elem1No),
-                                     *pfes->GetFaceNbrFE(Elem2NbrNo),
-                                     *T, elemmat);
+         interior_face_integs[k]->
+         AssembleFaceMatrix(*pfes->GetFE(T->Elem1No),
+                            *pfes->GetFaceNbrFE(Elem2NbrNo),
+                            *T, elemmat);
          if (keep_nbr_block)
          {
             mat->AddSubMatrix(vdofs_all, vdofs_all, elemmat, skip_zeros);
@@ -233,7 +234,7 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
 
 void ParBilinearForm::Assemble(int skip_zeros)
 {
-   if (fbfi.Size())
+   if (interior_face_integs.Size())
    {
       pfes->ExchangeFaceNbrData();
       if (!ext && mat == NULL)
@@ -244,7 +245,7 @@ void ParBilinearForm::Assemble(int skip_zeros)
 
    BilinearForm::Assemble(skip_zeros);
 
-   if (!ext && fbfi.Size() > 0)
+   if (!ext && interior_face_integs.Size() > 0)
    {
       AssembleSharedFaces(skip_zeros);
    }
@@ -316,7 +317,8 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const double a)
 const
 {
-   MFEM_VERIFY(fbfi.Size() == 0, "the case of interior face integrators is not"
+   MFEM_VERIFY(interior_face_integs.Size() == 0,
+               "the case of interior face integrators is not"
                " implemented");
 
    if (X.ParFESpace() != pfes)
@@ -471,7 +473,7 @@ void ParBilinearForm::RecoverFEMSolution(
    else
    {
       // Apply conforming prolongation
-      x.SetSize(P.Height());
+      x.SetSize(P.Height(), GetHypreMemoryType());
       P.Mult(X, x);
    }
 }

fem/pfespace.cpp

@@ -515,7 +515,7 @@ const FiniteElement *ParFiniteElementSpace::GetFE(int i) const
    else { return FiniteElementSpace::GetFE(i); }
 }
 
-const Operator *ParFiniteElementSpace::GetFaceRestriction(
+const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
    ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
 {
    const bool is_dg_space = IsDGSpace();
@@ -529,7 +529,7 @@ const Operator *ParFiniteElementSpace::GetFaceRestriction(
    }
    else
    {
-      Operator* res;
+      FaceRestriction *res;
       if (is_dg_space)
       {
          res = new ParL2FaceRestriction(*this, e_ordering, type, m);
@@ -2885,19 +2885,10 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
 
    HypreParMatrix* R;
    R = new HypreParMatrix(MyComm, dof_offsets[nrk], old_dof_offsets[nrk],
-                          dof_offsets, old_dof_offsets, diag, offd, cmap);
+                          dof_offsets, old_dof_offsets, diag, offd, cmap,
+                          true);
 
-#ifndef HYPRE_BIGINT
-   diag->LoseData();
-   offd->LoseData();
-#else
-   diag->SetDataOwner(false);
-   offd->SetDataOwner(false);
-#endif
-   delete diag;
-   delete offd;
-
-   R->SetOwnerFlags(3, 3, 1);
+   R->SetOwnerFlags(R->OwnsDiag(), R->OwnsOffd(), 1);
 
    return R;
 }

fem/pfespace.hpp

@@ -291,7 +291,7 @@ public:
    /** Returns pointer to the FiniteElement in the FiniteElementCollection
        associated with i'th element in the mesh object. If @a i is greater than
        or equal to the number of local mesh elements, @a i will be interpreted
-       as a shifted index of a face neigbor element. */
+       as a shifted index of a face neighbor element. */
    virtual const FiniteElement *GetFE(int i) const;
 
    /** Returns an Operator that converts L-vectors to E-vectors on each face.
@@ -299,7 +299,7 @@ public:
        presence of shared faces. Shared faces are treated as interior faces,
        the returned operator handles the communication needed to get the
        shared face values from other MPI ranks */
-   virtual const Operator *GetFaceRestriction(
+   virtual const FaceRestriction *GetFaceRestriction(
       ElementDofOrdering e_ordering, FaceType type,
       L2FaceValues mul = L2FaceValues::DoubleValued) const;
 

fem/plinearform.cpp

@@ -47,7 +47,7 @@ void ParLinearForm::Assemble()
 {
    LinearForm::Assemble();
 
-   if (iflfi.Size())
+   if (interior_face_integs.Size())
    {
       pfes->ExchangeFaceNbrData();
       AssembleSharedFaces();
@@ -59,10 +59,10 @@ void ParLinearForm::AssembleSharedFaces()
    Array<int> vdofs;
    Vector elemvect;
 
-   if (iflfi.Size())
+   if (interior_face_integs.Size())
    {
       ParMesh *pmesh = pfes->GetParMesh();
-      for (int k = 0; k < iflfi.Size(); k++)
+      for (int k = 0; k < interior_face_integs.Size(); k++)
       {
          for (int i = 0; i < pmesh->GetNSharedFaces(); i++)
          {
@@ -73,9 +73,10 @@ void ParLinearForm::AssembleSharedFaces()
             {
                int Elem2Nbr = tr->Elem2No - pmesh->GetNE();
                fes -> GetElementVDofs (tr -> Elem1No, vdofs);
-               iflfi[0] -> AssembleRHSElementVect (*fes->GetFE(tr -> Elem1No),
-                                                   *pfes->GetFaceNbrFE(Elem2Nbr),
-                                                   *tr, elemvect);
+               interior_face_integs[k]->
+               AssembleRHSElementVect(*fes->GetFE(tr->Elem1No),
+                                      *pfes->GetFaceNbrFE(Elem2Nbr),
+                                      *tr, elemvect);
                AddElementVector (vdofs, elemvect);
             }
          }

fem/pnonlinearform.cpp

@@ -218,7 +218,8 @@ void ParBlockNonlinearForm::SetEssentialBC(const
 
 double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s = 0; s < fes.Size(); ++s)
@@ -237,8 +238,9 @@ double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 
 void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
-   ys_true.Update(y.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
+   ys_true.Update(y, block_trueOffsets);
    xs.Update(block_offsets);
    ys.Update(block_offsets);
 
@@ -262,13 +264,17 @@ void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 
       ys_true.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
    }
+
+   ys_true.SyncFromBlocks();
+   y.SyncMemory(ys_true);
 }
 
 /// Return the local gradient matrix for the given true-dof vector x
 const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
    const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s=0; s<fes.Size(); ++s)
@@ -277,7 +283,8 @@ const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
          xs_true.GetBlock(s), xs.GetBlock(s));
    }
 
-   BlockNonlinearForm::ComputeGradientBlocked(xs); // (re)assemble Grad with b.c.
+   // (re)assemble Grad without b.c. into 'Grads'
+   BlockNonlinearForm::ComputeGradientBlocked(xs);
 
    delete BlockGrad;
    BlockGrad = new BlockOperator(block_offsets);

fem/restriction.cpp

@@ -13,6 +13,7 @@
 #include "gridfunc.hpp"
 #include "fespace.hpp"
 #include "../general/forall.hpp"
+#include <climits>
 
 namespace mfem
 {
@@ -267,35 +268,25 @@ void ElementRestriction::FillSparseMatrix(const Vector &mat_ea,
    FillJAndData(mat_ea, mat);
 }
 
-template <int MaxNbNbr>
 static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
                                       const int *nbr_elts, const int nbrNbElts)
 {
-   // Building the intersection
-   int inter[MaxNbNbr];
-   int cpt = 0;
+   // Find the minimal element index found in both my_elts[] and nbr_elts[]
+   int min_el = INT_MAX;
    for (int i = 0; i < nbElts; i++)
    {
       const int e_i = my_elts[i];
+      if (e_i >= min_el) { continue; }
       for (int j = 0; j < nbrNbElts; j++)
       {
          if (e_i==nbr_elts[j])
          {
-            inter[cpt] = e_i;
-            cpt++;
+            min_el = e_i; // we already know e_i < min_el
+            break;
          }
       }
    }
-   // Finding the minimum
-   int min = inter[0];
-   for (int i = 1; i < cpt; i++)
-   {
-      if (inter[i] < min)
-      {
-         min = inter[i];
-      }
-   }
-   return min;
+   return min_el;
 }
 
 /** Returns the index where a non-zero entry should be added and increment the
@@ -355,7 +346,7 @@ int ElementRestriction::FillI(SparseMatrix &mat) const
                   const int elt = j_E/elt_dofs;
                   j_elts[e_j] = elt;
                }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
                if (e == min_e) // add the nnz only once
                {
                   GetAndIncrementNnzIndex(i_L, I);
@@ -434,7 +425,7 @@ void ElementRestriction::FillJAndData(const Vector &ea_data,
                   j_elts[e_j] = elt;
                   j_B[e_j]    = j_E%elt_dofs;
                }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
                if (e == min_e) // add the nnz only once
                {
                   double val = 0.0;
@@ -847,7 +838,7 @@ void H1FaceRestriction::Mult(const Vector& x, Vector& y) const
    });
 }
 
-void H1FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
+void H1FaceRestriction::AddMultTranspose(const Vector& x, Vector& y) const
 {
    // Assumes all elements have the same number of dofs
    const int nd = dof;
@@ -856,7 +847,7 @@ void H1FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
    auto d_offsets = offsets.Read();
    auto d_indices = gather_indices.Read();
    auto d_x = Reshape(x.Read(), nd, vd, nf);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
    MFEM_FORALL(i, ndofs,
    {
       const int offset = d_offsets[i];
@@ -1267,7 +1258,7 @@ void L2FaceRestriction::Mult(const Vector& x, Vector& y) const
    }
 }
 
-void L2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
+void L2FaceRestriction::AddMultTranspose(const Vector& x, Vector& y) const
 {
    // Assumes all elements have the same number of dofs
    const int nd = dof;
@@ -1280,7 +1271,7 @@ void L2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
    if (m == L2FaceValues::DoubleValued)
    {
       auto d_x = Reshape(x.Read(), nd, vd, 2, nf);
-      auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+      auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
       MFEM_FORALL(i, ndofs,
       {
          const int offset = d_offsets[i];
@@ -1304,7 +1295,7 @@ void L2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
    else
    {
       auto d_x = Reshape(x.Read(), nd, vd, nf);
-      auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+      auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
       MFEM_FORALL(i, ndofs,
       {
          const int offset = d_offsets[i];

fem/restriction.hpp

@@ -21,10 +21,6 @@ namespace mfem
 class FiniteElementSpace;
 enum class ElementDofOrdering;
 
-/** An enum type to specify if only e1 value is requested (SingleValued) or both
-    e1 and e2 (DoubleValued). */
-enum class L2FaceValues : bool {SingleValued, DoubleValued};
-
 /// Operator that converts FiniteElementSpace L-vectors to E-vectors.
 /** Objects of this type are typically created and owned by FiniteElementSpace
     objects, see FiniteElementSpace::GetElementRestriction(). */
@@ -104,10 +100,75 @@ public:
    void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
 };
 
-/// Operator that extracts Face degrees of freedom.
+/** An enum type to specify if only e1 value is requested (SingleValued) or both
+    e1 and e2 (DoubleValued). */
+enum class L2FaceValues : bool {SingleValued, DoubleValued};
+
+/** @brief Base class for operators that extracts Face degrees of freedom.
+
+    In order to compute quantities on the faces of a mesh, it is often useful to
+    extract the degrees of freedom on the faces of the elements. This class
+    provides an interface for such operations.
+
+    If the FiniteElementSpace is ordered by Ordering::byVDIM, then the expected
+    format for the L-vector is (vdim x ndofs), otherwise if Ordering::byNODES
+    the expected format is (ndofs x vdim), where ndofs is the total number of
+    degrees of freedom.
+    Since FiniteElementSpace can either be continuous or discontinuous, the
+    degrees of freedom on a face can either be single valued or double valued,
+    this is what we refer to as the multiplicity and is represented by the
+    L2FaceValues enum type.
+    The format of the output face E-vector of degrees of freedom is
+    (face_dofs x vdim x multiplicity x nfaces), where face_dofs is the number of
+    degrees of freedom on each face, and nfaces the number of faces of the
+    requested FaceType (see FiniteElementSpace::GetNFbyType).
+
+    @note Objects of this type are typically created and owned by
+    FiniteElementSpace objects, see FiniteElementSpace::GetFaceRestriction(). */
+class FaceRestriction : public Operator
+{
+public:
+   FaceRestriction(): Operator() { }
+
+   FaceRestriction(int h, int w): Operator(h, w) { }
+
+   virtual ~FaceRestriction() { }
+
+   /** @brief Extract the face degrees of freedom from @a x into @a y.
+
+       @param[in]  x The L-vector of degrees of freedom.
+       @param[out] y The degrees of freedom on the face, corresponding to a face
+                     E-vector.
+   */
+   void Mult(const Vector &x, Vector &y) const override = 0;
+
+   /** @brief Add the face degrees of freedom @a x to the element degrees of
+       freedom @a y.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   virtual void AddMultTranspose(const Vector &x, Vector &y) const = 0;
+
+   /** @brief Set the face degrees of freedom in the element degrees of freedom
+       @a y to the values given in @a x.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   void MultTranspose(const Vector &x, Vector &y) const override
+   {
+      y = 0.0;
+      AddMultTranspose(x, y);
+   }
+};
+
+/// Operator that extracts Face degrees of freedom for H1 FiniteElementSpaces.
 /** Objects of this type are typically created and owned by FiniteElementSpace
     objects, see FiniteElementSpace::GetFaceRestriction(). */
-class H1FaceRestriction : public Operator
+class H1FaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
@@ -122,16 +183,42 @@ protected:
    Array<int> gather_indices;
 
 public:
-   H1FaceRestriction(const FiniteElementSpace&, const ElementDofOrdering,
-                     const FaceType);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
+   /** @brief Constructor for a H1FaceRestriction.
+
+       @param[in] fes The FiniteElementSpace on which this H1FaceRestriction
+                      operates.
+       @param[in] ordering The requested output ordering of the
+                           H1FaceRestriction, either Native or Lexicographic.
+       @param[in] type The requested type of faces on which this operator
+                       extracts the degrees of freedom, either Interior or
+                       Boundary.
+   */
+   H1FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type);
+
+   /** @brief Extract the face degrees of freedom from @a x into @a y.
+
+       @param[in]  x The L-vector of degrees of freedom.
+       @param[out] y The degrees of freedom on the face, corresponding to a face
+                     E-vector.
+   */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Add the face degrees of freedom @a x to the element degrees of
+       freedom @a y.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
 };
 
-/// Operator that extracts Face degrees of freedom.
+/// Operator that extracts Face degrees of freedom on L2 FiniteElementSpaces.
 /** Objects of this type are typically created and owned by FiniteElementSpace
     objects, see FiniteElementSpace::GetFaceRestriction(). */
-class L2FaceRestriction : public Operator
+class L2FaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
@@ -154,19 +241,38 @@ protected:
                      const L2FaceValues m = L2FaceValues::DoubleValued);
 
 public:
-   L2FaceRestriction(const FiniteElementSpace&, const ElementDofOrdering,
+   L2FaceRestriction(const FiniteElementSpace&,
+                     const ElementDofOrdering,
                      const FaceType,
                      const L2FaceValues m = L2FaceValues::DoubleValued);
-   virtual void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
+
+   /** @brief Extract the face degrees of freedom from @a x into @a y.
+
+       @param[in]  x The L-vector of degrees of freedom.
+       @param[out] y The degrees of freedom on the face, corresponding to a face
+                     E-vector.
+   */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Add the face degrees of freedom @a x to the element degrees of
+       freedom @a y.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
        given by this L2FaceRestriction. */
    virtual void FillI(SparseMatrix &mat, const bool keep_nbr_block = false) const;
+
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
        pattern given by this L2FaceRestriction, and the values of ea_data. */
    virtual void FillJAndData(const Vector &ea_data,
                              SparseMatrix &mat,
                              const bool keep_nbr_block = false) const;
+
    /// This methods adds the DG face matrices to the element matrices.
    void AddFaceMatricesToElementMatrices(Vector &fea_data,
                                          Vector &ea_data) const;

fem/tmop.cpp

@@ -34,6 +34,7 @@ void TMOP_Combo_QualityMetric::EvalP(const DenseMatrix &Jpt,
                                      DenseMatrix &P) const
 {
    DenseMatrix Pt(P.Size());
+   P = 0.0;
    for (int i = 0; i < tmop_q_arr.Size(); i++)
    {
       tmop_q_arr[i]->EvalP(Jpt, Pt);
@@ -50,6 +51,7 @@ void TMOP_Combo_QualityMetric::AssembleH(const DenseMatrix &Jpt,
    DenseMatrix At(A.Size());
    for (int i = 0; i < tmop_q_arr.Size(); i++)
    {
+      At = 0.0;
       tmop_q_arr[i]->AssembleH(Jpt, DS, weight, At);
       At *= wt_arr[i];
       A += At;

fem/tmop.hpp

@@ -371,6 +371,8 @@ public:
       AddQualityMetric(sh_metric, 1.-gamma_);
       AddQualityMetric(sz_metric, gamma_);
    }
+   virtual int Id() const { return 80; }
+   double GetGamma() const { return gamma; }
 
    virtual ~TMOP_Metric_080() { delete sh_metric; delete sz_metric; }
 };
@@ -590,6 +592,52 @@ public:
    virtual int Id() const { return 321; }
 };
 
+/// 3D barrier Shape+Size (VS) metric (polyconvex).
+class TMOP_Metric_332 : public TMOP_Combo_QualityMetric
+{
+protected:
+   double gamma;
+   TMOP_QualityMetric *sh_metric, *sz_metric;
+
+public:
+   TMOP_Metric_332(double gamma_) : gamma(gamma_),
+      sh_metric(new TMOP_Metric_302),
+      sz_metric(new TMOP_Metric_315)
+   {
+      // (1-gamma) mu_302 + gamma mu_315
+      AddQualityMetric(sh_metric, 1.-gamma_);
+      AddQualityMetric(sz_metric, gamma_);
+   }
+
+   virtual int Id() const { return 332; }
+   double GetGamma() const { return gamma; }
+
+   virtual ~TMOP_Metric_332() { delete sh_metric; delete sz_metric; }
+};
+
+/// 3D barrier Shape+Size (VS) metric (polyconvex).
+class TMOP_Metric_333 : public TMOP_Combo_QualityMetric
+{
+protected:
+   double gamma;
+   TMOP_QualityMetric *sh_metric, *sz_metric;
+
+public:
+   TMOP_Metric_333(double gamma_) : gamma(gamma_),
+      sh_metric(new TMOP_Metric_302),
+      sz_metric(new TMOP_Metric_316)
+   {
+      // (1-gamma) mu_302 + gamma mu_316
+      AddQualityMetric(sh_metric, 1.-gamma_);
+      AddQualityMetric(sz_metric, gamma_);
+   }
+
+   virtual int Id() const { return 333; }
+   double GetGamma() const { return gamma; }
+
+   virtual ~TMOP_Metric_333() { delete sh_metric; delete sz_metric; }
+};
+
 /// Shifted barrier form of 3D metric 16 (volume, ideal barrier metric), 3D
 class TMOP_Metric_352 : public TMOP_QualityMetric
 {

fem/tmop/tmop_pa_h2s.cpp

@@ -150,9 +150,49 @@ void EvalH_077(const int e, const int qx, const int qy,
    }
 }
 
+static MFEM_HOST_DEVICE inline
+void EvalH_080(const int e, const int qx, const int qy,
+               const double weight, const double gamma, const double *Jpt,
+               DeviceTensor<7,double> H)
+{
+   // h_80 = (1-gamma) h_2 + gamma h_77.
+
+   constexpr int DIM = 2;
+   double ddI1[4], ddI1b[4], dI2[4], dI2b[4], ddI2[4];
+   kernels::InvariantsEvaluator2D ie(Args()
+                                     .J(Jpt)
+                                     .dI2(dI2)
+                                     .ddI1(ddI1)
+                                     .ddI1b(ddI1b)
+                                     .dI2b(dI2b)
+                                     .ddI2(ddI2));
+
+   const double I2 = ie.Get_I2(), I2inv_sq = 1.0 / (I2 * I2);
+   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               H(r,c,i,j,qx,qy,e) =
+                  (1.0 - gamma) * 0.5 * weight * ddi1b(r,c) +
+                  gamma * ( weight * 0.5 * (1.0 - I2inv_sq) * ddi2(r,c) +
+                            weight * (I2inv_sq / I2) * di2(r,c) * di2(i,j) );
+            }
+         }
+      }
+   }
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
                            const Vector &x_,
                            const double metric_normal,
+                           const double metric_param,
                            const int mid,
                            const int NE,
                            const Array<double> &w_,
@@ -163,7 +203,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
                            const int d1d,
                            const int q1d)
 {
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77 || mid == 80,
                "Metric not yet implemented!");
 
    constexpr int DIM = 2;
@@ -222,6 +262,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
             if (mid ==  2) { EvalH_002(e,qx,qy,weight,Jpt,H); }
             if (mid ==  7) { EvalH_007(e,qx,qy,weight,Jpt,H); }
             if (mid == 77) { EvalH_077(e,qx,qy,weight,Jpt,H); }
+            if (mid == 80) { EvalH_080(e,qx,qy,weight,metric_param,Jpt,H); }
          } // qx
       } // qy
    });
@@ -241,7 +282,10 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
    const Array<double> &G = PA.maps->G;
    Vector &H = PA.H;
 
-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,M,N,W,B,G,J,H);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_080 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,mp,M,N,W,B,G,J,H);
 }
 
 } // namespace mfem

fem/tmop/tmop_pa_h3s.cpp

@@ -181,8 +181,58 @@ void EvalH_321(const int e, const int qx, const int qy, const int qz,
    }
 }
 
+// H_332 = (1-gamma) H_302 + gamma H_315
+static MFEM_HOST_DEVICE inline
+void EvalH_332(const int e, const int qx, const int qy, const int qz,
+               const double weight, const double gamma,
+               const double *J, DeviceTensor<8,double> dP)
+{
+   double B[9];
+   double         dI1b[9],          ddI1b[9];
+   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
+   double         dI3b[9],          ddI3b[9];
+   constexpr int DIM = 3;
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b).ddI1b(ddI1b)
+                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
+                                     .dI3b(dI3b).ddI3b(ddI3b));
+   double sign_detJ;
+   const double c1 = weight/9.;
+   const double I1b = ie.Get_I1b();
+   const double I2b = ie.Get_I2b();
+   const double I3b = ie.Get_I3b(sign_detJ);
+   ConstDeviceMatrix di1b(ie.Get_dI1b(),DIM,DIM);
+   ConstDeviceMatrix di2b(ie.Get_dI2b(),DIM,DIM);
+   ConstDeviceMatrix di3b(ie.Get_dI3b(sign_detJ),DIM,DIM);
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi2b(ie.Get_ddI2b(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi3b(ie.Get_ddI3b(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double dp_302 =
+                  (di2b(r,c)*di1b(i,j) + di1b(r,c)*di2b(i,j))
+                  + ddi2b(r,c)*I1b
+                  + ddi1b(r,c)*I2b;
+               const double dp_315 = 2.0 * weight * (I3b - 1.0) * ddi3b(r,c) +
+                                     2.0 * weight * di3b(r,c) * di3b(i,j);
+               dP(r,c,i,j,qx,qy,qz,e) = (1.0 - gamma) * c1 * dp_302 +
+                                        gamma * dp_315;
+            }
+         }
+      }
+   }
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                            const double metric_normal,
+                           const double metric_param,
                            const int mid,
                            const Vector &x_,
                            const int NE,
@@ -194,8 +244,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                            const int d1d,
                            const int q1d)
 {
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 ||
+               mid == 321 || mid == 332, "3D metric not yet implemented!");
 
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
@@ -255,6 +305,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                if (mid == 303) { EvalH_303(e,qx,qy,qz,weight,Jpt,H); }
                if (mid == 315) { EvalH_315(e,qx,qy,qz,weight,Jpt,H); }
                if (mid == 321) { EvalH_321(e,qx,qy,qz,weight,Jpt,H); }
+               if (mid == 332) { EvalH_332(e,qx,qy,qz,weight,metric_param,Jpt,H); }
             } // qx
          } // qy
       } // qz
@@ -275,7 +326,10 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
    const Array<double> &G = PA.maps->G;
    Vector &H = PA.H;
 
-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,M,X,N,W,B,G,J,H);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_332 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,mp,M,X,N,W,B,G,J,H);
 }
 
 } // namespace mfem

fem/tmop/tmop_pa_p2.cpp

@@ -58,8 +58,24 @@ void EvalP_077(const double *Jpt, double *P)
    kernels::Set(2,2, 0.5 * (1.0 - 1.0 / (I2 * I2)), ie.Get_dI2(), P);
 }
 
+static MFEM_HOST_DEVICE inline
+void EvalP_080(const double *Jpt, double gamma, double *P)
+{
+   // p_80 = (1-gamma) p_2 + gamma p_77.
+
+   double dI1b[4], dI2[4], dI2b[4];
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).
+                                     dI1b(dI1b).dI2(dI2).dI2b(dI2b));
+
+   kernels::Set(2,2, (1.0 - gamma) * 1./2., ie.Get_dI1b(), P);
+
+   const double I2 = ie.Get_I2();
+   kernels::Add(2,2, gamma * 0.5 * (1.0 - 1.0 / (I2 * I2)), ie.Get_dI2(), P);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
                            const double metric_normal,
+                           const double metric_param,
                            const int mid,
                            const int NE,
                            const DenseTensor &j_,
@@ -71,7 +87,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
                            const int d1d,
                            const int q1d)
 {
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77 || mid == 80,
                "Metric not yet implemented!");
 
    constexpr int DIM = 2;
@@ -132,6 +148,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
             if (mid ==  2) { EvalP_002(Jpt, P); }
             if (mid ==  7) { EvalP_007(Jpt, P); }
             if (mid == 77) { EvalP_077(Jpt, P); }
+            if (mid == 80) { EvalP_080(Jpt, metric_param, P); }
             for (int i = 0; i < 4; i++) { P[i] *= weight; }
 
             // PMatO += DS . P^t += DSh . (Jrt . P^t)
@@ -160,7 +177,10 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
    const Array<double> &G = PA.maps->G;
    const double mn = metric_normal;
 
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,M,N,J,W,B,G,X,Y);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_080 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,mp,M,N,J,W,B,G,X,Y);
 }
 
 } // namespace mfem

fem/tmop/tmop_pa_p3.cpp

@@ -75,8 +75,29 @@ void EvalP_321(const double *J, double *P)
    kernels::Add(3,3, ie.Get_dI1(), P);
 }
 
+// P_332 = (1-gamma) P_302 + gamma P_315.
+static MFEM_HOST_DEVICE inline
+void EvalP_332(const double *J, double gamma, double *P)
+{
+   double B[9];
+   double dI1b[9], dI2[9], dI2b[9], dI3b[9];
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b)
+                                     .dI2(dI2).dI2b(dI2b)
+                                     .dI3b(dI3b));
+   const double alpha = (1.0 - gamma) * ie.Get_I1b()/9.;
+   const double beta = (1.0 - gamma) * ie.Get_I2b()/9.;
+   kernels::Add(3,3, alpha, ie.Get_dI2b(), beta, ie.Get_dI1b(), P);
+
+   double sign_detJ;
+   const double I3b = ie.Get_I3b(sign_detJ);
+   kernels::Add(3,3, gamma * 2.0 * (I3b - 1.0), ie.Get_dI3b(sign_detJ), P);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
                            const double metric_normal,
+                           double metric_param,
                            const int mid,
                            const int NE,
                            const DenseTensor &j_,
@@ -88,8 +109,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
                            const int d1d,
                            const int q1d)
 {
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 ||
+               mid == 321 || mid == 332, "3D metric not yet implemented!");
 
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
@@ -146,10 +167,11 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
 
                // metric->EvalP(Jpt, P);
                double P[9];
-               if (mid == 302) { EvalP_302(Jpt,P); }
-               if (mid == 303) { EvalP_303(Jpt,P); }
-               if (mid == 315) { EvalP_315(Jpt,P); }
-               if (mid == 321) { EvalP_321(Jpt,P); }
+               if (mid == 302) { EvalP_302(Jpt, P); }
+               if (mid == 303) { EvalP_303(Jpt, P); }
+               if (mid == 315) { EvalP_315(Jpt, P); }
+               if (mid == 321) { EvalP_321(Jpt, P); }
+               if (mid == 332) { EvalP_332(Jpt, metric_param, P); }
                for (int i = 0; i < 9; i++) { P[i] *= weight; }
 
                // Y += DS . P^t += DSh . (Jrt . P^t)
@@ -180,7 +202,10 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
    const Array<double> &G = PA.maps->G;
    const double mn = metric_normal;
 
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,M,N,J,W,B,G,X,Y);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_332 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,mp,M,N,J,W,B,G,X,Y);
 }
 
 } // namespace mfem

fem/tmop/tmop_pa_w2.cpp

@@ -50,8 +50,15 @@ double EvalW_077(const double *Jpt)
    return 0.5*(I2b*I2b + 1./(I2b*I2b) - 2.);
 }
 
+static MFEM_HOST_DEVICE inline
+double EvalW_080(const double *Jpt, double gamma)
+{
+   return (1.0 - gamma) * EvalW_002(Jpt) + gamma * EvalW_077(Jpt);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
                            const double metric_normal,
+                           const double metric_param,
                            const int mid,
                            const int NE,
                            const DenseTensor &j_,
@@ -64,7 +71,7 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
                            const int d1d,
                            const int q1d)
 {
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77 || mid == 80,
                "2D metric not yet implemented!");
 
    constexpr int DIM = 2;
@@ -125,7 +132,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
             mid ==  1 ? EvalW_001(Jpt) :
             mid ==  2 ? EvalW_002(Jpt) :
             mid ==  7 ? EvalW_007(Jpt) :
-            mid == 77 ? EvalW_077(Jpt) : 0.0;
+            mid == 77 ? EvalW_077(Jpt) :
+            mid == 80 ? EvalW_080(Jpt, metric_param) : 0.0;
 
             E(qx,qy,e) = weight * EvalW;
          }
@@ -141,7 +149,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
    const int D1D = PA.maps->ndof;
    const int Q1D = PA.maps->nqpt;
    const int id = (D1D << 4 ) | Q1D;
-   const double m = metric_normal;
+   const double mn = metric_normal;
    const DenseTensor &J = PA.Jtr;
    const Array<double> &W = PA.ir->GetWeights();
    const Array<double> &B = PA.maps->B;
@@ -149,7 +157,10 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
    const Vector &O = PA.O;
    Vector &E = PA.E;
 
-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,m,M,N,J,W,B,G,X,O,E);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_080 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,mp,M,N,J,W,B,G,X,O,E);
 }
 
 } // namespace mfem

fem/tmop/tmop_pa_w3.cpp

@@ -58,8 +58,15 @@ double EvalW_321(const double *J)
    return ie.Get_I1() + ie.Get_I2()/ie.Get_I3() - 6.0;
 }
 
+static MFEM_HOST_DEVICE inline
+double EvalW_332(const double *J, double gamma)
+{
+   return (1.0 - gamma) * EvalW_302(J) + gamma * EvalW_315(J);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                            const double metric_normal,
+                           const double metric_param,
                            const int mid,
                            const int NE,
                            const DenseTensor &j_,
@@ -72,8 +79,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                            const int d1d,
                            const int q1d)
 {
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 ||
+               mid == 321 || mid == 332, "3D metric not yet implemented!");
 
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
@@ -134,7 +141,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                mid == 302 ? EvalW_302(Jpt) :
                mid == 303 ? EvalW_303(Jpt) :
                mid == 315 ? EvalW_315(Jpt) :
-               mid == 321 ? EvalW_321(Jpt) : 0.0;
+               mid == 321 ? EvalW_321(Jpt) :
+               mid == 332 ? EvalW_332(Jpt, metric_param) : 0.0;
 
                E(qx,qy,qz,e) = weight * EvalW;
             }
@@ -159,7 +167,10 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
    const Vector &O = PA.O;
    Vector &E = PA.E;
 
-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,M,N,J,W,B,G,O,X,E);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_332 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,mp,M,N,J,W,B,G,O,X,E);
 }
 
 } // namespace mfem

general/array.hpp

@@ -339,6 +339,10 @@ inline bool operator!=(const Array<T> &LHS, const Array<T> &RHS)
 }
 
 
+/// Utility function similar to std::as_const in c++17.
+template <typename T> const T &AsConst(T &a) { return a; }
+
+
 template <class T>
 class Array2D;
 

general/binaryio.cpp

@@ -71,7 +71,7 @@ static const unsigned char b64table[] =
    255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255
 };
 
-void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf)
+void DecodeBase64(const char *src, size_t len, std::vector<char> &buf)
 {
    const unsigned char *in = (const unsigned char *)src;
    buf.clear();
@@ -79,7 +79,7 @@ void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf)
    for (size_t i=0; i<len; ++i) { if (b64table[in[i]] != 255) { ++count; } }
    if (count % 4 != 0) { return; }
    buf.resize(3*len/4);
-   unsigned char *out = buf.data();
+   unsigned char *out = (unsigned char *)buf.data();
    count = 0;
    int pad = 0;
    unsigned char c[4];
@@ -97,8 +97,10 @@ void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf)
          count = pad = 0;
       }
    }
-   buf.resize(out - buf.data());
+   buf.resize(out - (unsigned char *)buf.data());
 }
 
+size_t NumBase64Chars(size_t nbytes) { return ((4*nbytes/3) + 3) & ~3; }
+
 } // namespace mfem::bin_io
 } // namespace mfem

general/binaryio.hpp

@@ -50,6 +50,7 @@ inline T read(const char *buf)
    return value;
 }
 
+/// Append the binary representation of @a val to the byte buffer @a vec.
 template <typename T>
 void AppendBytes(std::vector<char> &vec, const T &val)
 {
@@ -57,9 +58,17 @@ void AppendBytes(std::vector<char> &vec, const T &val)
    vec.insert(vec.end(), ptr, ptr + sizeof(T));
 }
 
-void WriteBase64(std::ostream &out, const void *bytes, size_t length);
+/// Given a buffer @a buf of length @a nbytes, encode the data in base-64
+/// format, and write the encoded data to the output stream @a out.
+void WriteBase64(std::ostream &out, const void *bytes, size_t nbytes);
 
-void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf);
+/// Decode @a len base-64 encoded characters in the buffer @a src, and store the
+/// resulting decoded data in @a buf. @a buf will be resized as needed.
+void DecodeBase64(const char *src, size_t len, std::vector<char> &buf);
+
+/// Return the number of characters needed to encode @a nbytes in base-64. This
+/// is equal to 4*nbytes/3, rounded up to the nearest multiple of 4.
+size_t NumBase64Chars(size_t nbytes);
 
 } // namespace mfem::bin_io
 

general/error.cpp

@@ -92,7 +92,7 @@ void mfem_backtrace(int mode, int depth)
    int err = unw_getcontext(&uc);
    err = err ? err : unw_init_local(&cursor, &uc);
 
-   Array<unw_word_t> addrs;
+   Array<unw_word_t> addrs(MemoryType::HOST);
    while (unw_step(&cursor) > 0 && addrs.Size() != depth)
    {
       err = err ? err : unw_get_proc_name(&cursor, name, UNW_NAME_LEN, &offp);

general/mem_manager.cpp

@@ -45,6 +45,9 @@
 #endif
 #endif // MFEM_USE_UMPIRE
 
+// Internal debug option, useful for tracking some memory manager operations.
+// #define MFEM_TRACK_MEM_MANAGER
+
 namespace mfem
 {
 
@@ -63,6 +66,27 @@ MemoryType GetMemoryType(MemoryClass mc)
 }
 
 
+bool MemoryClassContainsType(MemoryClass mc, MemoryType mt)
+{
+   switch (mc)
+   {
+      case MemoryClass::HOST: return IsHostMemory(mt);
+      case MemoryClass::HOST_32:
+         return (mt == MemoryType::HOST_32 ||
+                 mt == MemoryType::HOST_64 ||
+                 mt == MemoryType::HOST_DEBUG);
+      case MemoryClass::HOST_64:
+         return (mt == MemoryType::HOST_64 ||
+                 mt == MemoryType::HOST_DEBUG);
+      case MemoryClass::DEVICE: return IsDeviceMemory(mt);
+      case MemoryClass::MANAGED:
+         return (mt == MemoryType::MANAGED);
+   }
+   MFEM_ABORT("invalid MemoryClass");
+   return false;
+}
+
+
 static void MFEM_VERIFY_TYPES(const MemoryType h_mt, const MemoryType d_mt)
 {
    MFEM_VERIFY(IsHostMemory(h_mt), "h_mt = " << (int)h_mt);
@@ -147,10 +171,12 @@ struct Memory
 /// Alias class that holds the base memory region and the offset
 struct Alias
 {
-   Memory *const mem;
-   const size_t offset, bytes;
+   Memory *mem;
+   size_t offset;
    size_t counter;
-   const MemoryType h_mt;
+   // 'h_mt' is already stored in 'mem', however, we use this field for type
+   // checking since the alias may be dangling, i.e. 'mem' may be invalid.
+   MemoryType h_mt;
 };
 
 /// Maps for the Memory and the Alias classes
@@ -558,7 +584,7 @@ public:
 #ifdef MFEM_USE_HIP
       return HipMemcpyHtoD(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
    void *DtoD(void* dst, const void* src, size_t bytes) override
    {
@@ -568,7 +594,7 @@ public:
 #ifdef MFEM_USE_HIP
       return HipMemcpyDtoD(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
    void *DtoH(void *dst, const void *src, size_t bytes) override
    {
@@ -578,7 +604,7 @@ public:
 #ifdef MFEM_USE_HIP
       return HipMemcpyDtoH(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
 };
 #else
@@ -759,7 +785,7 @@ void *MemoryManager::New_(void *h_tmp, size_t bytes, MemoryType h_mt,
    // mm.InsertDevice(nullptr, h_ptr, bytes, h_mt, d_mt); // non-lazy dev alloc
 
    // MFEM_VERIFY_TYPES(h_mt, mt); // done by mm.Insert() above
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, false);
 
    return h_ptr;
 }
@@ -770,7 +796,7 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
 {
    MFEM_CONTRACT_VAR(alias);
    MFEM_ASSERT(exists, "Internal error!");
-   MFEM_ASSERT(!alias, "Cannot register an alias!");
+   MFEM_VERIFY(!alias, "Cannot register an alias!");
    const bool is_host_mem = IsHostMemory(mt);
    const MemType h_mt = is_host_mem ? mt : GetDualMemoryType(mt);
    const MemType d_mt = is_host_mem ? MemoryType::DEFAULT : mt;
@@ -798,20 +824,21 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
    }
    else // DEVICE TYPES
    {
-      MFEM_VERIFY(ptr, "cannot register NULL device pointer");
+      MFEM_VERIFY(ptr || bytes == 0,
+                  "cannot register NULL device pointer with bytes = " << bytes);
       if (h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
       else { h_ptr = h_tmp; }
       mm.InsertDevice(ptr, h_ptr, bytes, h_mt, d_mt);
       flags = own ? flags | Mem::OWNS_DEVICE : flags & ~Mem::OWNS_DEVICE;
       flags |= (Mem::OWNS_HOST | Mem::VALID_DEVICE);
    }
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, alias);
    return h_ptr;
 }
 
-void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,
-                              MemoryType h_mt, MemoryType d_mt,
-                              bool own, bool alias, unsigned &flags)
+void MemoryManager::Register2_(void *h_ptr, void *d_ptr, size_t bytes,
+                               MemoryType h_mt, MemoryType d_mt,
+                               bool own, bool alias, unsigned &flags)
 {
    MFEM_CONTRACT_VAR(alias);
    MFEM_ASSERT(exists, "Internal error!");
@@ -826,12 +853,14 @@ void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,
 
    flags |= Mem::REGISTERED | Mem::OWNS_INTERNAL;
 
+   MFEM_VERIFY(d_ptr || bytes == 0,
+               "cannot register NULL device pointer with bytes = " << bytes);
    mm.InsertDevice(d_ptr, h_ptr, bytes, h_mt, d_mt);
    flags = (own ? flags | (Mem::OWNS_HOST | Mem::OWNS_DEVICE) :
             flags & ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE)) |
            Mem::VALID_HOST;
 
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, alias);
 }
 
 void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
@@ -871,37 +900,40 @@ void MemoryManager::SetDeviceMemoryType_(void *h_ptr, unsigned flags,
    }
 }
 
-MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType mt, unsigned flags)
+MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType h_mt, unsigned flags)
 {
    const bool alias = flags & Mem::ALIAS;
    const bool registered = flags & Mem::REGISTERED;
    const bool owns_host = flags & Mem::OWNS_HOST;
    const bool owns_device = flags & Mem::OWNS_DEVICE;
    const bool owns_internal = flags & Mem::OWNS_INTERNAL;
-   MFEM_ASSERT(registered || IsHostMemory(mt),"");
+   MFEM_ASSERT(IsHostMemory(h_mt), "invalid h_mt = " << (int)h_mt);
+   // MFEM_ASSERT(registered || IsHostMemory(h_mt),"");
    MFEM_ASSERT(!owns_device || owns_internal, "invalid Memory state");
-   if (!mm.exists || !registered) { return mt; }
+   MFEM_ASSERT(registered || !(owns_host || owns_device || owns_internal),
+               "invalid Memory state");
+   if (!mm.exists || !registered) { return h_mt; }
    if (alias)
    {
       if (owns_internal)
       {
-         const MemoryType h_mt = maps->aliases.at(h_ptr).h_mt;
-         MFEM_ASSERT(mt == h_mt,"");
+         MFEM_ASSERT(mm.IsAlias(h_ptr), "");
+         MFEM_ASSERT(h_mt == maps->aliases.at(h_ptr).h_mt, "");
          mm.EraseAlias(h_ptr);
-         return h_mt;
       }
    }
    else // Known
    {
-      const MemoryType h_mt = mt;
-      MFEM_ASSERT(!owns_internal ||
-                  mt == maps->memories.at(h_ptr).h_mt,"");
       if (owns_host && (h_mt != MemoryType::HOST))
       { ctrl->Host(h_mt)->Dealloc(h_ptr); }
-      if (owns_internal) { mm.Erase(h_ptr, owns_device); }
-      return h_mt;
+      if (owns_internal)
+      {
+         MFEM_ASSERT(mm.IsKnown(h_ptr), "");
+         MFEM_ASSERT(h_mt == maps->memories.at(h_ptr).h_mt, "");
+         mm.Erase(h_ptr, owns_device);
+      }
    }
-   return mt;
+   return h_mt;
 }
 
 void MemoryManager::DeleteDevice_(void *h_ptr, unsigned & flags)
@@ -923,15 +955,19 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
       MFEM_VERIFY(bytes == 0, "Trying to access NULL with size " << bytes);
       return true;
    }
-
-   const bool known = mm.IsKnown(h_ptr);
-   const bool alias = mm.IsAlias(h_ptr);
-   const bool check = known || ((flags & Mem::ALIAS) && alias);
-   MFEM_VERIFY(check, "Unknown host pointer: " << h_ptr);
-   const internal::Memory &mem =
-      (flags & Mem::ALIAS) ?
-      *maps->aliases.at(h_ptr).mem : maps->memories.at(h_ptr);
-   MemoryType d_mt = mem.d_mt;
+   MemoryType d_mt;
+   if (!(flags & Mem::ALIAS))
+   {
+      auto iter = maps->memories.find(h_ptr);
+      MFEM_VERIFY(iter != maps->memories.end(), "internal error");
+      d_mt = iter->second.d_mt;
+   }
+   else
+   {
+      auto iter = maps->aliases.find(h_ptr);
+      MFEM_VERIFY(iter != maps->aliases.end(), "internal error");
+      d_mt = iter->second.mem->d_mt;
+   }
    if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
    switch (mc)
    {
@@ -969,7 +1005,7 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
 void *MemoryManager::ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                                 size_t bytes, unsigned &flags)
 {
-   MemoryManager::CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
    if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
    MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
    if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -993,7 +1029,7 @@ void *MemoryManager::ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
 const void *MemoryManager::Read_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                                  size_t bytes, unsigned &flags)
 {
-   CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
    if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
    MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
    if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -1017,7 +1053,7 @@ const void *MemoryManager::Read_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
 void *MemoryManager::Write_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                             size_t bytes, unsigned &flags)
 {
-   CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
    if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
    MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
    if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -1061,22 +1097,20 @@ void MemoryManager::SyncAlias_(const void *base_h_ptr, void *alias_h_ptr,
                  (base_flags & (Mem::VALID_HOST | Mem::VALID_DEVICE));
 }
 
-MemoryType MemoryManager::GetDeviceMemoryType_(void *h_ptr)
+MemoryType MemoryManager::GetDeviceMemoryType_(void *h_ptr, bool alias)
 {
    if (mm.exists)
    {
-      const bool known = mm.IsKnown(h_ptr);
-      if (known)
+      if (!alias)
       {
-         internal::Memory &mem = maps->memories.at(h_ptr);
-         return mem.d_mt;
-      }
-      const bool alias = mm.IsAlias(h_ptr);
-      if (alias)
-      {
-         internal::Memory *mem = maps->aliases.at(h_ptr).mem;
-         return mem->d_mt;
+         auto iter = maps->memories.find(h_ptr);
+         MFEM_ASSERT(iter != maps->memories.end(), "internal error");
+         return iter->second.d_mt;
       }
+      // alias == true
+      auto iter = maps->aliases.find(h_ptr);
+      MFEM_ASSERT(iter != maps->aliases.end(), "internal error");
+      return iter->second.mem->d_mt;
    }
    MFEM_ABORT("internal error");
    return MemoryManager::host_mem_type;
@@ -1086,7 +1120,7 @@ MemoryType MemoryManager::GetHostMemoryType_(void *h_ptr)
 {
    if (!mm.exists) { return MemoryManager::host_mem_type; }
    if (mm.IsKnown(h_ptr)) { return maps->memories.at(h_ptr).h_mt; }
-   if (mm.IsAlias(h_ptr)) { return maps->aliases.at(h_ptr).mem->h_mt; }
+   if (mm.IsAlias(h_ptr)) { return maps->aliases.at(h_ptr).h_mt; }
    return MemoryManager::host_mem_type;
 }
 
@@ -1137,7 +1171,7 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
       {
          if (dst_h_ptr != src_d_ptr && bytes != 0)
          {
-            internal::Memory &src_d_base = maps->memories.at(src_d_ptr);
+            internal::Memory &src_d_base = maps->memories.at(src_h_ptr);
             MemoryType src_d_mt = src_d_base.d_mt;
             ctrl->Device(src_d_mt)->DtoH(dst_h_ptr, src_d_ptr, bytes);
          }
@@ -1240,6 +1274,10 @@ bool MemoryManager::IsAlias_(const void *h_ptr)
 void MemoryManager::Insert(void *h_ptr, size_t bytes,
                            MemoryType h_mt, MemoryType d_mt)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: registering h_ptr: " << h_ptr
+             << ", bytes: " << bytes << std::endl;
+#endif
    if (h_ptr == NULL)
    {
       MFEM_VERIFY(bytes == 0, "Trying to add NULL with size " << bytes);
@@ -1254,8 +1292,14 @@ void MemoryManager::Insert(void *h_ptr, size_t bytes,
    if (res.second == false)
    {
       auto &m = res.first->second;
-      MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt && m.d_mt == d_mt,
+      MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt &&
+                  (m.d_mt == d_mt || (d_mt == MemoryType::DEFAULT &&
+                                      m.d_mt == GetDualMemoryType(h_mt))),
                   "Address already present with different attributes!");
+#ifdef MFEM_TRACK_MEM_MANAGER
+      mfem::out << "[mfem memory manager]: repeated registration of h_ptr: "
+                << h_ptr << std::endl;
+#endif
    }
 #endif
 }
@@ -1267,7 +1311,7 @@ void MemoryManager::InsertDevice(void *d_ptr, void *h_ptr, size_t bytes,
    MFEM_ASSERT(h_ptr != NULL, "internal error");
    Insert(h_ptr, bytes, h_mt, d_mt);
    internal::Memory &mem = maps->memories.at(h_ptr);
-   if (d_ptr == NULL) { ctrl->Device(d_mt)->Alloc(mem); }
+   if (d_ptr == NULL && bytes != 0) { ctrl->Device(d_mt)->Alloc(mem); }
    else { mem.d_ptr = d_ptr; }
 }
 
@@ -1276,6 +1320,11 @@ void MemoryManager::InsertAlias(const void *base_ptr, void *alias_ptr,
 {
    size_t offset = static_cast<size_t>(static_cast<const char*>(alias_ptr) -
                                        static_cast<const char*>(base_ptr));
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: registering alias of base_ptr: "
+             << base_ptr << ", offset: " << offset << ", bytes: " << bytes
+             << ", base is alias: " << base_is_alias << std::endl;
+#endif
    if (!base_ptr)
    {
       MFEM_VERIFY(offset == 0,
@@ -1288,26 +1337,33 @@ void MemoryManager::InsertAlias(const void *base_ptr, void *alias_ptr,
       MFEM_ASSERT(alias.mem,"");
       base_ptr = alias.mem->h_ptr;
       offset += alias.offset;
+#ifdef MFEM_TRACK_MEM_MANAGER
+      mfem::out << "[mfem memory manager]: real base_ptr: " << base_ptr
+                << std::endl;
+#endif
    }
    internal::Memory &mem = maps->memories.at(base_ptr);
+   MFEM_VERIFY(offset + bytes <= mem.bytes, "invalid alias");
    auto res =
       maps->aliases.emplace(alias_ptr,
-                            internal::Alias{&mem, offset, bytes, 1, mem.h_mt});
+                            internal::Alias{&mem, offset, 1, mem.h_mt});
    if (res.second == false) // alias_ptr was already in the map
    {
-      if (res.first->second.mem != &mem || res.first->second.offset != offset)
-      {
-         mfem_error("alias already exists with different base/offset!");
-      }
-      else
-      {
-         res.first->second.counter++;
-      }
+      internal::Alias &alias = res.first->second;
+      // Update the alias data in case the existing alias is dangling
+      alias.mem = &mem;
+      alias.offset = offset;
+      alias.h_mt = mem.h_mt;
+      alias.counter++;
    }
 }
 
 void MemoryManager::Erase(void *h_ptr, bool free_dev_ptr)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: un-registering h_ptr: " << h_ptr
+             << std::endl;
+#endif
    if (!h_ptr) { return; }
    auto mem_map_iter = maps->memories.find(h_ptr);
    if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
@@ -1321,10 +1377,6 @@ void MemoryManager::EraseDevice(void *h_ptr)
    if (!h_ptr) { return; }
    auto mem_map_iter = maps->memories.find(h_ptr);
    if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
-   if (maps->aliases.find(h_ptr) != maps->aliases.end())
-   {
-      mfem_error("cannot delete aliased obj!");
-   }
    internal::Memory &mem = mem_map_iter->second;
    if (mem.d_ptr) { ctrl->Device(mem.d_mt)->Dealloc(mem);}
    mem.d_ptr = nullptr;
@@ -1332,6 +1384,10 @@ void MemoryManager::EraseDevice(void *h_ptr)
 
 void MemoryManager::EraseAlias(void *alias_ptr)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: un-registering alias_ptr: " << alias_ptr
+             << std::endl;
+#endif
    if (!alias_ptr) { return; }
    auto alias_map_iter = maps->aliases.find(alias_ptr);
    if (alias_map_iter == maps->aliases.end()) { mfem_error("Unknown alias!"); }
@@ -1355,14 +1411,14 @@ void *MemoryManager::GetDevicePtr(const void *h_ptr, size_t bytes,
    if (!mem.d_ptr)
    {
       if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
+      if (mem.bytes) { ctrl->Device(d_mt)->Alloc(mem); }
    }
    // Aliases might have done some protections
-   ctrl->Device(d_mt)->Unprotect(mem);
+   if (mem.d_ptr) { ctrl->Device(d_mt)->Unprotect(mem); }
    if (copy_data)
    {
       MFEM_ASSERT(bytes <= mem.bytes, "invalid copy size");
-      ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes);
+      if (bytes) { ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes); }
    }
    ctrl->Host(h_mt)->Protect(mem, bytes);
    return mem.d_ptr;
@@ -1388,16 +1444,17 @@ void *MemoryManager::GetAliasDevicePtr(const void *alias_ptr, size_t bytes,
    if (!mem.d_ptr)
    {
       if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
+      if (mem.bytes) { ctrl->Device(d_mt)->Alloc(mem); }
    }
    void *alias_h_ptr = static_cast<char*>(mem.h_ptr) + offset;
    void *alias_d_ptr = static_cast<char*>(mem.d_ptr) + offset;
    MFEM_ASSERT(alias_h_ptr == alias_ptr, "internal error");
-   MFEM_ASSERT(bytes <= alias.bytes, "internal error");
+   MFEM_ASSERT(offset + bytes <= mem.bytes, "internal error");
    mem.d_rw = mem.h_rw = false;
-   ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes);
+   if (mem.d_ptr) { ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes); }
    ctrl->Host(h_mt)->AliasUnprotect(alias_ptr, bytes);
-   if (copy) { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
+   if (copy && mem.d_ptr)
+   { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
    ctrl->Host(h_mt)->AliasProtect(alias_ptr, bytes);
    return alias_d_ptr;
 }
@@ -1504,6 +1561,23 @@ void MemoryManager::Configure(const MemoryType host_mt,
 void MemoryManager::Destroy()
 {
    MFEM_VERIFY(exists, "MemoryManager has already been destroyed!");
+#ifdef MFEM_TRACK_MEM_MANAGER
+   size_t num_memories = maps->memories.size();
+   size_t num_aliases = maps->aliases.size();
+   if (num_memories != 0 || num_aliases != 0)
+   {
+      MFEM_WARNING("...\n\t number of registered pointers: " << num_memories
+                   << "\n\t number of registered aliases : " << num_aliases);
+   }
+#endif
+   // Keep for debugging purposes:
+#if 0
+   mfem::out << "Destroying the MemoryManager ...\n"
+             << "remaining registered pointers : "
+             << maps->memories.size() << '\n'
+             << "remaining registered aliases  : "
+             << maps->aliases.size() << '\n';
+#endif
    for (auto& n : maps->memories)
    {
       internal::Memory &mem = n.second;
@@ -1554,7 +1628,6 @@ int MemoryManager::PrintAliases(std::ostream &out)
       out << "\nalias: key " << n.first << ", "
           << "h_ptr " << alias.mem->h_ptr << ", "
           << "offset " << alias.offset << ", "
-          << "bytes  " << alias.bytes << ", "
           << "counter " << alias.counter;
       n_out++;
    }
@@ -1569,7 +1642,13 @@ int MemoryManager::CompareHostAndDevice_(void *h_ptr, size_t size,
                  mm.GetAliasDevicePtr(h_ptr, size, false) :
                  mm.GetDevicePtr(h_ptr, size, false);
    char *h_buf = new char[size];
+#ifdef MFEM_USE_CUDA
    CuMemcpyDtoH(h_buf, d_ptr, size);
+#elif MFE_USE_HIP
+   HipMemcpyDtoH(h_buf, d_ptr, size);
+#else
+   std::memcpy(h_buf, d_ptr, size);
+#endif
    int res = std::memcmp(h_ptr, h_buf, size);
    delete [] h_buf;
    return res;
@@ -1591,13 +1670,24 @@ void MemoryPrintFlags(unsigned flags)
          << std::endl;
 }
 
-void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr)
+void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr,
+                                         bool alias)
 {
    if (!mm.exists) {return;}
-   const bool known = mm.IsKnown(h_ptr);
-   const bool alias = mm.IsAlias(h_ptr);
-   if (known) { MFEM_VERIFY(h_mt == maps->memories.at(h_ptr).h_mt,""); }
-   if (alias) { MFEM_VERIFY(h_mt == maps->aliases.at(h_ptr).mem->h_mt,""); }
+   if (!alias)
+   {
+      auto it = maps->memories.find(h_ptr);
+      MFEM_VERIFY(it != maps->memories.end(),
+                  "host pointer is not registered: h_ptr = " << h_ptr);
+      MFEM_VERIFY(h_mt == it->second.h_mt, "host pointer MemoryType mismatch");
+   }
+   else
+   {
+      auto it = maps->aliases.find(h_ptr);
+      MFEM_VERIFY(it != maps->aliases.end(),
+                  "alias pointer is not registered: h_ptr = " << h_ptr);
+      MFEM_VERIFY(h_mt == it->second.h_mt, "alias pointer MemoryType mismatch");
+   }
 }
 
 MemoryManager mm;

general/mem_manager.hpp

@@ -17,6 +17,9 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
+#ifdef MFEM_USE_MPI
+#include <HYPRE_config.h> // HYPRE_USING_CUDA
+#endif
 
 namespace mfem
 {
@@ -90,6 +93,9 @@ inline bool IsDeviceMemory(MemoryType mt)
 /// Return a suitable MemoryType for a given MemoryClass.
 MemoryType GetMemoryType(MemoryClass mc);
 
+/// Return true iff the MemoryType @a mt is contained in the MemoryClass @a mc.
+bool MemoryClassContainsType(MemoryClass mc, MemoryType mt);
+
 /// Return a suitable MemoryClass from a pair of MemoryClass%es.
 /** Note: this operation is commutative, i.e. a*b = b*a, associative, i.e.
     (a*b)*c = a*(b*c), and has an identity element: MemoryClass::HOST.
@@ -463,6 +469,13 @@ public:
        returned. */
    inline MemoryType GetMemoryType() const;
 
+   /// Return the host MemoryType of the Memory object.
+   inline MemoryType GetHostMemoryType() const { return h_mt; }
+
+   /** @brief Return the device MemoryType of the Memory object. If the device
+       MemoryType is not set, return MemoryType::DEFAULT. */
+   inline MemoryType GetDeviceMemoryType() const;
+
    /** @brief Return true if host pointer is valid */
    inline bool HostIsValid() const;
 
@@ -596,9 +609,9 @@ private: // Static methods used by the Memory<T> class
                           bool own, bool alias, unsigned &flags);
 
    /// Register a pair of external host and device pointers
-   static void Register_(void *h_ptr, void *d_ptr, size_t bytes,
-                         MemoryType h_mt, MemoryType d_mt,
-                         bool own, bool alias, unsigned &flags);
+   static void Register2_(void *h_ptr, void *d_ptr, size_t bytes,
+                          MemoryType h_mt, MemoryType d_mt,
+                          bool own, bool alias, unsigned &flags);
 
    /// Register an alias. Note: base_h_ptr may be an alias.
    static void Alias_(void *base_h_ptr, size_t offset, size_t bytes,
@@ -635,13 +648,13 @@ private: // Static methods used by the Memory<T> class
 
    /// Return the type the of the currently valid memory.
    /// If more than one types are valid, return a device type.
-   static MemoryType GetDeviceMemoryType_(void *h_ptr);
+   static MemoryType GetDeviceMemoryType_(void *h_ptr, bool alias);
 
    /// Return the type the of the host memory.
    static MemoryType GetHostMemoryType_(void *h_ptr);
 
    /// Verify that h_mt and h_ptr's h_mt (memory or alias) are equal.
-   static void CheckHostMemoryType_(MemoryType h_mt, void *h_ptr);
+   static void CheckHostMemoryType_(MemoryType h_mt, void *h_ptr, bool alias);
 
    /// Copy entries from valid memory type to valid memory type.
    ///  Both dest_h_ptr and src_h_ptr are registered host pointers.
@@ -851,15 +864,21 @@ inline void Memory<T>::Wrap(T *ptr, int size, bool own)
 {
    h_ptr = ptr;
    capacity = size;
-   const size_t bytes = size*sizeof(T);
    flags = (own ? OWNS_HOST : 0) | VALID_HOST;
    h_mt = MemoryManager::GetHostMemoryType();
 #ifdef MFEM_DEBUG
    if (own && MemoryManager::Exists())
-   { MFEM_VERIFY(h_mt == MemoryManager::GetHostMemoryType_(h_ptr),""); }
+   {
+      MemoryType h_ptr_mt = MemoryManager::GetHostMemoryType_(h_ptr);
+      MFEM_VERIFY(h_mt == h_ptr_mt,
+                  "h_mt = " << (int)h_mt << ", h_ptr_mt = " << (int)h_ptr_mt);
+   }
 #endif
    if (own && h_mt != MemoryType::HOST)
-   { MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags); }
+   {
+      const size_t bytes = size*sizeof(T);
+      MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags);
+   }
 }
 
 template <typename T>
@@ -880,7 +899,7 @@ inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
    else
    {
       h_mt = MemoryManager::GetDualMemoryType(mt);
-      h_ptr = (h_mt == MemoryType::HOST) ? new T[size] : nullptr;
+      h_ptr = (h_mt == MemoryType::HOST) ? NewHOST(size) : nullptr;
    }
    flags = 0;
    h_ptr = (T*)MemoryManager::Register_(ptr, h_ptr, size*sizeof(T), mt,
@@ -897,7 +916,8 @@ inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
    MFEM_ASSERT(IsHostMemory(h_mt),"");
    const size_t bytes = size*sizeof(T);
    const MemoryType d_mt = MemoryManager::GetDualMemoryType(h_mt);
-   MemoryManager::Register_(h_ptr, d_ptr, bytes, h_mt, d_mt, own, false, flags);
+   MemoryManager::Register2_(h_ptr, d_ptr, bytes, h_mt, d_mt,
+                             own, false, flags);
 }
 
 template <typename T>
@@ -907,13 +927,35 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
    h_mt = base.h_mt;
    h_ptr = base.h_ptr + offset;
    if (!(base.flags & REGISTERED))
-   { flags = (base.flags | ALIAS) & ~(OWNS_HOST | OWNS_DEVICE); }
-   else
    {
-      const size_t s_bytes = size*sizeof(T);
-      const size_t o_bytes = offset*sizeof(T);
-      MemoryManager::Alias_(base.h_ptr, o_bytes, s_bytes, base.flags, flags);
+      if (
+#ifndef HYPRE_USING_CUDA
+         // If the following condition is true then MemoryManager::Exists()
+         // should also be true:
+         IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
+#else
+         // When HYPRE_USING_CUDA is defined we always register the 'base' if
+         // the MemoryManager::Exists():
+         MemoryManager::Exists()
+#endif
+      )
+      {
+         // Register 'base':
+         MemoryManager::Register_(base.h_ptr, nullptr, base.capacity*sizeof(T),
+                                  base.h_mt, base.flags & OWNS_HOST,
+                                  base.flags & ALIAS, base.flags);
+      }
+      else
+      {
+         // Copy the flags from 'base', setting the ALIAS flag to true, and
+         // setting both OWNS_HOST and OWNS_DEVICE to false:
+         flags = (base.flags | ALIAS) & ~(OWNS_HOST | OWNS_DEVICE);
+         return;
+      }
    }
+   const size_t s_bytes = size*sizeof(T);
+   const size_t o_bytes = offset*sizeof(T);
+   MemoryManager::Alias_(base.h_ptr, o_bytes, s_bytes, base.flags, flags);
 }
 
 template <typename T>
@@ -1069,7 +1111,14 @@ template <typename T>
 inline MemoryType Memory<T>::GetMemoryType() const
 {
    if (!(flags & VALID_DEVICE)) { return h_mt; }
-   return MemoryManager::GetDeviceMemoryType_(h_ptr);
+   return MemoryManager::GetDeviceMemoryType_(h_ptr, flags & ALIAS);
+}
+
+template <typename T>
+inline MemoryType Memory<T>::GetDeviceMemoryType() const
+{
+   if (!(flags & REGISTERED)) { return MemoryType::DEFAULT; }
+   return MemoryManager::GetDeviceMemoryType_(h_ptr, flags & ALIAS);
 }
 
 template <typename T>

general/version.cpp

@@ -82,6 +82,9 @@ const char *GetConfigStr()
 #ifdef MFEM_USE_CUDA
       "MFEM_USE_CUDA\n"
 #endif
+#ifdef MFEM_USE_EPIC
+      "MFEM_USE_EPIC\n"
+#endif
 #ifdef MFEM_USE_EXCEPTIONS
       "MFEM_USE_EXCEPTIONS\n"
 #endif