Modify ex26 with anisotropic diffusion and ILU smoothing

Variable order transfer operator

.github/workflows/mfem-analysis.yml

@@ -82,9 +82,9 @@ jobs:
       uses: mfem/github-actions/build-mfem@v2.0
       with:
         os: ${{ runner.os }}
-        target: optim
+        target: opt
         codecov: NO
-        mpi: parallel
+        mpi: par
         build-system: make
         hypre-dir: ${{ env.HYPRE_TOP_DIR }}
         metis-dir: ${{ env.METIS_TOP_DIR }}

.gitignore

@@ -225,6 +225,14 @@ miniapps/mtop/ParHeat*
 miniapps/mtop/seqheat
 miniapps/mtop/SeqHeat*
 
+miniapps/autodiff/paradiff
+miniapps/autodiff/seqadiff
+miniapps/autodiff/seqtest
+miniapps/autodiff/par_example
+miniapps/autodiff/seq_example
+miniapps/autodiff/seq_test
+miniapps/autodiff/Exampl*
+
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
 miniapps/navier/navier_kovasznay_vs
@@ -251,6 +259,8 @@ miniapps/performance/sol.*
 
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
+miniapps/shifted/extrapolate
+miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf

.gitlab-ci.yml

@@ -29,12 +29,34 @@ stages:
 
 variables:
   CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
+  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
+  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
+  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
+  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
+  # used in '.gitlab/configs/<machine>-config.yml':
+  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
+
+# Defines the default choice for updating the saved baseline results. By default
+# the baseline can only be updated from the master branch. This variable offers
+# the option to manually ask for rebaselining from another branch if necessary.
+  REBASELINE: "NO"
+  AUTOTEST: "NO"
+  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
+  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
+  #   files to the MFEM/autotest repo.
+  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
+  #   files to the MFEM/autotest repo. Instead they will just show the contents
+  #   of the report files and remove them.
+  AUTOTEST_COMMIT: "YES"
 
 # Trigger subpipelines:
 quartz-build-and-test:
   stage: sub-pipelines
   variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
     include: .gitlab/quartz-build-and-test.yml
     strategy: depend
@@ -42,7 +64,11 @@ quartz-build-and-test:
 quartz-baseline:
   stage: sub-pipelines
   variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    REBASELINE: "${REBASELINE}"
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
     include: .gitlab/quartz-baseline.yml
     strategy: depend
@@ -50,7 +76,10 @@ quartz-baseline:
 lassen-build-and-test:
   stage: sub-pipelines
   variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
     include: .gitlab/lassen-build-and-test.yml
     strategy: depend
@@ -58,7 +87,10 @@ lassen-build-and-test:
 corona-build-and-test:
   stage: sub-pipelines
   variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
     include: .gitlab/corona-build-and-test.yml
     strategy: depend

.gitlab/configs/common.yml

@@ -18,19 +18,13 @@ variables:
 # the pipeline, preventing any form of concurrency with other pipelines. This
 # also means that the BUILD_ROOT directory will never be cleaned.
 # TODO: add a clean-up mechanism
-  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
 
 # On LLNL's quartz, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
 
-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-  _REBASELINE: "NO"
-  _AUTOTEST: "NO"
-
 # Git repositories used in the pipeline
   TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
   TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
@@ -40,5 +34,3 @@ variables:
 # Directory used to place artifacts.
   ARTIFACTS_DIR: artifacts
   SLURM_OVERLAP: 1
-
-

.gitlab/configs/corona-config.yml

@@ -26,17 +26,20 @@ variables:
     - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
       when: never
     # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
       when: on_success
     # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
       when: on_failure
     # Always release resource
     - if: '$CI_JOB_NAME =~ /release_resource/'
       when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
     # Default is to run if previous stage succeeded
     - when: on_success
 
@@ -46,9 +49,11 @@ variables:
   extends: [.on_corona]
   stage: build_and_test
   script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
     - export THREADS=12
     - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
     - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
-
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/lassen-config.yml

@@ -21,14 +21,17 @@ variables:
     - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
       when: never
     # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
       when: on_success
     # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
       when: on_failure
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
     - when: on_success
 
 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
@@ -39,5 +42,8 @@ variables:
   extends: [.on_lassen]
   stage: build_and_test
   script:
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
+    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
   needs: [setup]

.gitlab/configs/quartz-config.yml

@@ -22,17 +22,20 @@ variables:
     - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
       when: never
     # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
       when: on_success
     # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
       when: on_failure
     # Always release resource
     - if: '$CI_JOB_NAME =~ /release_resource/'
       when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
     # Default is to run if previous stage succeeded
     - when: on_success
 
@@ -42,9 +45,11 @@ variables:
   extends: [.on_quartz]
   stage: build_and_test
   script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
     - export THREADS=12
     - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
     - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
-
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/report-build-and-test.yml

@@ -0,0 +1,81 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+.report_job_success:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # Report SUCCESS while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
+
+.report_job_failure:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # Report FAILURE while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock

.gitlab/configs/setup-baseline.yml

@@ -9,13 +9,6 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
-# when call the runtest script in MFEM test repo.
-# Note: the value must be consistent with what setup_baseline does.
-variables:
-  TPLS_DIR: ${BUILD_ROOT}/tpls
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}_baseline
-
 # The setup_baseline job doesn't rely on MFEM git repo. It prepares a
 # pipeline-wide working directory downloading/updating external repos.
 # TODO:
@@ -30,13 +23,50 @@ setup_baseline:
   variables:
     GIT_STRATEGY: none
   script:
+    #
+    # Setup ${BUILD_ROOT}/tpls and ${BUILD_ROOT}/tests:
+    #
+    - echo "MACHINE_NAME = ${MACHINE_NAME}"
+    - echo "REBASELINE = ${REBASELINE}"
+    - echo "AUTOTEST = ${AUTOTEST}"
+    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
     - echo "BUILD_ROOT ${BUILD_ROOT}"
     - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
     - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
     - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
     - cd tpls && git pull && cd ..
     - cd tests && git pull origin && cd ..
+    #
+    # Setup ${AUTOTEST_ROOT}/autotest:
+    #
     - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
     - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
+    - command -v flock || echo "Required command 'flock' not found"
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # clone/update the autotest repo while holding the file lock on
+        # 'autotest.lock'
+        err=0
+        if [[ ! -d "autotest" ]]; then
+          git clone ${AUTOTEST_REPO}
+        else
+          cd autotest && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock

.gitlab/configs/setup-build-and-test.yml

@@ -9,13 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-variables:
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}_build_and_test
-
-# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
-# then symlinks the repo to the parent directory of the MFEM source directory.
-# Unit tests that depend on the mfem/data repo will then detect that this
-# directory is present and be enabled.
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
 setup:
   tags:
     - shell
@@ -24,11 +21,74 @@ setup:
   variables:
     GIT_STRATEGY: none
   script:
-    - echo "BUILD_ROOT ${BUILD_ROOT}"
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+    #
+    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
+    # and '.gitlab/configs/<machine>-config.yml'
+    #
+    - echo "MACHINE_NAME = ${MACHINE_NAME}"
+    - echo "AUTOTEST = ${AUTOTEST}"
+    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
+    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
+    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
+    - command -v flock || echo "Required command 'flock' not found"
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/mfem-data.lock' ..."
+        # try to get an excusive lock on fd 9 (mfem-data.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/mfem-data.lock'"
+        date
+        # clone/update the mfem/data repo while holding the file lock on
+        # 'mfem-data.lock'
+        err=0
+        if [[ ! -d "mfem-data" ]]; then
+          git clone ${MFEM_DATA_REPO} "mfem-data"
+        else
+          cd "mfem-data" && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> mfem-data.lock
+    #
+    # Setup ${AUTOTEST_ROOT}/autotest:
+    #
     - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
     - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
-
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # clone/update the autotest repo while holding the file lock on
+        # 'autotest.lock'
+        err=0
+        if [[ ! -d "autotest" ]]; then
+          git clone ${AUTOTEST_REPO}
+        else
+          cd autotest && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock

.gitlab/corona-build-and-test.yml

@@ -22,6 +22,7 @@ allocate_resource:
   extends: .on_corona
   stage: allocate_resource
   script:
+    - echo ${ALLOC_NAME}
     - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
   timeout: 6h
   needs: [setup]
@@ -40,24 +41,27 @@ release_resource:
   extends: .on_corona
   stage: release_resource_and_report
   script:
+    - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
     - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
   needs: [rocm_gcc_8.3.1]
 
 # Jobs report
 report_job_success:
-  extends: .on_corona
   stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_success
+  extends:
+    - .on_corona
+    - .report_job_success
 
 report_job_failure:
-  extends: .on_corona
   stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_failure
+  extends:
+    - .on_corona
+    - .report_job_failure
 
 include:
   - local: .gitlab/configs/common.yml
   - local: .gitlab/configs/corona-config.yml
   - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/lassen-build-and-test.yml

@@ -19,20 +19,26 @@ opt_mpi_cuda_xl_16_1_1_8:
     SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
+opt_mpi_cuda_hypre_cuda_xl:
+  variables:
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+  extends: .build_and_test_on_lassen
+
 # Jobs report
 report_job_success:
-  extends: .on_lassen
   stage: report
-  script:
-    - .gitlab/scripts/report_build_and_test_success
+  extends:
+    - .on_lassen
+    - .report_job_success
 
 report_job_failure:
-  extends: .on_lassen
   stage: report
-  script:
-    - .gitlab/scripts/report_build_and_test_failure
+  extends:
+    - .on_lassen
+    - .report_job_failure
 
 include:
   - local: .gitlab/configs/common.yml
   - local: .gitlab/configs/lassen-config.yml
   - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/quartz-baseline.yml

@@ -16,12 +16,26 @@ stages:
   - setup
   - baseline_check
   - baseline_report
+  - cleanup
   - baseline_publish
 
 baselinecheck_mfem_intel_quartz:
   extends: [.on_quartz]
   stage: baseline_check
+  variables:
+    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
+    # when call the runtest script in MFEM test repo.
+    # Note: the value must be consistent with the setup performed in
+    #       .gitlab/configs/setup-baseline.yml.
+    TPLS_DIR: ${BUILD_ROOT}/tpls
   script:
+    - echo ${BUILD_ROOT}
+    - echo ${TPLS_DIR}
+    # Used by the tests in MFEM/tests:
+    - export MFEM_TEST_NP=32
+    # The next script uses the following environment variables:
+    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
+    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
     - .gitlab/scripts/baseline
   artifacts:
     when: always
@@ -29,33 +43,88 @@ baselinecheck_mfem_intel_quartz:
       - ${ARTIFACTS_DIR}
   allow_failure: true
 
+cleanup:
+  extends: .on_quartz
+  stage: cleanup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - echo "BUILD_ROOT=${BUILD_ROOT}"
+    - rm -rf "${BUILD_ROOT}" || true
+
 report_baseline:
   extends: [.on_quartz]
   stage: baseline_report
   script:
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${MACHINE_NAME}
-    - rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
-    - rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
     - |
-      if [[ -f ${rundir}/*.err ]]
-      then
-        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
-        cp ${rundir}/*.err ${rundir}/autotest-email.html
-      fi
-    - git add ${rundir}
-    - git commit -am "GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} with intel ($(date +%Y-%m-%d))"
-    - git push origin master
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # ----------------------
+        cd ${AUTOTEST_ROOT}/autotest || \
+          { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+        mkdir -p ${MACHINE_NAME}
+        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
+        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
+          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
+        fi
+        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
+          >> ${rundir}/pipeline.txt
+        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+          git pull && \
+          git add ${rundir} && \
+          git commit -m "${msg}" && \
+          git push origin master
+        else
+          for file in ${rundir}/*; do
+            echo "------------------------------"
+            echo "Content of '$file'"
+            echo "******************************"
+            cat $file
+            echo "******************************"
+          done
+          rm -rf ${rundir} || true
+        fi
+        err=$?
+        # ----------------------
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
 
 baselinepublish_mfem_quartz:
   extends: [.on_quartz]
   stage: baseline_publish
   rules:
-    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+    - if: '$REBASELINE == "YES"'
       when: manual
   script:
+    - echo ${BUILD_ROOT}
+    - echo ${PWD}
+    - echo ${ARTIFACTS_DIR}
+    - ls -lA ${ARTIFACTS_DIR}
     - .gitlab/scripts/rebaseline
 
 include:

.gitlab/quartz-build-and-test.yml

@@ -22,6 +22,7 @@ allocate_resource:
   extends: .on_quartz
   stage: allocate_resource
   script:
+    - echo ${ALLOC_NAME}
     - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
   timeout: 6h
 
@@ -58,7 +59,7 @@ opt_par_gcc_6_1_0_sundials:
 
 opt_par_gcc_6_1_0_petsc:
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
   extends: .build_and_test_on_quartz
 
 opt_par_gcc_6_1_0_pumi:
@@ -73,23 +74,26 @@ release_resource:
   extends: .on_quartz
   stage: release_resource_and_report
   script:
+    - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
     - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
 
 # Jobs report
 report_job_success:
-  extends: .on_quartz
   stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_success
+  extends:
+    - .on_quartz
+    - .report_job_success
 
 report_job_failure:
-  extends: .on_quartz
   stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_failure
+  extends:
+    - .on_quartz
+    - .report_job_failure
 
 include:
   - local: .gitlab/configs/common.yml
   - local: .gitlab/configs/quartz-config.yml
   - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -20,7 +20,8 @@ base_out=${base}.out
 artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
 
 # prepare
-cd ${BUILD_ROOT}
+cd ${BUILD_ROOT} || \
+  { echo "Invalid BUILD_ROOT=$BUILD_ROOT"; exit 1; }
 ln -snf ${CI_PROJECT_DIR} mfem
 cd tests
 [[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
@@ -28,11 +29,14 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
   lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+else
+  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
+  exit 1
 fi
 
 # post
@@ -60,6 +64,10 @@ then
   cp ${base_out} ${artifacts_path}/${base_out}
 fi
 
+if [[ -f ${BASELINE_TEST}.out ]]; then
+  cp ${BASELINE_TEST}.out ${artifacts_path}
+fi
+
 # base_diff won't even exist if there is no difference.
 if [[ -f ${base_diff} ]]
 then

.gitlab/scripts/report_build_and_test_failure

@@ -13,20 +13,33 @@
 
 echo "Runs if there was at least one failure on ${MACHINE_NAME}"
 
-cd ${AUTOTEST_ROOT}/autotest && git pull
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
 mkdir -p ${MACHINE_NAME}
 
 rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
 rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
 
-echo "There was an error while running CI on ${MACHINE_NAME}" > ${rundir}/gitlab.err
-echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
 
 msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
 
+# Create 'autotest-email.html' to indicate failure:
 cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
 
-git pull
-git add ${rundir}
-git commit -am "${msg}"
-git push origin master
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  git push origin master
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi

.gitlab/scripts/report_build_and_test_success

@@ -13,18 +13,30 @@
 
 echo "Can only run if all the ${MACHINE_NAME} jobs passed"
 
-cd ${AUTOTEST_ROOT}/autotest && git pull
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
 mkdir -p ${MACHINE_NAME}
 
 rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
 rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
 
-echo "The ${MACHINE_NAME} jobs were successful" > ${rundir}/gitlab.out
-echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
 
 msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
 
-git pull
-git add ${rundir}
-git commit -am "${msg}"
-git push origin master
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  git push origin master
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi

CHANGELOG

@@ -10,6 +10,28 @@
 
 Version 4.3.1 (development)
 ===========================
+
+- Add hipSPARSE support for sparse mat-vec multiplications.
+
+- Added support for using the HYPRE library built with HIP support. Similar to
+  the HYPRE + CUDA support added earlier, most of the MFEM examples and miniapps
+  work transparently with HYPRE + HIP builds. This includes the BoomerAMG, AMS,
+  and ADS solvers.
+
+- More explicit and consistent formating of the output of iterative solvers
+  with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
+
+- Added a miniapp for PDE-based extrapolation of finite element functions. See
+  miniapps/shifted/extrapolate.cpp.
+
+- Added support for automatic differentiation. Users can select between native
+  implementation and external library implementation during configuration. One
+  parallel and two serial examples are implemented in the miniapps/autodiff/
+  directory.
+
+- GridFunctionCoefficient (and the related vector, gradient, divergence, and
+  curl classes) now work properly with LORDiscretization and LORSolver.
+
 - Added support for mesh preprocessing to resolve fine scale problem data
   before simulation. This feature uses adaptive mesh refinement to control the
   associated data oscillation error. See the new Example 30/30p.
@@ -67,6 +89,18 @@ Version 4.3.1 (development)
 - Added initial TMOP-based capabilities for surface fitting and tangential
   relaxation in the mesh-optimizer and pmesh-optimizer miniapps.
 
+- Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
+  Blueprint MFEM wrapper functions in ConduitDataCollection.
+
+- `HypreParVector` and `Vector` now support move semantics, and the copy
+  constructor for `HypreParVector` now copies the local vector data.
+
+- The HPC versions of ex1 and ex1p (in miniapps/performance) now support
+  runtime selection of either 2D or 3D meshes.
+
+- Added ParaView visualization of `QuadratureFunction` fields, through both
+  `QuadratureFunction::SaveVTU` and `ParaViewDataCollection::RegisterQField`.
+
 
 Version 4.3, released on July 29, 2021
 ======================================

CMakeLists.txt

@@ -16,9 +16,6 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO)
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -84,6 +81,9 @@ if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
+if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14)
+endif()
 
 # Include xSDK default CMake file.
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
@@ -180,6 +180,19 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
+# AMD HIP
+if (MFEM_USE_HIP)
+  if (HIP_ARCH)
+    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
+    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
+  endif()
+  if (ROCM_PATH)
+    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
+  endif()
+  find_package(HIP REQUIRED)
+  find_package(HIPSPARSE REQUIRED)
+endif()
+
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -252,6 +265,11 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
   endif()
   find_package(OpenMP REQUIRED)
   set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
+  if(APPLE)
+    # On macOS, the compiler needs additional help to find the <omp.h> header.
+    # See issue #2642 for more information.
+    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
+  endif(APPLE)
 endif()
 
 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -349,7 +367,7 @@ endif()
 if (MFEM_USE_PUMI)
   # If PUMI_DIR was specified, only link to that directory,
   # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+  find_package(SCOREC 2.2.6 REQUIRED OPTIONAL_COMPONENTS gmi_sim
     CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
   if (SCOREC_FOUND)
     # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
@@ -367,6 +385,12 @@ if (MFEM_USE_HIOP)
   # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()
 
+# CoDiPack package
+if (MFEM_USE_CODIPACK)
+  find_package(CODIPACK REQUIRED)
+  # find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
+endif()
+
 # OCCA
 if (MFEM_USE_OCCA)
    find_package(OCCA REQUIRED)
@@ -392,18 +416,6 @@ if (MFEM_USE_CALIPER)
   find_package(Caliper REQUIRED)
 endif()
 
-# AMD HIP
-if (MFEM_USE_HIP)
-  find_package(HIP REQUIRED)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-    if (MFEM_USE_GINKGO)
-      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
-    endif()
-  endif()
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
@@ -443,10 +455,11 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER BENCHMARK PARELAG MPI_CXX)
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
+    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
+    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
+    MPI_CXX HIP HIPSPARSE)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -489,8 +502,6 @@ endforeach()
 
 if (MFEM_USE_CUDA)
   set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
-elseif(MFEM_USE_HIP)
-  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 
 add_subdirectory(config)

CODE_OF_CONDUCT.md

@@ -0,0 +1,133 @@
+# MFEM Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at mfem@llnl.gov.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident. Anyone involved in the reported behavior will recuse
+themselves from the investigation and decision making about the resolution of
+the complaint.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -21,6 +21,9 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.
 
+Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
+in the MFEM community, you agree to abide by its rules.
+
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -123,6 +126,7 @@ The MFEM source code has the following structure:
   ├── mesh
   ├── miniapps
   │   ├── adjoint
+  │   ├── autodiff
   │   ├── common
   │   ├── electromagnetics
   │   ├── gslib

INSTALL

@@ -37,7 +37,7 @@ as CUDA, HIP, OCCA, OpenMP and RAJA.
   https://developer.nvidia.com/cuda-toolkit
 
 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
+  https://rocmdocs.amd.com
 
 - OCCA support requires the OCCA library
   https://libocca.org
@@ -58,7 +58,8 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
+- Conda-forge, https://conda-forge.org (pre-built binaries linked with
+  OpenMPI/MPICH, hypre, and METIS)
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
 
 We also recommend downloading and building the MFEM-based GLVis visualization
@@ -78,7 +79,7 @@ Parallel build:
 
 CUDA build:
    make cuda -j 4
-   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')
+   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_70')
 
 HIP build:
    make hip -j 4
@@ -467,6 +468,14 @@ MFEM_USE_HIOP = YES/NO
    Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
    HPC solver for nonlinear optimization problems.
 
+MFEM_USE_CODIPACK = YES/NO
+  Enable automatic differentiation using the CoDiPack library.
+  www.scicomp.uni-kl.de/codi/
+
+MFEM_USE_ADFORWARD = YES/NO
+  Enable forward mode for AD packages. This option is valid
+  only if the AD package supports two modes (backward/forward).
+
 MFEM_USE_CUDA = YES/NO
    Enables support for CUDA devices in MFEM. CUDA is a parallel computing
    platform and programming model for general computing on graphical processing
@@ -558,7 +567,7 @@ The specific libraries and their options are:
   Options: HYPRE_OPT, HYPRE_LIB.
   Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
             HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
+            HYPRE >= 2.22.1   (HYPRE built with CUDA or HIP)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -631,7 +640,8 @@ The specific libraries and their options are:
   C++ compiler that supports the C++-14 standard. For additional requirements
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
+  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
+           Debug).
   Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
@@ -696,13 +706,18 @@ The specific libraries and their options are:
   URL: https://scorec.rpi.edu/pumi
        https://github.com/SCOREC/core
   Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI == 2.2.3.
+  Versions: PUMI >= 2.2.6.
 
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
   Options: HIOP_OPT, HIOP_LIB.
   Versions: HIOP >= 0.4.6.
 
+- CoDiPack (optiobal), used with MFEM_USE_CODIPACK = YES
+  URL: https://www.scicomp.uni-kl.de/codi/
+  Options: CODIPACK_OPT
+  Versions: 1.9.3
+
 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
   built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
   the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
@@ -724,7 +739,7 @@ The specific libraries and their options are:
   Versions: CUDA >= 10.1.168.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
-  URL: https://rocm.github.io/ROCmInstall.html
+  URL: https://rocmdocs.amd.com
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
 
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
@@ -908,7 +923,10 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
+MFEM_USE_CODIPACK
+MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
+MFEM_USE_HIP
 MFEM_USE_OCCA
 MFEM_USE_CEED
 MFEM_USE_RAJA
@@ -967,6 +985,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - POSIXCLOCKS
  - PUMI
  - HIOP
+ - CoDiPack
  - OCCA
  - RAJA
  - UMPIRE

README

@@ -12,6 +12,9 @@ to enable high-performance scalable finite element discretization research and
 application development on a wide variety of platforms, ranging from laptops to
 supercomputers.
 
+We welcome contributions and feedback from the community. Please see the file
+CONTRIBUTING.md for additional details about our development process.
+
 * For building instructions, see the file INSTALL, or type "make help".
 
 * Copyright and licensing information can be found in files LICENSE and NOTICE.
@@ -19,9 +22,6 @@ supercomputers.
 * The best starting point for new users interested in MFEM's features is to
   review the examples and miniapps at https://mfem.org/examples.
 
-* Developers interested in contributing to the library, should read the
-  instructions and documentation in the CONTRIBUTING.md file.
-
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support

config/cmake/MFEMConfig.cmake.in

@@ -54,6 +54,8 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
+set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
+set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)

config/cmake/config.hpp.in

@@ -175,6 +175,12 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO
 
+// Use forward mode for automatic differentiation
+#cmakedefine MFEM_USE_ADFORWARD
+
+// Enable the use of the CoDiPack library for AD
+#cmakedefine MFEM_USE_CODIPACK
+
 // Enable MFEM functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK
 

config/cmake/modules/FindCODIPACK.cmake

@@ -0,0 +1,24 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Automatic differentiation using the CoDiPack library.
+#  www.scicomp.uni-kl.de/codi/
+# Sets the following variables:
+#   - CODIPACK_FOUND
+#   - CODIPACK_INCLUDE_DIRS
+#   - CODIPACK_LIBRARIES
+
+include(MfemCmakeUtilities)
+mfem_find_package(CODIPACK CODIPACK CODIPACK_DIR
+  "include" "codi.h"
+  "lib" ""
+  "Paths to headers required by CODIPACK."
+  "Libraries required by CODIPACK.")

config/cmake/modules/FindHIP.cmake

@@ -1,692 +0,0 @@
-###############################################################################
-# FindHIP.cmake
-###############################################################################
-include(CheckCXXCompilerFlag)
-###############################################################################
-# SET: Variable defaults
-###############################################################################
-# User defined flags
-set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
-set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
-set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
-set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
-mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
-
-set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
-list(REMOVE_DUPLICATES _hip_configuration_types)
-foreach(config ${_hip_configuration_types})
-    string(TOUPPER ${config} config_upper)
-    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
-    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
-    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
-    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
-    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
-endforeach()
-option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
-option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
-mark_as_advanced(HIP_HOST_COMPILATION_CPP)
-
-###############################################################################
-# FIND: HIP and associated helper binaries
-###############################################################################
-
-get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
-
-# HIP is supported on Linux only
-if(UNIX AND NOT APPLE AND NOT CYGWIN)
-    # Search for HIP installation
-    if(NOT HIP_ROOT_DIR)
-        # Search in user specified path first
-        find_path(
-            HIP_ROOT_DIR
-            NAMES bin/hipconfig
-            PATHS
-            "$ENV{ROCM_PATH}/hip"
-            ENV HIP_PATH
-            ${_IMPORT_PREFIX}
-            /opt/rocm/hip
-            DOC "HIP installed location"
-            NO_DEFAULT_PATH
-            )
-        if(NOT EXISTS ${HIP_ROOT_DIR})
-            if(HIP_FIND_REQUIRED)
-                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
-            elseif(NOT HIP_FIND_QUIETLY)
-                message("HIP_ROOT_DIR not found or specified")
-            endif()
-        endif()
-        # And push it back to the cache
-        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
-    endif()
-
-    # Find HIPCC executable
-    find_program(
-        HIP_HIPCC_EXECUTABLE
-        NAMES hipcc
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCC_EXECUTABLE hipcc)
-    endif()
-    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
-
-    # Find HIPCONFIG executable
-    find_program(
-        HIP_HIPCONFIG_EXECUTABLE
-        NAMES hipconfig
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCONFIG_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
-    endif()
-    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
-
-    # Find HIPCC_CMAKE_LINKER_HELPER executable
-    find_program(
-        HIP_HIPCC_CMAKE_LINKER_HELPER
-        NAMES hipcc_cmake_linker_helper
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
-        # Now search in default paths
-        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
-    endif()
-    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
-        # Compute the version
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
-            OUTPUT_VARIABLE _hip_version
-            ERROR_VARIABLE _hip_error
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE
-            )
-        if(NOT _hip_error)
-            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
-        else()
-            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
-        endif()
-        mark_as_advanced(HIP_VERSION)
-    endif()
-    if(HIP_VERSION)
-        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
-        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
-        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
-        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
-        set(HIP_VERSION_STRING "${HIP_VERSION}")
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
-        # Compute the platform
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
-            OUTPUT_VARIABLE _hip_platform
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
-        mark_as_advanced(HIP_PLATFORM)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
-        # Compute the compiler
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
-            OUTPUT_VARIABLE _hip_compiler
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
-        mark_as_advanced(HIP_COMPILER)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
-        # Compute the runtime
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
-            OUTPUT_VARIABLE _hip_runtime
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
-        mark_as_advanced(HIP_RUNTIME)
-    endif()
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(
-    HIP
-    REQUIRED_VARS
-    HIP_ROOT_DIR
-    HIP_HIPCC_EXECUTABLE
-    HIP_HIPCONFIG_EXECUTABLE
-    HIP_PLATFORM
-    HIP_COMPILER
-    HIP_RUNTIME
-    VERSION_VAR HIP_VERSION
-    )
-
-###############################################################################
-# Set HIP CMAKE Flags
-###############################################################################
-# Copy the invocation styles from CXX to HIP
-set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
-set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
-set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
-set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
-#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
-set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
-
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
-set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
-
-if("${HIP_COMPILER}" STREQUAL "nvcc")
-    # Set the CMake Flags to use the nvcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "hcc")
-    # Set the CMake Flags to use the hcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "clang")
-    #Number of parallel jobs by default is 1
-    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
-      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
-    endif()
-    #Add support for parallel build and link
-    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
-      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
-    endif()
-    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
-      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
-        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-      else()
-        message("clang compiler doesn't support parallel jobs")
-      endif()
-    endif()
-
-    # Set the CMake Flags to use the HIP-Clang Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-
-    if("${HIP_RUNTIME}" STREQUAL "rocclr")
-      if(TARGET host)
-        message(STATUS "host interface - found")
-        set(HIP_HOST_INTERFACE host)
-      endif()
-    endif()
-endif()
-
-###############################################################################
-# MACRO: Locate helper files
-###############################################################################
-macro(HIP_FIND_HELPER_FILE _name _extension)
-    set(_hip_full_name "${_name}.${_extension}")
-    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
-    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
-    if(NOT EXISTS "${HIP_${_name}}")
-        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
-        if(HIP_FIND_REQUIRED)
-            message(FATAL_ERROR "${error_message}")
-        else()
-            if(NOT HIP_FIND_QUIETLY)
-                message(STATUS "${error_message}")
-            endif()
-        endif()
-    endif()
-    # Set this variable as internal, so the user isn't bugged with it.
-    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
-endmacro()
-
-###############################################################################
-hip_find_helper_file(run_make2cmake cmake)
-hip_find_helper_file(run_hipcc cmake)
-###############################################################################
-
-###############################################################################
-# MACRO: Reset compiler flags
-###############################################################################
-macro(HIP_RESET_FLAGS)
-    unset(HIP_HIPCC_FLAGS)
-    unset(HIP_HCC_FLAGS)
-    unset(HIP_CLANG_FLAGS)
-    unset(HIP_NVCC_FLAGS)
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        unset(HIP_HIPCC_FLAGS_${config_upper})
-        unset(HIP_HCC_FLAGS_${config_upper})
-        unset(HIP_CLANG_FLAGS_${config_upper})
-        unset(HIP_NVCC_FLAGS_${config_upper})
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Separate the options from the sources
-###############################################################################
-macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
-    set(${_sources})
-    set(${_cmake_options})
-    set(${_hipcc_options})
-    set(${_hcc_options})
-    set(${_clang_options})
-    set(${_nvcc_options})
-    set(_hipcc_found_options FALSE)
-    set(_hcc_found_options FALSE)
-    set(_clang_found_options FALSE)
-    set(_nvcc_found_options FALSE)
-    foreach(arg ${ARGN})
-        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
-            set(_hipcc_found_options TRUE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options TRUE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options TRUE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options TRUE)
-        elseif(
-                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
-                "x${arg}" STREQUAL "xSTATIC" OR
-                "x${arg}" STREQUAL "xSHARED" OR
-                "x${arg}" STREQUAL "xMODULE"
-                )
-            list(APPEND ${_cmake_options} ${arg})
-        else()
-            if(_hipcc_found_options)
-                list(APPEND ${_hipcc_options} ${arg})
-            elseif(_hcc_found_options)
-                list(APPEND ${_hcc_options} ${arg})
-            elseif(_clang_found_options)
-                list(APPEND ${_clang_options} ${arg})
-            elseif(_nvcc_found_options)
-                list(APPEND ${_nvcc_options} ${arg})
-            else()
-                # Assume this is a file
-                list(APPEND ${_sources} ${arg})
-            endif()
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Add include directories to pass to the hipcc command
-###############################################################################
-set(HIP_HIPCC_INCLUDE_ARGS_USER "")
-macro(HIP_INCLUDE_DIRECTORIES)
-    foreach(dir ${ARGN})
-        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
-    endforeach()
-endmacro()
-
-###############################################################################
-# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
-###############################################################################
-function(HIP_COMPUTE_BUILD_PATH path build_path)
-    # Convert to cmake style paths
-    file(TO_CMAKE_PATH "${path}" bpath)
-    if(IS_ABSOLUTE "${bpath}")
-        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
-        if(_binary_dir_pos EQUAL 0)
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
-        else()
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
-        endif()
-    endif()
-
-    # Remove leading /
-    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
-    # Avoid absolute paths by removing ':'
-    string(REPLACE ":" "_" bpath "${bpath}")
-    # Avoid relative paths that go up the tree
-    string(REPLACE "../" "__/" bpath "${bpath}")
-    # Avoid spaces
-    string(REPLACE " " "_" bpath "${bpath}")
-    # Strip off the filename
-    get_filename_component(bpath "${bpath}" PATH)
-
-    set(${build_path} "${bpath}" PARENT_SCOPE)
-endfunction()
-
-###############################################################################
-# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
-###############################################################################
-macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
-    set(_hip_found_config)
-    foreach(arg ${ARGN})
-        # Determine if we are dealing with a per-configuration flag
-        foreach(config ${_hip_configuration_types})
-            string(TOUPPER ${config} config_upper)
-            if(arg STREQUAL "${config_upper}")
-                set(_hip_found_config _${arg})
-                # Clear arg to prevent it from being processed anymore
-                set(arg)
-            endif()
-        endforeach()
-        if(arg)
-            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Try and include dependency file if it exists
-###############################################################################
-macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
-    set(HIP_HIPCC_DEPEND)
-    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
-
-    # Create the dependency file if it doesn't exist
-    if(NOT EXISTS ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-    # Include the dependency file
-    include(${dependency_file})
-
-    # Verify the existence of all the included files
-    if(HIP_HIPCC_DEPEND)
-        foreach(f ${HIP_HIPCC_DEPEND})
-            if(NOT EXISTS ${f})
-                # If they aren't there, regenerate the file again
-                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-            endif()
-        endforeach()
-    else()
-        # No dependencies, so regenerate the file
-        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-    endif()
-
-    # Regenerate the dependency file if needed
-    if(HIP_HIPCC_DEPEND_REGENERATE)
-        set(HIP_HIPCC_DEPEND ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-endmacro()
-
-###############################################################################
-# MACRO: Prepare cmake commands for the target
-###############################################################################
-macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
-    set(_hip_flags "")
-    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
-    if(HIP_HOST_COMPILATION_CPP)
-        set(HIP_C_OR_CXX CXX)
-    else()
-        set(HIP_C_OR_CXX C)
-    endif()
-    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
-
-    # Initialize list of includes with those specified by the user. Append with
-    # ones specified to cmake directly.
-    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
-
-    # Add the include directories
-    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
-    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
-
-    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
-    list(REMOVE_DUPLICATES _hip_include_directories)
-    if(_hip_include_directories)
-        foreach(dir ${_hip_include_directories})
-            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
-        endforeach()
-    endif()
-
-    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
-
-    # Add the compile definitions
-    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
-    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
-
-    # Check if we are building shared library.
-    set(_hip_build_shared_libs FALSE)
-    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
-    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
-    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
-        set(_hip_build_shared_libs TRUE)
-    endif()
-    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
-    if(_hip_found_STATIC GREATER -1)
-        set(_hip_build_shared_libs FALSE)
-    endif()
-
-    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
-    if(_hip_build_shared_libs)
-        list(APPEND HIP_HCC_FLAGS "-fPIC")
-        list(APPEND HIP_CLANG_FLAGS "-fPIC")
-        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
-    endif()
-
-    # Set host compiler
-    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
-
-    # Set compiler flags
-    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
-    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
-    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
-    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
-    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
-        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
-        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
-        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
-        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
-    endforeach()
-
-    # Reset the output variable
-    set(_hip_generated_files "")
-    set(_hip_source_files "")
-
-    # Iterate over all arguments and create custom commands for all source files
-    foreach(file ${ARGN})
-        # Ignore any file marked as a HEADER_FILE_ONLY
-        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
-        # Allow per source file overrides of the format. Also allows compiling non .cu files.
-        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
-        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
-            set(host_flag FALSE)
-        else()
-            set(host_flag TRUE)
-        endif()
-
-        if(NOT host_flag)
-            # Determine output directory
-            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
-            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
-
-            get_filename_component(basename ${file} NAME)
-            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
-            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
-
-            # Set file names
-            set(generated_file "${generated_file_path}/${generated_file_basename}")
-            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
-            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
-            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
-
-            # Set properties for object files
-            set_source_files_properties("${generated_file}"
-                PROPERTIES
-                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
-                )
-
-            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
-            get_filename_component(file_path "${file}" PATH)
-            if(IS_ABSOLUTE "${file_path}")
-                set(source_file "${file}")
-            else()
-                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
-            endif()
-
-            # Bring in the dependencies
-            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
-
-            # Configure the build script
-            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
-            file(GENERATE
-                OUTPUT "${custom_target_script}"
-                INPUT "${custom_target_script_pregen}"
-                )
-            set(main_dep DEPENDS ${source_file})
-            if(CMAKE_GENERATOR MATCHES "Makefiles")
-                set(verbose_output "$(VERBOSE)")
-            elseif(HIP_VERBOSE_BUILD)
-                set(verbose_output ON)
-            else()
-                set(verbose_output OFF)
-            endif()
-
-            # Create up the comment string
-            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
-            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
-
-            # Build the generated file and dependency file
-            add_custom_command(
-                OUTPUT ${generated_file}
-                # These output files depend on the source_file and the contents of cmake_dependency_file
-                ${main_dep}
-                DEPENDS ${HIP_HIPCC_DEPEND}
-                DEPENDS ${custom_target_script}
-                # Make sure the output directory exists before trying to write to it.
-                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
-                COMMAND ${CMAKE_COMMAND} ARGS
-                -D verbose:BOOL=${verbose_output}
-                -D build_configuration:STRING=${_hip_build_configuration}
-                -D "generated_file:STRING=${generated_file}"
-                -P "${custom_target_script}"
-                WORKING_DIRECTORY "${hip_compile_output_dir}"
-                COMMENT "${hip_build_comment_string}"
-                )
-
-            # Make sure the build system knows the file is generated
-            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
-            list(APPEND _hip_generated_files ${generated_file})
-            list(APPEND _hip_source_files ${file})
-        endif()
-    endforeach()
-
-    # Set the return parameter
-    set(${_generated_files} ${_hip_generated_files})
-    set(${_source_files} ${_hip_source_files})
-endmacro()
-
-###############################################################################
-# HIP_ADD_EXECUTABLE
-###############################################################################
-macro(HIP_ADD_EXECUTABLE hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if("${HIP_COMPILER}" STREQUAL "hcc")
-        if("x${HCC_HOME}" STREQUAL "x")
-            if (DEFINED ENV{ROCM_PATH})
-                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
-            else()
-                set(HCC_HOME "/opt/rocm/hcc")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    elseif("${HIP_COMPILER}" STREQUAL "clang")
-        if("x${HIP_CLANG_PATH}" STREQUAL "x")
-            if(DEFINED ENV{HIP_CLANG_PATH})
-                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
-            elseif(DEFINED ENV{ROCM_PATH})
-                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
-            else()
-                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    else()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-###############################################################################
-# HIP_ADD_LIBRARY
-###############################################################################
-macro(HIP_ADD_LIBRARY hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_hipcc.cmake

@@ -1,182 +0,0 @@
-###############################################################################
-# Runs commands using HIPCC
-###############################################################################
-
-###############################################################################
-# This file runs the hipcc commands to produce the desired output file
-# along with the dependency file needed by CMake to compute dependencies.
-#
-# Input variables:
-#
-# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
-#                               ON : Describe each step
-# build_configuration:STRING=<> Build configuration. Defaults to Debug.
-# generated_file:STRING=<>      File to generate. Mandatory argument.
-
-if(NOT build_configuration)
-    set(build_configuration Debug)
-endif()
-if(NOT generated_file)
-    message(FATAL_ERROR "You must specify generated_file on the command line")
-endif()
-
-# Set these up as variables to make reading the generated file easier
-set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
-set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
-set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
-set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
-set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
-set(HCC_HOME "@HCC_HOME@") #path
-set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
-
-@HIP_HOST_FLAGS@
-@_HIP_HIPCC_FLAGS@
-@_HIP_HCC_FLAGS@
-@_HIP_CLANG_FLAGS@
-@_HIP_NVCC_FLAGS@
-#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
-set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
-
-set(cmake_dependency_file "@cmake_dependency_file@") # path
-set(source_file "@source_file@") # path
-set(host_flag "@host_flag@") # bool
-
-# Determine compiler and compiler flags
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
-if(NOT host_flag)
-    set(__CC ${HIP_HIPCC_EXECUTABLE})
-    if("${HIP_PLATFORM}" STREQUAL "amd")
-        if("${HIP_COMPILER}" STREQUAL "hcc")
-            if(NOT "x${HCC_HOME}" STREQUAL "x")
-                set(ENV{HCC_HOME} ${HCC_HOME})
-            endif()
-            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
-        elseif("${HIP_COMPILER}" STREQUAL "clang")
-            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
-                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
-            endif()
-            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
-            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
-        endif()
-    else()
-        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
-    endif()
-else()
-    set(__CC ${HIP_HOST_COMPILER})
-    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
-endif()
-set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
-
-# hip_execute_process - Executes a command with optional command echo and status message.
-#   status     - Status message to print if verbose is true
-#   command    - COMMAND argument from the usual execute_process argument structure
-#   ARGN       - Remaining arguments are the command with arguments
-#   HIP_result - Return value from running the command
-macro(hip_execute_process status command)
-    set(_command ${command})
-    if(NOT "x${_command}" STREQUAL "xCOMMAND")
-        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
-    endif()
-    if(verbose)
-        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
-        # Build command string to print
-        set(hip_execute_process_string)
-        foreach(arg ${ARGN})
-            # Escape quotes if any
-            string(REPLACE "\"" "\\\"" arg ${arg})
-            # Surround args with spaces with quotes
-            if(arg MATCHES " ")
-                list(APPEND hip_execute_process_string "\"${arg}\"")
-            else()
-                list(APPEND hip_execute_process_string ${arg})
-            endif()
-        endforeach()
-        # Echo the command
-        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
-    endif()
-    # Run the command
-    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
-endmacro()
-
-# Delete the target file
-hip_execute_process(
-    "Removing ${generated_file}"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-    )
-
-# Generate the dependency file
-hip_execute_process(
-    "Generating dependency file: ${cmake_dependency_file}.pre"
-    COMMAND "${__CC}"
-    -M
-    "${source_file}"
-    -o "${cmake_dependency_file}.pre"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the cmake readable dependency file to a temp file
-hip_execute_process(
-    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
-    COMMAND "${CMAKE_COMMAND}"
-    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
-    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
-    -D "verbose=${verbose}"
-    -P "${HIP_run_make2cmake}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Copy the file if it is different
-hip_execute_process(
-    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
-    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Delete the temporary file
-hip_execute_process(
-    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the output file
-hip_execute_process(
-    "Generating ${generated_file}"
-    COMMAND "${__CC}"
-    -c
-    "${source_file}"
-    -o "${generated_file}"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    # Make sure that we delete the output file
-    hip_execute_process(
-        "Removing ${generated_file}"
-        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-        )
-    message(FATAL_ERROR "Error generating file ${generated_file}")
-else()
-    if(verbose)
-        message("Generated ${generated_file} successfully.")
-    endif()
-endif()
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_make2cmake.cmake

@@ -1,50 +0,0 @@
-###############################################################################
-# Computes dependencies using HIPCC
-###############################################################################
-
-###############################################################################
-# This file converts dependency files generated using hipcc to a format that
-# cmake can understand.
-
-# Input variables:
-#
-# input_file:STRING=<> Dependency file to parse. Required argument
-# output_file:STRING=<> Output file to generate. Required argument
-
-if(NOT input_file OR NOT output_file)
-    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
-endif()
-
-file(READ ${input_file} depend_text)
-
-if (NOT "${depend_text}" STREQUAL "")
-    string(REPLACE " /" "\n/" depend_text ${depend_text})
-    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
-    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
-
-    set(dependency_list "")
-
-    foreach(file ${depend_text})
-        string(REGEX REPLACE "^ +" "" file ${file})
-        if(NOT EXISTS "${file}")
-            message(WARNING " Removing non-existent dependency file: ${file}")
-            set(file "")
-        endif()
-
-        if(NOT IS_DIRECTORY "${file}")
-            get_filename_component(file_absolute "${file}" ABSOLUTE)
-            list(APPEND dependency_list "${file_absolute}")
-        endif()
-    endforeach()
-endif()
-
-# Remove the duplicate entries and sort them.
-list(REMOVE_DUPLICATES dependency_list)
-list(SORT dependency_list)
-
-foreach(file ${dependency_list})
-    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
-endforeach()
-
-file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHYPRE.cmake

@@ -14,10 +14,33 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
+#   - HYPRE_USING_HIP (internal)
+
+if (HYPRE_FOUND)
+  if (HYPRE_USING_HIP)
+    find_package(rocsparse REQUIRED)
+    find_package(rocrand REQUIRED)
+  endif()
+  return()
+endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
-  "Paths to headers required by HYPRE." "Libraries required by HYPRE.")
+  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
+  CHECK_BUILD HYPRE_USING_HIP FALSE
+  "
+#undef HYPRE_USING_HIP
+#include <HYPRE_config.h>
+
+#ifndef HYPRE_USING_HIP
+#error HYPRE is built without HIP.
+#endif
+
+int main()
+{
+   return 0;
+}
+")
 
 if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
   try_run(HYPRE_VERSION_RUN_RESULT HYPRE_VERSION_COMPILE_RESULT
@@ -33,3 +56,12 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
     message(FATAL_ERROR "Unable to determine HYPRE version.")
   endif()
 endif()
+
+if (HYPRE_FOUND AND HYPRE_USING_HIP)
+  find_package(rocsparse REQUIRED)
+  find_package(rocrand REQUIRED)
+  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
+  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
+      "HYPRE libraries + dependencies." FORCE)
+  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
+endif()

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -46,8 +46,7 @@ endfunction()
 # Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
   if (MFEM_USE_HIP)
-    hip_add_executable(${NAME} ${ARGN})
-    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
+    add_executable(${NAME} ${ARGN})
   else()
     add_executable(${NAME} ${ARGN})
   endif()
@@ -56,7 +55,7 @@ endmacro()
 # Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
   if (MFEM_USE_HIP)
-    hip_add_library(${NAME} ${ARGN})
+    add_library(${NAME} ${ARGN})
   else()
     add_library(${NAME} ${ARGN})
   endif()
@@ -92,8 +91,6 @@ macro(add_mfem_examples EXE_SRCS)
     # If CUDA is enabled, tag source files to be compiled with nvcc.
     if (MFEM_USE_CUDA)
       set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
-    elseif(MFEM_USE_HIP)
-      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
     endif()
 
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
@@ -159,8 +156,6 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
       endforeach()
       set(EXTRA_OPTIONS_LIST ${LIST_})
     endif()
-  elseif(MFEM_USE_HIP)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
   endif()
 
   # Actually add the executable
@@ -534,12 +529,15 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             if (NOT ImportConfig)
               set(ImportConfig RELEASE)
             endif()
+            set(ImportConfigSuffix "_${ImportConfig}")
             get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
             list(FIND ImpConfigs ${ImportConfig} _Index)
-            if (_Index EQUAL -1)
-              message(FATAL_ERROR " *** ${ReqPack}: configuration "
-                "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
-                "from the list: ${ImpConfigs}.")
+            if ((_Index EQUAL -1) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
+              set(ImportConfig "NO_CONFIG")
+              set(ImportConfigSuffix "")
+              # message(FATAL_ERROR " *** ${ReqPack}: configuration "
+              #   "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
+              #   "from the list: ${ImpConfigs}.")
             endif()
           endif()
           # Set _Pack_LIBS
@@ -551,8 +549,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           else()
             # Set _Pack_LIBS from the target properties for ImportConfig
-            foreach (_prop IMPORTED_LOCATION_${ImportConfig}
-                IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig})
+            foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
+                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix})
               get_target_property(_value ${TargetName} ${_prop})
               if (_value)
                 list(APPEND _Pack_LIBS ${_value})
@@ -564,7 +562,7 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           endif()
           # Set _Pack_INCS
-          foreach (_prop INCLUDE_DIRECTORIES)
+          foreach (_prop INCLUDE_DIRECTORIES INTERFACE_INCLUDE_DIRECTORIES)
             get_target_property(_value ${TargetName} ${_prop})
             if (_value)
               list(APPEND _Pack_INCS ${_value})
@@ -742,6 +740,133 @@ function(mfem_find_library Name Prefix Lib LibDoc CheckVar CheckSrc)
 endfunction(mfem_find_library)
 
 
+#
+# Extract compile and link options needed by the given target.
+#
+function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
+
+  if (NOT TARGET ${Target})
+    return()
+  endif()
+
+  # CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG -> '-Wl,-rpath,'
+  set(shared_link_flag ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
+  if (NOT shared_link_flag)
+    set(shared_link_flag "-Wl,-rpath,")
+  endif()
+
+  set(tgt "${Target}")
+  unset(CompileOpts)
+  unset(LinkOpts)
+  get_target_property(IsImported ${tgt} IMPORTED)
+  # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
+  # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
+  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
+  get_target_property(type ${tgt} TYPE)
+  # message(STATUS "${tgt}[TYPE]: ${type}")
+  unset(ImportConfig)
+  get_target_property(ImportConfigs ${tgt} IMPORTED_CONFIGURATIONS)
+  if (ImportConfigs)
+    list(GET ImportConfigs 0 ImportConfig)
+  endif()
+  if (NOT ImportConfig)
+    set(ImportConfig RELEASE)
+  endif()
+  # message(STATUS "${tgt}[ImportConfig]: ${ImportConfig}")
+  # List all properties with: cmake --help-property-list
+  get_target_property(Defs ${tgt} INTERFACE_COMPILE_DEFINITIONS)
+  if (Defs)
+    list(REMOVE_DUPLICATES Defs)
+    foreach(Def ${Defs})
+      list(APPEND CompileOpts "-D${Def}")
+    endforeach()
+  endif()
+  get_target_property(Opts ${tgt} INTERFACE_COMPILE_OPTIONS)
+  if (Opts)
+    foreach(Opt ${Opts})
+      list(APPEND CompileOpts "${Opt}")
+    endforeach()
+  endif()
+  get_target_property(Dirs ${tgt} INTERFACE_INCLUDE_DIRECTORIES)
+  if (Dirs)
+    list(REMOVE_DUPLICATES Dirs)
+    foreach(Dir ${Dirs})
+      list(APPEND CompileOpts "-I\"${Dir}\"")
+    endforeach()
+  endif()
+  get_target_property(SysDirs ${tgt} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+  if (SysDirs)
+    list(REMOVE_DUPLICATES SysDirs)
+    foreach(SysDir ${SysDirs})
+      list(APPEND CompileOpts "-isystem \"${SysDir}\"")
+    endforeach()
+  endif()
+  if ("${type}" STREQUAL "STATIC_LIBRARY")
+    get_target_property(Location ${tgt} LOCATION)
+    if (Location)
+      list(APPEND LinkOpts "\"${Location}\"")
+    else()
+      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
+    endif()
+  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
+    get_target_property(Location ${tgt} LOCATION)
+    if (Location)
+      get_filename_component(Dir ${Location} DIRECTORY)
+      get_filename_component(NameWE ${Location} NAME_WE)
+      string(REGEX REPLACE "^lib" "" LibName ${NameWE})
+      list(APPEND LinkOpts
+        "-L\"${Dir}\""
+        "${shared_link_flag}\"${Dir}\""
+        "-l${LibName}")
+    else()
+      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
+    endif()
+  elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
+    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
+    if (Libs)
+      foreach(Lib ${Libs})
+        if (NOT (TARGET ${Lib}))
+          list(APPEND LinkOpts "${Lib}")
+        else()
+          mfem_get_target_options(${Lib} COpts LOpts)
+          list(APPEND CompileOpts ${COpts})
+          list(APPEND LinkOpts ${LOpts})
+        endif()
+      endforeach()
+    endif()
+    # Other properties we may need to handle:
+    # INTERFACE_LINK_DEPENDS
+    # INTERFACE_LINK_DIRECTORIES
+    # INTERFACE_LINK_OPTIONS
+  else()
+    message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
+      " which is not supported!")
+  endif()
+
+  # Other potentially relevant properties:
+  # - For all target types:
+  # IMPORTED_LIBNAME
+  # IMPORTED_LIBNAME_${ImportConfig}
+  # INTERFACE_AUTOUIC_OPTIONS
+  # INTERFACE_COMPILE_FEATURES
+  # INTERFACE_POSITION_INDEPENDENT_CODE
+  # INTERFACE_SOURCES
+  # INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+  # - For non-"INTERFACE_LIBRARY" target types only:
+  # IMPORTED_LOCATION
+  # IMPORTED_LOCATION_${ImportConfig}
+  # IMPORTED_LINK_INTERFACE_LIBRARIES
+  # IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig}
+  # LINK_FLAGS
+  # LINK_FLAGS_${ImportConfig}
+  # LOCATION_${ImportConfig})
+
+  set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
+  set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)
+
+endfunction(mfem_get_target_options)
+
+
 #
 #   Function that creates 'config.mk' from 'config.mk.in' for the both the
 #   build- and the install-locations and define install rules for 'config.mk'
@@ -760,13 +885,15 @@ function(mfem_export_mk_files)
   # Convert Boolean vars to YES/NO without writing the values to cache
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
+      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
-      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
-      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
+      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
+      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
+      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
+      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
+      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
+      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -868,8 +995,18 @@ function(mfem_export_mk_files)
     get_filename_component(suffix ${lib} EXT)
     # handle interfaces (e.g., SCOREC::apf)
     if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
-    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
-      message(FATAL_ERROR "***** interface lib found ... exiting *****")
+    elseif (TARGET "${lib}")
+      mfem_get_target_options(${lib} CompileOpts LinkOpts)
+      # Removing duplicates may lead to issues:
+      # list(REMOVE_DUPLICATES CompileOpts)
+      # list(REMOVE_DUPLICATES LinkOpts)
+      string(REPLACE ";" " " COpts "${CompileOpts}")
+      string(REPLACE ";" " " LOpts "${LinkOpts}")
+      # message(STATUS "${lib}[COpts]: '${COpts}'")
+      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
+      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
+      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
+      # message(FATAL_ERROR "***** interface lib found ... exiting *****")
       # handle static and shared libs
     elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
       get_filename_component(dir ${lib} DIRECTORY)

config/config.hpp.in

@@ -180,6 +180,12 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO
 
+// Use forward mode for automatic differentiation
+// #define MFEM_USE_ADFORWARD
+
+// Enable the use of the CoDiPack library for AD
+// #define MFEM_USE_CODIPACK
+
 // Enable functionality based on the Google Benchmark library.
 // #define MFEM_USE_BENCHMARK
 

config/config.mk.in

@@ -58,6 +58,8 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
+MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
+MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
 
@@ -89,7 +91,7 @@ MFEM_MPIEXEC_NP = @MFEM_MPIEXEC_NP@
 MFEM_MPI_NP     = @MFEM_MPI_NP@
 
 # The NVCC compiler cannot link with -x=cu
-MFEM_LINK_FLAGS := $(filter-out -x=cu, $(MFEM_FLAGS))
+MFEM_LINK_FLAGS := $(filter-out -x=cu -xhip, $(MFEM_FLAGS))
 
 # Optional extra configuration
 @MFEM_CONFIG_EXTRA@

config/defaults.cmake

@@ -50,6 +50,7 @@ option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
+option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
@@ -58,6 +59,8 @@ option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
+option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
+option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
 
@@ -105,6 +108,7 @@ if (MFEM_USE_CUDA)
    set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
        "Libraries that HYPRE depends on.")
 endif()
+# HIP dependency for HYPRE is handled in FindHYPRE.cmake.
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -224,6 +228,11 @@ set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
+# If RAJA is built with external CAMP:
+# set(RAJA_REQUIRED_PACKAGES "camp"
+#     CACHE STRING "Packages that RAJA depends on.")
+# set(camp_DIR "${MFEM_DIR}/../camp/lib/cmake/camp"
+#     CACHE PATH "Path to CAMP CMake files.")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
@@ -243,6 +252,10 @@ set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")
 
+set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/include" CACHE STRING
+    "Path to CoDiPack headers.")
+set(CODIPACK_LIBRARIES "")
+
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules

config/defaults.mk

@@ -59,6 +59,9 @@ HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
 HIP_XCOMPILER =
 HIP_XLINKER   = -Wl,
 
+# Flags for generating dependencies.
+DEP_FLAGS = -MM -MT
+
 ifneq ($(NOTMAC),)
    AR      = ar
    ARFLAGS = crv
@@ -86,6 +89,9 @@ else
    BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup
    INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR)))
    INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup
+   # Silence unused command line argument warnings when generating dependencies
+   # with mpicxx and clang
+   DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
 endif
 
 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -151,6 +157,8 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_ADFORWARD     = NO
+MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
 
@@ -164,6 +172,20 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
    MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
 endif
 
+# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
+# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
+ifeq ($(MFEM_USE_HIP),YES)
+   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
+   HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR)))
+   ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
+      HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null)
+      ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
+         $(error Unable to determine HIP_DIR. Please set it manually.)
+      endif
+   endif
+endif
+
 # Compile and link options for zlib.
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
@@ -181,6 +203,11 @@ ifeq (YES,$(MFEM_USE_CUDA))
    # This is only necessary when hypre is built with cuda:
    HYPRE_LIB += -lcusparse -lcurand
 endif
+ifeq (YES,$(MFEM_USE_HIP))
+   # This is only necessary when hypre is built with hip:
+   HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\
+ -lrocsparse -lrocrand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -408,6 +435,11 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)
 
+# CoDiPack
+CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
+CODIPACK_OPT = -I$(CODIPACK_DIR)
+CODIPACK_LIB =
+
 # GSLIB library
 GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
@@ -417,9 +449,9 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 CUDA_OPT =
 CUDA_LIB = -lcusparse
 
-# HIP library configuration (currently not needed)
+# HIP library configuration
 HIP_OPT =
-HIP_LIB =
+HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa

config/test.mk

@@ -58,14 +58,14 @@ ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
 # Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 255 means the test was skipped, anything else means error
+# 0 means success, 242 means the test was skipped, anything else means error
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 242 ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
    rm -f $(1).stderr; exit $$err
 
@@ -76,7 +76,7 @@ mfem-test-file = \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 242 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
    rm -f $(1).stderr; exit $$err
 

doc/CodeDocumentation.conf.in

@@ -781,6 +781,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \
                          @MFEM_SOURCE_DIR@/miniapps/mtop \
+                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                          @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
                          @MFEM_SOURCE_DIR@/miniapps/performance \

doc/CodeDocumentation.dox

@@ -194,6 +194,8 @@ namespace mfem {
  * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="par__example_8cpp_source.html">Parallel pLaplacian example using AD</a>: Parallel pLaplacian example
+ * - <a class="el" href="seq__example_8cpp_source.html">Serial pLaplacian example using AD</a>: Serial pLaplacian example
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

examples/CMakeLists.txt

@@ -82,78 +82,80 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 add_mfem_examples(ALL_EXE_SRCS)
 
 # Add a test for each example
-foreach(SRC_FILE ${ALL_EXE_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+if (MFEM_ENABLE_TESTING)
+  foreach(SRC_FILE ${ALL_EXE_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  if (${TEST_NAME} MATCHES "ex0p?")
-    set(THIS_TEST_OPTIONS)
-  endif()
-  if (${TEST_NAME} MATCHES "ex10p*")
-    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
-  elseif(${TEST_NAME} MATCHES "ex15p*")
-    list(APPEND THIS_TEST_OPTIONS "-e" "1")
-  elseif(${TEST_NAME} MATCHES "ex27p*")
-    list(APPEND THIS_TEST_OPTIONS "-dg")
-  endif()
+    set(THIS_TEST_OPTIONS "-no-vis")
+    if (${TEST_NAME} MATCHES "ex0p?")
+      set(THIS_TEST_OPTIONS)
+    endif()
+    if (${TEST_NAME} MATCHES "ex10p*")
+      list(APPEND THIS_TEST_OPTIONS "-tf" "5")
+    elseif(${TEST_NAME} MATCHES "ex15p*")
+      list(APPEND THIS_TEST_OPTIONS "-e" "1")
+    elseif(${TEST_NAME} MATCHES "ex27p*")
+      list(APPEND THIS_TEST_OPTIONS "-dg")
+    endif()
 
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
-
-# Add CUDA/HIP tests.
-set(DEVICE_EXAMPLES
-    # serial examples with device support:
-    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-    # parallel examples with device support:
-    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
-set(MFEM_TEST_DEVICE)
-if (MFEM_USE_CUDA)
-  set(MFEM_TEST_DEVICE "cuda")
-elseif (MFEM_USE_HIP)
-  set(MFEM_TEST_DEVICE "hip")
-endif()
-if (MFEM_TEST_DEVICE)
-  foreach(TEST_NAME ${DEVICE_EXAMPLES})
-    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
     if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+      add_test(NAME ${TEST_NAME}_ser
         COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    elseif (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+    else()
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
         ${MPIEXEC_PREFLAGS}
         $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
         ${MPIEXEC_POSTFLAGS})
     endif()
   endforeach()
-endif()
 
-# If STRUMPACK is enabled, add a test run that uses it.
-if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
-    ${MPIEXEC_POSTFLAGS})
-endif()
+  # Add CUDA/HIP tests.
+  set(DEVICE_EXAMPLES
+      # serial examples with device support:
+      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+      # parallel examples with device support:
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+  set(MFEM_TEST_DEVICE)
+  if (MFEM_USE_CUDA)
+    set(MFEM_TEST_DEVICE "cuda")
+  elseif (MFEM_USE_HIP)
+    set(MFEM_TEST_DEVICE "hip")
+  endif()
+  if (MFEM_TEST_DEVICE)
+    foreach(TEST_NAME ${DEVICE_EXAMPLES})
+      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (NOT (${TEST_NAME} MATCHES ".*p$"))
+        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+      elseif (MFEM_USE_MPI)
+        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+          COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+          ${MPIEXEC_PREFLAGS}
+          $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+          ${MPIEXEC_POSTFLAGS})
+      endif()
+    endforeach()
+  endif()
 
-# If SuperLU_DIST is enabled, add a test run that uses it.
-if (MFEM_USE_SUPERLU)
-  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-    ${MPIEXEC_POSTFLAGS})
+  # If STRUMPACK is enabled, add a test run that uses it.
+  if (MFEM_USE_STRUMPACK)
+    add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+
+  # If SuperLU_DIST is enabled, add a test run that uses it.
+  if (MFEM_USE_SUPERLU)
+    add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
 endif()
 
 # Include the examples/amgx directory if AmgX is enabled

examples/amgx/CMakeLists.txt

@@ -50,30 +50,32 @@ add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
 # which builds the examples and runs:
 #   ctest -R amgx
 
-# Command line options for the tests.
-# Example 1/1p:
-set(EX1_TEST_OPTS)
-set(EX1P_TEST_OPTS)
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 1/1p:
+  set(EX1_TEST_OPTS)
+  set(EX1P_TEST_OPTS)
 
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
 
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

examples/caliper/CMakeLists.txt

@@ -30,22 +30,22 @@ set(PREFIX caliper_)
 add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})
 
 # Add a test for each example
-foreach(SRC_FILE ${CALIPER_EXE_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+if (MFEM_ENABLE_TESTING)
+  foreach(SRC_FILE ${CALIPER_EXE_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
+    set(THIS_TEST_OPTIONS "-no-vis")
 
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

examples/ex10.cpp

@@ -149,7 +149,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -376,10 +376,10 @@ int main(int argc, char *argv[])
 }
 
 
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -389,24 +389,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
       }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
    }
-   out << flush;
+   os << flush;
 }
 
 

examples/ex10p.cpp

@@ -154,7 +154,8 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -438,10 +439,11 @@ int main(int argc, char *argv[])
    return 0;
 }
 
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -451,25 +453,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks()
+      << " " << mesh->GetMyRank() << "\n";
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
       }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
    }
-   out << flush;
+   os << flush;
 }
 
 

examples/ex18.hpp

@@ -32,7 +32,7 @@ private:
    mutable DenseTensor flux;
    mutable Vector z;
 
-   void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;
+   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;
 
 public:
    FE_Evolution(FiniteElementSpace &vfes_,
@@ -256,26 +256,26 @@ inline double ComputeMaxCharSpeed(const Vector &state, const int dim)
 }
 
 // Compute the flux at solution nodes.
-void FE_Evolution::GetFlux(const DenseMatrix &x, DenseTensor &flux) const
+void FE_Evolution::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
 {
-   const int dof = flux.SizeI();
-   const int dim = flux.SizeJ();
+   const int flux_dof = flux_.SizeI();
+   const int flux_dim = flux_.SizeJ();
 
-   for (int i = 0; i < dof; i++)
+   for (int i = 0; i < flux_dof; i++)
    {
-      for (int k = 0; k < num_equation; k++) { state(k) = x(i, k); }
-      ComputeFlux(state, dim, f);
+      for (int k = 0; k < num_equation; k++) { state(k) = x_(i, k); }
+      ComputeFlux(state, flux_dim, f);
 
-      for (int d = 0; d < dim; d++)
+      for (int d = 0; d < flux_dim; d++)
       {
          for (int k = 0; k < num_equation; k++)
          {
-            flux(i, d, k) = f(k, d);
+            flux_(i, d, k) = f(k, d);
          }
       }
 
       // Update max char speed
-      const double mcs = ComputeMaxCharSpeed(state, dim);
+      const double mcs = ComputeMaxCharSpeed(state, flux_dim);
       if (mcs > max_char_speed) { max_char_speed = mcs; }
    }
 }

examples/ex19.cpp

@@ -171,7 +171,7 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -542,10 +542,10 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -555,23 +555,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
       }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
    }
-   out << flush;
+   os << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex19p.cpp

@@ -185,7 +185,8 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -196,10 +197,10 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
+        << "is NOT supported with the GPU version of hypre.\n\n";
+   return 242;
 #endif
 
    // 1. Initialize MPI
@@ -486,8 +487,8 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
 
       if (!spaces[0]->GetParMesh()->Nonconforming())
       {
-#ifndef HYPRE_USING_CUDA
-         // Not available yet when hypre is built with CUDA
+#if !defined(HYPRE_USING_GPU)
+         // Not available yet when hypre is built with GPU support
          stiff_prec_amg->SetElasticityOptions(spaces[0]);
 #endif
       }
@@ -617,10 +618,11 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -630,24 +632,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() <<
+      "\n";
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
       }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
    }
-   out << flush;
+   os << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex26.cpp

@@ -34,6 +34,43 @@
 using namespace std;
 using namespace mfem;
 
+MatrixConstantCoefficient AnisotropicCoefficient(int dim, double anisotropy)
+{
+   DenseMatrix coeff(dim, dim);
+   coeff = 0.0;
+   coeff(0,0) = anisotropy;
+   for (int d = 1; d < dim; ++d)
+   {
+      coeff(d, d) = 1.0;
+   }
+   return coeff;
+}
+
+class SymmetricILUSmoother : public Solver
+{
+   BlockILU ilu;
+   double alpha;
+
+public:
+   SymmetricILUSmoother(Operator &op, double alpha_)
+      : ilu(op), alpha(alpha_)
+   { }
+
+   void Mult(const Vector &b, Vector &x) const
+   {
+      ilu.Mult(b, x);
+      x *= alpha;
+   }
+
+   void MultTranspose(const Vector &b, Vector &x) const
+   {
+      ilu.Mult(b, x);
+      x *= alpha;
+   }
+
+   void SetOperator(const Operator &op) { }
+};
+
 // Class for constructing a multigrid preconditioner for the diffusion operator.
 // This example multigrid preconditioner class demonstrates the creation of the
 // diffusion bilinear forms and operators using partial assembly for all spaces
@@ -43,13 +80,18 @@ using namespace mfem;
 class DiffusionMultigrid : public GeometricMultigrid
 {
 private:
-   ConstantCoefficient one;
+   MatrixConstantCoefficient coeff;
+   bool use_ilu;
 
 public:
    // Constructs a diffusion multigrid for the given FiniteElementSpaceHierarchy
    // and the array of essential boundaries
-   DiffusionMultigrid(FiniteElementSpaceHierarchy& fespaces, Array<int>& ess_bdr)
-      : GeometricMultigrid(fespaces), one(1.0)
+   DiffusionMultigrid(FiniteElementSpaceHierarchy& fespaces, Array<int>& ess_bdr,
+                      double anisotropy, bool use_ilu_)
+      : GeometricMultigrid(fespaces),
+        coeff(AnisotropicCoefficient(fespaces.GetFinestFESpace().GetMesh()->Dimension(),
+                                     anisotropy)),
+        use_ilu(use_ilu_)
    {
       ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);
 
@@ -63,8 +105,7 @@ private:
    void ConstructBilinearForm(FiniteElementSpace& fespace, Array<int>& ess_bdr)
    {
       BilinearForm* form = new BilinearForm(&fespace);
-      form->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      form->AddDomainIntegrator(new DiffusionIntegrator(one));
+      form->AddDomainIntegrator(new DiffusionIntegrator(coeff));
       form->Assemble();
       bfs.Append(form);
 
@@ -78,18 +119,13 @@ private:
       ConstructBilinearForm(coarse_fespace, ess_bdr);
 
       OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
+      opr.SetType(Operator::MFEM_SPARSEMAT);
       bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
       opr.SetOperatorOwner(false);
 
-      CGSolver* pcg = new CGSolver();
-      pcg->SetPrintLevel(-1);
-      pcg->SetMaxIter(200);
-      pcg->SetRelTol(sqrt(1e-4));
-      pcg->SetAbsTol(0.0);
-      pcg->SetOperator(*opr.Ptr());
+      UMFPackSolver *coarse_solver = new UMFPackSolver(*opr.As<SparseMatrix>());
 
-      AddLevel(opr.Ptr(), pcg, true, true);
+      AddLevel(opr.Ptr(), coarse_solver, false, true);
    }
 
    void ConstructOperatorAndSmoother(FiniteElementSpace& fespace,
@@ -98,16 +134,25 @@ private:
       ConstructBilinearForm(fespace, ess_bdr);
 
       OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
+      opr.SetType(Operator::MFEM_SPARSEMAT);
       bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
       opr.SetOperatorOwner(false);
 
-      Vector diag(fespace.GetTrueVSize());
-      bfs.Last()->AssembleDiagonal(diag);
+      Solver *smoother;
 
-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
-                                                       *essentialTrueDofs.Last(), 2);
-      AddLevel(opr.Ptr(), smoother, true, true);
+      if (use_ilu)
+      {
+         smoother = new SymmetricILUSmoother(*opr, 0.5);
+      }
+      else
+      {
+         Vector diag(fespace.GetTrueVSize());
+         bfs.Last()->AssembleDiagonal(diag);
+         smoother = new OperatorChebyshevSmoother(
+            *opr, diag, *essentialTrueDofs.Last(), 2);
+      }
+
+      AddLevel(opr.Ptr(), smoother, false, true);
    }
 };
 
@@ -120,6 +165,8 @@ int main(int argc, char *argv[])
    int order_refinements = 2;
    const char *device_config = "cpu";
    bool visualization = true;
+   double anisotropy = 1.0;
+   bool use_ilu = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -128,8 +175,11 @@ int main(int argc, char *argv[])
                   "Number of geometric refinements done prior to order refinements.");
    args.AddOption(&order_refinements, "-or", "--order-refinements",
                   "Number of order refinements. Finest level in the hierarchy has order 2^{or}.");
+   args.AddOption(&anisotropy, "-a", "--anisotropy", "Anisotropy coefficient.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
+   args.AddOption(&use_ilu, "-i", "--use-ilu", "-no-i", "--no-ilu",
+                  "Use ILU smoothing?");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -210,7 +260,7 @@ int main(int argc, char *argv[])
    Array<int> ess_bdr(mesh->bdr_attributes.Max());
    ess_bdr = 1;
 
-   DiffusionMultigrid M(fespaces, ess_bdr);
+   DiffusionMultigrid M(fespaces, ess_bdr, anisotropy, use_ilu);
    M.SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1);
 
    OperatorPtr A;

examples/ex27.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);
 
 int main(int argc, char *argv[])
 {
@@ -302,26 +302,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
 
       bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
 
       bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -330,25 +330,26 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error;
+      double avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
 
    // 14. Save the refined mesh and the solution. This output can be viewed
@@ -637,11 +638,11 @@ Mesh * GenerateSerialMesh(int ref)
 
 double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
                    double alpha, double beta, double gamma,
-                   double &err)
+                   double &error)
 {
    double nrm = 0.0;
    double avg = 0.0;
-   err = 0.0;
+   error = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -705,20 +706,20 @@ double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
       }
    }
 
    // Normalize by the length of the boundary
    if (std::abs(nrm) > 0.0)
    {
-      err /= nrm;
+      error /= nrm;
       avg /= nrm;
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
-   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);
+   // quadrature weights may produce negative 'error')
+   error = (error >= 0.0) ? sqrt(error) : -sqrt(-error);
 
    // Return the average value of alpha * n.Grad(x) + beta * x
    return avg;

examples/ex27p.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);
 
 int main(int argc, char *argv[])
 {
@@ -321,26 +321,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
 
       bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
 
       bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -349,25 +349,26 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
+                                      error);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
 
    // 15. Save the refined mesh and the solution in parallel. This output can be
@@ -667,11 +668,11 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    double loc_vals[3];
    double &nrm = loc_vals[0];
    double &avg = loc_vals[1];
-   double &err = loc_vals[2];
+   double &error = loc_vals[2];
 
    nrm = 0.0;
    avg = 0.0;
-   err = 0.0;
+   error = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -735,7 +736,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
       }
    }
 
@@ -754,7 +755,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
+   // quadrature weights may produce negative 'error')
    glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);
 
    // Return the average value of alpha * n.Grad(x) + beta * x

examples/ex28p.cpp

@@ -81,10 +81,10 @@ Mesh * build_trapezoid_mesh(double offset)
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
+        << "is NOT supported with the GPU version of hypre.\n\n";
+   return 242;
 #endif
 
    // 1. Initialize MPI.

examples/ex29.cpp

@@ -167,9 +167,9 @@ int main(int argc, char *argv[])
 
    // 13. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);
 
-   cout << "|u - u_h|_2 = " << err << endl;
+   cout << "|u - u_h|_2 = " << error << endl;
 
    FiniteElementSpace flux_fespace(mesh, &fec, 3);
    GridFunction flux(&flux_fespace);

examples/ex29p.cpp

@@ -197,9 +197,9 @@ int main(int argc, char *argv[])
 
    // 15. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);
 
-   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }
+   if (myid == 0) { cout << "|u - u_h|_2 = " << error << endl; }
 
    ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
    ParGridFunction flux(&flux_fespace);

examples/ex3p.cpp

@@ -255,10 +255,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double err = x.ComputeL2Error(E);
+      double error = x.ComputeL2Error(E);
       if (myid == 0)
       {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| E_h - E ||_{L^2} = " << error << '\n' << endl;
       }
    }
 

examples/ex4p.cpp

@@ -256,10 +256,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double err = x.ComputeL2Error(F);
+      double error = x.ComputeL2Error(F);
       if (myid == 0)
       {
-         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| F_h - F ||_{L^2} = " << error << '\n' << endl;
       }
    }
 

examples/ex7p.cpp

@@ -282,10 +282,10 @@ int main(int argc, char *argv[])
    delete b;
 
    // 12. Compute and print the L^2 norm of the error.
-   double err = x.ComputeL2Error(sol_coef);
+   double error = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
-      cout << "\nL2 norm of error: " << err << endl;
+      cout << "\nL2 norm of error: " << error << endl;
    }
 
    // 13. Save the refined mesh and the solution. This output can be viewed

examples/ginkgo/CMakeLists.txt

@@ -31,29 +31,31 @@ add_mfem_examples(GINKGO_EXAMPLES_SRCS ${PFX} "" test_ginkgo)
 # which builds the examples and runs:
 #   ctest -R ginkgo
 
-# Command line options for the tests.
-set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
-set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
+  set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})
 
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
 
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

examples/hiop/CMakeLists.txt

@@ -33,31 +33,33 @@ add_mfem_examples(HIOP_EXAMPLES_SRCS ${PFX} "" test_hiop)
 # which builds the examples and runs:
 #   ctest -R hiop
 
-# Command line options for the tests.
-# Example 9:
-set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
-set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 9:
+  set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
+  set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
+  set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})
 
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
 
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

examples/petsc/CMakeLists.txt

@@ -94,30 +94,32 @@ if (MFEM_USE_SLEPC)
 endif()
 
 # Add the tests: one test per command-line-variable.
-set(TEST_OPTIONS_VARS
-    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
-if (MFEM_USE_SLEPC)
-    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-endif()
-
-foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
-  string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
-  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
-  string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
-  string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
-  set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})
-  set(TEST_NAME ${PFX}${TEST_NAME})
-  set(TEST_OPTIONS "-no-vis" ${${TEST_OPTIONS_VAR}})
-  # message(STATUS "${TEST_NAME_FULL} --> ${TEST_NAME} ${TEST_OPTIONS}")
-
-  # All PETSC tests are parallel.
-  if (MFEM_USE_MPI)
-    add_test(NAME ${TEST_NAME_FULL}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
+if (MFEM_ENABLE_TESTING)
+  set(TEST_OPTIONS_VARS
+      EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+      EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+      EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+  if (MFEM_USE_SLEPC)
+      list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
   endif()
-endforeach()
+
+  foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
+    string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
+    string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
+    string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
+    string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
+    set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})
+    set(TEST_NAME ${PFX}${TEST_NAME})
+    set(TEST_OPTIONS "-no-vis" ${${TEST_OPTIONS_VAR}})
+    # message(STATUS "${TEST_NAME_FULL} --> ${TEST_NAME} ${TEST_OPTIONS}")
+
+    # All PETSC tests are parallel.
+    if (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME_FULL}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

examples/pumi/CMakeLists.txt

@@ -37,37 +37,39 @@ add_mfem_examples(PUMI_EXAMPLES_SRCS ${PFX} "" test_pumi)
 # which builds the examples and runs:
 #   ctest -R pumi
 
-# Command line options for the tests.
-# TODO...
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # TODO...
 
-# Set the number of processors for the parallel examples. The value of
-# MFEM_MPI_NP is ignored.
-set(EX1_TEST_NP 1)
-set(EX1P_TEST_NP 8)
-set(EX2_TEST_NP 1)
-set(EX6P_TEST_NP 8)
+  # Set the number of processors for the parallel examples. The value of
+  # MFEM_MPI_NP is ignored.
+  set(EX1_TEST_NP 1)
+  set(EX1P_TEST_NP 8)
+  set(EX2_TEST_NP 1)
+  set(EX6P_TEST_NP 8)
 
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
 
-  # All PUMI examples require MPI
-  if (FALSE)
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
-    add_test(NAME ${TEST_NAME}_np=${TEST_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    # All PUMI examples require MPI
+    if (FALSE)
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
+      add_test(NAME ${TEST_NAME}_np=${TEST_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

examples/pumi/ex1p.cpp

@@ -145,7 +145,7 @@ int main(int argc, char *argv[])
    // Perform Uniform refinement
    if (ref_levels > 1)
    {
-      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+      auto uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
 
       if (geom_order > 1)
       {

examples/pumi/ex6p.cpp

@@ -150,7 +150,7 @@ int main(int argc, char *argv[])
 
    if (ref_levels > 1)
    {
-      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+      auto uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
 
       if ( geom_order > 1)
       {
@@ -345,9 +345,7 @@ int main(int argc, char *argv[])
       apf::destroyField(ipfield);
 
       // 18. Perform MesAdapt.
-      ma::Input* erinput = ma::configure(pumi_mesh, sizefield);
-      erinput->shouldFixShape = true;
-      erinput->maximumIterations = 2;
+      auto erinput = ma::configure(pumi_mesh, sizefield);
       if ( geom_order > 1)
       {
          crv::adapt(erinput);

examples/sundials/CMakeLists.txt

@@ -41,36 +41,38 @@ add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)
 # which builds the examples and runs:
 #   ctest -R sundials
 
-# Command line options for the tests.
-# Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
-set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
-set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
-set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
-# Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
-set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
-set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
-set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
-# Example 16: use the default options
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
+  set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
+  set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
+  set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
+  # Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
+  set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
+  set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
+  set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
+  # Example 16: use the default options
 
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
 
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

examples/superlu/CMakeLists.txt

@@ -32,31 +32,32 @@ add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
 # The SuperLU tests can be run separately using the target "test_superlu"
 # which builds the examples and runs:
 #   ctest -R superlu
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 1: Test SuperLU on the simple Poisson problem
+  set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
+  set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
 
-# Command line options for the tests.
-# Example 1: Test SuperLU on the simple Poisson problem
-set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
-set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})
 
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()

fem/bilinearform_ext.cpp

@@ -514,6 +514,10 @@ void EABilinearFormExtension::Assemble()
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
    const int integratorCount = integrators.Size();
+   if ( integratorCount == 0 )
+   {
+      ea_data = 0.0;
+   }
    for (int i = 0; i < integratorCount; ++i)
    {
       integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);

fem/bilininteg_diffusion_pa.cpp

@@ -1556,7 +1556,7 @@ static void SmemPADiffusionApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1585,118 +1585,102 @@ static void SmemPADiffusionApply3D(const int NE,
       double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
       double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
       double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
          }
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+      }
+      if (MFEM_THREAD_ID(z) == 0)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            const int i = qi(qx,dy,Q1D);
-            const int j = dj(qx,dy,D1D);
-            const int k = qk(qx,dy,Q1D);
-            const int l = dl(qx,dy,D1D);
-            B[i][j] = b(qx,dy);
-            G[k][l] = g(qx,dy) * sign(qx,dy);
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const int i = qi(qx,dy,Q1D);
+               const int j = dj(qx,dy,D1D);
+               const int k = qk(qx,dy,Q1D);
+               const int l = dl(qx,dy,D1D);
+               B[i][j] = b(qx,dy);
+               G[k][l] = g(qx,dy) * sign(qx,dy);
+            }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            double u[D1D], v[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
+               double u = 0.0, v = 0.0;
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const int i = qi(qx,dx,Q1D);
+                  const int j = dj(qx,dx,D1D);
+                  const int k = qk(qx,dx,Q1D);
+                  const int l = dl(qx,dx,D1D);
+                  const double s = sign(qx,dx);
+                  const double coords = X[dz][dy][dx];
+                  u += coords * B[i][j];
+                  v += coords * G[k][l] * s;
+               }
+               DDQ0[dz][dy][qx] = u;
+               DDQ1[dz][dy][qx] = v;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
+               MFEM_UNROLL(MD1)
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  const int i = qi(qy,dy,Q1D);
+                  const int j = dj(qy,dy,D1D);
+                  const int k = qk(qy,dy,Q1D);
+                  const int l = dl(qy,dy,D1D);
+                  const double s = sign(qy,dy);
+                  u += DDQ1[dz][dy][qx] * B[i][j];
+                  v += DDQ0[dz][dy][qx] * G[k][l] * s;
+                  w += DDQ0[dz][dy][qx] * B[i][j];
+               }
+               DQQ0[dz][qy][qx] = u;
+               DQQ1[dz][qy][qx] = v;
+               DQQ2[dz][qy][qx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MD1)
                for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const double coords = X[dz][dy][dx];
-                  u[dz] += coords * B[i][j];
-                  v[dz] += coords * G[k][l] * s;
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ0[dz][dy][qx] = u[dz];
-               DDQ1[dz][dy][qx] = v[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
-               {
-                  u[dz] += DDQ1[dz][dy][qx] * B[i][j];
-                  v[dz] += DDQ0[dz][dy][qx] * G[k][l] * s;
-                  w[dz] += DDQ0[dz][dy][qx] * B[i][j];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ0[dz][qy][qx] = u[dz];
-               DQQ1[dz][qy][qx] = v[dz];
-               DQQ2[dz][qy][qx] = w[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
                {
                   const int i = qi(qz,dz,Q1D);
                   const int j = dj(qz,dz,D1D);
                   const int k = qk(qz,dz,Q1D);
                   const int l = dl(qz,dz,D1D);
                   const double s = sign(qz,dz);
-                  u[qz] += DQQ0[dz][qy][qx] * B[i][j];
-                  v[qz] += DQQ1[dz][qy][qx] * B[i][j];
-                  w[qz] += DQQ2[dz][qy][qx] * G[k][l] * s;
+                  u += DQQ0[dz][qy][qx] * B[i][j];
+                  v += DQQ1[dz][qy][qx] * B[i][j];
+                  w += DQQ2[dz][qy][qx] * G[k][l] * s;
                }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
                const double O11 = d(qx,qy,qz,0,e);
                const double O12 = d(qx,qy,qz,1,e);
                const double O13 = d(qx,qy,qz,2,e);
@@ -1706,9 +1690,9 @@ static void SmemPADiffusionApply3D(const int NE,
                const double O31 = symmetric ? O13 : d(qx,qy,qz,6,e);
                const double O32 = symmetric ? O23 : d(qx,qy,qz,7,e);
                const double O33 = symmetric ? d(qx,qy,qz,5,e) : d(qx,qy,qz,8,e);
-               const double gX = u[qz];
-               const double gY = v[qz];
-               const double gZ = w[qz];
+               const double gX = u;
+               const double gY = v;
+               const double gZ = w;
                QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
                QQQ1[qz][qy][qx] = (O21*gX) + (O22*gY) + (O23*gZ);
                QQQ2[qz][qy][qx] = (O31*gX) + (O32*gY) + (O33*gZ);
@@ -1716,112 +1700,94 @@ static void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      if (MFEM_THREAD_ID(z) == 0)
       {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            const int i = qi(q,d,Q1D);
-            const int j = dj(q,d,D1D);
-            const int k = qk(q,d,Q1D);
-            const int l = dl(q,d,D1D);
-            Bt[j][i] = b(q,d);
-            Gt[l][k] = g(q,d) * sign(q,d);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               const int i = qi(q,d,Q1D);
+               const int j = dj(q,d,D1D);
+               const int k = qk(q,d,Q1D);
+               const int l = dl(q,d,D1D);
+               Bt[j][i] = b(q,d);
+               Gt[l][k] = g(q,d) * sign(q,d);
+            }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
+               double u = 0.0, v = 0.0, w = 0.0;
+               MFEM_UNROLL(MQ1)
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const int i = qi(qx,dx,Q1D);
+                  const int j = dj(qx,dx,D1D);
+                  const int k = qk(qx,dx,Q1D);
+                  const int l = dl(qx,dx,D1D);
+                  const double s = sign(qx,dx);
+                  u += QQQ0[qz][qy][qx] * Gt[l][k] * s;
+                  v += QQQ1[qz][qy][qx] * Bt[j][i];
+                  w += QQQ2[qz][qy][qx] * Bt[j][i];
+               }
+               QQD0[qz][qy][dx] = u;
+               QQD1[qz][qy][dx] = v;
+               QQD2[qz][qy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const int i = qi(qy,dy,Q1D);
+                  const int j = dj(qy,dy,D1D);
+                  const int k = qk(qy,dy,Q1D);
+                  const int l = dl(qy,dy,D1D);
+                  const double s = sign(qy,dy);
+                  u += QQD0[qz][qy][dx] * Bt[j][i];
+                  v += QQD1[qz][qy][dx] * Gt[l][k] * s;
+                  w += QQD2[qz][qy][dx] * Bt[j][i];
+               }
+               QDD0[qz][dy][dx] = u;
+               QDD1[qz][dy][dx] = v;
+               QDD2[qz][dy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MQ1)
                for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQQ0[qz][qy][qx] * Gt[l][k] * s;
-                  v[qz] += QQQ1[qz][qy][qx] * Bt[j][i];
-                  w[qz] += QQQ2[qz][qy][qx] * Bt[j][i];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD0[qz][qy][dx] = u[qz];
-               QQD1[qz][qy][dx] = v[qz];
-               QQD2[qz][qy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD0[qz][qy][dx] * Bt[j][i];
-                  v[qz] += QQD1[qz][qy][dx] * Gt[l][k] * s;
-                  w[qz] += QQD2[qz][qy][dx] * Bt[j][i];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD0[qz][dy][dx] = u[qz];
-               QDD1[qz][dy][dx] = v[qz];
-               QDD2[qz][dy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
                {
                   const int i = qi(qz,dz,Q1D);
                   const int j = dj(qz,dz,D1D);
                   const int k = qk(qz,dz,Q1D);
                   const int l = dl(qz,dz,D1D);
                   const double s = sign(qz,dz);
-                  u[dz] += QDD0[qz][dy][dx] * Bt[j][i];
-                  v[dz] += QDD1[qz][dy][dx] * Bt[j][i];
-                  w[dz] += QDD2[qz][dy][dx] * Gt[l][k] * s;
+                  u += QDD0[qz][dy][dx] * Bt[j][i];
+                  v += QDD1[qz][dy][dx] * Bt[j][i];
+                  w += QDD2[qz][dy][dx] * Gt[l][k] * s;
                }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += (u[dz] + v[dz] + w[dz]);
+               y(dx,dy,dz,e) += (u + v + w);
             }
          }
       }

fem/coefficient.cpp

@@ -21,6 +21,33 @@ namespace mfem
 
 using namespace std;
 
+// Given an ElementTransformation and IntegrationPoint in a refined mesh,
+// return the ElementTransformation of the parent coarse element, and set
+// coarse_ip to the location of the original ip within the coarse element.
+ElementTransformation *RefinedToCoarse(
+   Mesh &coarse_mesh, const ElementTransformation &T,
+   const IntegrationPoint &ip, IntegrationPoint &coarse_ip)
+{
+   Mesh &fine_mesh = *T.mesh;
+   // Get the element transformation of the coarse element containing the
+   // fine element.
+   int fine_element = T.ElementNo;
+   const CoarseFineTransformations &cf = fine_mesh.GetRefinementTransforms();
+   int coarse_element = cf.embeddings[fine_element].parent;
+   ElementTransformation *coarse_T = coarse_mesh.GetElementTransformation(
+                                        coarse_element);
+   // Transform the integration point from fine element coordinates to coarse
+   // element coordinates.
+   Geometry::Type geom = T.GetGeometryType();
+   IntegrationPointTransformation fine_to_coarse;
+   IsoparametricTransformation &emb_tr = fine_to_coarse.Transf;
+   emb_tr.SetIdentityTransformation(geom);
+   emb_tr.SetPointMat(cf.point_matrices[geom](cf.embeddings[fine_element].matrix));
+   fine_to_coarse.Transform(ip, coarse_ip);
+   coarse_T->SetIntPoint(&coarse_ip);
+   return coarse_T;
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -28,6 +55,52 @@ double PWConstCoefficient::Eval(ElementTransformation & T,
    return (constants(att-1));
 }
 
+void PWCoefficient::InitMap(const Array<int> & attr,
+                            const Array<Coefficient*> & coefs)
+{
+   MFEM_VERIFY(attr.Size() == coefs.Size(),
+               "PWCoefficient:  "
+               "Attribute and coefficient arrays have incompatible "
+               "dimensions.");
+
+   for (int i=0; i<attr.Size(); i++)
+   {
+      if (coefs[i] != NULL)
+      {
+         UpdateCoefficient(attr[i], *coefs[i]);
+      }
+   }
+}
+
+void PWCoefficient::SetTime(double t)
+{
+   Coefficient::SetTime(t);
+
+   std::map<int, Coefficient*>::iterator p = pieces.begin();
+   for (; p != pieces.end(); p++)
+   {
+      if (p->second != NULL)
+      {
+         p->second->SetTime(t);
+      }
+   }
+}
+
+double PWCoefficient::Eval(ElementTransformation &T,
+                           const IntegrationPoint &ip)
+{
+   const int att = T.Attribute;
+   std::map<int, Coefficient*>::const_iterator p = pieces.find(att);
+   if (p != pieces.end())
+   {
+      if ( p->second != NULL)
+      {
+         return p->second->Eval(T, ip);
+      }
+   }
+   return 0.0;
+}
+
 double FunctionCoefficient::Eval(ElementTransformation & T,
                                  const IntegrationPoint & ip)
 {
@@ -49,7 +122,17 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
-   return GridF -> GetValue (T, ip, Component);
+   Mesh *gf_mesh = GridF->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      return GridF->GetValue(T, ip, Component);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      return GridF->GetValue(*coarse_T, coarse_ip, Component);
+   }
 }
 
 void TransformedCoefficient::SetTime(double t)
@@ -120,6 +203,63 @@ void VectorCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
    }
 }
 
+void PWVectorCoefficient::InitMap(const Array<int> & attr,
+                                  const Array<VectorCoefficient*> & coefs)
+{
+   MFEM_VERIFY(attr.Size() == coefs.Size(),
+               "PWVectorCoefficient:  "
+               "Attribute and coefficient arrays have incompatible "
+               "dimensions.");
+
+   for (int i=0; i<attr.Size(); i++)
+   {
+      if (coefs[i] != NULL)
+      {
+         UpdateCoefficient(attr[i], *coefs[i]);
+      }
+   }
+}
+
+void PWVectorCoefficient::UpdateCoefficient(int attr, VectorCoefficient & coef)
+{
+   MFEM_VERIFY(coef.GetVDim() == vdim,
+               "PWVectorCoefficient::UpdateCoefficient:  "
+               "VectorCoefficient has incompatible dimension.");
+   pieces[attr] = &coef;
+}
+
+void PWVectorCoefficient::SetTime(double t)
+{
+   VectorCoefficient::SetTime(t);
+
+   std::map<int, VectorCoefficient*>::iterator p = pieces.begin();
+   for (; p != pieces.end(); p++)
+   {
+      if (p->second != NULL)
+      {
+         p->second->SetTime(t);
+      }
+   }
+}
+
+void PWVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
+                               const IntegrationPoint &ip)
+{
+   const int att = T.Attribute;
+   std::map<int, VectorCoefficient*>::const_iterator p = pieces.find(att);
+   if (p != pieces.end())
+   {
+      if ( p->second != NULL)
+      {
+         p->second->Eval(V, T, ip);
+         return;
+      }
+   }
+
+   V.SetSize(vdim);
+   V = 0.0;
+}
+
 void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
@@ -202,13 +342,30 @@ void VectorGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
-   GridFunc->GetVectorValue(T, ip, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetVectorValue(T, ip, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetVectorValue(*coarse_T, coarse_ip, V);
+   }
 }
 
 void VectorGridFunctionCoefficient::Eval(
    DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   GridFunc->GetVectorValues(T, ir, M);
+   if (T.mesh == GridFunc->FESpace()->GetMesh())
+   {
+      GridFunc->GetVectorValues(T, ir, M);
+   }
+   else
+   {
+      VectorCoefficient::Eval(M, T, ir);
+   }
 }
 
 GradientGridFunctionCoefficient::GradientGridFunctionCoefficient (
@@ -228,13 +385,30 @@ void GradientGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void GradientGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                            const IntegrationPoint &ip)
 {
-   GridFunc->GetGradient(T, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetGradient(T, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetGradient(*coarse_T, V);
+   }
 }
 
 void GradientGridFunctionCoefficient::Eval(
    DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   GridFunc->GetGradients(T, ir, M);
+   if (T.mesh == GridFunc->FESpace()->GetMesh())
+   {
+      GridFunc->GetGradients(T, ir, M);
+   }
+   else
+   {
+      VectorCoefficient::Eval(M, T, ir);
+   }
 }
 
 CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
@@ -260,7 +434,17 @@ void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
-   GridFunc->GetCurl(T, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetCurl(T, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetCurl(*coarse_T, V);
+   }
 }
 
 DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
@@ -272,7 +456,17 @@ DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
 double DivergenceGridFunctionCoefficient::Eval(ElementTransformation &T,
                                                const IntegrationPoint &ip)
 {
-   return GridFunc->GetDivergence(T);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      return GridFunc->GetDivergence(T);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      return GridFunc->GetDivergence(*coarse_T);
+   }
 }
 
 void VectorDeltaCoefficient::SetTime(double t)
@@ -331,6 +525,72 @@ void VectorRestrictedCoefficient::Eval(
    }
 }
 
+void PWMatrixCoefficient::InitMap(const Array<int> & attr,
+                                  const Array<MatrixCoefficient*> & coefs)
+{
+   MFEM_VERIFY(attr.Size() == coefs.Size(),
+               "PWMatrixCoefficient:  "
+               "Attribute and coefficient arrays have incompatible "
+               "dimensions.");
+
+   for (int i=0; i<attr.Size(); i++)
+   {
+      if (coefs[i] != NULL)
+      {
+         UpdateCoefficient(attr[i], *coefs[i]);
+      }
+   }
+}
+
+void PWMatrixCoefficient::UpdateCoefficient(int attr, MatrixCoefficient & coef)
+{
+   MFEM_VERIFY(coef.GetHeight() == height,
+               "PWMatrixCoefficient::UpdateCoefficient:  "
+               "MatrixCoefficient has incompatible height.");
+   MFEM_VERIFY(coef.GetWidth() == width,
+               "PWMatrixCoefficient::UpdateCoefficient:  "
+               "MatrixCoefficient has incompatible width.");
+   if (symmetric)
+   {
+      MFEM_VERIFY(coef.IsSymmetric(),
+                  "PWMatrixCoefficient::UpdateCoefficient:  "
+                  "MatrixCoefficient has incompatible symmetry.");
+   }
+   pieces[attr] = &coef;
+}
+
+void PWMatrixCoefficient::SetTime(double t)
+{
+   MatrixCoefficient::SetTime(t);
+
+   std::map<int, MatrixCoefficient*>::iterator p = pieces.begin();
+   for (; p != pieces.end(); p++)
+   {
+      if (p->second != NULL)
+      {
+         p->second->SetTime(t);
+      }
+   }
+}
+
+void PWMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                               const IntegrationPoint &ip)
+{
+   const int att = T.Attribute;
+   std::map<int, MatrixCoefficient*>::const_iterator p = pieces.find(att);
+   if (p != pieces.end())
+   {
+      if ( p->second != NULL)
+      {
+         p->second->Eval(K, T, ip);
+         return;
+      }
+   }
+
+   K.SetSize(height, width);
+   K = 0.0;
+}
+
 void MatrixFunctionCoefficient::SetTime(double t)
 {
    if (Q) { Q->SetTime(t); }

fem/coefficient.hpp

@@ -125,6 +125,87 @@ public:
                        const IntegrationPoint &ip);
 };
 
+/** @brief A piecewise coefficient with the pieces keyed off the element
+    attribute numbers.
+
+    A value of zero will be returned for any missing attribute numbers.
+
+    This object will not assume ownership of any Coefficient objects
+    passed to it. Consequently, the caller must ensure that the
+    individual Coefficient objects are not deleted while this
+    PWCoefficient is still in use.
+
+    \note The keys may either be domain attribute numbers or boundary
+    attribute numbers. If the PWCoefficient is used with a domain
+    integrator the keys are assumed to be domain attribute
+    numbers. Similarly, if the PWCoefficient is used with a boundary
+    integrator the keys are assumed to be boundary attribute numbers.
+*/
+class PWCoefficient : public Coefficient
+{
+private:
+   /** Internal data structure to store pointers to the appropriate
+       coefficients for different regions of the mesh. The keys used
+       in the map are the mesh attribute numbers (either element
+       attribute or boundary element attribute depending upon
+       context). The values returned for any missing attributes will
+       be zero. The coefficient pointers may be NULL in which case a
+       value of zero is returned.
+
+       The Coefficient objects contained in this map are NOT owned by
+       this PWCoefficient object. This means that they will not be
+       deleted when this object is deleted also the caller must ensure
+       that the various Coefficient objects are not deleted while this
+       PWCoefficient is still needed.
+   */
+   std::map<int, Coefficient*> pieces;
+
+   /** Convenience function to check for compatible array lengths,
+       loop over the arrays, and add their attribute/Coefficient pairs
+       to the internal data structure.
+   */
+   void InitMap(const Array<int> & attr,
+                const Array<Coefficient*> & coefs);
+
+public:
+
+   /// Constructs a piecewise coefficient
+   explicit PWCoefficient() {}
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of Coefficient pointers
+       Any missing attributes or NULL coefficient pointers will result in a
+       value of zero being returned for that attribute.
+
+       \note Ownership of the Coefficient objects will NOT be
+       transferred to this object.
+   */
+   PWCoefficient(const Array<int> & attr,
+                 const Array<Coefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Set the time for time dependent coefficients
+   virtual void SetTime(double t);
+
+   /// Replace a set of coefficients
+   void UpdateCoefficients(const Array<int> & attr,
+                           const Array<Coefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Replace a single Coefficient for a particular attribute
+   void UpdateCoefficient(int attr, Coefficient & coef)
+   { pieces[attr] = &coef; }
+
+   /// Remove a single Coefficient for a particular attribute
+   void ZeroCoefficient(int attr)
+   { pieces.erase(attr); }
+
+   /// Evaluate the coefficient.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
 /// A general function coefficient
 class FunctionCoefficient : public Coefficient
 {
@@ -413,6 +494,89 @@ public:
    const Vector& GetVec() { return vec; }
 };
 
+/** @brief A piecewise vector-valued coefficient with the pieces keyed off the
+    element attribute numbers.
+
+    A value of zero will be returned for any missing attribute numbers.
+
+    This object will not assume ownership of any VectorCoefficient
+    objects passed to it. Consequently, the caller must ensure that
+    the individual VectorCoefficient objects are not deleted while
+    this PWVectorCoefficient is still in use.
+
+    \note The keys may either be domain attribute numbers or boundary
+    attribute numbers. If the PWVectorCoefficient is used with a
+    domain integrator the keys are assumed to be domain attribute
+    numbers. Similarly, if the PWVectorCoefficient is used with a
+    boundary integrator the keys are assumed to be boundary attribute
+    numbers.
+*/
+class PWVectorCoefficient : public VectorCoefficient
+{
+private:
+   /** Internal data structure to store pointers to the appropriate
+       coefficients for different regions of the mesh. The keys used
+       in the map are the mesh attribute numbers (either element
+       attribute or boundary element attribute depending upon
+       context). The values returned for any missing attributes will
+       be zero. The coefficient pointers may be NULL in which case a
+       value of zero is returned.
+
+       The VectorCoefficient objects contained in this map are NOT
+       owned by this PWVectorCoefficient object. This means that they
+       will not be deleted when this object is deleted also the caller
+       must ensure that the various VectorCoefficient objects are not
+       deleted while this PWVectorCoefficient is still needed.
+   */
+   std::map<int, VectorCoefficient*> pieces;
+
+   /** Convenience function to check for compatible array lengths,
+       loop over the arrays, and add their attribute/VectorCoefficient
+       pairs to the internal data structure.
+   */
+   void InitMap(const Array<int> & attr,
+                const Array<VectorCoefficient*> & coefs);
+
+public:
+
+   /// Constructs a piecewise vector coefficient of dimension vd
+   explicit PWVectorCoefficient(int vd): VectorCoefficient(vd) {}
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param vd - dimension of the vector-valued result
+       \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of VectorCoefficient pointers
+       Any missing attributes or NULL coefficient pointers will result in a
+       zero vector being returned for that attribute.
+
+       \note Ownership of the VectorCoefficient objects will NOT be
+       transferred to this object.
+   */
+   PWVectorCoefficient(int vd, const Array<int> & attr,
+                       const Array<VectorCoefficient*> & coefs)
+      : VectorCoefficient(vd) { InitMap(attr, coefs); }
+
+   /// Set the time for time dependent coefficients
+   virtual void SetTime(double t);
+
+   /// Replace a set of coefficients
+   void UpdateCoefficients(const Array<int> & attr,
+                           const Array<VectorCoefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Replace a single Coefficient for a particular attribute
+   void UpdateCoefficient(int attr, VectorCoefficient & coef);
+
+   /// Remove a single VectorCoefficient for a particular attribute
+   void ZeroCoefficient(int attr)
+   { pieces.erase(attr); }
+
+   /// Evaluate the coefficient.
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   using VectorCoefficient::Eval;
+};
+
 /// A general vector function coefficient
 class VectorFunctionCoefficient : public VectorCoefficient
 {
@@ -782,6 +946,112 @@ public:
 };
 
 
+/** @brief A piecewise matrix-valued coefficient with the pieces keyed off the
+    element attribute numbers.
+
+    A value of zero will be returned for any missing attribute numbers.
+
+    This object will not assume ownership of any MatrixCoefficient
+    objects passed to it. Consequently, the caller must ensure that
+    the individual MatrixCoefficient objects are not deleted while
+    this PWMatrixCoefficient is still in use.
+
+    \note The keys may either be domain attribute numbers or boundary
+    attribute numbers. If the PWMatrixCoefficient is used with a
+    domain integrator the keys are assumed to be domain attribute
+    numbers. Similarly, if the PWMatrixCoefficient is used with a
+    boundary integrator the keys are assumed to be boundary attribute
+    numbers.
+*/
+class PWMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   /** Internal data structure to store pointers to the appropriate
+       coefficients for different regions of the mesh. The keys used
+       in the map are the mesh attribute numbers (either element
+       attribute or boundary element attribute depending upon
+       context). The values returned for any missing attributes will
+       be zero. The coefficient pointers may be NULL in which case a
+       value of zero is returned.
+
+       The MatrixCoefficient objects contained in this map are NOT
+       owned by this PWMatrixCoefficient object. This means that they
+       will not be deleted when this object is deleted also the caller
+       must ensure that the various MatrixCoefficient objects are not
+       deleted while this PWMatrixCoefficient is still needed.
+   */
+   std::map<int, MatrixCoefficient*> pieces;
+
+   /** Convenience function to check for compatible array lengths,
+       loop over the arrays, and add their attribute/MatrixCoefficient
+       pairs to the internal data structure.
+   */
+   void InitMap(const Array<int> & attr,
+                const Array<MatrixCoefficient*> & coefs);
+
+public:
+
+   /// Constructs a piecewise matrix coefficient of dimension dim by dim
+   explicit PWMatrixCoefficient(int dim, bool symm = false)
+      : MatrixCoefficient(dim, symm) {}
+
+   /// Constructs a piecewise matrix coefficient of dimension h by w
+   explicit PWMatrixCoefficient(int h, int w, bool symm = false)
+      : MatrixCoefficient(h, w, symm) {}
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param dim - size of the square matrix-valued result
+       \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of MatrixCoefficient pointers
+       \param symm - true if the result will be symmetric, false otherwise
+       Any missing attributes or NULL coefficient pointers will result in a
+       zero matrix being returned.
+
+       \note Ownership of the MatrixCoefficient objects will NOT be
+       transferred to this object.
+   */
+   PWMatrixCoefficient(int dim, const Array<int> & attr,
+                       const Array<MatrixCoefficient*> & coefs,
+                       bool symm=false)
+      : MatrixCoefficient(dim, symm) { InitMap(attr, coefs); }
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param h - height of the matrix-valued result
+       \param w - width of the matrix-valued result
+       \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of MatrixCoefficient pointers
+       \param symm - true if the result will be symmetric, false otherwise
+       Any missing attributes or NULL coefficient pointers will result in a
+       zero matrix being returned for that attribute.
+
+       \note Ownership of the MatrixCoefficient objects will NOT be
+       transferred to this object.
+   */
+   PWMatrixCoefficient(int h, int w, const Array<int> & attr,
+                       const Array<MatrixCoefficient*> & coefs,
+                       bool symm=false)
+      : MatrixCoefficient(h, w, symm) { InitMap(attr, coefs); }
+
+   /// Set the time for time dependent coefficients
+   virtual void SetTime(double t);
+
+   /// Replace a set of coefficients
+   void UpdateCoefficients(const Array<int> & attr,
+                           const Array<MatrixCoefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Replace a single coefficient for a particular attribute
+   void UpdateCoefficient(int attr, MatrixCoefficient & coef);
+
+   /// Remove a single MatrixCoefficient for a particular attribute
+   void ZeroCoefficient(int attr)
+   { pieces.erase(attr); }
+
+   /// Evaluate the coefficient.
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+};
+
 /** @brief A matrix coefficient with an optional scalar coefficient multiplier
     \a q.  The matrix function can either be represented by a std function or
     a constant matrix provided when constructing this object.  */

fem/complex_fem.cpp

@@ -195,6 +195,15 @@ ComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
    if ( lfi_imag ) { lfi->AddDomainIntegrator(lfi_imag); }
 }
 
+void
+ComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                                       LinearFormIntegrator *lfi_imag,
+                                       Array<int> &elem_attr_marker)
+{
+   if ( lfi_real ) { lfr->AddDomainIntegrator(lfi_real, elem_attr_marker); }
+   if ( lfi_imag ) { lfi->AddDomainIntegrator(lfi_imag, elem_attr_marker); }
+}
+
 void
 ComplexLinearForm::AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                                          LinearFormIntegrator *lfi_imag)
@@ -317,6 +326,14 @@ void SesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
    if (bfi_imag) { blfi->AddDomainIntegrator(bfi_imag); }
 }
 
+void SesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                                           BilinearFormIntegrator *bfi_imag,
+                                           Array<int> & elem_marker)
+{
+   if (bfi_real) { blfr->AddDomainIntegrator(bfi_real, elem_marker); }
+   if (bfi_imag) { blfi->AddDomainIntegrator(bfi_imag, elem_marker); }
+}
+
 void
 SesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                                         BilinearFormIntegrator *bfi_imag)
@@ -879,6 +896,15 @@ ParComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
    if ( lfi_imag ) { plfi->AddDomainIntegrator(lfi_imag); }
 }
 
+void
+ParComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                                          LinearFormIntegrator *lfi_imag,
+                                          Array<int> &elem_attr_marker)
+{
+   if ( lfi_real ) { plfr->AddDomainIntegrator(lfi_real, elem_attr_marker); }
+   if ( lfi_imag ) { plfi->AddDomainIntegrator(lfi_imag, elem_attr_marker); }
+}
+
 void
 ParComplexLinearForm::AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                                             LinearFormIntegrator *lfi_imag)
@@ -1040,6 +1066,14 @@ void ParSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
    if (bfi_imag) { pblfi->AddDomainIntegrator(bfi_imag); }
 }
 
+void ParSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                                              BilinearFormIntegrator *bfi_imag,
+                                              Array<int> & elem_marker)
+{
+   if (bfi_real) { pblfr->AddDomainIntegrator(bfi_real, elem_marker); }
+   if (bfi_imag) { pblfi->AddDomainIntegrator(bfi_imag, elem_marker); }
+}
+
 void
 ParSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                                            BilinearFormIntegrator *bfi_imag)
@@ -1209,7 +1243,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
-#ifndef HYPRE_USING_CUDA
+#if !defined(HYPRE_USING_GPU)
          ess_tdof_list.HostRead();
          for (int k = 0; k < n; k++)
          {
@@ -1222,7 +1256,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
             ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
          const int *d_diag_i = Aih->diag->i;
          double *d_diag_data = Aih->diag->data;
-         CuWrap1D(n, [=] MFEM_DEVICE (int k)
+         MFEM_GPU_FORALL(k, n,
          {
             const int j = d_ess_tdof_list[k];
             d_diag_data[d_diag_i[j]] = 0.0;

fem/complex_fem.hpp

@@ -128,6 +128,11 @@ public:
    void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
                             LinearFormIntegrator *lfi_imag);
 
+   /// Adds new Domain Integrator, restricted to the given attributes.
+   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                            LinearFormIntegrator *lfi_imag,
+                            Array<int> &elem_attr_marker);
+
    /// Adds new Boundary Integrator.
    void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                               LinearFormIntegrator *lfi_imag);
@@ -260,6 +265,11 @@ public:
    void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
                             BilinearFormIntegrator *bfi_imag);
 
+   /// Adds new Domain Integrator, restricted to the given attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                            BilinearFormIntegrator *bfi_imag,
+                            Array<int> &elem_marker);
+
    /// Adds new Boundary Integrator.
    void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                               BilinearFormIntegrator *bfi_imag);
@@ -464,6 +474,11 @@ public:
    void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
                             LinearFormIntegrator *lfi_imag);
 
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                            LinearFormIntegrator *lfi_imag,
+                            Array<int> &elem_attr_marker);
+
    /// Adds new Boundary Integrator.
    void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                               LinearFormIntegrator *lfi_imag);
@@ -598,6 +613,11 @@ public:
    void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
                             BilinearFormIntegrator *bfi_imag);
 
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                            BilinearFormIntegrator *bfi_imag,
+                            Array<int> &elem_marker);
+
    /// Adds new Boundary Integrator.
    void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                               BilinearFormIntegrator *bfi_imag);

fem/conduitdatacollection.cpp

@@ -330,7 +330,7 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
          }
          else
          {
-            Node &(n_bndry_conn_conv) =
+            Node &n_bndry_conn_conv =
                n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
             n_bndry_conn.to_int_array(n_bndry_conn_conv);
             bndry_indices = (n_bndry_conn_conv).value();
@@ -645,7 +645,8 @@ ConduitDataCollection::MeshToBlueprintMesh(Mesh *mesh,
                                            Node &n_mesh,
                                            const std::string &coordset_name,
                                            const std::string &main_topology_name,
-                                           const std::string &boundary_topology_name)
+                                           const std::string &boundary_topology_name,
+                                           const std::string &main_adjset_name)
 {
    int dim = mesh->SpaceDimension();
 
@@ -815,6 +816,83 @@ ConduitDataCollection::MeshToBlueprintMesh(Mesh *mesh,
          bndry_att_vals[i] = mesh->GetBdrAttribute(i);
       }
    }
+
+   ////////////////////////////////////////////
+   // Setup adjsets
+   ////////////////////////////////////////////
+
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh = dynamic_cast<ParMesh*>(mesh);
+   if (pmesh)
+   {
+      ////////////////////////////////////////////
+      // Setup main adjset
+      ////////////////////////////////////////////
+
+      Node &n_adjset = n_mesh["adjsets"][main_adjset_name];
+
+      n_adjset["association"] = "vertex";
+      n_adjset["topology"] = main_topology_name;
+      n_adjset["groups"].set(DataType::object());
+
+      const GroupTopology &pmesh_gtopo = pmesh->gtopo;
+      const int local_rank = pmesh->GetMyRank();
+      const int num_groups = pmesh_gtopo.NGroups();
+      // NOTE: skip the first group since its the local-only group
+      for (int i = 1; i < num_groups; i++)
+      {
+         const int num_group_nbrs = pmesh_gtopo.GetGroupSize(i);
+         const int *group_nbrs = pmesh_gtopo.GetGroup(i);
+         const int num_group_verts = pmesh->GroupNVertices(i);
+
+         // NOTE: 'neighbor' values are local to this processor, but Blueprint
+         // expects global domain identifiers, so we collapse this layer of
+         // indirection
+         Array<int> group_ranks(num_group_nbrs);
+         std::string group_name = "group";
+         {
+            for (int j = 0; j < num_group_nbrs; j++)
+            {
+               group_ranks[j] = pmesh_gtopo.GetNeighborRank(group_nbrs[j]);
+            }
+            group_ranks.Sort();
+            for (int j = 0; j < num_group_nbrs; j++)
+            {
+               group_name += "_" + std::to_string(group_ranks[j]);
+            }
+
+            // NOTE: Blueprint only wants remote ranks in its neighbor list,
+            // so we remove the local rank after the canonicalized Blueprint
+            // group name is formed
+            group_ranks.DeleteFirst(local_rank);
+         }
+         Node &n_group = n_adjset["groups"][group_name];
+
+         n_group["neighbors"].set(group_ranks.GetData(), group_ranks.Size());
+         n_group["values"].set(DataType::c_int(num_group_verts));
+
+         int_array group_vals = n_group["values"].value();
+         for (int j = 0; j < num_group_verts; j++)
+         {
+            group_vals[j] = pmesh->GroupVertex(i, j);
+         }
+      }
+
+      // NOTE: We don't create an adjset for face neighbor data because
+      // these faces aren't listed in the 'boundary_topology_name' topology
+      // (this topology only covers the faces between 'main_topology_name'
+      // elements and void). To include a face neighbor data adjset, this
+      // function would need to export a topology with either (1) all faces
+      // in the mesh topology or (2) all boundary faces, including neighbors.
+
+      ////////////////////////////////////////////
+      // Setup distributed state
+      ////////////////////////////////////////////
+
+      Node &n_domid = n_mesh["state/domain_id"];
+      n_domid.set(local_rank);
+   }
+#endif
 }
 
 //---------------------------------------------------------------------------//

fem/conduitdatacollection.hpp

@@ -166,7 +166,8 @@ public:
                                    conduit::Node &out,
                                    const std::string &coordset_name = "coords",
                                    const std::string &main_topology_name = "main",
-                                   const std::string &boundary_topology_name = "boundary");
+                                   const std::string &boundary_topology_name = "boundary",
+                                   const std::string &main_adjset_name = "main_adjset");
 
    /// Describes a MFEM grid function using the mesh blueprint
    /** Sets up passed conduit::Node out to describe the given grid function

fem/datacollection.cpp

@@ -11,6 +11,7 @@
 
 #include "fem.hpp"
 #include "../mesh/nurbs.hpp"
+#include "../mesh/vtk.hpp"
 #include "../general/binaryio.hpp"
 #include "../general/text.hpp"
 #include "picojson.h"
@@ -787,53 +788,44 @@ void ParaViewDataCollection::Load(int )
 
 std::string ParaViewDataCollection::GenerateCollectionPath()
 {
-   std::string out = "";
-   out = prefix_path + DataCollection::GetCollectionName();
-   return out;
+   return prefix_path + DataCollection::GetCollectionName();
 }
 
 std::string ParaViewDataCollection::GeneratePVTUPath()
 {
-   std::string out = "Cycle" + to_padded_string(cycle,pad_digits_cycle);
-   return out;
+   return "Cycle" + to_padded_string(cycle,pad_digits_cycle);
 }
 
 std::string ParaViewDataCollection::GenerateVTUPath()
 {
-   std::string out = GeneratePVTUPath();
-   return out;
+   return GeneratePVTUPath();
 }
 
 std::string ParaViewDataCollection::GeneratePVDFileName()
 {
-   std::string out = GetCollectionName()+".pvd";
-   return out;
+   return GetCollectionName() + ".pvd";
 }
 
-std::string ParaViewDataCollection::GeneratePVTUFileName()
+std::string ParaViewDataCollection::GeneratePVTUFileName(
+   const std::string &prefix)
 {
-   std::string out = "data.pvtu";
-   return out;
+   return prefix + ".pvtu";
 }
 
-std::string ParaViewDataCollection::GenerateVTUFileName()
+std::string ParaViewDataCollection::GenerateVTUFileName(
+   const std::string &prefix, int rank)
 {
-   std::string out = "proc" + to_padded_string(myid,pad_digits_rank)+".vtu";
-   return out;
-}
-std::string ParaViewDataCollection::GenerateVTUFileName(int crank)
-{
-   std::string out = "proc" + to_padded_string(crank,pad_digits_rank)+".vtu";
-   return out;
+   return prefix + to_padded_string(rank, pad_digits_rank) + ".vtu";
 }
 
 void ParaViewDataCollection::Save()
 {
    // add a new collection to the PDV file
 
+   std::string col_path = GenerateCollectionPath();
    // check if the directories are created
    {
-      std::string path = GenerateCollectionPath()+"/"+GenerateVTUPath();
+      std::string path = col_path + "/" + GenerateVTUPath();
       int err = create_directory(path, mesh, myid);
       if (err)
       {
@@ -850,8 +842,7 @@ void ParaViewDataCollection::Save()
 
    if (myid == 0 && !pvd_stream.is_open())
    {
-      std::string dpath=GenerateCollectionPath();
-      std::string pvdname=dpath+"/"+GeneratePVDFileName();
+      std::string pvdname = col_path + "/" + GeneratePVDFileName();
 
       bool write_header = true;
       std::ifstream pvd_in;
@@ -915,80 +906,87 @@ void ParaViewDataCollection::Save()
       }
    }
 
-   // define the vtu file
+   std::string vtu_prefix = col_path + "/" + GenerateVTUPath() + "/";
+
+   // Save the local part of the mesh and grid functions fields to the local
+   // VTU file
    {
-      std::string fname = GenerateCollectionPath()+"/"+GenerateVTUPath()+"/"
-                          +GenerateVTUFileName();
-      std::fstream out(fname, std::ios::out);
+      std::ofstream out(vtu_prefix + GenerateVTUFileName("proc", myid));
       out.precision(precision);
-      SaveDataVTU(out,levels_of_detail);
-      out.close();
+      SaveDataVTU(out, levels_of_detail);
    }
 
-   // define the pvtu file only on process 0
-   if (myid==0)
+   // Save the local part of the quadrature function fields
+   for (const auto &qfield : q_field_map)
    {
-      std::string fname = GenerateCollectionPath()+"/"+GeneratePVTUPath()+"/"
-                          +GeneratePVTUFileName();
-      std::fstream out(fname, std::ios::out);
+      const std::string &field_name = qfield.first;
+      std::ofstream out(vtu_prefix + GenerateVTUFileName(field_name, myid));
+      qfield.second->SaveVTU(out, pv_data_format, compression);
+   }
 
-      out << "<?xml version=\"1.0\"?>\n";
-      out << "<VTKFile type=\"PUnstructuredGrid\"";
-      out << " version =\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
-      out << "<PUnstructuredGrid GhostLevel=\"0\">\n";
-
-      out << "<PPoints>\n";
-      out << "\t<PDataArray type=\"" << GetDataTypeString() << "\" ";
-      out << " Name=\"Points\" NumberOfComponents=\"3\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PPoints>\n";
-
-      out << "<PCells>\n";
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"connectivity\" NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"offsets\"      NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "\t<PDataArray type=\"UInt8\" ";
-      out << " Name=\"types\"        NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PCells>\n";
-
-      out << "<PPointData>\n";
-      for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it)
+   // MPI rank 0 also creates a "PVTU" file that points to all of the separately
+   // written VTU files.
+   // This file path is then appended to the PVD file.
+   if (myid == 0)
+   {
+      // Create the main PVTU file
       {
-         int vec_dim=it->second->VectorDim();
-         out << "<PDataArray type=\"" << GetDataTypeString()
-             << "\" Name=\"" << it->first
-             << "\" NumberOfComponents=\"" << vec_dim << "\" "
-             << "format=\"" << GetDataFormatString() << "\" />\n";
+         std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data"));
+         WritePVTUHeader(pvtu_out);
+
+         // Grid function fields
+         pvtu_out << "<PPointData>\n";
+         for (auto &field_it : field_map)
+         {
+            int vec_dim = field_it.second->VectorDim();
+            pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                     << "\" Name=\"" << field_it.first
+                     << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                     << "format=\"" << GetDataFormatString() << "\" />\n";
+         }
+         pvtu_out << "</PPointData>\n";
+         // Element attributes
+         pvtu_out << "<PCellData>\n";
+         pvtu_out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
+                  << "\" NumberOfComponents=\"1\""
+                  << " format=\"" << GetDataFormatString() << "\"/>\n";
+         pvtu_out << "</PCellData>\n";
+
+         WritePVTUFooter(pvtu_out, "proc");
       }
-      out << "</PPointData>\n";
 
-      // CELL DATA
-      out << "<PCellData>\n";
-      out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
-          << "\" NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PCellData>\n";
+      // Add the latest PVTU to the PVD
+      pvd_stream << "<DataSet timestep=\"" << GetTime()
+                 << "\" group=\"\" part=\"" << 0 << "\" file=\""
+                 << GeneratePVTUPath() + "/" + GeneratePVTUFileName("data")
+                 << "\" name=\"mesh\"/>\n";
 
-      for (int ii=0; ii<num_procs; ii++)
+      // Create PVTU files for each quadrature field and add them to the PVD
+      // file
+      for (auto &q_field : q_field_map)
       {
-         // this one is generated without the path
-         std::string nfname=GenerateVTUFileName(ii);
-         out << "<Piece Source=\"" << nfname << "\"/>\n";
-      }
-      out << "</PUnstructuredGrid>\n";
-      out << "</VTKFile>\n";
-      out.close();
+         const std::string &q_field_name = q_field.first;
+         std::string q_fname = GeneratePVTUPath() + "/"
+                               + GeneratePVTUFileName(q_field_name);
 
-      fname = GeneratePVTUPath()+"/"+GeneratePVTUFileName();
-      // add the pvtu file to the pvd_stream
-      pvd_stream << "<DataSet timestep=\"" << GetTime();  // GetCycle();
-      pvd_stream << "\" group=\"\" part=\"" << 0 << "\" file=\"";
-      pvd_stream << fname << "\"/>\n";
+         std::ofstream pvtu_out(col_path + "/" + q_fname);
+         WritePVTUHeader(pvtu_out);
+         int vec_dim = q_field.second->GetVDim();
+         pvtu_out << "<PPointData>\n";
+         pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                  << "\" Name=\"" << q_field_name
+                  << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                  << "format=\"" << GetDataFormatString() << "\" />\n";
+         pvtu_out << "</PPointData>\n";
+         WritePVTUFooter(pvtu_out, q_field_name);
+
+         pvd_stream << "<DataSet timestep=\"" << GetTime()
+                    << "\" group=\"\" part=\"" << 0 << "\" file=\""
+                    << q_fname << "\" name=\"" << q_field_name << "\"/>\n";
+      }
       pvd_stream.flush();
+      // Move the insertion point before the closing collection tag, so that
+      // the PVD file is valid even when writing incrementally.
       std::fstream::pos_type pos = pvd_stream.tellp();
       pvd_stream << "</Collection>\n";
       pvd_stream << "</VTKFile>" << std::endl;
@@ -996,6 +994,44 @@ void ParaViewDataCollection::Save()
    }
 }
 
+void ParaViewDataCollection::WritePVTUHeader(std::ostream &out)
+{
+   out << "<?xml version=\"1.0\"?>\n";
+   out << "<VTKFile type=\"PUnstructuredGrid\"";
+   out << " version =\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
+   out << "<PUnstructuredGrid GhostLevel=\"0\">\n";
+
+   out << "<PPoints>\n";
+   out << "\t<PDataArray type=\"" << GetDataTypeString() << "\" ";
+   out << " Name=\"Points\" NumberOfComponents=\"3\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "</PPoints>\n";
+
+   out << "<PCells>\n";
+   out << "\t<PDataArray type=\"Int32\" ";
+   out << " Name=\"connectivity\" NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "\t<PDataArray type=\"Int32\" ";
+   out << " Name=\"offsets\"      NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "\t<PDataArray type=\"UInt8\" ";
+   out << " Name=\"types\"        NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "</PCells>\n";
+}
+
+void ParaViewDataCollection::WritePVTUFooter(std::ostream &out,
+                                             const std::string &vtu_prefix)
+{
+   for (int ii=0; ii<num_procs; ii++)
+   {
+      std::string vtu_filename = GenerateVTUFileName(vtu_prefix, ii);
+      out << "<Piece Source=\"" << vtu_filename << "\"/>\n";
+   }
+   out << "</PUnstructuredGrid>\n";
+   out << "</VTKFile>\n";
+}
+
 void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
 {
    out << "<VTKFile type=\"UnstructuredGrid\"";
@@ -1015,16 +1051,6 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
    {
       SaveGFieldVTU(out,ref,it);
    }
-   // iterate over all quadrature functions
-   // if the Quadrature functions are dumped as cell data
-   // the cycle should be moved before the grid functions
-   // and the PrintVTU CellData section should be open in the mesh dump
-   for (QFieldMapIterator it=q_field_map.begin(); it!=q_field_map.end(); ++it)
-   {
-      // save the quadrature functions
-      // this one is not implemented yet
-      SaveQFieldVTU(out,ref,it);
-   }
    out << "</PointData>\n";
    // close the mesh
    out << "</Piece>\n"; // close the piece open in the PrintVTU method
@@ -1032,27 +1058,21 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
    out << "</VTKFile>" << std::endl;
 }
 
-void ParaViewDataCollection::SaveQFieldVTU(std::ostream &out, int ref,
-                                           const QFieldMapIterator& it )
-{
-   MFEM_WARNING("SaveQFieldVTU is not currently implemented - field name:"<<it->second);
-}
-
 void ParaViewDataCollection::SaveGFieldVTU(std::ostream &out, int ref_,
-                                           const FieldMapIterator& it)
+                                           const FieldMapIterator &it)
 {
    RefinedGeometry *RefG;
    Vector val;
    DenseMatrix vval, pmat;
    std::vector<char> buf;
    int vec_dim = it->second->VectorDim();
+   out << "<DataArray type=\"" << GetDataTypeString()
+       << "\" Name=\"" << it->first;
+   out << "\" NumberOfComponents=\"" << vec_dim << "\""
+       << " format=\"" << GetDataFormatString() << "\" >" << '\n';
    if (vec_dim == 1)
    {
       // scalar data
-      out << "<DataArray type=\"" << GetDataTypeString()
-          << "\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"1\" format=\""
-          << GetDataFormatString() << "\" >\n";
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
@@ -1060,51 +1080,23 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &out, int ref_,
          it->second->GetValues(i, RefG->RefPts, val, pmat);
          for (int j = 0; j < val.Size(); j++)
          {
-            if (pv_data_format == VTKFormat::ASCII)
-            {
-               out << ZeroSubnormal(val(j)) << '\n';
-            }
-            else if (pv_data_format == VTKFormat::BINARY)
-            {
-               bin_io::AppendBytes(buf, val(j));
-            }
-            else
-            {
-               bin_io::AppendBytes<float>(buf, float(val(j)));
-            }
+            WriteBinaryOrASCII(out, buf, val(j), "\n", pv_data_format);
          }
       }
    }
    else
    {
       // vector data
-      out << "<DataArray type=\"" << GetDataTypeString()
-          << "\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"" << vec_dim << "\""
-          << " format=\"" << GetDataFormatString() << "\" >" << '\n';
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
                    mesh->GetElementBaseGeometry(i), ref_, 1);
-
          it->second->GetVectorValues(i, RefG->RefPts, vval, pmat);
-
          for (int jj = 0; jj < vval.Width(); jj++)
          {
             for (int ii = 0; ii < vval.Height(); ii++)
             {
-               if (pv_data_format == VTKFormat::ASCII)
-               {
-                  out << ZeroSubnormal(vval(ii,jj)) << ' ';
-               }
-               else if (pv_data_format == VTKFormat::BINARY)
-               {
-                  bin_io::AppendBytes(buf, vval(ii,jj));
-               }
-               else
-               {
-                  bin_io::AppendBytes<float>(buf, float(vval(ii,jj)));
-               }
+               WriteBinaryOrASCII(out, buf, vval(ii,jj), " ", pv_data_format);
             }
             if (pv_data_format == VTKFormat::ASCII) { out << '\n'; }
          }

fem/datacollection.hpp

@@ -491,19 +491,20 @@ private:
    bool restart_mode;
 
 protected:
+   void WritePVTUHeader(std::ostream &out);
+   void WritePVTUFooter(std::ostream &out, const std::string &vtu_prefix);
    void SaveDataVTU(std::ostream &out, int ref);
    void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
-   void SaveQFieldVTU(std::ostream &out, int ref, const QFieldMapIterator& it);
    const char *GetDataFormatString() const;
    const char *GetDataTypeString() const;
 
-   std::string  GenerateCollectionPath();
-   std::string  GenerateVTUFileName();
-   std::string  GenerateVTUFileName(int rank);
-   std::string  GenerateVTUPath();
-   std::string  GeneratePVDFileName();
-   std::string  GeneratePVTUFileName();
-   std::string  GeneratePVTUPath();
+   std::string GenerateCollectionPath();
+   std::string GenerateVTUFileName(const std::string &prefix, int rank);
+   std::string GenerateVTUPath();
+   std::string GeneratePVDFileName();
+   std::string GeneratePVTUFileName(const std::string &prefix);
+   std::string GeneratePVTUPath();
+
 
 public:
    /// Constructor. The collection name is used when saving the data.

fem/doftrans.cpp

@@ -85,6 +85,11 @@ void TransformPrimal(const DofTransformation *ran_dof_trans,
    }
 }
 
+void DofTransformation::InvTransformDual(Vector &v) const
+{
+   InvTransformDual(v.GetData());
+}
+
 void TransformDual(const DofTransformation *ran_dof_trans,
                    const DofTransformation *dom_dof_trans,
                    DenseMatrix &elmat)
@@ -195,6 +200,35 @@ void VDofTransformation::TransformDual(double *v) const
    }
 }
 
+void VDofTransformation::InvTransformDual(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->InvTransformDual(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->InvTransformDual(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
 const double ND_DofTransformation::T_data[24] =
 {
    1.0,  0.0,  0.0,  1.0,
@@ -291,6 +325,26 @@ ND_TriDofTransformation::TransformDual(double *v) const
    }
 }
 
+void
+ND_TriDofTransformation::InvTransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
 ND_TetDofTransformation::ND_TetDofTransformation(int p)
    : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
 {
@@ -355,4 +409,24 @@ ND_TetDofTransformation::TransformDual(double *v) const
    }
 }
 
+void
+ND_TetDofTransformation::InvTransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
 } // namespace mfem

fem/doftrans.hpp

@@ -102,6 +102,10 @@ public:
    virtual void TransformDual(double *v) const = 0;
    virtual void TransformDual(Vector &v) const;
 
+   /** Inverse Transform dual DoFs */
+   virtual void InvTransformDual(double *v) const = 0;
+   virtual void InvTransformDual(Vector &v) const;
+
    /** Transform a matrix of dual DoFs entries as computed by a
        BilinearFormIntegrator before summing into a BilinearForm object. */
    virtual void TransformDual(DenseMatrix &V) const;
@@ -183,10 +187,12 @@ public:
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;
 
    void TransformPrimal(double *v) const;
    void InvTransformPrimal(double *v) const;
    void TransformDual(double *v) const;
+   void InvTransformDual(double *v) const;
 };
 
 /** Abstract base class for high-order Nedelec spaces on elements with
@@ -235,6 +241,9 @@ public:
    void InvTransformPrimal(double *v) const;
 
    void TransformDual(double *v) const;
+
+   void InvTransformDual(double *v) const;
+   using DofTransformation::InvTransformDual;
 };
 
 /// DoF transformation implementation for the Nedelec basis on tetrahedra
@@ -246,12 +255,15 @@ public:
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;
 
    void TransformPrimal(double *v) const;
 
    void InvTransformPrimal(double *v) const;
 
    void TransformDual(double *v) const;
+
+   void InvTransformDual(double *v) const;
 };
 
 /// DoF transformation implementation for the Nedelec basis on wedge elements
@@ -264,12 +276,16 @@ public:
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;
 
    void TransformPrimal(double *v) const;
 
    void InvTransformPrimal(double *v) const;
 
    void TransformDual(double *v) const;
+
+   void InvTransformDual(double *v) const;
+
 };
 
 } // namespace mfem

fem/eltrans.cpp

@@ -21,7 +21,8 @@ ElementTransformation::ElementTransformation()
      EvalState(0),
      geom(Geometry::INVALID),
      Attribute(-1),
-     ElementNo(-1)
+     ElementNo(-1),
+     mesh(nullptr)
 { }
 
 double ElementTransformation::EvalWeight()

fem/eltrans.hpp

@@ -75,6 +75,12 @@ public:
 
    int Attribute, ElementNo, ElementType;
 
+   /// The Mesh object containing the element.
+   /** If the element transformation belongs to a mesh, this will point to the
+       containing Mesh object. ElementNo will be the number of the element in
+       this Mesh. This will be NULL if the element does not belong to a mesh. */
+   class Mesh *mesh;
+
    ElementTransformation();
 
    /** @brief Force the reevaluation of the Jacobian in the next call. */

fem/fespace.cpp

@@ -894,9 +894,10 @@ int FiniteElementSpace::GetDegenerateFaceDofs(int index, Array<int> &dofs,
 int FiniteElementSpace::GetNumBorderDofs(Geometry::Type geom, int order) const
 {
    // return the number of vertex and edge DOFs that precede inner DOFs
-   int nv = fec->GetNumDof(Geometry::POINT, order);
-   int ne = fec->GetNumDof(Geometry::SEGMENT, order);
-   return Geometry::NumVerts[geom] * (nv + ne);
+   const int nv = fec->GetNumDof(Geometry::POINT, order);
+   const int ne = fec->GetNumDof(Geometry::SEGMENT, order);
+
+   return Geometry::NumVerts[geom] * (geom == Geometry::SEGMENT ? nv : (nv + ne));
 }
 
 int FiniteElementSpace::GetEntityDofs(int entity, int index, Array<int> &dofs,
@@ -1709,8 +1710,7 @@ void FiniteElementSpace::RefinementOperator
             fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
 
             x.GetSubVector(f_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
-
+            doftrans->InvTransformDual(subX);
             for (int p = 0; p < f_dofs.Size(); ++p)
             {
                if (processed[DecodeDof(f_dofs[p])])
@@ -1719,9 +1719,9 @@ void FiniteElementSpace::RefinementOperator
                }
             }
 
-            lP.MultTranspose(subX, subY);
-            doftrans->TransformPrimal(subY);
-            y.AddElementVector(c_vdofs, subY);
+            lP.MultTranspose(subX, subYt);
+            old_DoFTrans[geom]->TransformDual(subYt);
+            y.AddElementVector(c_vdofs, subYt);
          }
 
          if (vdoftrans)

fem/fmsconvert.cpp

@@ -112,7 +112,7 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
    // NOTE: transplanted from the FmsMeshToMesh function
    //       We should do this work once and save it.
    //--------------------------------------------------
-   FmsInt dim, n_vert, n_elem, space_dim;
+   FmsInt dim, n_elem, space_dim;
 
    // Find the first component that has coordinates - that will be the new mfem
    // mesh.
@@ -144,7 +144,6 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
          n_ents[et] += num_ents;
       }
    }
-   n_vert = n_ents[FMS_VERTEX];
    //--------------------------------------------------
 
    // Interrogate the field.

fem/gridfunc.cpp

@@ -3948,6 +3948,129 @@ std::ostream &operator<<(std::ostream &out, const QuadratureFunction &qf)
    return out;
 }
 
+void QuadratureFunction::SaveVTU(std::ostream &out, VTKFormat format,
+                                 int compression_level) const
+{
+   out << R"(<VTKFile type="UnstructuredGrid" version="0.1")";
+   if (compression_level != 0)
+   {
+      out << R"( compressor="vtkZLibDataCompressor")";
+   }
+   out << " byte_order=\"" << VTKByteOrder() << "\">\n";
+   out << "<UnstructuredGrid>\n";
+
+   const char *fmt_str = (format == VTKFormat::ASCII) ? "ascii" : "binary";
+   const char *type_str = (format != VTKFormat::BINARY32) ? "Float64" : "Float32";
+   std::vector<char> buf;
+
+   int np = qspace->GetSize();
+   int ne = qspace->GetNE();
+   int sdim = qspace->GetMesh()->SpaceDimension();
+
+   // For quadrature functions, each point is a vertex cell, so number of cells
+   // is equal to number of points
+   out << "<Piece NumberOfPoints=\"" << np
+       << "\" NumberOfCells=\"" << np << "\">\n";
+
+   // print out the points
+   out << "<Points>\n";
+   out << "<DataArray type=\"" << type_str
+       << "\" NumberOfComponents=\"3\" format=\"" << fmt_str << "\">\n";
+
+   Vector pt(sdim);
+   for (int i = 0; i < ne; i++)
+   {
+      ElementTransformation &T = *qspace->GetMesh()->GetElementTransformation(i);
+      const IntegrationRule &ir = GetElementIntRule(i);
+      for (int j = 0; j < ir.Size(); j++)
+      {
+         T.Transform(ir[j], pt);
+         WriteBinaryOrASCII(out, buf, pt[0], " ", format);
+         if (sdim > 1) { WriteBinaryOrASCII(out, buf, pt[1], " ", format); }
+         else { WriteBinaryOrASCII(out, buf, 0.0, " ", format); }
+         if (sdim > 2) { WriteBinaryOrASCII(out, buf, pt[2], "", format); }
+         else { WriteBinaryOrASCII(out, buf, 0.0, "", format); }
+         if (format == VTKFormat::ASCII) { out << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</Points>\n";
+
+   // Write cells (each cell is just a vertex)
+   out << "<Cells>\n";
+   // Connectivity
+   out << R"(<DataArray type="Int32" Name="connectivity" format=")"
+       << fmt_str << "\">\n";
+
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(out, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   // Offsets
+   out << R"(<DataArray type="Int32" Name="offsets" format=")"
+       << fmt_str << "\">\n";
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(out, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   // Types
+   out << R"(<DataArray type="UInt8" Name="types" format=")"
+       << fmt_str << "\">\n";
+   for (int i = 0; i < np; i++)
+   {
+      uint8_t vtk_cell_type = VTKGeometry::POINT;
+      WriteBinaryOrASCII(out, buf, vtk_cell_type, "\n", format);
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</Cells>\n";
+
+   out << "<PointData>\n";
+   out << "<DataArray type=\"" << type_str << "\" Name=\"u\" format=\""
+       << fmt_str << "\" NumberOfComponents=\"" << vdim << "\">\n";
+   for (int i = 0; i < ne; i++)
+   {
+      DenseMatrix vals;
+      GetElementValues(i, vals);
+      for (int j = 0; j < vals.Size(); ++j)
+      {
+         for (int vd = 0; vd < vdim; ++vd)
+         {
+            WriteBinaryOrASCII(out, buf, vals(vd, j), " ", format);
+         }
+         if (format == VTKFormat::ASCII) { out << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</PointData>\n";
+
+   out << "</Piece>\n";
+   out << "</UnstructuredGrid>\n";
+   out << "</VTKFile>" << std::endl;
+}
+
+void QuadratureFunction::SaveVTU(const std::string &filename, VTKFormat format,
+                                 int compression_level) const
+{
+   std::ofstream f(filename + ".vtu");
+   SaveVTU(f, format, compression_level);
+}
+
 
 double ZZErrorEstimator(BilinearFormIntegrator &blfi,
                         GridFunction &u,

fem/gridfunc.hpp

@@ -902,6 +902,22 @@ public:
 
    /// Write the QuadratureFunction to the stream @a out.
    void Save(std::ostream &out) const;
+
+   /// @brief Write the QuadratureFunction to @a out in VTU (ParaView) format.
+   ///
+   /// The data will be uncompressed if @a compression_level is zero, or if the
+   /// format is VTKFormat::ASCII. Otherwise, zlib compression will be used for
+   /// binary data.
+   void SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
+
+   /// @brief Save the QuadratureFunction to a VTU (ParaView) file.
+   ///
+   /// The extension ".vtu" will be appended to @a filename.
+   /// @sa SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+   ///             int compression_level=0)
+   void SaveVTU(const std::string &filename, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
 };
 
 /// Overload operator<< for std::ostream and QuadratureFunction.

fem/gslib.cpp

@@ -78,6 +78,8 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
    MFEM_VERIFY(m.GetNumGeometries(m.Dimension()) == 1,
                "Mixed meshes are not currently supported in FindPointsGSLIB.");
+   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
+               "Variable order mesh is not currently supported.");
 
    // call FreeData if FindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -590,7 +592,8 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
    const L2_FECollection *fec_l2 = dynamic_cast<const L2_FECollection *>(fec_in);
 
    if (fec_h1 && gf_order == mesh_order &&
-       fec_h1->GetBasisType() == BasisType::GaussLobatto)
+       fec_h1->GetBasisType() == BasisType::GaussLobatto &&
+       !field_in.FESpace()->IsVariableOrder())
    {
       InterpolateH1(field_in, field_out);
       return;
@@ -857,6 +860,8 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
    MFEM_VERIFY(m.GetNumGeometries(m.Dimension()) == 1,
                "Mixed meshes are not currently supported in FindPointsGSLIB.");
+   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
+               "Variable order mesh is not currently supported.");
 
    // FreeData if OversetFindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }

fem/pfespace.cpp

@@ -1010,12 +1010,9 @@ void ParFiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
 {
    FiniteElementSpace::GetEssentialVDofs(bdr_attr_is_ess, ess_dofs, component);
 
-   if (Conforming())
-   {
-      // Make sure that processors without boundary elements mark
-      // their boundary dofs (if they have any).
-      Synchronize(ess_dofs);
-   }
+   // Make sure that processors without boundary elements mark
+   // their boundary dofs (if they have any).
+   Synchronize(ess_dofs);
 }
 
 void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
@@ -1041,7 +1038,8 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
    {
       if (bool(ted[i]) != bool(true_ess_dofs2[i])) { counter++; }
    }
-   MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter);
+   MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter
+               << ", rank = " << MyRank);
 #endif
 
    MarkerToList(true_ess_dofs, ess_tdof_list);

fem/qinterp/eval.hpp

@@ -65,13 +65,13 @@ static void Values2D(const int NE,
       MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
       MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       ConstDeviceMatrix B(sB, D1D,Q1D);
       DeviceMatrix DD(sm0[tidz], MD1, MD1);
       DeviceMatrix DQ(sm1[tidz], MD1, MQ1);
       DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
       for (int c = 0; c < VDIM; c++)
       {
          kernels::internal::LoadX(e,D1D,c,x,DD);
@@ -126,14 +126,14 @@ static void Values3D(const int NE,
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
       MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       ConstDeviceMatrix B(sB, D1D,Q1D);
       DeviceCube DDD(sm0, MD1,MD1,MD1);
       DeviceCube DDQ(sm1, MD1,MD1,MQ1);
       DeviceCube DQQ(sm0, MD1,MQ1,MQ1);
       DeviceCube QQQ(sm1, MQ1,MQ1,MQ1);
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
       for (int c = 0; c < VDIM; c++)
       {
          kernels::internal::LoadX(e,D1D,c,x,DDD);

fem/staticcond.cpp

@@ -40,8 +40,6 @@ StaticCondensation::StaticCondensation(FiniteElementSpace *fespace)
 #endif
    S = S_e = NULL;
    symm = false;
-   A_data.Reset();
-   A_ipiv.Reset();
 
    Array<int> vdofs;
    const int NE = fes->GetNE();

fem/tmop.hpp

@@ -1133,7 +1133,7 @@ protected:
 
    // Evaluation of the discrete target specification on different meshes.
    // Owned.
-   AdaptivityEvaluator *adapt_eval;
+   AdaptivityEvaluator *adapt_lim_eval;
 
    void SetDiscreteTargetBase(const GridFunction &tspec_);
    void SetTspecAtIndex(int idx, const GridFunction &tspec_);
@@ -1156,7 +1156,7 @@ public:
 #endif
         amr_el(-1), lim_min_size(-0.1),
         good_tspec(false), good_tspec_grad(false), good_tspec_hess(false),
-        adapt_eval(NULL) { }
+        adapt_lim_eval(NULL) { }
 
    virtual ~DiscreteAdaptTC();
 
@@ -1232,8 +1232,8 @@ public:
 
    void SetAdaptivityEvaluator(AdaptivityEvaluator *ae)
    {
-      if (adapt_eval) { delete adapt_eval; }
-      adapt_eval = ae;
+      if (adapt_lim_eval) { delete adapt_lim_eval; }
+      adapt_lim_eval = ae;
    }
 
    const Vector &GetTspecPert1H()   { return tspec_pert1h; }
@@ -1315,15 +1315,15 @@ protected:
    int integ_order;
 
    // Weight Coefficient multiplying the quality metric term.
-   Coefficient *coeff1; // not owned, if NULL -> coeff1 is 1.
+   Coefficient *metric_coeff; // not owned, if NULL -> metric_coeff is 1.
    // Normalization factor for the metric term.
    double metric_normal;
 
    // Nodes and weight Coefficient used for "limiting" the TMOP_Integrator.
    // These are both NULL when there is no limiting.
-   // The class doesn't own nodes0 and coeff0.
-   const GridFunction *nodes0;
-   Coefficient *coeff0;
+   // The class doesn't own lim_nodes0 and lim_coeff.
+   const GridFunction *lim_nodes0;
+   Coefficient *lim_coeff;
    // Limiting reference distance. Not owned.
    const GridFunction *lim_dist;
    // Limiting function. Owned.
@@ -1332,20 +1332,21 @@ protected:
    double lim_normal;
 
    // Adaptive limiting.
-   const GridFunction *zeta_0;       // Not owned.
+   const GridFunction *adapt_lim_gf0;    // Not owned.
 #ifdef MFEM_USE_MPI
-   const ParGridFunction *pzeta_0;
+   const ParGridFunction *adapt_lim_pgf0;
 #endif
-   GridFunction *zeta;               // Owned. Updated by adapt_eval.
-   Coefficient *coeff_zeta;          // Not owned.
-   AdaptivityEvaluator *adapt_eval;  // Not owned.
+   GridFunction *adapt_lim_gf;           // Owned. Updated by adapt_lim_eval.
+   Coefficient *adapt_lim_coeff;         // Not owned.
+   AdaptivityEvaluator *adapt_lim_eval;  // Not owned.
 
    // Surface fitting.
-   GridFunction *sigma, *sigma_bar;  // Owned. Updated by sigma_eval.
-   const Array<bool> *sigma_marker;  // Not owned.
-   Coefficient *coeff_sigma;         // Not owned.
-   AdaptivityEvaluator *sigma_eval;  // Not owned.
-   double sigma_normal;
+   GridFunction *surf_fit_gf,
+                *surf_fit_gf_bar;       // Owned, Updated by surf_fit_eval.
+   const Array<bool> *surf_fit_marker;  // Not owned.
+   Coefficient *surf_fit_coeff;         // Not owned.
+   AdaptivityEvaluator *surf_fit_eval;  // Not owned.
+   double surf_fit_normal;
 
    DiscreteAdaptTC *discr_tc;
 
@@ -1416,7 +1417,7 @@ protected:
 
    void ComputeNormalizationEnergies(const GridFunction &x,
                                      double &metric_energy, double &lim_energy,
-                                     double &sigma_energy);
+                                     double &surf_fit_gf_energy);
 
    void AssembleElementVectorExact(const FiniteElement &el,
                                    ElementTransformation &T,
@@ -1471,7 +1472,7 @@ protected:
 
    void DisableLimiting()
    {
-      nodes0 = NULL; coeff0 = NULL; lim_dist = NULL;
+      lim_nodes0 = NULL; lim_coeff = NULL; lim_dist = NULL;
       delete lim_func; lim_func = NULL;
    }
 
@@ -1531,12 +1532,14 @@ public:
    TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc,
                    TMOP_QualityMetric *hm)
       : h_metric(hm), metric(m), targetC(tc), IntegRules(NULL),
-        integ_order(-1), coeff1(NULL), metric_normal(1.0),
-        nodes0(NULL), coeff0(NULL),
+        integ_order(-1), metric_coeff(NULL), metric_normal(1.0),
+        lim_nodes0(NULL), lim_coeff(NULL),
         lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
-        zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
-        sigma(NULL), sigma_bar(NULL), sigma_marker(NULL), coeff_sigma(NULL),
-        sigma_eval(NULL), sigma_normal(1.0),
+        adapt_lim_gf0(NULL), adapt_lim_gf(NULL), adapt_lim_coeff(NULL),
+        adapt_lim_eval(NULL),
+        surf_fit_gf(NULL), surf_fit_gf_bar(NULL), surf_fit_marker(NULL),
+        surf_fit_coeff(NULL),
+        surf_fit_eval(NULL), surf_fit_normal(1.0),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
    { PA.enabled = false; }
@@ -1564,7 +1567,7 @@ public:
 
        Note that the Coefficient is evaluated in the physical configuration and
        not in the target configuration which may be undefined. */
-   void SetCoefficient(Coefficient &w1) { coeff1 = &w1; }
+   void SetCoefficient(Coefficient &w1) { metric_coeff = &w1; }
 
    /** @brief Limiting of the mesh displacements (general version).
 
@@ -1631,7 +1634,7 @@ public:
    void GetSurfaceFittingErrors(double &err_avg, double &err_max);
 
    /// Update the original/reference nodes used for limiting.
-   void SetLimitingNodes(const GridFunction &n0) { nodes0 = &n0; }
+   void SetLimitingNodes(const GridFunction &n0) { lim_nodes0 = &n0; }
 
    /** @brief Computes the integral of W(Jacobian(Trt)) over a target zone.
        @param[in] el     Type of FiniteElement.

fem/tmop/tmop_pa.cpp

@@ -38,13 +38,13 @@ void TMOP_Integrator::AssembleGradPA(const Vector &xe,
    if (PA.dim == 2)
    {
       AssembleGradPA_2D(xe);
-      if (coeff0) { AssembleGradPA_C0_2D(xe); }
+      if (lim_coeff) { AssembleGradPA_C0_2D(xe); }
    }
 
    if (PA.dim == 3)
    {
       AssembleGradPA_3D(xe);
-      if (coeff0) { AssembleGradPA_C0_3D(xe); }
+      if (lim_coeff) { AssembleGradPA_C0_3D(xe); }
    }
 }
 
@@ -53,8 +53,8 @@ void TMOP_Integrator::AssemblePA_Limiting()
    const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
                          Device::GetDeviceMemoryType() : pa_mt;
    // Return immediately if limiting is not enabled
-   if (coeff0 == nullptr) { return; }
-   MFEM_VERIFY(nodes0, "internal error");
+   if (lim_coeff == nullptr) { return; }
+   MFEM_VERIFY(lim_nodes0, "internal error");
 
    MFEM_VERIFY(PA.enabled, "AssemblePA_Limiting but PA is not enabled!");
    MFEM_VERIFY(lim_func, "No TMOP_LimiterFunction specification!")
@@ -68,14 +68,14 @@ void TMOP_Integrator::AssemblePA_Limiting()
 
    const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
 
-   // H0 for coeff0, (dim x dim) Q-vector
+   // H0 for lim_coeff, (dim x dim) Q-vector
    PA.H0.UseDevice(true);
    PA.H0.SetSize(PA.dim * PA.dim * PA.nq * NE, mt);
 
-   // coeff0 -> PA.C0 (Q-vector)
+   // lim_coeff -> PA.C0 (Q-vector)
    PA.C0.UseDevice(true);
    if (ConstantCoefficient* cQ =
-          dynamic_cast<ConstantCoefficient*>(coeff0))
+          dynamic_cast<ConstantCoefficient*>(lim_coeff))
    {
       PA.C0.SetSize(1, Device::GetMemoryType());
       PA.C0.HostWrite();
@@ -90,17 +90,17 @@ void TMOP_Integrator::AssemblePA_Limiting()
          ElementTransformation& T = *fes->GetElementTransformation(e);
          for (int q = 0; q < ir.GetNPoints(); ++q)
          {
-            C0(q,e) = coeff0->Eval(T, ir.IntPoint(q));
+            C0(q,e) = lim_coeff->Eval(T, ir.IntPoint(q));
          }
       }
    }
 
-   // nodes0 -> PA.X0 (E-vector)
-   MFEM_VERIFY(nodes0->FESpace() == fes, "");
+   // lim_nodes0 -> PA.X0 (E-vector)
+   MFEM_VERIFY(lim_nodes0->FESpace() == fes, "");
    const Operator *n0_R = fes->GetElementRestriction(ordering);
    PA.X0.SetSize(n0_R->Height(), Device::GetMemoryType());
    PA.X0.UseDevice(true);
-   n0_R->Mult(*nodes0, PA.X0);
+   n0_R->Mult(*lim_nodes0, PA.X0);
 
    // Limiting distances: lim_dist -> PA.LD (E-vector)
    // TODO: remove the hack for the case lim_dist == NULL.
@@ -217,8 +217,8 @@ void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
    PA.Jtr_needs_update = true;
    PA.Jtr_debug_grad = false;
 
-   // Limiting: coeff0 -> PA.C0, nodes0 -> PA.X0, lim_dist -> PA.LD, PA.H0
-   if (coeff0) { AssemblePA_Limiting(); }
+   // Limiting: lim_coeff -> PA.C0, lim_nodes0 -> PA.X0, lim_dist -> PA.LD, PA.H0
+   if (lim_coeff) { AssemblePA_Limiting(); }
 }
 
 void TMOP_Integrator::AssembleGradDiagonalPA(Vector &de) const
@@ -236,13 +236,13 @@ void TMOP_Integrator::AssembleGradDiagonalPA(Vector &de) const
    if (PA.dim == 2)
    {
       AssembleDiagonalPA_2D(de);
-      if (coeff0) { AssembleDiagonalPA_C0_2D(de); }
+      if (lim_coeff) { AssembleDiagonalPA_C0_2D(de); }
    }
 
    if (PA.dim == 3)
    {
       AssembleDiagonalPA_3D(de);
-      if (coeff0) { AssembleDiagonalPA_C0_3D(de); }
+      if (lim_coeff) { AssembleDiagonalPA_C0_3D(de); }
    }
 }
 
@@ -258,13 +258,13 @@ void TMOP_Integrator::AddMultPA(const Vector &xe, Vector &ye) const
    if (PA.dim == 2)
    {
       AddMultPA_2D(xe,ye);
-      if (coeff0) { AddMultPA_C0_2D(xe,ye); }
+      if (lim_coeff) { AddMultPA_C0_2D(xe,ye); }
    }
 
    if (PA.dim == 3)
    {
       AddMultPA_3D(xe,ye);
-      if (coeff0) { AddMultPA_C0_3D(xe,ye); }
+      if (lim_coeff) { AddMultPA_C0_3D(xe,ye); }
    }
 }
 
@@ -283,13 +283,13 @@ void TMOP_Integrator::AddMultGradPA(const Vector &re, Vector &ce) const
    if (PA.dim == 2)
    {
       AddMultGradPA_2D(re,ce);
-      if (coeff0) { AddMultGradPA_C0_2D(re,ce); }
+      if (lim_coeff) { AddMultGradPA_C0_2D(re,ce); }
    }
 
    if (PA.dim == 3)
    {
       AddMultGradPA_3D(re,ce);
-      if (coeff0) { AddMultGradPA_C0_3D(re,ce); }
+      if (lim_coeff) { AddMultGradPA_C0_3D(re,ce); }
    }
 }
 
@@ -307,13 +307,13 @@ double TMOP_Integrator::GetLocalStateEnergyPA(const Vector &xe) const
    if (PA.dim == 2)
    {
       energy = GetLocalStateEnergyPA_2D(xe);
-      if (coeff0) { energy += GetLocalStateEnergyPA_C0_2D(xe); }
+      if (lim_coeff) { energy += GetLocalStateEnergyPA_C0_2D(xe); }
    }
 
    if (PA.dim == 3)
    {
       energy = GetLocalStateEnergyPA_3D(xe);
-      if (coeff0) { energy += GetLocalStateEnergyPA_C0_3D(xe); }
+      if (lim_coeff) { energy += GetLocalStateEnergyPA_C0_3D(xe); }
    }
 
    return energy;

fem/tmop/tmop_pa.hpp

@@ -181,9 +181,9 @@ template<int T_D1D, int T_Q1D, int T_MAX> return_t kernel(__VA_ARGS__)
 if (K##kernel.Find(id)) { return K##kernel.At(id)(__VA_ARGS__,0,0); }\
 else {\
    constexpr int T_MAX = 4;\
-   const int D1D = (id>>4)&0xF, Q1D = id&0xF;\
-   MFEM_VERIFY(D1D <= MAX_D1D && Q1D <= MAX_Q1D, "Max size error!");\
-   return kernel<0,0,T_MAX>(__VA_ARGS__,D1D,Q1D); }
+   const int d1d = (id>>4)&0xF, q1d = id&0xF;\
+   MFEM_VERIFY(d1d <= MAX_D1D && q1d <= MAX_Q1D, "Max size error!");\
+   return kernel<0,0,T_MAX>(__VA_ARGS__,d1d,q1d); }
 
 } // namespace kernels
 

fem/tmop/tmop_pa_da3.cpp

@@ -55,6 +55,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
       MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       ConstDeviceMatrix B(sB, D1D, Q1D);
       DeviceCube DDD(sm0, MD1,MD1,MD1);
       DeviceCube DDQ(sm1, MD1,MD1,MQ1);
@@ -88,7 +90,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
       }
       min = min_size[0];
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
       kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
       kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);

fem/tmop/tmop_pa_h2d.cpp

@@ -97,9 +97,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_2D,
                   const double *Jtr = &J(0,0,qx,qy,e);
 
                   // Jrt = Jtr^{-1}
-                  double j[4];
-                  ConstDeviceMatrix Jrt(j,2,2);
-                  kernels::CalcInverse<2>(Jtr, j);
+                  double jrt_data[4];
+                  ConstDeviceMatrix Jrt(jrt_data,2,2);
+                  kernels::CalcInverse<2>(Jtr, jrt_data);
 
                   const double gg = G(qy,dy) * G(qy,dy);
                   const double gb = G(qy,dy) * B(qy,dy);

fem/tmop/tmop_pa_h2m_c0.cpp

@@ -68,8 +68,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
             double Xh[2];
             kernels::internal::PullEval<MQ1,NBZ>(Q1D,qx,qy,QQ,Xh);
 
-            double B[4];
-            DeviceMatrix H(B,2,2);
+            double H_data[4];
+            DeviceMatrix H(H_data,2,2);
             for (int i = 0; i < DIM; i++)
             {
                for (int j = 0; j < DIM; j++)
@@ -78,9 +78,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
                }
             }
 
-            // p2 = B . Xh
+            // p2 = H . Xh
             double p2[2];
-            kernels::Mult(2,2,B,Xh,p2);
+            kernels::Mult(2,2,H_data,Xh,p2);
             kernels::internal::PushEval<MQ1,NBZ>(Q1D,qx,qy,p2,QQ);
          }
       }

fem/tmop/tmop_pa_h3m_c0.cpp

@@ -70,8 +70,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
                double Xh[3];
                kernels::internal::PullEval<MQ1>(Q1D,qx,qy,qz,QQQ,Xh);
 
-               double B[9];
-               DeviceMatrix H(B,3,3);
+               double H_data[9];
+               DeviceMatrix H(H_data,3,3);
                for (int i = 0; i < DIM; i++)
                {
                   for (int j = 0; j < DIM; j++)
@@ -80,9 +80,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
                   }
                }
 
-               // p2 = B . Xh
+               // p2 = H . Xh
                double p2[3];
-               kernels::Mult(3,3,B,Xh,p2);
+               kernels::Mult(3,3,H_data,Xh,p2);
                kernels::internal::PushEval<MQ1>(Q1D,qx,qy,qz,p2,QQQ);
             }
          }

fem/tmop/tmop_pa_h3s_c0.cpp

@@ -55,6 +55,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
       constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
 
       MFEM_SHARED double sBLD[MQ1*MD1];
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -66,8 +67,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
 
       kernels::internal::LoadX(e,D1D,LD,DDD);
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
-
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);
       kernels::internal::EvalZ(D1D,Q1D,BLD,DQQ,QQQ);

fem/tmop/tmop_pa_p3_c0.cpp

@@ -62,6 +62,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
 
       MFEM_SHARED double B[MQ1*MD1];
       MFEM_SHARED double sBLD[MQ1*MD1];
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -86,7 +87,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
       kernels::internal::LoadX<MD1>(e,D1D,X1,DDD1);
 
       kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
 
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);

fem/tmop/tmop_pa_w3_c0.cpp

@@ -63,6 +63,7 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
 
       MFEM_SHARED double B[MQ1*MD1];
       MFEM_SHARED double sBLD[MQ1*MD1];
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -87,7 +88,6 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
       kernels::internal::LoadX<MD1>(e,D1D,X1,DDD1);
 
       kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
 
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);

fem/tmop_amr.cpp

@@ -837,26 +837,26 @@ void TMOPHRSolver::ParUpdate()
 }
 #endif
 
-void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
+void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh_, NonlinearForm *nlf_)
 {
-   const FiniteElementSpace &fes = *mesh->GetNodalFESpace();
+   const FiniteElementSpace &fes = *mesh_->GetNodalFESpace();
 
    // Update Nonlinear form and Set Essential BC
-   nlf->Update();
+   nlf_->Update();
    const int dim = fes.GetFE(0)->GetDim();
    if (move_bnd == false)
    {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      Array<int> ess_bdr(mesh_->bdr_attributes.Max());
       ess_bdr = 1;
-      nlf->SetEssentialBC(ess_bdr);
+      nlf_->SetEssentialBC(ess_bdr);
    }
    else
    {
       const int nd  = fes.GetBE(0)->GetDof();
       int n = 0;
-      for (int i = 0; i < mesh->GetNBE(); i++)
+      for (int i = 0; i < mesh_->GetNBE(); i++)
       {
-         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         const int attr = mesh_->GetBdrElement(i)->GetAttribute();
          MFEM_VERIFY(!(dim == 2 && attr == 3),
                      "Boundary attribute 3 must be used only for 3D meshes. "
                      "Adjust the attributes (1/2/3/4 for fixed x/y/z/all "
@@ -866,9 +866,9 @@ void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
       }
       Array<int> ess_vdofs(n), vdofs;
       n = 0;
-      for (int i = 0; i < mesh->GetNBE(); i++)
+      for (int i = 0; i < mesh_->GetNBE(); i++)
       {
-         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         const int attr = mesh_->GetBdrElement(i)->GetAttribute();
          fes.GetBdrElementVDofs(i, vdofs);
          if (attr == 1) // Fix x components.
          {
@@ -891,7 +891,7 @@ void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
             { ess_vdofs[n++] = vdofs[j]; }
          }
       }
-      nlf->SetEssentialVDofs(ess_vdofs);
+      nlf_->SetEssentialVDofs(ess_vdofs);
    }
 }
 

fem/tmop_tools.cpp

@@ -444,15 +444,19 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (untangling == false && min_detT_out < 0.0)
       {
          // No untangling, and detJ got negative -- no good.
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
+         if (print_options.iterations)
+         {
+            mfem::out << "Scale = " << scale << " Neg det(J) found.\n";
+         }
          scale *= detJ_factor; continue;
       }
       if (untangling == true && min_detT_out < *min_det_ptr)
       {
          // Untangling, and detJ got even more negative -- no good.
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Neg det(J) decreased.\n"; }
+         if (print_options.iterations)
+         {
+            mfem::out << "Scale = " << scale << " Neg det(J) decreased.\n";
+         }
          scale *= detJ_factor; continue;
       }
 
@@ -477,7 +481,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (energy_out > energy_in + 0.2*fabs(energy_in) ||
           std::isnan(energy_out) != 0)
       {
-         if (print_level >= 0)
+         if (print_options.iterations)
          {
             mfem::out << "Scale = " << scale << " Increasing energy: "
                       << energy_in << " --> " << energy_out << '\n';
@@ -492,7 +496,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
       if (norm_out > 1.2*norm_in)
       {
-         if (print_level >= 0)
+         if (print_options.iterations)
          {
             mfem::out << "Scale = " << scale << " Norm increased: "
                       << norm_in << " --> " << norm_out << '\n';
@@ -508,13 +512,15 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (min_detT_out > 0.0)
       {
          *min_det_ptr = 0.0;
-         if (print_level >= 0)
+         if (print_options.summary || print_options.iterations ||
+             print_options.first_and_last)
          { mfem::out << "The mesh has been untangled at the used points!\n"; }
       }
       else { *min_det_ptr = untangle_factor * min_detT_out; }
    }
 
-   if (print_level >= 0)
+   if (print_options.summary || print_options.iterations ||
+       print_options.first_and_last)
    {
       if (untangling)
       {
@@ -568,9 +574,9 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
    if (parallel)
    {
 #ifdef MFEM_USE_MPI
-      const ParNonlinearForm *nlf =
+      const ParNonlinearForm *pnlf =
          dynamic_cast<const ParNonlinearForm *>(oper);
-      const ParFiniteElementSpace *pfesc = nlf->ParFESpace();
+      const ParFiniteElementSpace *pfesc = pnlf->ParFESpace();
       Vector x_loc(pfesc->GetVSize());
       pfesc->GetProlongationMatrix()->Mult(x, x_loc);
       for (int i = 0; i < integs.Size(); i++)

fem/transfer.cpp

@@ -903,7 +903,8 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
                                    const FiniteElementSpace& hFESpace_)
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize())
 {
-   if (lFESpace_.FEColl() == hFESpace_.FEColl())
+   bool isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
+   if (lFESpace_.FEColl() == hFESpace_.FEColl() && !isvar_order)
    {
       OperatorPtr P(Operator::ANY_TYPE);
       hFESpace_.GetTransferOperator(lFESpace_, P);
@@ -912,8 +913,15 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
    }
    else if (lFESpace_.GetMesh()->GetNE() > 0
             && hFESpace_.GetMesh()->GetNE() > 0
+            && lFESpace_.GetVDim() == 1
+            && hFESpace_.GetVDim() == 1
             && dynamic_cast<const TensorBasisElement*>(lFESpace_.GetFE(0))
-            && dynamic_cast<const TensorBasisElement*>(hFESpace_.GetFE(0)))
+            && dynamic_cast<const TensorBasisElement*>(hFESpace_.GetFE(0))
+            && !isvar_order
+            && (hFESpace_.FEColl()->GetContType() ==
+                mfem::FiniteElementCollection::CONTINUOUS ||
+                hFESpace_.FEColl()->GetContType() ==
+                mfem::FiniteElementCollection::DISCONTINUOUS))
    {
       opr = new TensorProductPRefinementTransferOperator(lFESpace_, hFESpace_);
    }
@@ -941,6 +949,7 @@ PRefinementTransferOperator::PRefinementTransferOperator(
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize()), lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
+   isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
 }
 
 PRefinementTransferOperator::~PRefinementTransferOperator() {}
@@ -961,11 +970,11 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
 
    for (int i = 0; i < mesh->GetNE(); i++)
    {
-      hFESpace.GetElementDofs(i, h_dofs);
-      lFESpace.GetElementDofs(i, l_dofs);
+      DofTransformation * doftrans_h = hFESpace.GetElementDofs(i, h_dofs);
+      DofTransformation * doftrans_l = lFESpace.GetElementDofs(i, l_dofs);
 
       const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
-      if (geom != cached_geom)
+      if (geom != cached_geom || isvar_order)
       {
          h_fe = hFESpace.GetFE(i);
          l_fe = lFESpace.GetFE(i);
@@ -982,7 +991,15 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
          h_dofs.Copy(h_vdofs);
          hFESpace.DofsToVDofs(vd, h_vdofs);
          x.GetSubVector(l_vdofs, subX);
+         if (doftrans_l)
+         {
+            doftrans_l->InvTransformPrimal(subX);
+         }
          loc_prol.Mult(subX, subY);
+         if (doftrans_h)
+         {
+            doftrans_h->TransformPrimal(subY);
+         }
          y.SetSubVector(h_vdofs, subY);
       }
    }
@@ -1010,11 +1027,11 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
 
    for (int i = 0; i < mesh->GetNE(); i++)
    {
-      hFESpace.GetElementDofs(i, h_dofs);
-      lFESpace.GetElementDofs(i, l_dofs);
+      DofTransformation * doftrans_h = hFESpace.GetElementDofs(i, h_dofs);
+      DofTransformation * doftrans_l = lFESpace.GetElementDofs(i, l_dofs);
 
       const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
-      if (geom != cached_geom)
+      if (geom != cached_geom || isvar_order)
       {
          h_fe = hFESpace.GetFE(i);
          l_fe = lFESpace.GetFE(i);
@@ -1033,6 +1050,10 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
          hFESpace.DofsToVDofs(vd, h_vdofs);
 
          x.GetSubVector(h_vdofs, subX);
+         if (doftrans_h)
+         {
+            doftrans_h->InvTransformDual(subX);
+         }
          for (int p = 0; p < h_dofs.Size(); ++p)
          {
             if (processed[lFESpace.DecodeDof(h_dofs[p])])
@@ -1042,6 +1063,10 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
          }
 
          loc_prol.Mult(subX, subY);
+         if (doftrans_l)
+         {
+            doftrans_l->TransformDual(subY);
+         }
          y.AddElementVector(l_vdofs, subY);
       }
 
@@ -1085,7 +1110,8 @@ TensorProductPRefinementTransferOperator(
    // must be sorted in lexicographical order
    for (int i = 0; i < ir.GetNPoints(); ++i)
    {
-      irLex.IntPoint(i) = ir.IntPoint(hdofmap[i]);
+      int j = hdofmap[i] >=0 ? hdofmap[i] : -1 - hdofmap[i];
+      irLex.IntPoint(i) = ir.IntPoint(j);
    }
 
    NE = lFESpace.GetNE();
@@ -1403,20 +1429,36 @@ void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
    elem_restrict_lex_l->MultTranspose(localL, y);
 }
 
-#ifdef MFEM_USE_MPI
-TrueTransferOperator::TrueTransferOperator(const
-                                           ParFiniteElementSpace& lFESpace_,
-                                           const ParFiniteElementSpace& hFESpace_)
+
+TrueTransferOperator::TrueTransferOperator(const FiniteElementSpace& lFESpace_,
+                                           const FiniteElementSpace& hFESpace_)
    : Operator(hFESpace_.GetTrueVSize(), lFESpace_.GetTrueVSize()),
      lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
    localTransferOperator = new TransferOperator(lFESpace_, hFESpace_);
 
-   tmpL.SetSize(lFESpace_.GetVSize());
-   tmpH.SetSize(hFESpace_.GetVSize());
+   P = lFESpace.GetProlongationMatrix();
+   R = hFESpace.IsVariableOrder() ? hFESpace.GetHpRestrictionMatrix() :
+       hFESpace.GetRestrictionMatrix();
 
-   hFESpace.GetRestrictionMatrix()->BuildTranspose();
+   // P and R can be both null
+   // P can be null and R not null
+   // If P is not null it is assumed that R is not null as well
+   if (P) { MFEM_VERIFY(R, "Both P and R have to be not NULL") }
+
+   if (P)
+   {
+      tmpL.SetSize(lFESpace_.GetVSize());
+      tmpH.SetSize(hFESpace_.GetVSize());
+      R->BuildTranspose();
+   }
+   // P can be null and R not null
+   else if (R)
+   {
+      tmpH.SetSize(hFESpace_.GetVSize());
+      R->BuildTranspose();
+   }
 }
 
 TrueTransferOperator::~TrueTransferOperator()
@@ -1426,17 +1468,40 @@ TrueTransferOperator::~TrueTransferOperator()
 
 void TrueTransferOperator::Mult(const Vector& x, Vector& y) const
 {
-   lFESpace.GetProlongationMatrix()->Mult(x, tmpL);
-   localTransferOperator->Mult(tmpL, tmpH);
-   hFESpace.GetRestrictionMatrix()->Mult(tmpH, y);
+   if (P)
+   {
+      P->Mult(x, tmpL);
+      localTransferOperator->Mult(tmpL, tmpH);
+      R->Mult(tmpH, y);
+   }
+   else if (R)
+   {
+      localTransferOperator->Mult(x, tmpH);
+      R->Mult(tmpH, y);
+   }
+   else
+   {
+      localTransferOperator->Mult(x, y);
+   }
 }
 
 void TrueTransferOperator::MultTranspose(const Vector& x, Vector& y) const
 {
-   hFESpace.GetRestrictionMatrix()->MultTranspose(x, tmpH);
-   localTransferOperator->MultTranspose(tmpH, tmpL);
-   lFESpace.GetProlongationMatrix()->MultTranspose(tmpL, y);
+   if (P)
+   {
+      R->MultTranspose(x, tmpH);
+      localTransferOperator->MultTranspose(tmpH, tmpL);
+      P->MultTranspose(tmpL, y);
+   }
+   else if (R)
+   {
+      R->MultTranspose(x, tmpH);
+      localTransferOperator->MultTranspose(tmpH, y);
+   }
+   else
+   {
+      localTransferOperator->MultTranspose(x, y);
+   }
 }
-#endif
 
 } // namespace mfem

fem/transfer.hpp

@@ -387,6 +387,7 @@ class PRefinementTransferOperator : public Operator
 private:
    const FiniteElementSpace& lFESpace;
    const FiniteElementSpace& hFESpace;
+   bool isvar_order;
 
 public:
    /// @brief Constructs a transfer operator from \p lFESpace to \p hFESpace
@@ -452,14 +453,15 @@ public:
    virtual void MultTranspose(const Vector& x, Vector& y) const override;
 };
 
-#ifdef MFEM_USE_MPI
 /// @brief Matrix-free transfer operator between finite element spaces working
 /// on true degrees of freedom
 class TrueTransferOperator : public Operator
 {
 private:
-   const ParFiniteElementSpace& lFESpace;
-   const ParFiniteElementSpace& hFESpace;
+   const FiniteElementSpace& lFESpace;
+   const FiniteElementSpace& hFESpace;
+   const Operator * P = nullptr;
+   const SparseMatrix * R = nullptr;
    TransferOperator* localTransferOperator;
    mutable Vector tmpL;
    mutable Vector tmpH;
@@ -467,8 +469,8 @@ private:
 public:
    /// @brief Constructs a transfer operator working on true degrees of freedom
    /// from \p lFESpace to \p hFESpace
-   TrueTransferOperator(const ParFiniteElementSpace& lFESpace_,
-                        const ParFiniteElementSpace& hFESpace_);
+   TrueTransferOperator(const FiniteElementSpace& lFESpace_,
+                        const FiniteElementSpace& hFESpace_);
 
    /// Destructor
    ~TrueTransferOperator();
@@ -484,7 +486,6 @@ public:
        the true dof vector \p y corresponding to the coarse space. */
    virtual void MultTranspose(const Vector& x, Vector& y) const override;
 };
-#endif
 
 } // namespace mfem