Merge branch 'master' into dpg-neumann

Fix CHANGELOG entries

.binder/environment.yml

@@ -0,0 +1,12 @@
+# extends https://github.com/jupyterhub/repo2docker/blob/main/repo2docker/buildpacks/conda/environment.yml
+# see https://mybinder.readthedocs.io/en/latest/using/config_files.html#environment-yml-install-a-conda-environment
+channels:
+- conda-forge
+dependencies:
+- xeus-cling=0.13.0
+- xwidgets=0.26.0
+# NOTE: it's possible these aren't needed for the lab frontend
+- widgetsnbextension=3.5.1
+- pip
+- pip:
+  - glvis==0.3.2

.binder/postBuild

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+# cling is installed here (in bin) and will look in {dir}/include and {dir}/lib
+# without extra intervention (jk it doesn't look in {dir}/lib unless something
+# has been #included from {dir}/include first...)
+install_dir=/srv/conda/envs/notebook
+mkdir -p $install_dir
+
+# build and install mfem, which is the directory we start in
+make serial SHARED=YES -j8
+make install PREFIX=$install_dir
+
+# install xeus-glvis
+git clone https://github.com/GLVis/xeus-glvis.git
+pushd xeus-glvis
+make install prefix=$install_dir
+popd
+
+# install jupyter-lab extension
+jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build
+jupyter labextension install glvis-jupyter
+
+# fixup kernelspec, we could probably do this from sh but ¯\_(ツ)_/¯
+python .binder/update_kernel_env.py

.binder/update_kernel_env.py

@@ -0,0 +1,14 @@
+# Update the LD_LIBRARY_PATH of the C++14 kernel so it can find mfem without
+# extra pragma cling statements
+
+import json
+
+kernelspec = "/srv/conda/envs/notebook/share/jupyter/kernels/xcpp14/kernel.json"
+
+with open(kernelspec, "r") as f:
+    obj = json.load(f)
+
+obj["env"] = {"LD_LIBRARY_PATH": "/srv/conda/envs/notebook/lib"}
+
+with open(kernelspec, "w") as f:
+    json.dump(obj, f)

.github/workflows/repo-check.yml

@@ -63,7 +63,7 @@ jobs:
         exit 1
 
   code-style:
-    runs-on: ubuntu-16.04 # needed for astyle 2.05.1
+    runs-on: ubuntu-18.04
 
     steps:
     - name: checkout mfem
@@ -71,7 +71,7 @@ jobs:
 
     - name: get astyle
       run: |
-        sudo apt-get install astyle=2.05.1-0ubuntu1
+        sudo apt-get install astyle=3.1-1ubuntu2
 
     - name: style check
       run: |
@@ -105,6 +105,9 @@ jobs:
 
     - name: branch-history
       run: |
-        git fetch origin master:master
+        # We override origin to make sure we point to the main repo.
+        # This is to have consistent test results on PRs from forks.
+        git remote remove origin
+        git remote add origin https://github.com/mfem/mfem.git
         git checkout -b gh-actions-branch-history
         ./config/githooks/pre-push --history

.gitignore

@@ -288,10 +288,15 @@ miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*
 
+miniapps/parelag/MultilevelHcurlHdivSolver
+miniapps/parelag/*.mesh
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
+tests/unit/cunit_tests
+tests/unit/pcunit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
@@ -299,6 +304,12 @@ tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
 tests/unit/debug_device_tests
 
+# Benchmark binaries
+tests/benchmarks/bench_ceed
+tests/benchmarks/bench_tmop
+tests/benchmarks/bench_vector
+tests/benchmarks/bench_virtuals
+
 # Test script output
 tests/scripts/*.err
 tests/scripts/*.out
@@ -315,3 +326,6 @@ build-*/*
 # PETSc automated build
 petsc-build/*
 pkg.gitcommit
+
+# Jupyter Notebook Checkpoints
+.ipynb_checkpoints

.gitlab-ci.yml

@@ -13,243 +13,52 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!
 
-# We define the following GitLab pipeline variables:
-#
-# BUILD_ROOT:
-# The path to the shared resources between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-#
-# REBASELINE:
-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-#
-# MFEM_ALLOC_NAME:
-# On LLNL's quartz, there is only one allocation shared among jobs in order to
-# save time and resources. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-#
-# TPLS_REPO & TESTS_REPO:
-# Git repositories used in the pipeline
-#
-# ARTIFACTS_DIR:
-# Directory used to place artifacts.
-
-variables:
-  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
-  REBASELINE: "NO"
-  AUTOTEST: "NO"
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
 
 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where quartz resources are allocated/released once for all.
+# - Allocate/Release is where quartz resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
-  - setup
-  - q_allocate_resources
-  - q_build_and_test
-  - q_release_resources
-  - l_build_and_test
-  - c_build_and_test
-  - setup_baseline
-  - baseline_check
-  - baseline_to_autotest
-  - baseline_publish
+  - sub-pipelines
 
-# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
-# then symlinks the repo to the parent directory of the MFEM source directory.
-# Unit tests that depend on the mfem/data repo will then detect that this
-# directory is present and be enabled.
-setup:
-  tags:
-    - shell
-    - quartz
-  stage: setup
+variables:
+  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
+
+# Trigger subpipelines:
+quartz-build-and-test:
+  stage: sub-pipelines
   variables:
-    GIT_STRATEGY: none
-  script:
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+    _AUTOTEST: $AUTOTEST
+  trigger:
+    include: .gitlab/quartz-build-and-test.yml
+    strategy: depend
 
-# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
-# It prepares a pipeline-wide working directory downloading/updating external
-# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
-# are now using unique directories so repo are never shared with another
-# pipeline. This is not memory efficient (we keep a lot of data), hence this
-# reminder.
-# Note: This job can start immediately.
-setup_baseline:
-  tags:
-    - shell
-    - quartz
-  stage: setup_baseline
+quartz-baseline:
+  stage: sub-pipelines
   variables:
-    GIT_STRATEGY: none
-  script:
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
-    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
-    - cd tpls && git pull && cd ..
-    - cd tests && git pull && cd ..
-    - cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
-  needs: []
+    _AUTOTEST: $AUTOTEST
+  trigger:
+    include: .gitlab/quartz-baseline.yml
+    strategy: depend
 
-.build_toss_3_x86_64_ib_script:
-  script:
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test
-
-.build_toss_3_x86_64_ib_corona_script:
-  script:
-    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
-
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
-# speed-up the allocation.  However this would not be scalable to multiple
-# builds.
-.build_blueos_3_ppc64le_ib_script:
-  script:
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
-
-# Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
-# differentiates between the two tests.
-.baseline_script: &baseline_script |
-  # locals
-  _glob_err=${BASELINE_TEST}.err
-  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
-  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
-  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
-  # prepare
-  cd ${BUILD_ROOT}
-  ln -snf ${CI_PROJECT_DIR} mfem
-  cd tests
-  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
-  # run
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
-  # post
-  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
-  if [[ -s ${_glob_err} ]]
-  then
-    echo "ERROR during ${BASELINE_TEST} execution";
-    echo "Here is the ${_glob_err} file content";
-    cat ${_glob_err}
-    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
-    exit 1;
-  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
-  then
-    echo "Something went WRONG in ${BASELINE_TEST}:";
-    echo "Either ${_base_patch} or ${_base_out} should exists";
-    exit 1;
-  elif [[ -f ${_base_patch} ]]
-  then
-    echo "${BASELINE_TEST}: Differences found, patch generated"
-    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
-  elif [[ -f ${_base_out} ]]
-  then
-    echo "${BASELINE_TEST}: Differences found, replacement file generated"
-    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
-  fi
-  # _base_diff won't even exist if there is no difference.
-  if [[ -f ${_base_diff} ]]
-  then
-    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
-    cat ${_base_diff}
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
-    # We create a .err file, because that's how we signal that there was a diff.
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
-  fi
-  if [[ ! -s ${_base_diff} ]]
-  then
-    echo "${BASELINE_TEST}: PASSED"
-    true
-  else
-    echo "${BASELINE_TEST}: FAILED"
-    false
-  fi
-
-# Actual templates for baseline checks
-.baselinecheck_mfem:
-  stage: baseline_check
+lassen-build-and-test:
+  stage: sub-pipelines
   variables:
-    BASELINE_TEST: baseline
-    ADDITIONAL_DIR: ${BUILD_ROOT}/tpls
-  script:
-    - *baseline_script
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
+    _AUTOTEST: $AUTOTEST
+  trigger:
+    include: .gitlab/lassen-build-and-test.yml
+    strategy: depend
 
-.samplebaselinecheck_mfem:
-  stage: baseline_check
+corona-build-and-test:
+  stage: sub-pipelines
   variables:
-    BASELINE_TEST: sample-runs-baseline
-    ADDITIONAL_DIR: ""
-  script:
-    - *baseline_script
-  timeout: 4h
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-# This job can only be manually triggered on a pipeline for master branch, or if
-# the pipeline was triggered with REBASELINE="YES"
-.rebaseline_mfem:
-  stage: baseline_publish
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-      when: manual
-  script:
-    - export PATCH_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.patch
-    - export FULL_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.out
-    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
-    - cd ${BUILD_ROOT}/tests
-    - |
-      if [[ ! -f "${DIFF_FILE}" ]]
-      then
-          echo "Nothing to be done: no relevant change in baseline"
-          exit 0
-      elif [[ -f "${PATCH_FILE}" ]]
-      then
-          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
-      elif [[ -f "${FULL_FILE}t" ]]
-      then
-          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
-      else
-          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
-          exit 1
-      fi
-    - git add baseline-${SYS_TYPE}.saved
-    - git commit -m "${SYS_TYPE} rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
-    - git push origin master
-
-# The list on jobs is defined in machine-specific files.
-include:
-  - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml
+    _AUTOTEST: $AUTOTEST
+  trigger:
+    include: .gitlab/corona-build-and-test.yml
+    strategy: depend

.gitlab/README.md

@@ -0,0 +1,94 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+
+This directory contains most of the GitLab CI configuration. MFEM runs both PR
+and nightly testing on GitLab.
+
+# Structure
+
+## Top level
+
+The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
+This file only defines one stage, in which we trigger several
+sub-pipelines.
+
+We use sub-pipelines to isolate the test for one combination of `machine`
+and `test type`.
+
+Machines typically include:
+
+* Quartz: Intel bi-socket x86
+* Lassen: Power9 + Nvidia GPU
+* Corona: AMD GPU
+
+Test types include:
+
+* Build and test: Spack driven build of dependencies, mfem build, mfem
+  test
+* Baseline: Script driven build of dependencies, thorough testing
+
+⚠️ The sub-pipeline design allows to add a new machine or a new test type without
+altering the scheduling, execution and displaying of the others.
+
+## Sub-pipelines
+
+Each file is this directory is the root configuration file for one
+sub-pipeline. The naming reflects the corresponding couple (`machine`,
+`test_type`).
+
+Those files define the *stages* and the *jobs* for the sub-pipeline. They
+also contain any configuration that cannot be shared. For the most part
+though, the configuration is shared and is placed in `.gitlab/configs`.
+
+We try to keep scripts out of the CI config and share them among similar
+jobs. They are gathered in `.gitlab/scripts`.
+
+## Scripts
+
+Scripts specific to the CI only are in `.gitlab/scripts`. It is best practice
+to keep scripts outside the CI configuration (no bash scripts embedded in a
+yaml file) because it helps with readability, maintenance and also with
+transition to another CI system.
+
+⚠️ Most of the scripts there are driven by environment variables and do not have a
+usage function. This should be improved.
+
+
+# More testing
+
+## Adding a new target to a build_and_test pipeline
+
+`build_and_test` pipelines rely on Spack to install dependencies. Spack is
+driven by Uberenv which helps freezing Spack configuration: the goal being to
+point to specific commit in Spack and isolate its configuration so that it is
+not influenced by the user environment. More documentation about this can be
+found in `tests/gitlab`.
+
+In the end, the MFEM target for which to build the dependencies is expressed
+with a spack spec of MFEM, within the limits permitted by the MFEM spack
+package.
+
+In any build-and-test sub-pipeline a job basically consists in defining the
+spack spec to use. Adding a job on quartz for example resumes to:
+
+```yaml
+<job_name>:
+  variables:
+    SPEC: "<spack_spec>"
+  extends: .build_and_test_on_quartz
+```
+
+The remaining and non trivial work is to make sure this spec is working. To
+test a spec before adding it, or reproduce a CI configuration, please refer to
+`tests/gitlab/reproduce-ci-jobs-interactively.md`.
+
+⚠️ It is assumed that the spack spec applies to `mfem@develop`. That's why in the
+CI all the specs start with the compiler or the variants to apply to mfem. The
+mechanism still works with a full spec.

.gitlab/configs/common.yml

@@ -0,0 +1,44 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# We define the following GitLab pipeline variables:
+variables:
+
+# The path to the shared resource between all jobs. For example, external
+# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
+# once for all targets, so that build happen here. The BUILD_ROOT is unique to
+# the pipeline, preventing any form of concurrency with other pipelines. This
+# also means that the BUILD_ROOT directory will never be cleaned.
+# TODO: add a clean-up mechanism
+  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
+# save time and resource. This allocation has to be uniquely named so that we
+# are sure to retrieve it.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Defines the default choice for updating the saved baseline results. By default
+# the baseline can only be updated from the master branch. This variable offers
+# the option to manually ask for rebaselining from another branch if necessary.
+  _REBASELINE: "NO"
+  _AUTOTEST: "NO"
+
+# Git repositories used in the pipeline
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
+
+# Directory used to place artifacts.
+  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1
+
+

.gitlab/configs/corona-config.yml

@@ -0,0 +1,54 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipeline configuration for the Corona machine at LLNL
+variables:
+  MACHINE_NAME: corona
+
+.on_corona:
+  tags:
+    - shell
+    - corona
+  rules:
+    # Don’t run corona jobs if...
+    # Note: This makes corona an "opt-in" machine. To activate builds on corona
+    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
+    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
+    # pipeline there.
+    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
+      when: never
+    # Don’t run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# Spack helped builds
+# Generic corona build job, extending build script
+.build_and_test_on_corona:
+  extends: [.on_corona]
+  stage: build_and_test
+  script:
+    - export THREADS=12
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
+

.gitlab/configs/lassen-config.yml

@@ -0,0 +1,43 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Lassen machine at LLNL
+variables:
+  MACHINE_NAME: lassen
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+      when: on_failure
+    - when: on_success
+
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen
+# to speed-up the allocation.  However this would not be scalable to
+# multiple builds.
+.build_and_test_on_lassen:
+  extends: [.on_lassen]
+  stage: build_and_test
+  script:
+    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
+  needs: [setup]

.gitlab/configs/quartz-config.yml

@@ -0,0 +1,50 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Quartz machine at LLNL
+variables:
+  MACHINE_NAME: quartz
+
+.on_quartz:
+  tags:
+    - shell
+    - quartz
+  rules:
+    # Don't run quartz jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# Spack helped builds
+# Generic quartz build job, extending build script
+.build_and_test_on_quartz:
+  extends: [.on_quartz]
+  stage: build_and_test
+  script:
+    - export THREADS=12
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
+

.gitlab/configs/setup-baseline.yml

@@ -0,0 +1,42 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
+# when call the runtest script in MFEM test repo.
+# Note: the value must be consistent with what setup_baseline does.
+variables:
+  TPLS_DIR: ${BUILD_ROOT}/tpls
+  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}_baseline
+
+# The setup_baseline job doesn't rely on MFEM git repo. It prepares a
+# pipeline-wide working directory downloading/updating external repos.
+# TODO:
+# updating tests and tpls is not necessary anymore since pipelines are now
+# using unique directories so repo are never shared with another pipeline. This
+# is not memory efficient (we keep a lot of data), hence this reminder.
+setup_baseline:
+  tags:
+    - shell
+    - quartz
+  stage: setup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - echo "BUILD_ROOT ${BUILD_ROOT}"
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
+    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
+    - cd tpls && git pull && cd ..
+    - cd tests && git pull origin && cd ..
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
+    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
+    - cd autotest && git pull && cd ..

.gitlab/configs/setup-build-and-test.yml

@@ -0,0 +1,34 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+variables:
+  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}_build_and_test
+
+# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
+# then symlinks the repo to the parent directory of the MFEM source directory.
+# Unit tests that depend on the mfem/data repo will then detect that this
+# directory is present and be enabled.
+setup:
+  tags:
+    - shell
+    - quartz
+  stage: setup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - echo "BUILD_ROOT ${BUILD_ROOT}"
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
+    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
+    - cd autotest && git pull && cd ..
+

.gitlab/corona-build-and-test.yml

@@ -0,0 +1,63 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - allocate_resource
+  - build_and_test
+  - release_resource_and_report
+
+# Slurm shared allocation
+allocate_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: allocate_resource
+  script:
+    - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+  needs: [setup]
+
+# Build and test jobs, simply provide a spec
+rocm_gcc_8.3.1:
+  variables:
+    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
+  extends: .build_and_test_on_corona
+  needs: [allocate_resource]
+
+# Release slurm allocation
+release_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: release_resource_and_report
+  script:
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+  needs: [rocm_gcc_8.3.1]
+
+# Jobs report
+report_job_success:
+  extends: .on_corona
+  stage: release_resource_and_report
+  script:
+    - .gitlab/scripts/report_build_and_test_success
+
+report_job_failure:
+  extends: .on_corona
+  stage: release_resource_and_report
+  script:
+    - .gitlab/scripts/report_build_and_test_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/corona-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml

.gitlab/lassen-build-and-test.yml

@@ -9,26 +9,30 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# GitLab pipelines configurations for the Lassen machine at LLNL
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    - when: on_success
-
-# Spack helped builds
-# Generic lassen build job, extending build script
-# Note: Lassen jobs can start as soon as the setup job is complete.
-.build_and_test_on_lassen:
-  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
-  stage: l_build_and_test
-  needs: [setup]
+stages:
+  - setup
+  - build_and_test
+  - report
 
 opt_mpi_cuda_xl_16_1_1_8:
   variables:
     SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
+
+# Jobs report
+report_job_success:
+  extends: .on_lassen
+  stage: report
+  script:
+    - .gitlab/scripts/report_build_and_test_success
+
+report_job_failure:
+  extends: .on_lassen
+  stage: report
+  script:
+    - .gitlab/scripts/report_build_and_test_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/lassen-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml

.gitlab/quartz-baseline.yml

@@ -0,0 +1,64 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+variables:
+  BASELINE_TEST: baseline
+
+stages:
+  - setup
+  - baseline_check
+  - baseline_report
+  - baseline_publish
+
+baselinecheck_mfem_intel_quartz:
+  extends: [.on_quartz]
+  stage: baseline_check
+  script:
+    - .gitlab/scripts/baseline
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+report_baseline:
+  extends: [.on_quartz]
+  stage: baseline_report
+  script:
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - mkdir -p ${MACHINE_NAME}
+    - rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
+    - rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
+    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+    - |
+      if [[ -f ${rundir}/*.err ]]
+      then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
+        cp ${rundir}/*.err ${rundir}/autotest-email.html
+      fi
+    - git add ${rundir}
+    - git commit -am "GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+baselinepublish_mfem_quartz:
+  extends: [.on_quartz]
+  stage: baseline_publish
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+      when: manual
+  script:
+    - .gitlab/scripts/rebaseline
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/quartz-config.yml
+  - local: .gitlab/configs/setup-baseline.yml

.gitlab/quartz-build-and-test.yml

@@ -0,0 +1,95 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - allocate_resource
+  - build_and_test
+  - release_resource_and_report
+
+# Allocate
+allocate_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: allocate_resource
+  script:
+    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+
+# GitLab jobs for the Quartz machine at LLNL
+debug_ser_gcc_4_9_3:
+  variables:
+    SPEC: "%gcc@4.9.3 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug+mpi"
+  extends: .build_and_test_on_quartz
+
+opt_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 ~mpi"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_sundials:
+  variables:
+    SPEC: "%gcc@6.1.0 +sundials"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_petsc:
+  variables:
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_pumi:
+  variables:
+    SPEC: "%gcc@6.1.0 +pumi"
+  extends: .build_and_test_on_quartz
+
+# Release
+release_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: release_resource_and_report
+  script:
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+
+# Jobs report
+report_job_success:
+  extends: .on_quartz
+  stage: release_resource_and_report
+  script:
+    - .gitlab/scripts/report_build_and_test_success
+
+report_job_failure:
+  extends: .on_quartz
+  stage: release_resource_and_report
+  script:
+    - .gitlab/scripts/report_build_and_test_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/quartz-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml

.gitlab/quartz.yml

@@ -1,184 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Quartz machine at LLNL
-
-.on_quartz:
-  tags:
-    - shell
-    - quartz
-  rules:
-    # Don't run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resources
-    - if: '$CI_JOB_NAME =~ /release_resources/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# This is a yaml anchor, it can be used to avoid duplication like here.
-# The code below will simply be pasted wherever the anchor is placed.
-.safe_create_rundir: &safe_create_rundir |
-  if ! mkdir ${rundir}; then
-    n=1
-    while ! mkdir ${rundir}_${n}
-    do
-      n=$((n+1))
-    done
-    rundir=${rundir}_${n}
-  fi
-
-# Allocate
-q_allocate_resources:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_allocate_resources
-  script:
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-
-# Release
-q_release_resources:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-
-# Release
-q_report_success:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Can only run if all the quartz jobs passed"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
-q_report_failure:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Runs if there was at least one failure on quartz"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
-    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
-  stage: q_build_and_test
-
-# Build MFEM
-debug_ser_gcc_4_9_3:
-  variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_ser_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_par_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
-  extends: .build_and_test_on_quartz
-
-opt_ser_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_sundials:
-  variables:
-    SPEC: "%gcc@6.1.0 +sundials"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_petsc:
-  variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_pumi:
-  variables:
-    SPEC: "%gcc@6.1.0 +pumi"
-  extends: .build_and_test_on_quartz
-
-# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
-# that "setup-baseline" can start immediately. Then, we have to use needs for
-# each one of the baseline jobs, otherwise they will wait for the rest of the
-# pipeline.
-
-baselinecheck_mfem_intel_quartz:
-  extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup_baseline]
-
-update_autotest:
-  extends: [.on_quartz]
-  needs: [baselinecheck_mfem_intel_quartz]
-  stage: baseline_to_autotest
-  script:
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-    - |
-      if [[ -f ${rundir}/*.err ]]
-      then
-        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
-        cp ${rundir}/*.err ${rundir}/autotest-email.html
-      fi
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
-baselinepublish_mfem_quartz:
-  extends: [.on_quartz, .rebaseline_mfem]
-  needs: [baselinecheck_mfem_intel_quartz]

.gitlab/scripts/baseline

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# locals
+glob_err=${BASELINE_TEST}.err
+base=${BASELINE_TEST}-${SYS_TYPE}
+base_diff=${base}.diff
+base_patch=${base}.patch
+base_out=${base}.out
+artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
+
+# prepare
+cd ${BUILD_ROOT}
+ln -snf ${CI_PROJECT_DIR} mfem
+cd tests
+[[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
+mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
+
+# run
+if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
+  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+elif [[ ${MACHINE_NAME} == "corona" ]]; then
+  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+elif [[ ${MACHINE_NAME} == "lassen" ]]; then
+  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+fi
+
+# post
+mkdir ${artifacts_path}
+
+if [[ -s ${glob_err} ]]
+then
+  echo "ERROR during ${BASELINE_TEST} execution";
+  echo "Here is the ${glob_err} file content";
+  cat ${glob_err}
+  cp ${glob_err} ${artifacts_path}/${glob_err}
+  exit 1;
+elif [[ ! -f ${base_patch} && ! -f ${base_out} ]]
+then
+  echo "Something went WRONG in ${BASELINE_TEST}:";
+  echo "Either ${base_patch} or ${base_out} should exists";
+  exit 1;
+elif [[ -f ${base_patch} ]]
+then
+  echo "${BASELINE_TEST}: Differences found, patch generated"
+  cp ${base_patch} ${artifacts_path}/${base_patch}
+elif [[ -f ${base_out} ]]
+then
+  echo "${BASELINE_TEST}: Differences found, replacement file generated"
+  cp ${base_out} ${artifacts_path}/${base_out}
+fi
+
+# base_diff won't even exist if there is no difference.
+if [[ -f ${base_diff} ]]
+then
+  echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
+  cat ${base_diff}
+  cp ${base_diff} ${artifacts_path}/${base_diff}
+  # We create a .err file, because that's how we signal that there was a diff.
+  cp ${base_diff} ${artifacts_path}/gitlab-${BASELINE_TEST}-${MACHINE_NAME}.err
+fi
+
+if [[ ! -s ${base_diff} ]]
+then
+  echo "${BASELINE_TEST}: PASSED"
+  true
+else
+  echo "${BASELINE_TEST}: FAILED"
+  false
+fi

.gitlab/scripts/rebaseline

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# There will be collision between corona and quartz baselines.
+# Once the corresponding files have been generated, we can switch to machine
+# specific ref.
+ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
+#ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}-${MACHINE_NAME}
+
+PATCH_FILE=${ARTIFACT_PATH}.patch
+FULL_FILE=${ARTIFACT_PATH}.out
+DIFF_FILE=${ARTIFACT_PATH}.diff
+
+# There will be collision between corona and quartz baselines.
+# Once the corresponding files have been generated, we can switch to machine
+# specific ref.
+SAVED_NAME=baseline-${SYS_TYPE}.saved
+#SAVED_NAME=baseline-${SYS_TYPE}-${MACHINE_NAME}.saved
+
+cd ${BUILD_ROOT}/tests
+
+if [[ ! -f "${DIFF_FILE}" ]]
+then
+    echo "Nothing to be done: no relevant change in baseline"
+    exit 0
+elif [[ -f "${PATCH_FILE}" ]]
+then
+    patch "${SAVED_NAME}" < "${PATCH_FILE}"
+elif [[ -f "${FULL_FILE}" ]]
+then
+    cp "${FULL_FILE}" "${SAVED_NAME}"
+else
+    echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+    exit 1
+fi
+
+git add "${SAVED_NAME}"
+git commit -m "${SYS_TYPE} (${MACHINE_NAME}) rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
+git push origin master

.gitlab/scripts/report_build_and_test_failure

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+echo "Runs if there was at least one failure on ${MACHINE_NAME}"
+
+cd ${AUTOTEST_ROOT}/autotest && git pull
+mkdir -p ${MACHINE_NAME}
+
+rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+echo "There was an error while running CI on ${MACHINE_NAME}" > ${rundir}/gitlab.err
+echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+
+msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+
+git pull
+git add ${rundir}
+git commit -am "${msg}"
+git push origin master

.gitlab/scripts/report_build_and_test_success

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+echo "Can only run if all the ${MACHINE_NAME} jobs passed"
+
+cd ${AUTOTEST_ROOT}/autotest && git pull
+mkdir -p ${MACHINE_NAME}
+
+rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+echo "The ${MACHINE_NAME} jobs were successful" > ${rundir}/gitlab.out
+echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+
+msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+git pull
+git add ${rundir}
+git commit -am "${msg}"
+git push origin master

.gitlab/scripts/safe_create_rundir

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# This script takes a seed for a directory name and appends it with a counter
+# incremented until it can create a new directory with it.
+
+# Usage:
+#
+# Expects 1 argument: a string that is use as a seed for the directory name.
+#
+# > rundir="desired_name"
+# > rundir=$(./safe_create_rundir $rundir)
+
+set -o errexit
+set -o nounset
+
+rundir=${1:-""}
+if [[ -z ${rundir} ]]; then
+  >&2 echo "The script expects a string as argument for directory creation."
+  exit 1
+fi
+
+if ! mkdir ${rundir}; then
+  n=1
+  while ! mkdir ${rundir}_${n}
+  do
+    n=$((n+1))
+  done
+  rundir=${rundir}_${n}
+fi
+
+echo $rundir

CHANGELOG

@@ -10,8 +10,27 @@
 
 Version 4.3.1 (development)
 ===========================
+- Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
+  formatting. See the "make style" target.
+
+- Split the fem/fe.?pp files into separate files in the new fem/fe/ directory
+  to simplify and clarify the organization of FiniteElement classes.
+
 - Added support for hr-adaptivity using TMOP-based error estimator.
 
+- Coefficient::SetTime now propagates the new time into internally stored
+  Coefficient objects.
+
+- Added initial support for google-benchmarks in the tests/benchmarks directory.
+  It can be enabled with MFEM_USE_BENCHMARK=YES.
+
+- Added Binder (mybinder.org) configuration files for C++ MFEM Jupyter Notebooks
+  with inline GLVis visualization as well as a new examples/jupyter/ directory
+  with a sample notebook based on Example 1. Implementation based on xeus-cling,
+  github.com/jupyter-xeus/xeus-cling + xeus-glvis, github.com/GLVis/xeus-glvis.
+
+- Added 'double' atomicAdd implementation for previous versions of CUDA.
+
 - Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
   shaped elements.
 
@@ -27,6 +46,19 @@ Version 4.3.1 (development)
   functions on wedges and pyramids which are not amenable to reordering. The
   ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
 
+- Gmsh meshes where all elements have zero physical tag (the default Gmsh
+  output format if no physical groups are defined) are now successfully loaded,
+  and elements are reassigned attribute number 1.
+
+- Added new miniapps that use the ParELAG library, its hybrid smoothers, and the
+  hierarchy of spaces created by the element-based AMG (AMGe) methodology in
+  ParELAG to build multigrid solvers for H(curl) and H(div) forms. See the
+  miniapps/parelag directory for more details.
+
+- Fixed several MinGW build issues on Windows.
+
+- Remove the 'u' flag in the ar command, to update all files in the archive,
+  avoiding file name collisions from different subdirectories.
 
 Version 4.3, released on July 29, 2021
 ======================================

CMakeLists.txt

@@ -90,6 +90,11 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 
 # Enable languages.
 enable_language(CXX)
+if (MINGW)
+   # MinGW GCC does not expose the functions jn/_jn, yn/_yn (used in Example
+   # 25/25p) unless we use '-std=gnu++11':
+   set(CMAKE_CXX_EXTENSIONS ON)
+endif()
 if (MFEM_USE_CUDA)
    if (MFEM_USE_HIP)
       message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
@@ -328,11 +333,11 @@ if (MFEM_USE_AMGX)
 endif()
 
 if (MFEM_USE_CONDUIT)
-    find_package(Conduit REQUIRED conduit relay blueprint )
+    find_package(Conduit REQUIRED conduit relay blueprint)
 endif()
 
 if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms )
+    find_package(FMS REQUIRED fms)
 endif()
 
 # Axom/Sidre
@@ -377,6 +382,11 @@ if (MFEM_USE_UMPIRE)
    find_package(UMPIRE REQUIRED)
 endif()
 
+# GOOGLE-BENCHMARK
+if (MFEM_USE_BENCHMARK)
+   find_package(Benchmark REQUIRED)
+endif()
+
 # Caliper
 if (MFEM_USE_CALIPER)
   find_package(Caliper REQUIRED)
@@ -405,6 +415,11 @@ if (MFEM_USE_MKL_CPARDISO)
   endif()
 endif()
 
+# PARELAG
+if (MFEM_USE_PARELAG)
+   find_package(PARELAG REQUIRED)
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -428,10 +443,10 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
     SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER)
+    CUSPARSE MKL_CPARDISO AMGX CALIPER BENCHMARK PARELAG MPI_CXX)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -496,13 +511,9 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-if (CMAKE_VERSION VERSION_GREATER 2.8.11)
-  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
-else()
-  target_link_libraries(mfem ${TPL_LIBRARIES})
-endif()
+target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
 if (MINGW)
-  target_link_libraries(mfem ws2_32)
+  target_link_libraries(mfem PRIVATE ws2_32)
 endif()
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
@@ -541,15 +552,21 @@ endif()
 set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")
 
 #-------------------------------------------------------------------------------
-# Examples, miniapps, and testing
+# Examples, miniapps, benchmarks and testing
 #-------------------------------------------------------------------------------
 
-# Enable testing if required
+# Enable testing and benchmarks if required
 if (MFEM_ENABLE_TESTING)
   enable_testing()
   set(MFEM_ALL_TESTS_TARGET_NAME tests)
   add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
   add_subdirectory(tests EXCLUDE_FROM_ALL)
+
+  # Create a target for all benchmarks and, optionally, enable it.
+  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
+  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
+                  ${MFEM_ENABLE_GOOGLE_BENCHMARKS})
+  add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
 endif()
 
 # Define a target that all examples and miniapps will depend on.
@@ -569,6 +586,7 @@ add_subdirectory(miniapps EXCLUDE_FROM_ALL)
 # Target to build all executables, i.e. everything.
 add_custom_target(exec)
 add_dependencies(exec
+  ${MFEM_ALL_BENCHMARKS_TARGET_NAME}
   ${MFEM_ALL_EXAMPLES_TARGET_NAME}
   ${MFEM_ALL_MINIAPPS_TARGET_NAME}
   ${MFEM_ALL_TESTS_TARGET_NAME})

CONTRIBUTING.md

@@ -105,12 +105,14 @@ The MFEM source code has the following structure:
   │   ├── caliper
   │   ├── ginkgo
   │   ├── hiop
+  │   ├── jupyter
   │   ├── petsc
   │   ├── pumi
   │   ├── sundials
   |   └── superlu
   ├── fem
   │   ├── ceed
+  │   ├── fe
   │   ├── qinterp
   │   └── tmop
   ├── general
@@ -126,6 +128,7 @@ The MFEM source code has the following structure:
   │   ├── mtop
   │   ├── navier
   │   ├── nurbs
+  │   ├── parelag
   │   ├── performance
   │   ├── shifted
   │   ├── solvers
@@ -327,7 +330,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
     documentation in source comments.
   - Consistent code styling is enforced with `make style` in the top-level
     directory. This requires [Artistic Style](http://astyle.sourceforge.net) (we
-    specifically use version 2.05.1). See also the file `config/mfem.astylerc`.
+    specifically use version 3.1). See also the file `config/mfem.astylerc`.
   - Use `mfem::out` and `mfem::err` instead of `std::cout` and `std::cerr` in
     internal library code. (You can use `std` in examples and miniapps.)
   - When manually resolving conflicts during a merge, make sure to mention the
@@ -579,10 +582,10 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   semi-automated manner.
 
 - This instance is meant to complete CI testing with tests on Livermore
-  Computing systems. Gitlab pipeline status is reported in the corresponding
+  Computing systems. GitLab pipeline status is reported in the corresponding
   GitHub pull request.
 
-- In Gitlab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
+- In GitLab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
 
 - No change to the MFEM repo can be made on this instance.
 

INSTALL

@@ -459,6 +459,10 @@ MFEM_USE_UMPIRE = YES/NO
    discovery, provision, and management of memory on machines with multiple
    memory devices like NUMA and GPUs.
 
+MFEM_USE_BENCHMARK = YES/NO
+   Enables support for Google Benchmark, a library to support the benchmarking
+   of functions, in the tests/benchmarks directory.
+
 MFEM_USE_HIOP = YES/NO
    Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
    HPC solver for nonlinear optimization problems.
@@ -523,6 +527,11 @@ MFEM_USE_FMS = YES/NO
    convetion routines between FMS's FmsDataCollection structure and MFEM's
    DataCollection class, see the header file fem/fmsconvert.hpp.
 
+MFEM_USE_PARELAG = YES/NO
+   Enables the miniapps that use the ParELAG library. MFEM does not currently
+   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
+   currently only concerns the miniapps.
+
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -692,7 +701,7 @@ The specific libraries and their options are:
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
   Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.
+  Versions: HIOP >= 0.4.6.
 
 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
   built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
@@ -730,10 +739,10 @@ The specific libraries and their options are:
   Versions: libCEED >= 0.8.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
+  Beginning with MFEM v4.3, only RAJA v0.14.0+ is supported.
   URL: https://github.com/LLNL/RAJA
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.13.0.
+  Versions: RAJA >= 0.14.0.
 
 - Caliper (optional), used when MFEM_USE_CALIPER = YES.
   URL: https://github.com/LLNL/Caliper
@@ -744,7 +753,12 @@ The specific libraries and their options are:
   Umpire requires camp when the Umpire version is >= 3.0.0.
   URL: https://github.com/LLNL/Umpire
   Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-  Versions: Umpire >= 2.0.0.
+  Versions: Umpire >= 3.0.0.
+
+- Benchmark, used when MFEM_USE_BENCHMARK = YES.
+  URL: https://github.com/google/benchmark
+  Options: BENCHMARK_DIR, BENCHMARK_LIB.
+  Versions: Benchmark >= 1.5.6.
 
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
@@ -766,6 +780,10 @@ The specific libraries and their options are:
   Options: FMS_OPT, FMS_LIB.
   Versions: FMS >= 0.2.
 
+- ParELAG, used when MFEM_USE_PARELAG = YES.
+  URL: https://github.com/LLNL/parelag
+  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -898,6 +916,8 @@ MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
 MFEM_USE_FMS
+MFEM_USE_BENCHMARK
+MFEM_USE_PARELAG
 
 The following options are CMake specific:
 
@@ -953,6 +973,8 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - AXOM - Used when MFEM_USE_SIDRE is enabled
  - CALIPER
  - FMS
+ - BENCHMARK
+ - ParELAG
 
 The following built-in CMake packages are also used:
 

config/XSDKDefaults.cmake

@@ -283,3 +283,11 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_UMPIRE)
   SET(MFEM_USE_UMPIRE ${TPL_ENABLE_UMPIRE} CACHE BOOL "Enable Umpire" FORCE)
 ENDIF()
+
+IF (DEFINED TPL_ENABLE_BENCHMARK)
+  SET(MFEM_USE_BENCHMARK ${TPL_ENABLE_BENCHMARK} CACHE BOOL "Enable Google-Benchmark" FORCE)
+ENDIF()
+
+IF (DEFINED TPL_ENABLE_PARELAG)
+  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
+ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -55,6 +55,8 @@ set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
+set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
+set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -175,4 +175,7 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO
 
+// Enable MFEM functionality based on the Google Benchmark library.
+#cmakedefine MFEM_USE_BENCHMARK
+
 #endif // MFEM_CONFIG_HEADER

config/cmake/modules/FindBenchmark.cmake

@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - BENCHMARK_FOUND
+#   - BENCHMARK_LIBRARIES
+#   - BENCHMARK_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(Benchmark BENCHMARK BENCHMARK_DIR
+        "include" "benchmark/benchmark.h"
+        "lib" "benchmark"
+        "Paths to headers required by Google Benchmark."
+        "Libraries required by Google Benchmark.")

config/cmake/modules/FindPARELAG.cmake

@@ -0,0 +1,19 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - PARELAG_FOUND
+#   - PARELAG_LIBRARIES
+#   - PARELAG_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(PARELAG PARELAG PARELAG_DIR "" "" "" ""
+  "Paths to headers required by ParELAG." "Libraries required by ParELAG.")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -764,7 +764,7 @@ function(mfem_export_mk_files)
       MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
       MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
       MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2)
+      MFEM_USE_ADIOS2 MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)

config/config.hpp.in

@@ -180,4 +180,7 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO
 
+// Enable functionality based on the Google Benchmark library.
+// #define MFEM_USE_BENCHMARK
+
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -58,6 +58,8 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
+MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
+MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -58,6 +58,8 @@ option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
+option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
+option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
 
 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -74,6 +76,7 @@ set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 option(MFEM_ENABLE_TESTING "Enable the ctest framework for testing" ON)
 option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
+option(MFEM_ENABLE_GOOGLE_BENCHMARKS "Build all of the Google benchmarks" OFF)
 
 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
@@ -224,6 +227,16 @@ set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
+set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
+    "Path to Google Benchmark")
+
+# Provide paths, since ParELAG is dependent on MFEM and MFEM needs to be
+# compiled (or at least cmake needs to succeed) before compiling ParELAG.
+set(PARELAG_DIR "${MFEM_DIR}/../parelag" CACHE PATH "Path to ParELAG")
+set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
+    STRING "Path to ParELAG headers.")
+set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
+    "The ParELAG library.")
 
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")

config/defaults.mk

@@ -61,7 +61,7 @@ HIP_XLINKER   = -Wl,
 
 ifneq ($(NOTMAC),)
    AR      = ar
-   ARFLAGS = cruv
+   ARFLAGS = crv
    RANLIB  = ranlib
    PICFLAG = $(XCOMPILER)-fPIC
    SO_EXT  = so
@@ -73,7 +73,7 @@ ifneq ($(NOTMAC),)
 else
    # Silence "has no symbols" warnings on Mac OS X
    AR      = ar
-   ARFLAGS = Scruv
+   ARFLAGS = Scrv
    RANLIB  = ranlib -no_warning_for_no_symbols
    PICFLAG = $(XCOMPILER)-fPIC
    SO_EXT  = dylib
@@ -151,6 +151,8 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_BENCHMARK     = NO
+MFEM_USE_PARELAG       = NO
 
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -429,6 +431,11 @@ CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
 CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
 
+# BENCHMARK library configuration
+BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
+BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
+BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread
+
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
 CEED_OPT = -I$(CEED_DIR)/include
@@ -459,6 +466,11 @@ MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
    -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
    -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
 
+# PARELAG library configuration
+PARELAG_DIR = @MFEM_DIR@/../parelag
+PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
+PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

config/githooks/pre-push

@@ -87,12 +87,12 @@ fi
 ## style check
 #if [[ "${option}" == "--style" || "${option}" == "" ]]; then
 if [[ "${option}" == "--style" ]]; then
-  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
+  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 3.1" ]]; then
     cd tests/scripts
     if ! ./runtest code-style; then code=1; fi
     cd -
   else
-    echo "Warning: astyle not found or version is not 2.05.1"
+    echo "Warning: astyle not found or version is not 3.1"
   fi
 fi
 

doc/CodeDocumentation.conf.in

@@ -765,6 +765,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/linalg \
                          @MFEM_SOURCE_DIR@/mesh \
                          @MFEM_SOURCE_DIR@/fem \
+                         @MFEM_SOURCE_DIR@/fem/fe \
                          @MFEM_SOURCE_DIR@/examples \
                          @MFEM_SOURCE_DIR@/examples/caliper \
                          @MFEM_SOURCE_DIR@/examples/amgx \
@@ -786,7 +787,8 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/shifted \
                          @MFEM_SOURCE_DIR@/miniapps/solvers \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/parelag
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

examples/ex1.cpp

@@ -2,62 +2,7 @@
 //
 // Compile with: make ex1
 //
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/octahedron.mesh -o 1
-//               ex1 -m ../data/periodic-annulus-sector.msh
-//               ex1 -m ../data/periodic-torus-sector.msh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//             * ex1 -pa -d raja-hip
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cpu -o 4 -a
-//             * ex1 -pa -d ceed-cuda
-//             * ex1 -pa -d ceed-hip
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
+// Sample runs:  ex1 -m ../data/inline-quad.mesh
 
 #include "mfem.hpp"
 #include <fstream>
@@ -69,7 +14,7 @@ using namespace mfem;
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
+   const char *mesh_file = "../data/inline-quad.mesh";
    int order = 1;
    bool static_cond = false;
    bool pa = false;
@@ -161,7 +106,9 @@ int main(int argc, char *argv[])
    if (mesh.bdr_attributes.Size())
    {
       Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
+      ess_bdr = 0;
+      ess_bdr[0] = 1;
+      ess_bdr[1] = 1;
       fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 

examples/ex14.cpp

@@ -135,8 +135,8 @@ int main(int argc, char *argv[])
    a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
    if (eta > 0)
    {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
    }
    a->Assemble();
    a->Finalize();

examples/ex14p.cpp

@@ -199,8 +199,8 @@ int main(int argc, char *argv[])
    a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
    if (eta > 0)
    {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
    }
    a->Assemble();
    a->Finalize();
@@ -221,7 +221,7 @@ int main(int argc, char *argv[])
    {
       HyprePCG pcg(*A);
       pcg.SetTol(1e-12);
-      pcg.SetMaxIter(200);
+      pcg.SetMaxIter(500);
       pcg.SetPrintLevel(2);
       pcg.SetPreconditioner(*amg);
       pcg.Mult(*B, *X);
@@ -232,7 +232,7 @@ int main(int argc, char *argv[])
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetAbsTol(0.0);
       gmres.SetRelTol(1e-12);
-      gmres.SetMaxIter(200);
+      gmres.SetMaxIter(500);
       gmres.SetKDim(10);
       gmres.SetPrintLevel(1);
       gmres.SetOperator(*A);

examples/ex25.cpp

@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML *, Vector &);
 public:
    PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                               Vector &),

examples/ex25p.cpp

@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML *, Vector &);
 public:
    PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                               Vector &),

examples/ex27.cpp

@@ -295,17 +295,7 @@ int main(int argc, char *argv[])
    //     element solution.
    a.RecoverFEMSolution(X, b, u);
 
-   // 13. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
-   //     normal.
-   BilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 14. Compute the various boundary integrals.
+   // 13. Compute the various boundary integrals.
    mfem::out << endl
              << "Verifying boundary conditions" << endl
              << "=============================" << endl;
@@ -361,7 +351,7 @@ int main(int argc, char *argv[])
                 << " error " << err << endl;
    }
 
-   // 15. Save the refined mesh and the solution. This output can be viewed
+   // 14. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
    {
       ofstream mesh_ofs("refined.mesh");
@@ -372,7 +362,7 @@ int main(int argc, char *argv[])
       u.Save(sol_ofs);
    }
 
-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       string title_str = h1 ? "H1" : "DG";
@@ -385,7 +375,7 @@ int main(int argc, char *argv[])
                << " keys 'mmc'" << flush;
    }
 
-   // 17. Free the used memory.
+   // 16. Free the used memory.
    delete fec;
    delete mesh;
 

examples/ex27p.cpp

@@ -314,17 +314,7 @@ int main(int argc, char *argv[])
    //     local finite element solution on each processor.
    a.RecoverFEMSolution(X, b, u);
 
-   // 14. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
-   //     normal.
-   ParBilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 15. Compute the various boundary integrals.
+   // 14. Compute the various boundary integrals.
    mfem::out << endl
              << "Verifying boundary conditions" << endl
              << "=============================" << endl;
@@ -380,7 +370,7 @@ int main(int argc, char *argv[])
                 << " error " << err << endl;
    }
 
-   // 16. Save the refined mesh and the solution in parallel. This output can be
+   // 15. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_name;
@@ -396,7 +386,7 @@ int main(int argc, char *argv[])
       u.Save(sol_ofs);
    }
 
-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       string title_str = h1 ? "H1" : "DG";
@@ -411,7 +401,7 @@ int main(int argc, char *argv[])
                << " keys 'mmc'" << flush;
    }
 
-   // 18. Free the used memory.
+   // 17. Free the used memory.
    delete fec;
 
    return 0;

examples/ex8.cpp

@@ -2,34 +2,7 @@
 //
 // Compile with: make ex8
 //
-// Sample runs:  ex8 -m ../data/square-disc.mesh
-//               ex8 -m ../data/star.mesh
-//               ex8 -m ../data/star-mixed.mesh
-//               ex8 -m ../data/escher.mesh
-//               ex8 -m ../data/fichera.mesh
-//               ex8 -m ../data/fichera-mixed.mesh
-//               ex8 -m ../data/square-disc-p2.vtk
-//               ex8 -m ../data/square-disc-p3.mesh
-//               ex8 -m ../data/star-surf.mesh -o 2
-//               ex8 -m ../data/mobius-strip.mesh
-//
-// Description:  This example code demonstrates the use of the Discontinuous
-//               Petrov-Galerkin (DPG) method in its primal 2x2 block form as a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = f with homogeneous Dirichlet boundary conditions. We
-//               use high-order continuous trial space, a high-order interfacial
-//               (trace) space, and a high-order discontinuous test space
-//               defining a local dual (H^{-1}) norm.
-//
-//               We use the primal form of DPG, see "A primal DPG method without
-//               a first-order reformulation", Demkowicz and Gopalakrishnan, CAM
-//               2013, DOI:10.1016/j.camwa.2013.06.029.
-//
-//               The example highlights the use of interfacial (trace) finite
-//               elements and spaces, trace face integrators and the definition
-//               of block operators and preconditioners.
-//
-//               We recommend viewing examples 1-5 before viewing this example.
+// Sample runs:  ex8 -m ../data/inline-quad.mesh
 
 #include "mfem.hpp"
 #include <fstream>
@@ -41,7 +14,7 @@ using namespace mfem;
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
+   const char *mesh_file = "../data/inline-quad.mesh";
    int order = 1;
    bool visualization = 1;
 
@@ -67,6 +40,10 @@ int main(int argc, char *argv[])
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
+   cout << "mesh attr max = " << mesh->bdr_attributes.Max() << endl;
+   cout << "mesh bdr elemens = " << mesh->GetNBE() << endl;
+
+
    // 3. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
    //    largest number that gives a final mesh with no more than 10,000
@@ -139,6 +116,20 @@ int main(int argc, char *argv[])
    BlockVector x(offsets), b(offsets);
    x = 0.;
 
+   // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
+   //    the mixed bilinear form for the interfacial unknowns, Bhat,
+   //    the inverse stiffness matrix on the discontinuous test space, Sinv,
+   //    and the stiffness matrix on the continuous trial space, S0.
+   Array<int> ess_bdr(mesh->bdr_attributes.Max());
+   Array<int> ess_hat_bdr(mesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+   ess_bdr[1] = 1;
+   ess_hat_bdr = 1;
+   ess_hat_bdr[0] = 0;
+   ess_hat_bdr[1] = 0;
+
+
    // 6. Set up the linear form F(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (f,phi_i) where f=1.0 and
    //    phi_i are the basis functions in the test finite element fespace.
@@ -147,17 +138,11 @@ int main(int argc, char *argv[])
    F.AddDomainIntegrator(new DomainLFIntegrator(one));
    F.Assemble();
 
-   // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
-   //    the mixed bilinear form for the interfacial unknowns, Bhat,
-   //    the inverse stiffness matrix on the discontinuous test space, Sinv,
-   //    and the stiffness matrix on the continuous trial space, S0.
-   Array<int> ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 1;
 
    MixedBilinearForm *B0 = new MixedBilinearForm(x0_space,test_space);
    B0->AddDomainIntegrator(new DiffusionIntegrator(one));
    B0->Assemble();
-   B0->EliminateTrialDofs(ess_bdr, x.GetBlock(x0_var), F);
+   // B0->EliminateTrialDofs(ess_bdr, x.GetBlock(x0_var), F); // will be taken care at the matrix level
    B0->Finalize();
 
    MixedBilinearForm *Bhat = new MixedBilinearForm(xhat_space,test_space);
@@ -165,6 +150,7 @@ int main(int argc, char *argv[])
    Bhat->Assemble();
    Bhat->Finalize();
 
+
    BilinearForm *Sinv = new BilinearForm(test_space);
    SumIntegrator *Sum = new SumIntegrator;
    Sum->AddIntegrator(new DiffusionIntegrator(one));
@@ -187,16 +173,40 @@ int main(int argc, char *argv[])
    // 8. Set up the 1x2 block Least Squares DPG operator, B = [B0  Bhat],
    //    the normal equation operator, A = B^t Sinv B, and
    //    the normal equation right-hand-size, b = B^t Sinv F.
-   BlockOperator B(offsets_test, offsets);
+   BlockMatrix B(offsets_test, offsets);
    B.SetBlock(0,0,&matB0);
    B.SetBlock(0,1,&matBhat);
-   RAPOperator A(B, matSinv, B);
+
+   SparseMatrix & B1 = *B.CreateMonolithic();
+   SparseMatrix & A = *RAP(B1,matSinv,B1);
+
    {
       Vector SinvF(s_test);
       matSinv.Mult(F,SinvF);
-      B.MultTranspose(SinvF, b);
+      B1.MultTranspose(SinvF, b);
    }
 
+   Array<int> ess_tdofs0;
+   Array<int> ess_tdofs1;
+   x0_space->GetEssentialTrueDofs(ess_bdr,ess_tdofs0);
+   xhat_space->GetEssentialTrueDofs(ess_hat_bdr,ess_tdofs1);
+
+
+   // Esential BC on the field variable
+   for (int i = 0; i<ess_tdofs0.Size(); i++)
+   {
+      int j = ess_tdofs0[i];
+      A.EliminateRowCol(j,x[j],b);
+   }
+
+   // Neuman BC on the field variable (equivalently essential BC on the flux variable)
+   for (int i = 0; i<ess_tdofs1.Size(); i++)
+   {
+      int j = ess_tdofs1[i] + x0_space->GetTrueVSize();
+      A.EliminateRowCol(j,x[j],b);
+   }
+
+
    // 9. Set up a block-diagonal preconditioner for the 2x2 normal equation
    //
    //        [ S0^{-1}     0     ]
@@ -204,28 +214,14 @@ int main(int argc, char *argv[])
    //
    //    corresponding to the primal (x0) and interfacial (xhat) unknowns.
    SparseMatrix * Shat = RAP(matBhat, matSinv, matBhat);
+   for (int i = 0; i<ess_tdofs1.Size(); i++)
+   {
+      int j = ess_tdofs1[i];
+      Shat->EliminateRowCol(j);
+   }
 
-#ifndef MFEM_USE_SUITESPARSE
-   const double prec_rtol = 1e-3;
-   const int prec_maxit = 200;
-   CGSolver *S0inv = new CGSolver;
-   S0inv->SetOperator(matS0);
-   S0inv->SetPrintLevel(-1);
-   S0inv->SetRelTol(prec_rtol);
-   S0inv->SetMaxIter(prec_maxit);
-   CGSolver *Shatinv = new CGSolver;
-   Shatinv->SetOperator(*Shat);
-   Shatinv->SetPrintLevel(-1);
-   Shatinv->SetRelTol(prec_rtol);
-   Shatinv->SetMaxIter(prec_maxit);
-   // Disable 'iterative_mode' when using CGSolver (or any IterativeSolver) as
-   // a preconditioner:
-   S0inv->iterative_mode = false;
-   Shatinv->iterative_mode = false;
-#else
    Operator *S0inv = new UMFPackSolver(matS0);
    Operator *Shatinv = new UMFPackSolver(*Shat);
-#endif
 
    BlockDiagonalPreconditioner P(offsets);
    P.SetDiagonalBlock(0, S0inv);
@@ -235,10 +231,9 @@ int main(int argc, char *argv[])
    //     Check the weighted norm of residual for the DPG least square problem.
    //     Wrap the primal variable in a GridFunction for visualization purposes.
    PCG(A, P, b, x, 1, 200, 1e-12, 0.0);
-
    {
       Vector LSres(s_test);
-      B.Mult(x, LSres);
+      B1.Mult(x, LSres);
       LSres -= F;
       double res = sqrt(matSinv.InnerProduct(LSres, LSres));
       cout << "\n|| B0*x0 + Bhat*xhat - F ||_{S^-1} = " << res << endl;

examples/jupyter/README.md

@@ -0,0 +1,31 @@
+# Jupyter Notebooks using xeus-cling
+
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/mfem/mfem/master?filepath=examples%2Fjupyter%2Fex.ipynb)
+
+[xeus-cling](https://github.com/jupyter-xeus/xeus-cling) is a C++ Jupyter Kernel based on [cling](https://github.com/root-project/cling),
+which can be used to create interactive C++ MFEM and GLVis notebooks.
+
+Click on the `binder` button above for an interactive example.
+
+## Installing Locally
+
+In order to run notebooks locally you will need `xeus-cling` along with `mfem` and `xglvis`. We recommend you use
+[miniconda](https://docs.conda.io/en/latest/miniconda.html) or, if you already have it installed,
+[conda](https://docs.conda.io/projects/conda/en/latest/).
+
+1. Follow the install steps on https://github.com/jupyter-xeus/xeus-cling to install the C++ kernels
+2. Build and install a _shared_ version of mfem
+    * for example: `make serial SHARED=YES`
+3. Install [pyglvis](https://github.com/glvis/pyglvis)
+    * for the widget frontend
+4. Get [xeus-glvis](https://github.com/glvis/xeus-glvis) and `cp` the header to `{PREFIX}/glvis/xglvis.hpp`
+    * (this could be improved)
+
+## Running Locally
+
+Once you've installed Jupyter, the C++ Kernel, mfem, and glvis start the notebook server (`jupyter-notebook`)
+and open an existing example or a new `C++ 1x` kernel.
+
+You will _always_ need to `#pragma cling load("mfem")` and you may need to point the `cling` runtime at your
+mfem and/or glvis installs, do this with the
+`#pragma cling` [statements](https://xeus-cling.readthedocs.io/en/latest/build_options.html#using-third-party-libraries).

examples/jupyter/ex.ipynb

@@ -0,0 +1,155 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "owned-extraction",
+   "metadata": {},
+   "source": [
+    "## Load the MFEM library\n",
+    "\n",
+    "Any non-default libraries must be loaded before you can `#include` files that use them. For more info see the [xeus-cling help](https://xeus-cling.readthedocs.io/en/latest/build_options.html)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "waiting-portrait",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#pragma cling load(\"mfem\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "foreign-recycling",
+   "metadata": {},
+   "source": [
+    "## MFEM Example 1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "public-white",
+   "metadata": {},
+   "source": [
+    "This is the simplest MFEM example and a good starting point for new users. The example demonstrates the use of MFEM to define and solve an $H^1$ finite element discretization of the Laplace problem\n",
+    "\n",
+    "$$\n",
+    "-\\Delta u = 1\n",
+    "$$\n",
+    "\n",
+    "with homogeneous Dirichlet boundary conditions $u=0$.\n",
+    "\n",
+    "The example illustrates the use of the basic MFEM classes for defining the mesh, finite element space, as well as linear and bilinear forms corresponding to the left-hand side and right-hand side of the discrete linear system.\n",
+    "\n",
+    "Compare with MFEM's [ex1.cpp](https://github.com/mfem/mfem/blob/master/examples/ex1.cpp) and PyMFEM's [ex1.py](https://github.com/mfem/PyMFEM/blob/master/examples/ex1.py)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "protective-darkness",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#include <fstream>\n",
+    "#include <iostream>\n",
+    "#include <sstream>\n",
+    "\n",
+    "#include <mfem.hpp>\n",
+    "#include <glvis/xglvis.hpp>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "falling-monkey",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "using namespace std;\n",
+    "using namespace mfem;\n",
+    "\n",
+    "Mesh mesh = Mesh::MakeCartesian2D(5, 5, Element::TRIANGLE);\n",
+    "mesh.UniformRefinement();\n",
+    "\n",
+    "H1_FECollection fec(2, mesh.Dimension());\n",
+    "\n",
+    "FiniteElementSpace fespace(&mesh, &fec);\n",
+    "cout << \"Number of finite element unknowns: \" << fespace.GetTrueVSize() << endl;\n",
+    "\n",
+    "Array<int> ess_tdof_list;\n",
+    "if (mesh.bdr_attributes.Size())\n",
+    "{\n",
+    "  Array<int> ess_bdr(mesh.bdr_attributes.Max());\n",
+    "  ess_bdr = 1;\n",
+    "  fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);\n",
+    "}\n",
+    "\n",
+    "LinearForm b(&fespace);\n",
+    "ConstantCoefficient one(1.0);\n",
+    "b.AddDomainIntegrator(new DomainLFIntegrator(one));\n",
+    "b.Assemble();\n",
+    "\n",
+    "GridFunction x(&fespace);\n",
+    "x = 0.0;\n",
+    "\n",
+    "BilinearForm a(&fespace);\n",
+    "a.AddDomainIntegrator(new DiffusionIntegrator(one));\n",
+    "a.Assemble();\n",
+    "\n",
+    "OperatorPtr A;\n",
+    "Vector B, X;\n",
+    "a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);\n",
+    "\n",
+    "cout << \"Size of linear system: \" << A->Height() << endl;\n",
+    "\n",
+    "GSSmoother M((SparseMatrix&)(*A));\n",
+    "PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);\n",
+    "a.RecoverFEMSolution(X, b, x);"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "hawaiian-republican",
+   "metadata": {},
+   "source": [
+    "## GLVis Visualization\n",
+    "\n",
+    "For now we save the computational mesh and finite element solution in a string and pass that to the glvis widget, see https://github.com/glvis/xeus-glvis for the widget backend and https://github.com/GLVis/pyglvis/tree/master/js for the widget frontend."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ordinary-equation",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "std::stringstream ss;\n",
+    "ss << \"solution\\n\" << mesh << x << flush;\n",
+    "\n",
+    "auto glv = glvis::glvis();\n",
+    "glv.plot(ss.str() + \"keys Rjml\"); // the `+ \"keys ....\"' is optional\n",
+    "glv"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "C++14",
+   "language": "C++14",
+   "name": "xcpp14"
+  },
+  "language_info": {
+   "codemirror_mode": "text/x-c++src",
+   "file_extension": ".cpp",
+   "mimetype": "text/x-c++src",
+   "name": "c++",
+   "version": "14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

fem/CMakeLists.txt

@@ -43,6 +43,15 @@ set(SRCS
   eltrans.cpp
   estimators.cpp
   fe.cpp
+  fe/fe_base.cpp
+  fe/fe_fixed_order.cpp
+  fe/fe_h1.cpp
+  fe/fe_l2.cpp
+  fe/fe_nd.cpp
+  fe/fe_nurbs.cpp
+  fe/fe_pos.cpp
+  fe/fe_rt.cpp
+  fe/fe_ser.cpp
   fe_coll.cpp
   fespace.cpp
   geom.cpp
@@ -124,6 +133,15 @@ set(HDRS
   eltrans.hpp
   estimators.hpp
   fe.hpp
+  fe/fe_base.hpp
+  fe/fe_fixed_order.hpp
+  fe/fe_h1.hpp
+  fe/fe_l2.hpp
+  fe/fe_nd.hpp
+  fe/fe_nurbs.hpp
+  fe/fe_pos.hpp
+  fe/fe_rt.hpp
+  fe/fe_ser.hpp
   fe_coll.hpp
   fem.hpp
   fespace.hpp

fem/bilinearform.cpp

@@ -969,6 +969,7 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
                                   const Vector &sol, Vector &rhs,
                                   DiagonalPolicy dpolicy)
 {
+   vdofs.HostRead();
    for (int i = 0; i < vdofs.Size(); i++)
    {
       int vdof = vdofs[i];

fem/bilininteg.cpp

@@ -747,6 +747,7 @@ void DiffusionIntegrator::AssembleElementMatrix
 #ifdef MFEM_THREAD_SAFE
    DenseMatrix dshape(nd, dim), dshapedxt(nd, spaceDim);
    DenseMatrix dshapedxt_m(nd, MQ ? spaceDim : 0);
+   DenseMatrix M(MQ ? spaceDim : 0);
    Vector D(VQ ? VQ->GetVDim() : 0);
 #else
    dshape.SetSize(nd, dim);

fem/bilininteg.hpp

@@ -1982,24 +1982,32 @@ private:
 
 public:
    /// Construct a diffusion integrator with coefficient Q = 1
-   DiffusionIntegrator()
-      : Q(NULL), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(const IntegrationRule *ir = nullptr)
+      : BilinearFormIntegrator(ir),
+        Q(NULL), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
 
    /// Construct a diffusion integrator with a scalar coefficient q
-   DiffusionIntegrator(Coefficient &q)
-      : Q(&q), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr)
+      : BilinearFormIntegrator(ir),
+        Q(&q), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
 
    /// Construct a diffusion integrator with a vector coefficient q
-   DiffusionIntegrator(VectorCoefficient &q)
-      : Q(NULL), VQ(&q), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(VectorCoefficient &q,
+                       const IntegrationRule *ir = nullptr)
+      : BilinearFormIntegrator(ir),
+        Q(NULL), VQ(&q), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
 
    /// Construct a diffusion integrator with a matrix coefficient q
-   DiffusionIntegrator(MatrixCoefficient &q)
-      : Q(NULL), VQ(NULL), MQ(&q), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(MatrixCoefficient &q,
+                       const IntegrationRule *ir = nullptr)
+      : BilinearFormIntegrator(ir),
+        Q(NULL), VQ(NULL), MQ(&q), SMQ(NULL), maps(NULL), geom(NULL) { }
 
    /// Construct a diffusion integrator with a symmetric matrix coefficient q
-   DiffusionIntegrator(SymmetricMatrixCoefficient &q)
-      : Q(NULL), VQ(NULL), MQ(NULL), SMQ(&q), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(SymmetricMatrixCoefficient &q,
+                       const IntegrationRule *ir = nullptr)
+      : BilinearFormIntegrator(ir),
+        Q(NULL), VQ(NULL), MQ(NULL), SMQ(&q), maps(NULL), geom(NULL) { }
 
    /** Given a particular Finite Element computes the element stiffness matrix
        elmat. */
@@ -2668,6 +2676,9 @@ public:
    VectorDiffusionIntegrator(Coefficient &q)
       : Q(&q) { }
 
+   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir)
+      : BilinearFormIntegrator(ir), Q(&q) { }
+
    /** \brief Integrator with scalar coefficient for caller-specified vector
        dimension.
 
@@ -2927,10 +2938,11 @@ public:
 
     sum_e eta (r_e([u]), r_e([v]))
 
-    where r_e is the lifting operator defined on each edge e. The parameter eta
-    can be chosen to be one to obtain a stable discretization. The constructor
-    for this integrator requires the finite element space because the lifting
-    operator depends on the element-wise inverse mass matrix.
+    where r_e is the lifting operator defined on each edge e (potentially
+    weighted by a coefficient Q). The parameter eta can be chosen to be one to
+    obtain a stable discretization. The constructor for this integrator requires
+    the finite element space because the lifting operator depends on the
+    element-wise inverse mass matrix.
 
     BR2 stands for the second method of Bassi and Rebay:
 
@@ -2953,14 +2965,28 @@ protected:
    Array<int> ipiv;
    Array<int> ipiv_offsets, Minv_offsets;
 
+   Coefficient *Q;
+
    Vector shape1, shape2;
 
    DenseMatrix R11, R12, R21, R22;
    DenseMatrix MinvR11, MinvR12, MinvR21, MinvR22;
    DenseMatrix Re, MinvRe;
 
+   /// Precomputes the inverses (LU factorizations) of the local mass matrices.
+   /** @a fes must be a DG space, so the mass matrix is block diagonal, and its
+       inverse can be computed locally. This is required for the computation of
+       the lifting operators @a r_e.
+   */
+   void PrecomputeMassInverse(class FiniteElementSpace &fes);
+
 public:
-   DGDiffusionBR2Integrator(class FiniteElementSpace *fes, double e = 1.0);
+   DGDiffusionBR2Integrator(class FiniteElementSpace &fes, double e = 1.0);
+   DGDiffusionBR2Integrator(class FiniteElementSpace &fes, Coefficient &Q_,
+                            double e = 1.0);
+   MFEM_DEPRECATED DGDiffusionBR2Integrator(class FiniteElementSpace *fes,
+                                            double e = 1.0);
+
    using BilinearFormIntegrator::AssembleFaceMatrix;
    virtual void AssembleFaceMatrix(const FiniteElement &el1,
                                    const FiniteElement &el2,

fem/bilininteg_br2.cpp

@@ -16,20 +16,39 @@
 namespace mfem
 {
 
-DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(FiniteElementSpace *fes,
-                                                   double e) : eta(e)
+DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(
+   FiniteElementSpace &fes, double e) : eta(e), Q(NULL)
 {
+   PrecomputeMassInverse(fes);
+}
+
+DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(
+   FiniteElementSpace &fes, Coefficient &Q_, double e) : eta(e), Q(&Q_)
+{
+   PrecomputeMassInverse(fes);
+}
+
+DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(
+   FiniteElementSpace *fes, double e) : eta(e), Q(NULL)
+{
+   PrecomputeMassInverse(*fes);
+}
+
+void DGDiffusionBR2Integrator::PrecomputeMassInverse(FiniteElementSpace &fes)
+{
+   MFEM_VERIFY(fes.IsDGSpace(),
+               "The BR2 integrator is only defined for DG spaces.");
    // Precompute local mass matrix inverses needed for the lifting operators
    // First compute offsets and total size needed (e.g. for mixed meshes or
    // p-refinement)
-   int nel = fes->GetNE();
+   int nel = fes.GetNE();
    Minv_offsets.SetSize(nel+1);
    ipiv_offsets.SetSize(nel+1);
    ipiv_offsets[0] = 0;
    Minv_offsets[0] = 0;
    for (int i=0; i<nel; ++i)
    {
-      int dof = fes->GetFE(i)->GetDof();
+      int dof = fes.GetFE(i)->GetDof();
       ipiv_offsets[i+1] = ipiv_offsets[i] + dof;
       Minv_offsets[i+1] = Minv_offsets[i] + dof*dof;
    }
@@ -37,7 +56,7 @@ DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(FiniteElementSpace *fes,
 #ifdef MFEM_USE_MPI
    // When running in parallel, we also need to compute the local mass matrices
    // of face neighbor elements
-   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace *>(fes);
+   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace *>(&fes);
    if (pfes != NULL)
    {
       ParMesh *pmesh = pfes->GetParMesh();
@@ -64,15 +83,15 @@ DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(FiniteElementSpace *fes,
    {
       const FiniteElement *fe = NULL;
       ElementTransformation *tr = NULL;
-      if (i < fes->GetNE())
+      if (i < fes.GetNE())
       {
-         fe = fes->GetFE(i);
-         tr = fes->GetElementTransformation(i);
+         fe = fes.GetFE(i);
+         tr = fes.GetElementTransformation(i);
       }
       else
       {
 #ifdef MFEM_USE_MPI
-         int inbr = i - fes->GetNE();
+         int inbr = i - fes.GetNE();
          fe = pfes->GetFaceNbrFE(inbr);
          tr = pfes->GetParMesh()->GetFaceNbrElementTransformation(inbr);
 #endif
@@ -151,21 +170,24 @@ void DGDiffusionBR2Integrator::AssembleFaceMatrix(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
+      Trans.SetAllIntPoints(&ip);
 
-      Trans.Loc1.Transform(ip, eip1);
+      const IntegrationPoint &eip1 = Trans.Elem1->GetIntPoint();
       el1.CalcShape(eip1, shape1);
+      double q = Q ? Q->Eval(*Trans.Elem1, eip1) : 1.0;
       if (ndof2)
       {
-         Trans.Loc2.Transform(ip, eip2);
+         const IntegrationPoint &eip2 = Trans.Elem2->GetIntPoint();
          el2.CalcShape(eip2, shape2);
+         // Set coefficient value q to the average of the values on either side
+         if (Q) { q = 0.5*(q + Q->Eval(*Trans.Elem2, eip2)); }
       }
-
-      double w = factor*sqrt(eta)*ip.weight*Trans.Face->Weight();
-      if (ndof2)
-      {
-         w /= 2;
-      }
+      // Take sqrt here because
+      //    eta (r_e([u]), r_e([v])) = (sqrt(eta) r_e([u]), sqrt(eta) r_e([v]))
+      double w = sqrt((factor + 1)*eta*q)*ip.weight*Trans.Face->Weight();
+      // r_e is defined by, (r_e([u]), tau) = <[u], {tau}>, so we pick up a
+      // factor of 0.5 on interior faces from the average term.
+      if (ndof2) { w *= 0.5; }
 
       for (int i = 0; i < ndof1; i++)
       {

fem/bilininteg_diffusion_pa.cpp

@@ -125,7 +125,7 @@ void PADiffusionSetup2D<2>(const int Q1D,
                D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
                D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
                D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
-               (J22*R12 - J12*R22)); // 2,2 or 1,2
+                                        (J22*R12 - J12*R22)); // 2,2 or 1,2
                if (!symmetric)
                {
                   D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2

fem/bilininteg_hcurl.cpp

@@ -186,7 +186,7 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
                   const double wy = (c == 1) ? Bo(qy,dy) : Bc(qy,dy);
 
                   mass[qx] += wy * wy * ((c == 0) ? op(qx,qy,0,e) :
-                  op(qx,qy,symmetric ? 2 : 3, e));
+                                         op(qx,qy,symmetric ? 2 : 3, e));
                }
             }
 
@@ -237,7 +237,7 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
          const int D1Dx = (c == 0) ? D1D - 1 : D1D;
 
          const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
-         (symmetric ? 5 : 8));
+                                         (symmetric ? 5 : 8));
 
          double mass[MAX_Q1D];
 

fem/ceed/solvers-atpmg.cpp

@@ -178,10 +178,10 @@ int CeedATPMGElemRestriction(int order,
                   {
                      left_in_edof = i*P1d + 0;
                      right_in_edof = i*P1d + (P1d - 1);
-                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]
-                                                + e*in_layout[2]] + rounding_guard;
-                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]
-                                                 + e*in_layout[2]] + rounding_guard;
+                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                    + rounding_guard;
+                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                     + rounding_guard;
                      coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
                      coarse_j = (left_in_ldof < right_in_ldof) ?
                                 coarse_1d_edof(j, P1d, coarse_P1d) : reverse_coarse_1d_edof(j, P1d, coarse_P1d);
@@ -190,10 +190,10 @@ int CeedATPMGElemRestriction(int order,
                   {
                      left_in_edof = 0*P1d + j;
                      right_in_edof = (P1d - 1)*P1d + j;
-                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]
-                                                + e*in_layout[2]] + rounding_guard;
-                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]
-                                                 + e*in_layout[2]] + rounding_guard;
+                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                    + rounding_guard;
+                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                     + rounding_guard;
                      coarse_i = (left_in_ldof < right_in_ldof) ?
                                 coarse_1d_edof(i, P1d, coarse_P1d) : reverse_coarse_1d_edof(i, P1d, coarse_P1d);
                      coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
@@ -234,8 +234,8 @@ int CeedATPMGElemRestriction(int order,
                   // Determine topology; is this edof on the outside of the element
                   // in the i, j, or k direction?
                   int in_edof = i*P1d*P1d + j*P1d + k;
-                  int in_ldof = in_elem_dof[in_edof*in_layout[0] +
-                                            e*in_layout[2]] + rounding_guard;
+                  int in_ldof = in_elem_dof[in_edof*in_layout[0]+e*in_layout[2]]
+                                + rounding_guard;
                   bool i_edge = (i == 0 || i == P1d - 1);
                   bool j_edge = (j == 0 || j == P1d - 1);
                   bool k_edge = (k == 0 || k == P1d - 1);
@@ -265,10 +265,10 @@ int CeedATPMGElemRestriction(int order,
                      {
                         left_in_edof = 0*P1d*P1d + j*P1d + k;
                         right_in_edof = (P1d - 1)*P1d*P1d + j*P1d + k;
-                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0] +
-                                                   e*in_layout[2]] + rounding_guard;
-                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                       + rounding_guard;
+                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                         coarse_i = (left_in_ldof < right_in_ldof) ?
                                    coarse_1d_edof(i, P1d, coarse_P1d) : reverse_coarse_1d_edof(i, P1d, coarse_P1d);
                         coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
@@ -278,10 +278,10 @@ int CeedATPMGElemRestriction(int order,
                      {
                         left_in_edof = i*P1d*P1d + 0*P1d + k;
                         right_in_edof = i*P1d*P1d + (P1d - 1)*P1d + k;
-                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0] +
-                                                   e*in_layout[2]] + rounding_guard;
-                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                       + rounding_guard;
+                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                         coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
                         coarse_j = (left_in_ldof < right_in_ldof) ?
                                    coarse_1d_edof(j, P1d, coarse_P1d) : reverse_coarse_1d_edof(j, P1d, coarse_P1d);
@@ -296,10 +296,10 @@ int CeedATPMGElemRestriction(int order,
                         }
                         left_in_edof = i*P1d*P1d + j*P1d + 0;
                         right_in_edof = i*P1d*P1d + j*P1d + (P1d - 1);
-                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0] +
-                                                   e*in_layout[2]] + rounding_guard;
-                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                       + rounding_guard;
+                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                         coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
                         coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
                         coarse_k = (left_in_ldof < right_in_ldof) ?
@@ -323,14 +323,14 @@ int CeedATPMGElemRestriction(int order,
                         bottom_right_edof = i*P1d*P1d + 0*P1d + (P1d - 1);
                         top_right_edof = i*P1d*P1d + (P1d - 1)*P1d + (P1d - 1);
                         top_left_edof = i*P1d*P1d + (P1d - 1)*P1d + 0;
-                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0] +
-                                                       e*in_layout[2]] + rounding_guard;
-                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0] +
-                                                        e*in_layout[2]] + rounding_guard;
-                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0] +
-                                                     e*in_layout[2]] + rounding_guard;
-                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0]+e*in_layout[2]]
+                                           + rounding_guard;
+                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0]+e*in_layout[2]]
+                                            + rounding_guard;
+                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0]+e*in_layout[2]]
+                                         + rounding_guard;
+                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                         int m = min4(bottom_left_ldof, bottom_right_ldof, top_right_ldof,
                                      top_left_ldof);
                         coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
@@ -361,14 +361,14 @@ int CeedATPMGElemRestriction(int order,
                         bottom_right_edof = 0*P1d*P1d + j*P1d + (P1d - 1);
                         top_right_edof = (P1d - 1)*P1d*P1d + j*P1d + (P1d - 1);
                         top_left_edof = (P1d - 1)*P1d*P1d + j*P1d + 0;
-                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0] +
-                                                       e*in_layout[2]] + rounding_guard;
-                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0] +
-                                                        e*in_layout[2]] + rounding_guard;
-                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0] +
-                                                     e*in_layout[2]] + rounding_guard;
-                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0]+e*in_layout[2]]
+                                           + rounding_guard;
+                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0]+e*in_layout[2]]
+                                            + rounding_guard;
+                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0]+e*in_layout[2]]
+                                         + rounding_guard;
+                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                         int m = min4(bottom_left_ldof, bottom_right_ldof, top_right_ldof,
                                      top_left_ldof);
                         coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
@@ -404,14 +404,14 @@ int CeedATPMGElemRestriction(int order,
                         bottom_right_edof = 0*P1d*P1d + (P1d - 1)*P1d + k;
                         top_right_edof = (P1d - 1)*P1d*P1d + (P1d - 1)*P1d + k;
                         top_left_edof = (P1d - 1)*P1d*P1d + 0*P1d + k;
-                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0] +
-                                                       e*in_layout[2]] + rounding_guard;
-                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0] +
-                                                        e*in_layout[2]] + rounding_guard;
-                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0] +
-                                                     e*in_layout[2]] + rounding_guard;
-                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0]+e*in_layout[2]]
+                                           + rounding_guard;
+                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0]+e*in_layout[2]]
+                                            + rounding_guard;
+                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0]+e*in_layout[2]]
+                                         + rounding_guard;
+                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                         int m = min4(bottom_left_ldof, bottom_right_ldof,
                                      top_right_ldof, top_left_ldof);
                         coarse_k = coarse_1d_edof(k, P1d, coarse_P1d);

fem/coefficient.cpp

@@ -52,6 +52,13 @@ double GridFunctionCoefficient::Eval (ElementTransformation &T,
    return GridF -> GetValue (T, ip, Component);
 }
 
+void TransformedCoefficient::SetTime(double t)
+{
+   if (Q1) { Q1->SetTime(t); }
+   if (Q2) { Q2->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
 double TransformedCoefficient::Eval(ElementTransformation &T,
                                     const IntegrationPoint &ip)
 {
@@ -66,6 +73,12 @@ double TransformedCoefficient::Eval(ElementTransformation &T,
    }
 }
 
+void DeltaCoefficient::SetTime(double t)
+{
+   if (weight) { weight->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
 void DeltaCoefficient::SetDeltaCenter(const Vector& vcenter)
 {
    MFEM_VERIFY(vcenter.Size() <= 3,
@@ -87,6 +100,12 @@ double DeltaCoefficient::EvalDelta(ElementTransformation &T,
    return weight ? weight->Eval(T, ip, GetTime())*w : w;
 }
 
+void RestrictedCoefficient::SetTime(double t)
+{
+   if (c) { c->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
 void VectorCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
                              const IntegrationRule &ir)
 {
@@ -134,6 +153,15 @@ VectorArrayCoefficient::VectorArrayCoefficient (int dim)
    }
 }
 
+void VectorArrayCoefficient::SetTime(double t)
+{
+   for (int i = 0; i < vdim; i++)
+   {
+      if (Coeff[i]) { Coeff[i]->SetTime(t); }
+   }
+   this->VectorCoefficient::SetTime(t);
+}
+
 void VectorArrayCoefficient::Set(int i, Coefficient *c, bool own)
 {
    if (ownCoeff[i]) { delete Coeff[i]; }
@@ -247,6 +275,12 @@ double DivergenceGridFunctionCoefficient::Eval(ElementTransformation &T,
    return GridFunc->GetDivergence(T);
 }
 
+void VectorDeltaCoefficient::SetTime(double t)
+{
+   d.SetTime(t);
+   this->VectorCoefficient::SetTime(t);
+}
+
 void VectorDeltaCoefficient::SetDirection(const Vector &d_)
 {
    dir = d_;
@@ -261,6 +295,12 @@ void VectorDeltaCoefficient::EvalDelta(
    V *= d.EvalDelta(T, ip);
 }
 
+void VectorRestrictedCoefficient::SetTime(double t)
+{
+   if (c) { c->SetTime(t); }
+   this->VectorCoefficient::SetTime(t);
+}
+
 void VectorRestrictedCoefficient::Eval(Vector &V, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
@@ -291,6 +331,12 @@ void VectorRestrictedCoefficient::Eval(
    }
 }
 
+void MatrixFunctionCoefficient::SetTime(double t)
+{
+   if (Q) { Q->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
@@ -371,6 +417,12 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
    }
 }
 
+void SymmetricMatrixFunctionCoefficient::SetTime(double t)
+{
+   if (Q) { Q->SetTime(t); }
+   this->SymmetricMatrixCoefficient::SetTime(t);
+}
+
 void SymmetricMatrixFunctionCoefficient::Eval(DenseSymmetricMatrix &K,
                                               ElementTransformation &T,
                                               const IntegrationPoint &ip)
@@ -413,6 +465,15 @@ MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
    }
 }
 
+void MatrixArrayCoefficient::SetTime(double t)
+{
+   for (int i=0; i < height*width; i++)
+   {
+      if (Coeff[i]) { Coeff[i]->SetTime(t); }
+   }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void MatrixArrayCoefficient::Set(int i, int j, Coefficient * c, bool own)
 {
    if (ownCoeff[i*width+j]) { delete Coeff[i*width+j]; }
@@ -431,6 +492,7 @@ MatrixArrayCoefficient::~MatrixArrayCoefficient ()
 void MatrixArrayCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                   const IntegrationPoint &ip)
 {
+   K.SetSize(height, width);
    for (int i = 0; i < height; i++)
    {
       for (int j = 0; j < width; j++)
@@ -440,6 +502,12 @@ void MatrixArrayCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
    }
 }
 
+void MatrixRestrictedCoefficient::SetTime(double t)
+{
+   if (c) { c->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void MatrixRestrictedCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
@@ -455,6 +523,33 @@ void MatrixRestrictedCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
    }
 }
 
+void SumCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+void ProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+void RatioCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+void PowerCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
 InnerProductCoefficient::InnerProductCoefficient(VectorCoefficient &A,
                                                  VectorCoefficient &B)
    : a(&A), b(&B)
@@ -464,6 +559,13 @@ InnerProductCoefficient::InnerProductCoefficient(VectorCoefficient &A,
                "Arguments have incompatible dimensions.");
 }
 
+void InnerProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
 double InnerProductCoefficient::Eval(ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
@@ -481,6 +583,13 @@ VectorRotProductCoefficient::VectorRotProductCoefficient(VectorCoefficient &A,
                "Arguments must have dimension equal to two.");
 }
 
+void VectorRotProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
 double VectorRotProductCoefficient::Eval(ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
@@ -497,6 +606,12 @@ DeterminantCoefficient::DeterminantCoefficient(MatrixCoefficient &A)
                "Argument must be a square matrix.");
 }
 
+void DeterminantCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
 double DeterminantCoefficient::Eval(ElementTransformation &T,
                                     const IntegrationPoint &ip)
 {
@@ -545,6 +660,15 @@ VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &A_,
                "Arguments must have the same dimension.");
 }
 
+void VectorSumCoefficient::SetTime(double t)
+{
+   if (ACoef) { ACoef->SetTime(t); }
+   if (BCoef) { BCoef->SetTime(t); }
+   if (alphaCoef) { alphaCoef->SetTime(t); }
+   if (betaCoef) { betaCoef->SetTime(t); }
+   this->VectorCoefficient::SetTime(t);
+}
+
 void VectorSumCoefficient::Eval(Vector &V, ElementTransformation &T,
                                 const IntegrationPoint &ip)
 {
@@ -568,6 +692,13 @@ ScalarVectorProductCoefficient::ScalarVectorProductCoefficient(
    : VectorCoefficient(B.GetVDim()), aConst(0.0), a(&A), b(&B)
 {}
 
+void ScalarVectorProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->VectorCoefficient::SetTime(t);
+}
+
 void ScalarVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
                                           const IntegrationPoint &ip)
 {
@@ -581,6 +712,12 @@ NormalizedVectorCoefficient::NormalizedVectorCoefficient(VectorCoefficient &A,
    : VectorCoefficient(A.GetVDim()), a(&A), tol(tol_)
 {}
 
+void NormalizedVectorCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   this->VectorCoefficient::SetTime(t);
+}
+
 void NormalizedVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
@@ -599,6 +736,13 @@ VectorCrossProductCoefficient::VectorCrossProductCoefficient(
                "Arguments must have dimension equal to three.");
 }
 
+void VectorCrossProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->VectorCoefficient::SetTime(t);
+}
+
 void VectorCrossProductCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
@@ -620,6 +764,13 @@ MatrixVectorProductCoefficient::MatrixVectorProductCoefficient(
                "Arguments have incompatible dimensions.");
 }
 
+void MatrixVectorProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->VectorCoefficient::SetTime(t);
+}
+
 void MatrixVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
                                           const IntegrationPoint &ip)
 {
@@ -649,6 +800,13 @@ MatrixSumCoefficient::MatrixSumCoefficient(MatrixCoefficient &A,
                "Arguments must have the same dimensions.");
 }
 
+void MatrixSumCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void MatrixSumCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
                                 const IntegrationPoint &ip)
 {
@@ -670,6 +828,13 @@ ScalarMatrixProductCoefficient::ScalarMatrixProductCoefficient(
    : MatrixCoefficient(B.GetHeight(), B.GetWidth()), aConst(0.0), a(&A), b(&B)
 {}
 
+void ScalarMatrixProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void ScalarMatrixProductCoefficient::Eval(DenseMatrix &M,
                                           ElementTransformation &T,
                                           const IntegrationPoint &ip)
@@ -683,6 +848,12 @@ TransposeMatrixCoefficient::TransposeMatrixCoefficient(MatrixCoefficient &A)
    : MatrixCoefficient(A.GetWidth(), A.GetHeight()), a(&A)
 {}
 
+void TransposeMatrixCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void TransposeMatrixCoefficient::Eval(DenseMatrix &M,
                                       ElementTransformation &T,
                                       const IntegrationPoint &ip)
@@ -699,6 +870,12 @@ InverseMatrixCoefficient::InverseMatrixCoefficient(MatrixCoefficient &A)
                "Argument must be a square matrix.");
 }
 
+void InverseMatrixCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void InverseMatrixCoefficient::Eval(DenseMatrix &M,
                                     ElementTransformation &T,
                                     const IntegrationPoint &ip)
@@ -713,6 +890,13 @@ OuterProductCoefficient::OuterProductCoefficient(VectorCoefficient &A,
      va(A.GetVDim()), vb(B.GetVDim())
 {}
 
+void OuterProductCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (b) { b->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void OuterProductCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
                                    const IntegrationPoint &ip)
 {
@@ -739,6 +923,13 @@ CrossCrossCoefficient::CrossCrossCoefficient(Coefficient &A,
      vk(K.GetVDim())
 {}
 
+void CrossCrossCoefficient::SetTime(double t)
+{
+   if (a) { a->SetTime(t); }
+   if (k) { k->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
 void CrossCrossCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
                                  const IntegrationPoint &ip)
 {

fem/coefficient.hpp

@@ -45,7 +45,7 @@ public:
    Coefficient() { time = 0.; }
 
    /// Set the time for time dependent coefficients
-   void SetTime(double t) { time = t; }
+   virtual void SetTime(double t) { time = t; }
 
    /// Get the time for time dependent coefficients
    double GetTime() { return time; }
@@ -217,6 +217,9 @@ public:
                            double (*F)(double,double))
       : Q1(q1), Q2(q2), Transform2(F) { Transform1 = 0; }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Evaluate the coefficient at @a ip.
    virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };
@@ -269,6 +272,9 @@ public:
       weight = NULL; sdim = 3; tdf = NULL;
    }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Set the center location of the delta function.
    void SetDeltaCenter(const Vector& center);
 
@@ -333,6 +339,9 @@ public:
    RestrictedCoefficient(Coefficient &c_, Array<int> &attr)
    { c = &c_; attr.Copy(active_attr); }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Evaluate the coefficient at @a ip.
    virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
    { return active_attr[T.Attribute-1] ? c->Eval(T, ip, GetTime()) : 0.0; }
@@ -350,7 +359,7 @@ public:
    VectorCoefficient(int vd) { vdim = vd; time = 0.; }
 
    /// Set the time for time dependent coefficients
-   void SetTime(double t) { time = t; }
+   virtual void SetTime(double t) { time = t; }
 
    /// Get the time for time dependent coefficients
    double GetTime() { return time; }
@@ -456,6 +465,9 @@ public:
        still need to be added with Set(). */
    explicit VectorArrayCoefficient(int dim);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Returns i'th coefficient.
    Coefficient* GetCoeff(int i) { return Coeff[i]; }
 
@@ -632,6 +644,9 @@ public:
                           double s)
       : VectorCoefficient(dir_.Size()), dir(dir_), d(x,y,z,s) { }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Replace the associated DeltaCoefficient with a new DeltaCoefficient.
    /** The new DeltaCoefficient cannot have a specified weight Coefficient, i.e.
        DeltaCoefficient::Weight() should return NULL. */
@@ -677,6 +692,9 @@ public:
       : VectorCoefficient(vc.GetVDim())
    { c = &vc; attr.Copy(active_attr); }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Evaluate the vector coefficient at @a ip.
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
@@ -708,7 +726,7 @@ public:
       height(h), width(w), time(0.), symmetric(symm) { }
 
    /// Set the time for time dependent coefficients
-   void SetTime(double t) { time = t; }
+   virtual void SetTime(double t) { time = t; }
 
    /// Get the time for time dependent coefficients
    double GetTime() { return time; }
@@ -817,6 +835,9 @@ public:
       : MatrixCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
    { }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip);
@@ -844,6 +865,9 @@ public:
        actual coefficients still need to be added with Set(). */
    explicit MatrixArrayCoefficient (int dim);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Get the coefficient located at (i,j) in the matrix.
    Coefficient* GetCoeff (int i, int j) { return Coeff[i*width+j]; }
 
@@ -881,6 +905,9 @@ public:
       : MatrixCoefficient(mc.GetHeight(), mc.GetWidth())
    { c = &mc; attr.Copy(active_attr); }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip);
@@ -911,6 +938,9 @@ public:
                   double alpha_ = 1.0, double beta_ = 1.0)
       : aConst(0.0), a(&A), b(&B), alpha(alpha_), beta(beta_) { }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first term in the linear combination as a constant
    void SetAConst(double A) { a = NULL; aConst = A; }
    /// Return the first term in the linear combination
@@ -959,7 +989,7 @@ public:
    { dim = dimension; time = 0.; }
 
    /// Set the time for time dependent coefficients
-   void SetTime(double t) { time = t; }
+   virtual void SetTime(double t) { time = t; }
 
    /// Get the time for time dependent coefficients
    double GetTime() { return time; }
@@ -1037,6 +1067,9 @@ public:
       : SymmetricMatrixCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
    { }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseSymmetricMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip);
@@ -1063,6 +1096,9 @@ public:
    ProductCoefficient(Coefficient &A, Coefficient &B)
       : aConst(0.0), a(&A), b(&B) { }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first term in the product as a constant
    void SetAConst(double A) { a = NULL; aConst = A; }
    /// Return the first term in the product
@@ -1108,6 +1144,9 @@ public:
    RatioCoefficient(Coefficient &A, double B)
       : aConst(0.0), bConst(B), a(&A), b(NULL) { }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the numerator in the ratio as a constant
    void SetAConst(double A) { a = NULL; aConst = A; }
    /// Return the numerator of the ratio
@@ -1151,6 +1190,9 @@ public:
    PowerCoefficient(Coefficient &A, double p_)
       : a(&A), p(p_) { }
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the base coefficient
    void SetACoef(Coefficient &A) { a = &A; }
    /// Return the base coefficient
@@ -1181,6 +1223,9 @@ public:
    /// Construct with the two vector coefficients.  Result is \f$ A \cdot B \f$.
    InnerProductCoefficient(VectorCoefficient &A, VectorCoefficient &B);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first vector in the inner product
    void SetACoef(VectorCoefficient &A) { a = &A; }
    /// Return the first vector coefficient in the inner product
@@ -1210,6 +1255,9 @@ public:
    /// Constructor with two vector coefficients.  Result is \f$ A_x B_y - A_y * B_x; \f$.
    VectorRotProductCoefficient(VectorCoefficient &A, VectorCoefficient &B);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first vector in the product
    void SetACoef(VectorCoefficient &A) { a = &A; }
    /// Return the first vector of the product
@@ -1237,6 +1285,9 @@ public:
    /// Construct with the matrix.
    DeterminantCoefficient(MatrixCoefficient &A);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the matrix coefficient
    void SetACoef(MatrixCoefficient &A) { a = &A; }
    /// Return the matrix coefficient
@@ -1280,6 +1331,9 @@ public:
    VectorSumCoefficient(VectorCoefficient &A_, VectorCoefficient &B_,
                         Coefficient &alpha_, Coefficient &beta_);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first vector coefficient
    void SetACoef(VectorCoefficient &A) { ACoef = &A; }
    /// Return the first vector coefficient
@@ -1341,6 +1395,9 @@ public:
    /// Constructor with two coefficients.  Result is A * B.
    ScalarVectorProductCoefficient(Coefficient &A, VectorCoefficient &B);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the scalar factor as a constant
    void SetAConst(double A) { a = NULL; aConst = A; }
    /// Return the scalar factor
@@ -1379,6 +1436,9 @@ public:
    */
    NormalizedVectorCoefficient(VectorCoefficient &A, double tol = 1e-6);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the vector coefficient
    void SetACoef(VectorCoefficient &A) { a = &A; }
    /// Return the vector coefficient
@@ -1404,6 +1464,9 @@ public:
    /// Construct with the two coefficients.  Result is A x B.
    VectorCrossProductCoefficient(VectorCoefficient &A, VectorCoefficient &B);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first term in the product
    void SetACoef(VectorCoefficient &A) { a = &A; }
    /// Return the first term in the product
@@ -1435,6 +1498,9 @@ public:
    /// Constructor with two coefficients.  Result is A*B.
    MatrixVectorProductCoefficient(MatrixCoefficient &A, VectorCoefficient &B);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the matrix coefficient
    void SetACoef(MatrixCoefficient &A) { a = &A; }
    /// Return the matrix coefficient
@@ -1487,6 +1553,9 @@ public:
    MatrixSumCoefficient(MatrixCoefficient &A, MatrixCoefficient &B,
                         double alpha_ = 1.0, double beta_ = 1.0);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first matrix coefficient
    void SetACoef(MatrixCoefficient &A) { a = &A; }
    /// Return the first matrix coefficient
@@ -1528,6 +1597,9 @@ public:
    /// Constructor with two coefficients.  Result is A*B.
    ScalarMatrixProductCoefficient(Coefficient &A, MatrixCoefficient &B);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the scalar factor as a constant
    void SetAConst(double A) { a = NULL; aConst = A; }
    /// Return the scalar factor
@@ -1558,6 +1630,9 @@ public:
    /// Construct with the matrix coefficient.  Result is \f$ A^T \f$.
    TransposeMatrixCoefficient(MatrixCoefficient &A);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the matrix coefficient
    void SetACoef(MatrixCoefficient &A) { a = &A; }
    /// Return the matrix coefficient
@@ -1578,6 +1653,9 @@ public:
    /// Construct with the matrix coefficient.  Result is \f$ A^{-1} \f$.
    InverseMatrixCoefficient(MatrixCoefficient &A);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the matrix coefficient
    void SetACoef(MatrixCoefficient &A) { a = &A; }
    /// Return the matrix coefficient
@@ -1602,6 +1680,9 @@ public:
    /// Construct with two vector coefficients.  Result is \f$ A B^T \f$.
    OuterProductCoefficient(VectorCoefficient &A, VectorCoefficient &B);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the first vector in the outer product
    void SetACoef(VectorCoefficient &A) { a = &A; }
    /// Return the first vector coefficient in the outer product
@@ -1637,6 +1718,9 @@ public:
    CrossCrossCoefficient(double A, VectorCoefficient &K);
    CrossCrossCoefficient(Coefficient &A, VectorCoefficient &K);
 
+   /// Set the time for internally stored coefficients
+   void SetTime(double t);
+
    /// Reset the scalar factor as a constant
    void SetAConst(double A) { a = NULL; aConst = A; }
    /// Return the scalar factor

fem/datacollection.cpp

@@ -482,7 +482,7 @@ void VisItDataCollection::SaveRootFile()
    std::string root_name = prefix_path + name + "_" +
                            to_padded_string(cycle, pad_digits_cycle) +
                            ".mfem_root";
-   std::ofstream root_file(root_name.c_str());
+   std::ofstream root_file(root_name);
    root_file << GetVisItRootString();
    if (!root_file)
    {
@@ -548,7 +548,7 @@ void VisItDataCollection::Load(int cycle_)
 
 void VisItDataCollection::LoadVisItRootFile(const std::string& root_name)
 {
-   std::ifstream root_file(root_name.c_str());
+   std::ifstream root_file(root_name);
    std::stringstream buffer;
    buffer << root_file.rdbuf();
    if (!buffer)
@@ -853,6 +853,7 @@ void ParaViewDataCollection::Save()
       std::string dpath=GenerateCollectionPath();
       std::string pvdname=dpath+"/"+GeneratePVDFileName();
 
+      bool write_header = true;
       std::ifstream pvd_in;
       if (restart_mode && (pvd_in.open(pvdname,std::ios::binary),pvd_in.good()))
       {
@@ -879,20 +880,34 @@ void ParaViewDataCollection::Save()
                pos_end = pvd_in.tellg();
             }
          }
+         // Since pvd_in is opened in binary mode, count will store the number
+         // of bytes from the beginning of the file until the desired insertion
+         // point (in text mode on Windows this is not the case).
          size_t count = pos_end - pos_begin;
-         std::vector<char> buf(count);
-         pvd_in.clear();
-         pvd_in.seekg(pos_begin);
-         pvd_in.read(buf.data(), count);
-         pvd_in.close();
-         pvd_stream.open(pvdname.c_str(),std::ios::out);
-         pvd_stream.write(buf.data(), count);
+         if (count != 0)
+         {
+            write_header = false;
+            std::vector<char> buf(count);
+            // Read the contents of the PVD file, from the beginning to the
+            // insertion point.
+            pvd_in.clear();
+            pvd_in.seekg(pos_begin);
+            pvd_in.read(buf.data(), count);
+            pvd_in.close();
+            // Open the PVD file in truncate mode to delete the previous
+            // contents. Open in binary mode to write the data buffer without
+            // converting \r\n to \r\r\n on Windows.
+            pvd_stream.open(pvdname,std::ios::out|std::ios::trunc|std::ios::binary);
+            pvd_stream.write(buf.data(), count);
+            // Close and reopen the file in text mode, appending to the end.
+            pvd_stream.close();
+            pvd_stream.open(pvdname,std::ios::in|std::ios::out|std::ios::ate);
+         }
       }
-      else
+      if (write_header)
       {
-         // initialize new pvd file
-         pvd_stream.open(pvdname.c_str(),std::ios::out);
-         // initialize the file
+         // Initialize new pvd file.
+         pvd_stream.open(pvdname,std::ios::out|std::ios::trunc);
          pvd_stream << "<?xml version=\"1.0\"?>\n";
          pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
          pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
@@ -904,7 +919,7 @@ void ParaViewDataCollection::Save()
    {
       std::string fname = GenerateCollectionPath()+"/"+GenerateVTUPath()+"/"
                           +GenerateVTUFileName();
-      std::fstream out(fname.c_str(), std::ios::out);
+      std::fstream out(fname, std::ios::out);
       out.precision(precision);
       SaveDataVTU(out,levels_of_detail);
       out.close();
@@ -915,7 +930,7 @@ void ParaViewDataCollection::Save()
    {
       std::string fname = GenerateCollectionPath()+"/"+GeneratePVTUPath()+"/"
                           +GeneratePVTUFileName();
-      std::fstream out(fname.c_str(), std::ios::out);
+      std::fstream out(fname, std::ios::out);
 
       out << "<?xml version=\"1.0\"?>\n";
       out << "<VTKFile type=\"PUnstructuredGrid\"";
@@ -973,6 +988,7 @@ void ParaViewDataCollection::Save()
       pvd_stream << "<DataSet timestep=\"" << GetTime();  // GetCycle();
       pvd_stream << "\" group=\"\" part=\"" << 0 << "\" file=\"";
       pvd_stream << fname << "\"/>\n";
+      pvd_stream.flush();
       std::fstream::pos_type pos = pvd_stream.tellp();
       pvd_stream << "</Collection>\n";
       pvd_stream << "</VTKFile>" << std::endl;

fem/fe/fe_h1.hpp

@@ -0,0 +1,153 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_H1
+#define MFEM_FE_H1
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+/// Arbitrary order H1 elements in 1D
+class H1_SegmentElement : public NodalTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, dshape_x, d2shape_x;
+#endif
+
+public:
+   /// Construct the H1_SegmentElement of order @a p and BasisType @a btype
+   H1_SegmentElement(const int p, const int btype = BasisType::GaussLobatto);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order H1 elements in 2D on a square
+class H1_QuadrilateralElement : public NodalTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
+#endif
+
+public:
+   /// Construct the H1_QuadrilateralElement of order @a p and BasisType @a btype
+   H1_QuadrilateralElement(const int p,
+                           const int btype = BasisType::GaussLobatto);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order H1 elements in 3D on a cube
+class H1_HexahedronElement : public NodalTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z,
+           d2shape_x, d2shape_y, d2shape_z;
+#endif
+
+public:
+   /// Construct the H1_HexahedronElement of order @a p and BasisType @a btype
+   H1_HexahedronElement(const int p, const int btype = BasisType::GaussLobatto);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order H1 elements in 2D on a triangle
+class H1_TriangleElement : public NodalFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_l, dshape_x, dshape_y, dshape_l, u;
+   mutable Vector ddshape_x, ddshape_y, ddshape_l;
+   mutable DenseMatrix du, ddu;
+#endif
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the H1_TriangleElement of order @a p and BasisType @a btype
+   H1_TriangleElement(const int p, const int btype = BasisType::GaussLobatto);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &ddshape) const;
+};
+
+
+/// Arbitrary order H1 elements in 3D  on a tetrahedron
+class H1_TetrahedronElement : public NodalFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_z, shape_l;
+   mutable Vector dshape_x, dshape_y, dshape_z, dshape_l, u;
+   mutable Vector ddshape_x, ddshape_y, ddshape_z, ddshape_l;
+   mutable DenseMatrix du, ddu;
+#endif
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the H1_TetrahedronElement of order @a p and BasisType @a btype
+   H1_TetrahedronElement(const int p,
+                         const int btype = BasisType::GaussLobatto);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &ddshape) const;
+};
+
+
+
+/// Arbitrary order H1 elements in 3D on a wedge
+class H1_WedgeElement : public NodalFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector t_shape, s_shape;
+   mutable DenseMatrix t_dshape, s_dshape;
+#endif
+   Array<int> t_dof, s_dof;
+
+   H1_TriangleElement TriangleFE;
+   H1_SegmentElement  SegmentFE;
+
+public:
+   /// Construct the H1_WedgeElement of order @a p and BasisType @a btype
+   H1_WedgeElement(const int p,
+                   const int btype = BasisType::GaussLobatto);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+} // namespace mfem
+
+#endif

fem/fe/fe_l2.cpp

@@ -0,0 +1,694 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// L2 Finite Element classes
+
+#include "fe_l2.hpp"
+#include "fe_h1.hpp"
+
+namespace mfem
+{
+
+using namespace std;
+
+L2_SegmentElement::L2_SegmentElement(const int p, const int btype)
+   : NodalTensorFiniteElement(1, p, VerifyOpen(btype), L2_DOF_MAP)
+{
+   const double *op = poly1d.OpenPoints(p, btype);
+
+#ifndef MFEM_THREAD_SAFE
+   shape_x.SetSize(p + 1);
+   dshape_x.SetDataAndSize(NULL, p + 1);
+#endif
+
+   for (int i = 0; i <= p; i++)
+   {
+      Nodes.IntPoint(i).x = op[i];
+   }
+}
+
+void L2_SegmentElement::CalcShape(const IntegrationPoint &ip,
+                                  Vector &shape) const
+{
+   basis1d.Eval(ip.x, shape);
+}
+
+void L2_SegmentElement::CalcDShape(const IntegrationPoint &ip,
+                                   DenseMatrix &dshape) const
+{
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(dof), dshape_x(dshape.Data(), dof);
+#else
+   dshape_x.SetData(dshape.Data());
+#endif
+   basis1d.Eval(ip.x, shape_x, dshape_x);
+}
+
+void L2_SegmentElement::ProjectDelta(int vertex, Vector &dofs) const
+{
+   const int p = order;
+   const double *op = poly1d.OpenPoints(p, b_type);
+
+   switch (vertex)
+   {
+      case 0:
+         for (int i = 0; i <= p; i++)
+         {
+            dofs(i) = poly1d.CalcDelta(p,(1.0 - op[i]));
+         }
+         break;
+
+      case 1:
+         for (int i = 0; i <= p; i++)
+         {
+            dofs(i) = poly1d.CalcDelta(p,op[i]);
+         }
+         break;
+   }
+}
+
+
+L2_QuadrilateralElement::L2_QuadrilateralElement(const int p, const int btype)
+   : NodalTensorFiniteElement(2, p, VerifyOpen(btype), L2_DOF_MAP)
+{
+   const double *op = poly1d.OpenPoints(p, b_type);
+
+#ifndef MFEM_THREAD_SAFE
+   shape_x.SetSize(p + 1);
+   shape_y.SetSize(p + 1);
+   dshape_x.SetSize(p + 1);
+   dshape_y.SetSize(p + 1);
+#endif
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i <= p; i++)
+      {
+         Nodes.IntPoint(o++).Set2(op[i], op[j]);
+      }
+}
+
+void L2_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
+                                        Vector &shape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1), shape_y(p+1);
+#endif
+
+   basis1d.Eval(ip.x, shape_x);
+   basis1d.Eval(ip.y, shape_y);
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i <= p; i++)
+      {
+         shape(o++) = shape_x(i)*shape_y(j);
+      }
+}
+
+void L2_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
+                                         DenseMatrix &dshape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1), shape_y(p+1), dshape_x(p+1), dshape_y(p+1);
+#endif
+
+   basis1d.Eval(ip.x, shape_x, dshape_x);
+   basis1d.Eval(ip.y, shape_y, dshape_y);
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i <= p; i++)
+      {
+         dshape(o,0) = dshape_x(i)* shape_y(j);
+         dshape(o,1) =  shape_x(i)*dshape_y(j);  o++;
+      }
+}
+
+void L2_QuadrilateralElement::ProjectDelta(int vertex, Vector &dofs) const
+{
+   const int p = order;
+   const double *op = poly1d.OpenPoints(p, b_type);
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1), shape_y(p+1);
+#endif
+
+   for (int i = 0; i <= p; i++)
+   {
+      shape_x(i) = poly1d.CalcDelta(p,(1.0 - op[i]));
+      shape_y(i) = poly1d.CalcDelta(p,op[i]);
+   }
+
+   switch (vertex)
+   {
+      case 0:
+         for (int o = 0, j = 0; j <= p; j++)
+            for (int i = 0; i <= p; i++)
+            {
+               dofs[o++] = shape_x(i)*shape_x(j);
+            }
+         break;
+      case 1:
+         for (int o = 0, j = 0; j <= p; j++)
+            for (int i = 0; i <= p; i++)
+            {
+               dofs[o++] = shape_y(i)*shape_x(j);
+            }
+         break;
+      case 2:
+         for (int o = 0, j = 0; j <= p; j++)
+            for (int i = 0; i <= p; i++)
+            {
+               dofs[o++] = shape_y(i)*shape_y(j);
+            }
+         break;
+      case 3:
+         for (int o = 0, j = 0; j <= p; j++)
+            for (int i = 0; i <= p; i++)
+            {
+               dofs[o++] = shape_x(i)*shape_y(j);
+            }
+         break;
+   }
+}
+
+
+L2_HexahedronElement::L2_HexahedronElement(const int p, const int btype)
+   : NodalTensorFiniteElement(3, p, VerifyOpen(btype), L2_DOF_MAP)
+{
+   const double *op = poly1d.OpenPoints(p, btype);
+
+#ifndef MFEM_THREAD_SAFE
+   shape_x.SetSize(p + 1);
+   shape_y.SetSize(p + 1);
+   shape_z.SetSize(p + 1);
+   dshape_x.SetSize(p + 1);
+   dshape_y.SetSize(p + 1);
+   dshape_z.SetSize(p + 1);
+#endif
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j <= p; j++)
+         for (int i = 0; i <= p; i++)
+         {
+            Nodes.IntPoint(o++).Set3(op[i], op[j], op[k]);
+         }
+}
+
+void L2_HexahedronElement::CalcShape(const IntegrationPoint &ip,
+                                     Vector &shape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1), shape_y(p+1), shape_z(p+1);
+#endif
+
+   basis1d.Eval(ip.x, shape_x);
+   basis1d.Eval(ip.y, shape_y);
+   basis1d.Eval(ip.z, shape_z);
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j <= p; j++)
+         for (int i = 0; i <= p; i++)
+         {
+            shape(o++) = shape_x(i)*shape_y(j)*shape_z(k);
+         }
+}
+
+void L2_HexahedronElement::CalcDShape(const IntegrationPoint &ip,
+                                      DenseMatrix &dshape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1),  shape_y(p+1),  shape_z(p+1);
+   Vector dshape_x(p+1), dshape_y(p+1), dshape_z(p+1);
+#endif
+
+   basis1d.Eval(ip.x, shape_x, dshape_x);
+   basis1d.Eval(ip.y, shape_y, dshape_y);
+   basis1d.Eval(ip.z, shape_z, dshape_z);
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j <= p; j++)
+         for (int i = 0; i <= p; i++)
+         {
+            dshape(o,0) = dshape_x(i)* shape_y(j)* shape_z(k);
+            dshape(o,1) =  shape_x(i)*dshape_y(j)* shape_z(k);
+            dshape(o,2) =  shape_x(i)* shape_y(j)*dshape_z(k);  o++;
+         }
+}
+
+void L2_HexahedronElement::ProjectDelta(int vertex, Vector &dofs) const
+{
+   const int p = order;
+   const double *op = poly1d.OpenPoints(p, b_type);
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1), shape_y(p+1);
+#endif
+
+   for (int i = 0; i <= p; i++)
+   {
+      shape_x(i) = poly1d.CalcDelta(p,(1.0 - op[i]));
+      shape_y(i) = poly1d.CalcDelta(p,op[i]);
+   }
+
+   switch (vertex)
+   {
+      case 0:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_x(i)*shape_x(j)*shape_x(k);
+               }
+         break;
+      case 1:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_y(i)*shape_x(j)*shape_x(k);
+               }
+         break;
+      case 2:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_y(i)*shape_y(j)*shape_x(k);
+               }
+         break;
+      case 3:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_x(i)*shape_y(j)*shape_x(k);
+               }
+         break;
+      case 4:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_x(i)*shape_x(j)*shape_y(k);
+               }
+         break;
+      case 5:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_y(i)*shape_x(j)*shape_y(k);
+               }
+         break;
+      case 6:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_y(i)*shape_y(j)*shape_y(k);
+               }
+         break;
+      case 7:
+         for (int o = 0, k = 0; k <= p; k++)
+            for (int j = 0; j <= p; j++)
+               for (int i = 0; i <= p; i++)
+               {
+                  dofs[o++] = shape_x(i)*shape_y(j)*shape_y(k);
+               }
+         break;
+   }
+}
+
+
+L2_TriangleElement::L2_TriangleElement(const int p, const int btype)
+   : NodalFiniteElement(2, Geometry::TRIANGLE, ((p + 1)*(p + 2))/2, p,
+                        FunctionSpace::Pk)
+{
+   const double *op = poly1d.OpenPoints(p, VerifyOpen(btype));
+
+#ifndef MFEM_THREAD_SAFE
+   shape_x.SetSize(p + 1);
+   shape_y.SetSize(p + 1);
+   shape_l.SetSize(p + 1);
+   dshape_x.SetSize(p + 1);
+   dshape_y.SetSize(p + 1);
+   dshape_l.SetSize(p + 1);
+   u.SetSize(dof);
+   du.SetSize(dof, dim);
+#else
+   Vector shape_x(p + 1), shape_y(p + 1), shape_l(p + 1);
+#endif
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i + j <= p; i++)
+      {
+         double w = op[i] + op[j] + op[p-i-j];
+         Nodes.IntPoint(o++).Set2(op[i]/w, op[j]/w);
+      }
+
+   DenseMatrix T(dof);
+   for (int k = 0; k < dof; k++)
+   {
+      IntegrationPoint &ip = Nodes.IntPoint(k);
+      poly1d.CalcBasis(p, ip.x, shape_x);
+      poly1d.CalcBasis(p, ip.y, shape_y);
+      poly1d.CalcBasis(p, 1. - ip.x - ip.y, shape_l);
+
+      for (int o = 0, j = 0; j <= p; j++)
+         for (int i = 0; i + j <= p; i++)
+         {
+            T(o++, k) = shape_x(i)*shape_y(j)*shape_l(p-i-j);
+         }
+   }
+
+   Ti.Factor(T);
+   // mfem::out << "L2_TriangleElement(" << p << ") : "; Ti.TestInversion();
+}
+
+void L2_TriangleElement::CalcShape(const IntegrationPoint &ip,
+                                   Vector &shape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p + 1), shape_y(p + 1), shape_l(p + 1), u(dof);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y, shape_l);
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i + j <= p; i++)
+      {
+         u(o++) = shape_x(i)*shape_y(j)*shape_l(p-i-j);
+      }
+
+   Ti.Mult(u, shape);
+}
+
+void L2_TriangleElement::CalcDShape(const IntegrationPoint &ip,
+                                    DenseMatrix &dshape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector  shape_x(p + 1),  shape_y(p + 1),  shape_l(p + 1);
+   Vector dshape_x(p + 1), dshape_y(p + 1), dshape_l(p + 1);
+   DenseMatrix du(dof, dim);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x, dshape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y, dshape_y);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y, shape_l, dshape_l);
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i + j <= p; i++)
+      {
+         int k = p - i - j;
+         du(o,0) = ((dshape_x(i)* shape_l(k)) -
+                    ( shape_x(i)*dshape_l(k)))*shape_y(j);
+         du(o,1) = ((dshape_y(j)* shape_l(k)) -
+                    ( shape_y(j)*dshape_l(k)))*shape_x(i);
+         o++;
+      }
+
+   Ti.Mult(du, dshape);
+}
+
+void L2_TriangleElement::ProjectDelta(int vertex, Vector &dofs) const
+{
+   switch (vertex)
+   {
+      case 0:
+         for (int i = 0; i < dof; i++)
+         {
+            const IntegrationPoint &ip = Nodes.IntPoint(i);
+            dofs[i] = pow(1.0 - ip.x - ip.y, order);
+         }
+         break;
+      case 1:
+         for (int i = 0; i < dof; i++)
+         {
+            const IntegrationPoint &ip = Nodes.IntPoint(i);
+            dofs[i] = pow(ip.x, order);
+         }
+         break;
+      case 2:
+         for (int i = 0; i < dof; i++)
+         {
+            const IntegrationPoint &ip = Nodes.IntPoint(i);
+            dofs[i] = pow(ip.y, order);
+         }
+         break;
+   }
+}
+
+
+L2_TetrahedronElement::L2_TetrahedronElement(const int p, const int btype)
+   : NodalFiniteElement(3, Geometry::TETRAHEDRON, ((p + 1)*(p + 2)*(p + 3))/6,
+                        p, FunctionSpace::Pk)
+{
+   const double *op = poly1d.OpenPoints(p, VerifyNodal(VerifyOpen(btype)));
+
+#ifndef MFEM_THREAD_SAFE
+   shape_x.SetSize(p + 1);
+   shape_y.SetSize(p + 1);
+   shape_z.SetSize(p + 1);
+   shape_l.SetSize(p + 1);
+   dshape_x.SetSize(p + 1);
+   dshape_y.SetSize(p + 1);
+   dshape_z.SetSize(p + 1);
+   dshape_l.SetSize(p + 1);
+   u.SetSize(dof);
+   du.SetSize(dof, dim);
+#else
+   Vector shape_x(p + 1), shape_y(p + 1), shape_z(p + 1), shape_l(p + 1);
+#endif
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j + k <= p; j++)
+         for (int i = 0; i + j + k <= p; i++)
+         {
+            double w = op[i] + op[j] + op[k] + op[p-i-j-k];
+            Nodes.IntPoint(o++).Set3(op[i]/w, op[j]/w, op[k]/w);
+         }
+
+   DenseMatrix T(dof);
+   for (int m = 0; m < dof; m++)
+   {
+      IntegrationPoint &ip = Nodes.IntPoint(m);
+      poly1d.CalcBasis(p, ip.x, shape_x);
+      poly1d.CalcBasis(p, ip.y, shape_y);
+      poly1d.CalcBasis(p, ip.z, shape_z);
+      poly1d.CalcBasis(p, 1. - ip.x - ip.y - ip.z, shape_l);
+
+      for (int o = 0, k = 0; k <= p; k++)
+         for (int j = 0; j + k <= p; j++)
+            for (int i = 0; i + j + k <= p; i++)
+            {
+               T(o++, m) = shape_x(i)*shape_y(j)*shape_z(k)*shape_l(p-i-j-k);
+            }
+   }
+
+   Ti.Factor(T);
+   // mfem::out << "L2_TetrahedronElement(" << p << ") : "; Ti.TestInversion();
+}
+
+void L2_TetrahedronElement::CalcShape(const IntegrationPoint &ip,
+                                      Vector &shape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p + 1), shape_y(p + 1), shape_z(p + 1), shape_l(p + 1);
+   Vector u(dof);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y);
+   poly1d.CalcBasis(p, ip.z, shape_z);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y - ip.z, shape_l);
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j + k <= p; j++)
+         for (int i = 0; i + j + k <= p; i++)
+         {
+            u(o++) = shape_x(i)*shape_y(j)*shape_z(k)*shape_l(p-i-j-k);
+         }
+
+   Ti.Mult(u, shape);
+}
+
+void L2_TetrahedronElement::CalcDShape(const IntegrationPoint &ip,
+                                       DenseMatrix &dshape) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector  shape_x(p + 1),  shape_y(p + 1),  shape_z(p + 1),  shape_l(p + 1);
+   Vector dshape_x(p + 1), dshape_y(p + 1), dshape_z(p + 1), dshape_l(p + 1);
+   DenseMatrix du(dof, dim);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x, dshape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y, dshape_y);
+   poly1d.CalcBasis(p, ip.z, shape_z, dshape_z);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y - ip.z, shape_l, dshape_l);
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j + k <= p; j++)
+         for (int i = 0; i + j + k <= p; i++)
+         {
+            int l = p - i - j - k;
+            du(o,0) = ((dshape_x(i)* shape_l(l)) -
+                       ( shape_x(i)*dshape_l(l)))*shape_y(j)*shape_z(k);
+            du(o,1) = ((dshape_y(j)* shape_l(l)) -
+                       ( shape_y(j)*dshape_l(l)))*shape_x(i)*shape_z(k);
+            du(o,2) = ((dshape_z(k)* shape_l(l)) -
+                       ( shape_z(k)*dshape_l(l)))*shape_x(i)*shape_y(j);
+            o++;
+         }
+
+   Ti.Mult(du, dshape);
+}
+
+void L2_TetrahedronElement::ProjectDelta(int vertex, Vector &dofs) const
+{
+   switch (vertex)
+   {
+      case 0:
+         for (int i = 0; i < dof; i++)
+         {
+            const IntegrationPoint &ip = Nodes.IntPoint(i);
+            dofs[i] = pow(1.0 - ip.x - ip.y - ip.z, order);
+         }
+         break;
+      case 1:
+         for (int i = 0; i < dof; i++)
+         {
+            const IntegrationPoint &ip = Nodes.IntPoint(i);
+            dofs[i] = pow(ip.x, order);
+         }
+         break;
+      case 2:
+         for (int i = 0; i < dof; i++)
+         {
+            const IntegrationPoint &ip = Nodes.IntPoint(i);
+            dofs[i] = pow(ip.y, order);
+         }
+         break;
+      case 3:
+         for (int i = 0; i < dof; i++)
+         {
+            const IntegrationPoint &ip = Nodes.IntPoint(i);
+            dofs[i] = pow(ip.z, order);
+         }
+         break;
+   }
+}
+
+
+L2_WedgeElement::L2_WedgeElement(const int p, const int btype)
+   : NodalFiniteElement(3, Geometry::PRISM, ((p + 1)*(p + 1)*(p + 2))/2,
+                        p, FunctionSpace::Qk),
+     TriangleFE(p, btype),
+     SegmentFE(p, btype)
+{
+#ifndef MFEM_THREAD_SAFE
+   t_shape.SetSize(TriangleFE.GetDof());
+   s_shape.SetSize(SegmentFE.GetDof());
+   t_dshape.SetSize(TriangleFE.GetDof(), 2);
+   s_dshape.SetSize(SegmentFE.GetDof(), 1);
+#endif
+
+   t_dof.SetSize(dof);
+   s_dof.SetSize(dof);
+
+   // Interior DoFs
+   int m=0;
+   for (int k=0; k<=p; k++)
+   {
+      int l=0;
+      for (int j=0; j<=p; j++)
+      {
+         for (int i=0; i<=j; i++)
+         {
+            t_dof[m] = l;
+            s_dof[m] = k;
+            l++; m++;
+         }
+      }
+   }
+
+   // Define Nodes
+   const IntegrationRule & t_Nodes = TriangleFE.GetNodes();
+   const IntegrationRule & s_Nodes = SegmentFE.GetNodes();
+   for (int i=0; i<dof; i++)
+   {
+      Nodes.IntPoint(i).x = t_Nodes.IntPoint(t_dof[i]).x;
+      Nodes.IntPoint(i).y = t_Nodes.IntPoint(t_dof[i]).y;
+      Nodes.IntPoint(i).z = s_Nodes.IntPoint(s_dof[i]).x;
+   }
+}
+
+void L2_WedgeElement::CalcShape(const IntegrationPoint &ip,
+                                Vector &shape) const
+{
+#ifdef MFEM_THREAD_SAFE
+   Vector t_shape(TriangleFE.GetDof());
+   Vector s_shape(SegmentFE.GetDof());
+#endif
+
+   IntegrationPoint ipz; ipz.x = ip.z; ipz.y = 0.0; ipz.z = 0.0;
+
+   TriangleFE.CalcShape(ip, t_shape);
+   SegmentFE.CalcShape(ipz, s_shape);
+
+   for (int i=0; i<dof; i++)
+   {
+      shape[i] = t_shape[t_dof[i]] * s_shape[s_dof[i]];
+   }
+}
+
+void L2_WedgeElement::CalcDShape(const IntegrationPoint &ip,
+                                 DenseMatrix &dshape) const
+{
+#ifdef MFEM_THREAD_SAFE
+   Vector      t_shape(TriangleFE.GetDof());
+   DenseMatrix t_dshape(TriangleFE.GetDof(), 2);
+   Vector      s_shape(SegmentFE.GetDof());
+   DenseMatrix s_dshape(SegmentFE.GetDof(), 1);
+#endif
+
+   IntegrationPoint ipz; ipz.x = ip.z; ipz.y = 0.0; ipz.z = 0.0;
+
+   TriangleFE.CalcShape(ip, t_shape);
+   TriangleFE.CalcDShape(ip, t_dshape);
+   SegmentFE.CalcShape(ipz, s_shape);
+   SegmentFE.CalcDShape(ipz, s_dshape);
+
+   for (int i=0; i<dof; i++)
+   {
+      dshape(i, 0) = t_dshape(t_dof[i],0) * s_shape[s_dof[i]];
+      dshape(i, 1) = t_dshape(t_dof[i],1) * s_shape[s_dof[i]];
+      dshape(i, 2) = t_shape[t_dof[i]] * s_dshape(s_dof[i],0);
+   }
+}
+
+}

fem/fe/fe_l2.hpp

@@ -0,0 +1,151 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_L2
+#define MFEM_FE_L2
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+/// Arbitrary order L2 elements in 1D on a segment
+class L2_SegmentElement : public NodalTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, dshape_x;
+#endif
+
+public:
+   /// Construct the L2_SegmentElement of order @a p and BasisType @a btype
+   L2_SegmentElement(const int p, const int btype = BasisType::GaussLegendre);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 2D on a square
+class L2_QuadrilateralElement : public NodalTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y;
+#endif
+
+public:
+   /// Construct the L2_QuadrilateralElement of order @a p and BasisType @a btype
+   L2_QuadrilateralElement(const int p,
+                           const int btype = BasisType::GaussLegendre);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectCurl_2D(fe, Trans, curl); }
+};
+
+
+/// Arbitrary order L2 elements in 3D on a cube
+class L2_HexahedronElement : public NodalTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z;
+#endif
+
+public:
+   /// Construct the L2_HexahedronElement of order @a p and BasisType @a btype
+   L2_HexahedronElement(const int p,
+                        const int btype = BasisType::GaussLegendre);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 2D on a triangle
+class L2_TriangleElement : public NodalFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_l, dshape_x, dshape_y, dshape_l, u;
+   mutable DenseMatrix du;
+#endif
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the L2_TriangleElement of order @a p and BasisType @a btype
+   L2_TriangleElement(const int p,
+                      const int btype = BasisType::GaussLegendre);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectCurl_2D(fe, Trans, curl); }
+};
+
+
+/// Arbitrary order L2 elements in 3D on a tetrahedron
+class L2_TetrahedronElement : public NodalFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_z, shape_l;
+   mutable Vector dshape_x, dshape_y, dshape_z, dshape_l, u;
+   mutable DenseMatrix du;
+#endif
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the L2_TetrahedronElement of order @a p and BasisType @a btype
+   L2_TetrahedronElement(const int p,
+                         const int btype = BasisType::GaussLegendre);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 3D on a wedge
+class L2_WedgeElement : public NodalFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector t_shape, s_shape;
+   mutable DenseMatrix t_dshape, s_dshape;
+#endif
+   Array<int> t_dof, s_dof;
+
+   L2_TriangleElement TriangleFE;
+   L2_SegmentElement  SegmentFE;
+
+public:
+   /// Construct the L2_WedgeElement of order @a p and BasisType @a btype
+   L2_WedgeElement(const int p,
+                   const int btype = BasisType::GaussLegendre);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+} // namespace mfem
+
+#endif

fem/fe/fe_nd.hpp

@@ -0,0 +1,326 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_ND
+#define MFEM_FE_ND
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+/// Arbitrary order Nedelec elements in 3D on a cube
+class ND_HexahedronElement : public VectorTensorFiniteElement
+{
+   static const double tk[18];
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_cx, shape_ox, shape_cy, shape_oy, shape_cz, shape_oz;
+   mutable Vector dshape_cx, dshape_cy, dshape_cz;
+#endif
+   Array<int> dof2tk;
+   const double *cp;
+
+public:
+   /** @brief Construct the ND_HexahedronElement of order @a p and closed and
+       open BasisType @a cb_type and @a ob_type */
+   ND_HexahedronElement(const int p,
+                        const int cb_type = BasisType::GaussLobatto,
+                        const int ob_type = BasisType::GaussLegendre);
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_ND(*this, tk, dof2tk, Trans, I); }
+
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_ND(tk, dof2tk, Trans, R); }
+
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+
+   using FiniteElement::Project;
+
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   {
+      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
+      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   }
+
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
+
+   virtual void Project(const FiniteElement &fe,
+                        ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_ND(tk, dof2tk, fe, Trans, I); }
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const
+   { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectCurl_ND(tk, dof2tk, fe, Trans, curl); }
+
+protected:
+   void ProjectIntegrated(VectorCoefficient &vc,
+                          ElementTransformation &Trans,
+                          Vector &dofs) const;
+};
+
+
+/// Arbitrary order Nedelec elements in 2D on a square
+class ND_QuadrilateralElement : public VectorTensorFiniteElement
+{
+   static const double tk[8];
+
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_cx, shape_ox, shape_cy, shape_oy;
+   mutable Vector dshape_cx, dshape_cy;
+#endif
+   Array<int> dof2tk;
+   const double *cp;
+
+public:
+   /** @brief Construct the ND_QuadrilateralElement of order @a p and closed and
+       open BasisType @a cb_type and @a ob_type */
+   ND_QuadrilateralElement(const int p,
+                           const int cb_type = BasisType::GaussLobatto,
+                           const int ob_type = BasisType::GaussLegendre);
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_ND(*this, tk, dof2tk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_ND(tk, dof2tk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   {
+      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
+      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe,
+                        ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_ND(tk, dof2tk, fe, Trans, I); }
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const
+   { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
+
+protected:
+   void ProjectIntegrated(VectorCoefficient &vc,
+                          ElementTransformation &Trans,
+                          Vector &dofs) const;
+};
+
+
+/// Arbitrary order Nedelec elements in 3D on a tetrahedron
+class ND_TetrahedronElement : public VectorFiniteElement
+{
+   static const double tk[18], c;
+
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_z, shape_l;
+   mutable Vector dshape_x, dshape_y, dshape_z, dshape_l;
+   mutable DenseMatrix u;
+#endif
+   Array<int> dof2tk;
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the ND_TetrahedronElement of order @a p
+   ND_TetrahedronElement(const int p);
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_ND(*this, tk, dof2tk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_ND(tk, dof2tk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe,
+                        ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_ND(tk, dof2tk, fe, Trans, I); }
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const
+   { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectCurl_ND(tk, dof2tk, fe, Trans, curl); }
+};
+
+/// Arbitrary order Nedelec elements in 2D on a triangle
+class ND_TriangleElement : public VectorFiniteElement
+{
+   static const double tk[8], c;
+
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_l;
+   mutable Vector dshape_x, dshape_y, dshape_l;
+   mutable DenseMatrix u;
+   mutable Vector curlu;
+#endif
+   Array<int> dof2tk;
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the ND_TriangleElement of order @a p
+   ND_TriangleElement(const int p);
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_ND(*this, tk, dof2tk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_ND(tk, dof2tk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe,
+                        ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_ND(tk, dof2tk, fe, Trans, I); }
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const
+   { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
+};
+
+
+/// Arbitrary order Nedelec elements in 1D on a segment
+class ND_SegmentElement : public VectorTensorFiniteElement
+{
+   static const double tk[1];
+
+   Array<int> dof2tk;
+
+public:
+   /** @brief Construct the ND_SegmentElement of order @a p and open
+       BasisType @a ob_type */
+   ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const
+   { obasis1d.Eval(ip.x, shape); }
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   // virtual void CalcCurlShape(const IntegrationPoint &ip,
+   //                            DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_ND(*this, tk, dof2tk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_ND(tk, dof2tk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe,
+                        ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_ND(tk, dof2tk, fe, Trans, I); }
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const
+   { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
+};
+
+} // namespace mfem
+
+#endif

fem/fe/fe_nurbs.cpp

@@ -0,0 +1,405 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// H1 Finite Element classes utilizing the Bernstein basis
+
+#include "fe_nurbs.hpp"
+#include "../../mesh/nurbs.hpp"
+
+namespace mfem
+{
+
+using namespace std;
+
+void NURBS1DFiniteElement::SetOrder() const
+{
+   order = kv[0]->GetOrder();
+   dof = order + 1;
+
+   weights.SetSize(dof);
+   shape_x.SetSize(dof);
+}
+
+void NURBS1DFiniteElement::CalcShape(const IntegrationPoint &ip,
+                                     Vector &shape) const
+{
+   kv[0]->CalcShape(shape, ijk[0], ip.x);
+
+   double sum = 0.0;
+   for (int i = 0; i <= order; i++)
+   {
+      sum += (shape(i) *= weights(i));
+   }
+
+   shape /= sum;
+}
+
+void NURBS1DFiniteElement::CalcDShape(const IntegrationPoint &ip,
+                                      DenseMatrix &dshape) const
+{
+   Vector grad(dshape.Data(), dof);
+
+   kv[0]->CalcShape (shape_x, ijk[0], ip.x);
+   kv[0]->CalcDShape(grad,    ijk[0], ip.x);
+
+   double sum = 0.0, dsum = 0.0;
+   for (int i = 0; i <= order; i++)
+   {
+      sum  += (shape_x(i) *= weights(i));
+      dsum += (   grad(i) *= weights(i));
+   }
+
+   sum = 1.0/sum;
+   add(sum, grad, -dsum*sum*sum, shape_x, grad);
+}
+
+void NURBS1DFiniteElement::CalcHessian (const IntegrationPoint &ip,
+                                        DenseMatrix &hessian) const
+{
+   Vector grad(dof);
+   Vector hess(hessian.Data(), dof);
+
+   kv[0]->CalcShape (shape_x,  ijk[0], ip.x);
+   kv[0]->CalcDShape(grad,     ijk[0], ip.x);
+   kv[0]->CalcD2Shape(hess,    ijk[0], ip.x);
+
+   double sum = 0.0, dsum = 0.0, d2sum = 0.0;
+   for (int i = 0; i <= order; i++)
+   {
+      sum   += (shape_x(i) *= weights(i));
+      dsum  += (   grad(i) *= weights(i));
+      d2sum += (   hess(i) *= weights(i));
+   }
+
+   sum = 1.0/sum;
+   add(sum, hess, -2*dsum*sum*sum, grad, hess);
+   add(1.0, hess, (-d2sum + 2*dsum*dsum*sum)*sum*sum, shape_x, hess);
+}
+
+
+void NURBS2DFiniteElement::SetOrder() const
+{
+   orders[0] = kv[0]->GetOrder();
+   orders[1] = kv[1]->GetOrder();
+   shape_x.SetSize(orders[0]+1);
+   shape_y.SetSize(orders[1]+1);
+   dshape_x.SetSize(orders[0]+1);
+   dshape_y.SetSize(orders[1]+1);
+   d2shape_x.SetSize(orders[0]+1);
+   d2shape_y.SetSize(orders[1]+1);
+
+   order = max(orders[0], orders[1]);
+   dof = (orders[0] + 1)*(orders[1] + 1);
+   u.SetSize(dof);
+   du.SetSize(dof);
+   weights.SetSize(dof);
+}
+
+void NURBS2DFiniteElement::CalcShape(const IntegrationPoint &ip,
+                                     Vector &shape) const
+{
+   kv[0]->CalcShape(shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape(shape_y, ijk[1], ip.y);
+
+   double sum = 0.0;
+   for (int o = 0, j = 0; j <= orders[1]; j++)
+   {
+      const double sy = shape_y(j);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         sum += ( shape(o) = shape_x(i)*sy*weights(o) );
+      }
+   }
+
+   shape /= sum;
+}
+
+void NURBS2DFiniteElement::CalcDShape(const IntegrationPoint &ip,
+                                      DenseMatrix &dshape) const
+{
+   double sum, dsum[2];
+
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+
+   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
+   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
+
+   sum = dsum[0] = dsum[1] = 0.0;
+   for (int o = 0, j = 0; j <= orders[1]; j++)
+   {
+      const double sy = shape_y(j), dsy = dshape_y(j);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         sum += ( u(o) = shape_x(i)*sy*weights(o) );
+
+         dsum[0] += ( dshape(o,0) = dshape_x(i)*sy *weights(o) );
+         dsum[1] += ( dshape(o,1) =  shape_x(i)*dsy*weights(o) );
+      }
+   }
+
+   sum = 1.0/sum;
+   dsum[0] *= sum*sum;
+   dsum[1] *= sum*sum;
+
+   for (int o = 0; o < dof; o++)
+   {
+      dshape(o,0) = dshape(o,0)*sum - u(o)*dsum[0];
+      dshape(o,1) = dshape(o,1)*sum - u(o)*dsum[1];
+   }
+}
+
+void NURBS2DFiniteElement::CalcHessian (const IntegrationPoint &ip,
+                                        DenseMatrix &hessian) const
+{
+   double sum, dsum[2], d2sum[3];
+
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+
+   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
+   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
+
+   kv[0]->CalcD2Shape(d2shape_x, ijk[0], ip.x);
+   kv[1]->CalcD2Shape(d2shape_y, ijk[1], ip.y);
+
+   sum = dsum[0] = dsum[1] = 0.0;
+   d2sum[0] = d2sum[1] = d2sum[2] = 0.0;
+   for (int o = 0, j = 0; j <= orders[1]; j++)
+   {
+      const double sy = shape_y(j), dsy = dshape_y(j), d2sy = d2shape_y(j);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         const double sx = shape_x(i), dsx = dshape_x(i), d2sx = d2shape_x(i);
+         sum += ( u(o) = sx*sy*weights(o) );
+
+         dsum[0] += ( du(o,0) = dsx*sy*weights(o) );
+         dsum[1] += ( du(o,1) = sx*dsy*weights(o) );
+
+         d2sum[0] += ( hessian(o,0) = d2sx*sy*weights(o) );
+         d2sum[1] += ( hessian(o,1) = dsx*dsy*weights(o) );
+         d2sum[2] += ( hessian(o,2) = sx*d2sy*weights(o) );
+      }
+   }
+
+   sum = 1.0/sum;
+   dsum[0] *= sum;
+   dsum[1] *= sum;
+
+   d2sum[0] *= sum;
+   d2sum[1] *= sum;
+   d2sum[2] *= sum;
+
+   for (int o = 0; o < dof; o++)
+   {
+      hessian(o,0) = hessian(o,0)*sum
+                     - 2*du(o,0)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[0] - d2sum[0]);
+
+      hessian(o,1) = hessian(o,1)*sum
+                     - du(o,0)*sum*dsum[1]
+                     - du(o,1)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[1] - d2sum[1]);
+
+      hessian(o,2) = hessian(o,2)*sum
+                     - 2*du(o,1)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[2]);
+   }
+}
+
+
+void NURBS3DFiniteElement::SetOrder() const
+{
+   orders[0] = kv[0]->GetOrder();
+   orders[1] = kv[1]->GetOrder();
+   orders[2] = kv[2]->GetOrder();
+   shape_x.SetSize(orders[0]+1);
+   shape_y.SetSize(orders[1]+1);
+   shape_z.SetSize(orders[2]+1);
+
+   dshape_x.SetSize(orders[0]+1);
+   dshape_y.SetSize(orders[1]+1);
+   dshape_z.SetSize(orders[2]+1);
+
+   d2shape_x.SetSize(orders[0]+1);
+   d2shape_y.SetSize(orders[1]+1);
+   d2shape_z.SetSize(orders[2]+1);
+
+   order = max(max(orders[0], orders[1]), orders[2]);
+   dof = (orders[0] + 1)*(orders[1] + 1)*(orders[2] + 1);
+   u.SetSize(dof);
+   du.SetSize(dof);
+   weights.SetSize(dof);
+}
+
+void NURBS3DFiniteElement::CalcShape(const IntegrationPoint &ip,
+                                     Vector &shape) const
+{
+   kv[0]->CalcShape(shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape(shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape(shape_z, ijk[2], ip.z);
+
+   double sum = 0.0;
+   for (int o = 0, k = 0; k <= orders[2]; k++)
+   {
+      const double sz = shape_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const double sy_sz = shape_y(j)*sz;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            sum += ( shape(o) = shape_x(i)*sy_sz*weights(o) );
+         }
+      }
+   }
+
+   shape /= sum;
+}
+
+void NURBS3DFiniteElement::CalcDShape(const IntegrationPoint &ip,
+                                      DenseMatrix &dshape) const
+{
+   double sum, dsum[3];
+
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape ( shape_z, ijk[2], ip.z);
+
+   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
+   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
+   kv[2]->CalcDShape(dshape_z, ijk[2], ip.z);
+
+   sum = dsum[0] = dsum[1] = dsum[2] = 0.0;
+   for (int o = 0, k = 0; k <= orders[2]; k++)
+   {
+      const double sz = shape_z(k), dsz = dshape_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const double  sy_sz  =  shape_y(j)* sz;
+         const double dsy_sz  = dshape_y(j)* sz;
+         const double  sy_dsz =  shape_y(j)*dsz;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            sum += ( u(o) = shape_x(i)*sy_sz*weights(o) );
+
+            dsum[0] += ( dshape(o,0) = dshape_x(i)* sy_sz *weights(o) );
+            dsum[1] += ( dshape(o,1) =  shape_x(i)*dsy_sz *weights(o) );
+            dsum[2] += ( dshape(o,2) =  shape_x(i)* sy_dsz*weights(o) );
+         }
+      }
+   }
+
+   sum = 1.0/sum;
+   dsum[0] *= sum*sum;
+   dsum[1] *= sum*sum;
+   dsum[2] *= sum*sum;
+
+   for (int o = 0; o < dof; o++)
+   {
+      dshape(o,0) = dshape(o,0)*sum - u(o)*dsum[0];
+      dshape(o,1) = dshape(o,1)*sum - u(o)*dsum[1];
+      dshape(o,2) = dshape(o,2)*sum - u(o)*dsum[2];
+   }
+}
+
+void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
+                                        DenseMatrix &hessian) const
+{
+   double sum, dsum[3], d2sum[6];
+
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape ( shape_z, ijk[2], ip.z);
+
+   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
+   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
+   kv[2]->CalcDShape(dshape_z, ijk[2], ip.z);
+
+   kv[0]->CalcD2Shape(d2shape_x, ijk[0], ip.x);
+   kv[1]->CalcD2Shape(d2shape_y, ijk[1], ip.y);
+   kv[2]->CalcD2Shape(d2shape_z, ijk[2], ip.z);
+
+   sum = dsum[0] = dsum[1] = dsum[2] = 0.0;
+   d2sum[0] = d2sum[1] = d2sum[2] = d2sum[3] = d2sum[4] = d2sum[5] = 0.0;
+
+   for (int o = 0, k = 0; k <= orders[2]; k++)
+   {
+      const double sz = shape_z(k), dsz = dshape_z(k), d2sz = d2shape_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const double sy = shape_y(j), dsy = dshape_y(j), d2sy = d2shape_y(j);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            const double sx = shape_x(i), dsx = dshape_x(i), d2sx = d2shape_x(i);
+            sum += ( u(o) = sx*sy*sz*weights(o) );
+
+            dsum[0] += ( du(o,0) = dsx*sy*sz*weights(o) );
+            dsum[1] += ( du(o,1) = sx*dsy*sz*weights(o) );
+            dsum[2] += ( du(o,2) = sx*sy*dsz*weights(o) );
+
+            d2sum[0] += ( hessian(o,0) = d2sx*sy*sz*weights(o) );
+            d2sum[1] += ( hessian(o,1) = dsx*dsy*sz*weights(o) );
+            d2sum[2] += ( hessian(o,2) = dsx*sy*dsz*weights(o) );
+
+            d2sum[3] += ( hessian(o,3) = sx*dsy*dsz*weights(o) );
+
+            d2sum[4] += ( hessian(o,4) = sx*sy*d2sz*weights(o) );
+            d2sum[5] += ( hessian(o,5) = sx*d2sy*sz*weights(o) );
+         }
+      }
+   }
+
+   sum = 1.0/sum;
+   dsum[0] *= sum;
+   dsum[1] *= sum;
+   dsum[2] *= sum;
+
+   d2sum[0] *= sum;
+   d2sum[1] *= sum;
+   d2sum[2] *= sum;
+
+   d2sum[3] *= sum;
+   d2sum[4] *= sum;
+   d2sum[5] *= sum;
+
+   for (int o = 0; o < dof; o++)
+   {
+      hessian(o,0) = hessian(o,0)*sum
+                     - 2*du(o,0)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[0] - d2sum[0]);
+
+      hessian(o,1) = hessian(o,1)*sum
+                     - du(o,0)*sum*dsum[1]
+                     - du(o,1)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[1] - d2sum[1]);
+
+      hessian(o,2) = hessian(o,2)*sum
+                     - du(o,0)*sum*dsum[2]
+                     - du(o,2)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[2] - d2sum[2]);
+
+      hessian(o,3) = hessian(o,3)*sum
+                     - du(o,1)*sum*dsum[2]
+                     - du(o,2)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[2] - d2sum[3]);
+
+      hessian(o,4) = hessian(o,4)*sum
+                     - 2*du(o,2)*sum*dsum[2]
+                     + u[o]*sum*(2*dsum[2]*dsum[2] - d2sum[4]);
+
+      hessian(o,5) = hessian(o,5)*sum
+                     - 2*du(o,1)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[5]);
+
+   }
+}
+
+}

fem/fe/fe_nurbs.hpp

@@ -0,0 +1,153 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_NURBS
+#define MFEM_FE_NURBS
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+class KnotVector;
+
+/// An arbitrary order and dimension NURBS element
+class NURBSFiniteElement : public ScalarFiniteElement
+{
+protected:
+   mutable Array <const KnotVector*> kv;
+   mutable const int *ijk;
+   mutable int patch, elem;
+   mutable Vector weights;
+
+public:
+   /** @brief Construct NURBSFiniteElement with given
+       @param D    Reference space dimension
+       @param G    Geometry type (of type Geometry::Type)
+       @param Do   Number of degrees of freedom in the FiniteElement
+       @param O    Order/degree of the FiniteElement
+       @param F    FunctionSpace type of the FiniteElement
+    */
+   NURBSFiniteElement(int D, Geometry::Type G, int Do, int O, int F)
+      : ScalarFiniteElement(D, G, Do, O, F)
+   {
+      ijk = NULL;
+      patch = elem = -1;
+      kv.SetSize(dim);
+      weights.SetSize(dof);
+      weights = 1.0;
+   }
+
+   void                 Reset      ()         const { patch = elem = -1; }
+   void                 SetIJK     (const int *IJK) const { ijk = IJK; }
+   int                  GetPatch   ()         const { return patch; }
+   void                 SetPatch   (int p)    const { patch = p; }
+   int                  GetElement ()         const { return elem; }
+   void                 SetElement (int e)    const { elem = e; }
+   Array <const KnotVector*> &KnotVectors()   const { return kv; }
+   Vector              &Weights    ()         const { return weights; }
+   /// Update the NURBSFiniteElement according to the currently set knot vectors
+   virtual void         SetOrder   ()         const { }
+};
+
+
+/// An arbitrary order 1D NURBS element on a segment
+class NURBS1DFiniteElement : public NURBSFiniteElement
+{
+protected:
+   mutable Vector shape_x;
+
+public:
+   /// Construct the NURBS1DFiniteElement of order @a p
+   NURBS1DFiniteElement(int p)
+      : NURBSFiniteElement(1, Geometry::SEGMENT, p + 1, p, FunctionSpace::Qk),
+        shape_x(p + 1) { }
+
+   virtual void SetOrder() const;
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian (const IntegrationPoint &ip,
+                             DenseMatrix &hessian) const;
+};
+
+/// An arbitrary order 2D NURBS element on a square
+class NURBS2DFiniteElement : public NURBSFiniteElement
+{
+protected:
+   mutable Vector u, shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
+   mutable DenseMatrix du;
+
+public:
+   /// Construct the NURBS2DFiniteElement of order @a p
+   NURBS2DFiniteElement(int p)
+      : NURBSFiniteElement(2, Geometry::SQUARE, (p + 1)*(p + 1), p,
+                           FunctionSpace::Qk),
+        u(dof), shape_x(p + 1), shape_y(p + 1), dshape_x(p + 1),
+        dshape_y(p + 1), d2shape_x(p + 1), d2shape_y(p + 1), du(dof,2)
+   { orders[0] = orders[1] = p; }
+
+   /// Construct the NURBS2DFiniteElement with x-order @a px and y-order @a py
+   NURBS2DFiniteElement(int px, int py)
+      : NURBSFiniteElement(2, Geometry::SQUARE, (px + 1)*(py + 1),
+                           std::max(px, py), FunctionSpace::Qk),
+        u(dof), shape_x(px + 1), shape_y(py + 1), dshape_x(px + 1),
+        dshape_y(py + 1), d2shape_x(px + 1), d2shape_y(py + 1), du(dof,2)
+   { orders[0] = px; orders[1] = py; }
+
+   virtual void SetOrder() const;
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian (const IntegrationPoint &ip,
+                             DenseMatrix &hessian) const;
+};
+
+/// An arbitrary order 3D NURBS element on a cube
+class NURBS3DFiniteElement : public NURBSFiniteElement
+{
+protected:
+   mutable Vector u, shape_x, shape_y, shape_z;
+   mutable Vector dshape_x, dshape_y, dshape_z;
+   mutable Vector d2shape_x, d2shape_y, d2shape_z;
+   mutable DenseMatrix du;
+
+public:
+   /// Construct the NURBS3DFiniteElement of order @a p
+   NURBS3DFiniteElement(int p)
+      : NURBSFiniteElement(3, Geometry::CUBE, (p + 1)*(p + 1)*(p + 1), p,
+                           FunctionSpace::Qk),
+        u(dof), shape_x(p + 1), shape_y(p + 1), shape_z(p + 1),
+        dshape_x(p + 1), dshape_y(p + 1), dshape_z(p + 1),
+        d2shape_x(p + 1), d2shape_y(p + 1), d2shape_z(p + 1), du(dof,3)
+   { orders[0] = orders[1] = orders[2] = p; }
+
+   /// Construct the NURBS3DFiniteElement with x-order @a px and y-order @a py
+   /// and z-order @a pz
+   NURBS3DFiniteElement(int px, int py, int pz)
+      : NURBSFiniteElement(3, Geometry::CUBE, (px + 1)*(py + 1)*(pz + 1),
+                           std::max(std::max(px,py),pz), FunctionSpace::Qk),
+        u(dof), shape_x(px + 1), shape_y(py + 1), shape_z(pz + 1),
+        dshape_x(px + 1), dshape_y(py + 1), dshape_z(pz + 1),
+        d2shape_x(px + 1), d2shape_y(py + 1), d2shape_z(pz + 1), du(dof,3)
+   { orders[0] = px; orders[1] = py; orders[2] = pz; }
+
+   virtual void SetOrder() const;
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian (const IntegrationPoint &ip,
+                             DenseMatrix &hessian) const;
+};
+
+} // namespace mfem
+
+#endif

fem/fe/fe_pos.hpp

@@ -0,0 +1,372 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_POS
+#define MFEM_FE_POS
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+/** @brief Class for finite elements utilizing the
+    always positive Bernstein basis. */
+class PositiveFiniteElement : public ScalarFiniteElement
+{
+public:
+   /** @brief Construct PositiveFiniteElement with given
+       @param D    Reference space dimension
+       @param G    Geometry type (of type Geometry::Type)
+       @param Do   Number of degrees of freedom in the FiniteElement
+       @param O    Order/degree of the FiniteElement
+       @param F    FunctionSpace type of the FiniteElement
+   */
+   PositiveFiniteElement(int D, Geometry::Type G, int Do, int O,
+                         int F = FunctionSpace::Pk) :
+      ScalarFiniteElement(D, G, Do, O, F)
+   { }
+
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { ScalarLocalInterpolation(Trans, I, *this); }
+
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { ScalarLocalRestriction(Trans, R, *this); }
+
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { CheckScalarFE(fe).ScalarLocalInterpolation(Trans, I, *this); }
+
+   using FiniteElement::Project;
+
+   // Low-order monotone "projection" (actually it is not a projection): the
+   // dofs are set to be the Coefficient values at the nodes.
+   virtual void Project(Coefficient &coeff,
+                        ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                        DenseMatrix &I) const;
+};
+
+
+class PositiveTensorFiniteElement : public PositiveFiniteElement,
+   public TensorBasisElement
+{
+public:
+   PositiveTensorFiniteElement(const int dims, const int p,
+                               const DofMapType dmtype);
+
+   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
+                                 DofToQuad::Mode mode) const
+   {
+      return (mode == DofToQuad::FULL) ?
+             ScalarFiniteElement::GetDofToQuad(ir, mode) :
+             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
+   }
+};
+
+
+/// A 2D positive bi-quadratic element on a square utilizing the 2nd order
+/// Bernstein basis
+class BiQuadPos2DFiniteElement : public PositiveFiniteElement
+{
+public:
+   /// Construct the BiQuadPos2DFiniteElement
+   BiQuadPos2DFiniteElement();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const;
+   using FiniteElement::Project;
+   virtual void Project(Coefficient &coeff, ElementTransformation &Trans,
+                        Vector &dofs) const;
+   virtual void Project(VectorCoefficient &vc, ElementTransformation &Trans,
+                        Vector &dofs) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs = 0.; dofs(vertex) = 1.; }
+};
+
+
+/// A 1D quadratic positive element utilizing the 2nd order Bernstein basis
+class QuadPos1DFiniteElement : public PositiveFiniteElement
+{
+public:
+   /// Construct the QuadPos1DFiniteElement
+   QuadPos1DFiniteElement();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+
+/// Arbitrary order H1 elements in 1D utilizing the Bernstein basis
+class H1Pos_SegmentElement : public PositiveTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   // This is to share scratch space between invocations, which helps speed
+   // things up, but with OpenMP, we need one copy per thread. Right now, we
+   // solve this by allocating this space within each function call every time
+   // we call it. Alternatively, we should do some sort thread private thing.
+   // Brunner, Jan 2014
+   mutable Vector shape_x, dshape_x;
+#endif
+
+public:
+   /// Construct the H1Pos_SegmentElement of order @a p
+   H1Pos_SegmentElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order H1 elements in 2D utilizing the Bernstein basis on a square
+class H1Pos_QuadrilateralElement : public PositiveTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   // See comment in H1Pos_SegmentElement
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y;
+#endif
+
+public:
+   /// Construct the H1Pos_QuadrilateralElement of order @a p
+   H1Pos_QuadrilateralElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order H1 elements in 3D utilizing the Bernstein basis on a cube
+class H1Pos_HexahedronElement : public PositiveTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   // See comment in H1Pos_SegmentElement.
+   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z;
+#endif
+
+public:
+   /// Construct the H1Pos_HexahedronElement of order @a p
+   H1Pos_HexahedronElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order H1 elements in 2D utilizing the Bernstein basis on a triangle
+class H1Pos_TriangleElement : public PositiveFiniteElement
+{
+protected:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector m_shape, dshape_1d;
+   mutable DenseMatrix m_dshape;
+#endif
+   Array<int> dof_map;
+
+public:
+   /// Construct the H1Pos_TriangleElement of order @a p
+   H1Pos_TriangleElement(const int p);
+
+   // The size of shape is (p+1)(p+2)/2 (dof).
+   static void CalcShape(const int p, const double x, const double y,
+                         double *shape);
+
+   // The size of dshape_1d is p+1; the size of dshape is (dof x dim).
+   static void CalcDShape(const int p, const double x, const double y,
+                          double *dshape_1d, double *dshape);
+
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+
+/// Arbitrary order H1 elements in 3D utilizing the Bernstein basis on a
+/// tetrahedron
+class H1Pos_TetrahedronElement : public PositiveFiniteElement
+{
+protected:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector m_shape, dshape_1d;
+   mutable DenseMatrix m_dshape;
+#endif
+   Array<int> dof_map;
+
+public:
+   /// Construct the H1Pos_TetrahedronElement of order @a p
+   H1Pos_TetrahedronElement(const int p);
+
+   // The size of shape is (p+1)(p+2)(p+3)/6 (dof).
+   static void CalcShape(const int p, const double x, const double y,
+                         const double z, double *shape);
+
+   // The size of dshape_1d is p+1; the size of dshape is (dof x dim).
+   static void CalcDShape(const int p, const double x, const double y,
+                          const double z, double *dshape_1d, double *dshape);
+
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+
+/// Arbitrary order H1 elements in 3D utilizing the Bernstein basis on a wedge
+class H1Pos_WedgeElement : public PositiveFiniteElement
+{
+protected:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector t_shape, s_shape;
+   mutable DenseMatrix t_dshape, s_dshape;
+#endif
+   Array<int> t_dof, s_dof;
+
+   H1Pos_TriangleElement TriangleFE;
+   H1Pos_SegmentElement  SegmentFE;
+
+public:
+   /// Construct the H1Pos_WedgeElement of order @a p
+   H1Pos_WedgeElement(const int p);
+
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+
+/// Arbitrary order L2 elements in 1D utilizing the Bernstein basis on a segment
+class L2Pos_SegmentElement : public PositiveTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, dshape_x;
+#endif
+
+public:
+   /// Construct the L2Pos_SegmentElement of order @a p
+   L2Pos_SegmentElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 2D utilizing the Bernstein basis on a square
+class L2Pos_QuadrilateralElement : public PositiveTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y;
+#endif
+
+public:
+   /// Construct the L2Pos_QuadrilateralElement of order @a p
+   L2Pos_QuadrilateralElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 3D utilizing the Bernstein basis on a cube
+class L2Pos_HexahedronElement : public PositiveTensorFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z;
+#endif
+
+public:
+   /// Construct the L2Pos_HexahedronElement of order @a p
+   L2Pos_HexahedronElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 2D utilizing the Bernstein basis on a triangle
+class L2Pos_TriangleElement : public PositiveFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector dshape_1d;
+#endif
+
+public:
+   /// Construct the L2Pos_TriangleElement of order @a p
+   L2Pos_TriangleElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 3D utilizing the Bernstein basis on a
+/// tetrahedron
+class L2Pos_TetrahedronElement : public PositiveFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector dshape_1d;
+#endif
+
+public:
+   /// Construct the L2Pos_TetrahedronElement of order @a p
+   L2Pos_TetrahedronElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const;
+};
+
+
+/// Arbitrary order L2 elements in 3D utilizing the Bernstein basis on a wedge
+class L2Pos_WedgeElement : public PositiveFiniteElement
+{
+protected:
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector t_shape, s_shape;
+   mutable DenseMatrix t_dshape, s_dshape;
+#endif
+   Array<int> t_dof, s_dof;
+
+   L2Pos_TriangleElement TriangleFE;
+   L2Pos_SegmentElement  SegmentFE;
+
+public:
+   /// Construct the L2Pos_WedgeElement of order @a p
+   L2Pos_WedgeElement(const int p);
+
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+} // namespace mfem
+
+#endif

fem/fe/fe_rt.hpp

@@ -0,0 +1,268 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_RT
+#define MFEM_FE_RT
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+/// Arbitrary order Raviart-Thomas elements in 2D on a square
+class RT_QuadrilateralElement : public VectorTensorFiniteElement
+{
+private:
+   static const double nk[8];
+
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_cx, shape_ox, shape_cy, shape_oy;
+   mutable Vector dshape_cx, dshape_cy;
+#endif
+   Array<int> dof2nk;
+   const double *cp;
+
+public:
+   /** @brief Construct the RT_QuadrilateralElement of order @a p and closed and
+       open BasisType @a cb_type and @a ob_type */
+   RT_QuadrilateralElement(const int p,
+                           const int cb_type = BasisType::GaussLobatto,
+                           const int ob_type = BasisType::GaussLegendre);
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_RT(*this, nk, dof2nk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_RT(nk, dof2nk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_RT(CheckVectorFE(fe), nk, dof2nk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   {
+      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
+      else { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_RT(nk, dof2nk, fe, Trans, I); }
+   // Gradient + rotation = Curl: H1 -> H(div)
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
+   // Curl = Gradient + rotation: H1 -> H(div)
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+
+protected:
+   void ProjectIntegrated(VectorCoefficient &vc, ElementTransformation &Trans,
+                          Vector &dofs) const;
+};
+
+
+/// Arbitrary order Raviart-Thomas elements in 3D on a cube
+class RT_HexahedronElement : public VectorTensorFiniteElement
+{
+   static const double nk[18];
+
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_cx, shape_ox, shape_cy, shape_oy, shape_cz, shape_oz;
+   mutable Vector dshape_cx, dshape_cy, dshape_cz;
+#endif
+   Array<int> dof2nk;
+   const double *cp;
+
+public:
+   /** @brief Construct the RT_HexahedronElement of order @a p and closed and
+       open BasisType @a cb_type and @a ob_type */
+   RT_HexahedronElement(const int p,
+                        const int cb_type = BasisType::GaussLobatto,
+                        const int ob_type = BasisType::GaussLegendre);
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_RT(*this, nk, dof2nk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_RT(nk, dof2nk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_RT(CheckVectorFE(fe), nk, dof2nk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   {
+      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
+      else { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_RT(nk, dof2nk, fe, Trans, I); }
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+
+protected:
+   void ProjectIntegrated(VectorCoefficient &vc,
+                          ElementTransformation &Trans,
+                          Vector &dofs) const;
+};
+
+
+/// Arbitrary order Raviart-Thomas elements in 2D on a triangle
+class RT_TriangleElement : public VectorFiniteElement
+{
+   static const double nk[6], c;
+
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_l;
+   mutable Vector dshape_x, dshape_y, dshape_l;
+   mutable DenseMatrix u;
+   mutable Vector divu;
+#endif
+   Array<int> dof2nk;
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the RT_TriangleElement of order @a p
+   RT_TriangleElement(const int p);
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_RT(*this, nk, dof2nk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_RT(nk, dof2nk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_RT(CheckVectorFE(fe), nk, dof2nk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_RT(nk, dof2nk, fe, Trans, I); }
+   // Gradient + rotation = Curl: H1 -> H(div)
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
+   // Curl = Gradient + rotation: H1 -> H(div)
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+};
+
+
+/// Arbitrary order Raviart-Thomas elements in 3D on a tetrahedron
+class RT_TetrahedronElement : public VectorFiniteElement
+{
+   static const double nk[12], c;
+
+#ifndef MFEM_THREAD_SAFE
+   mutable Vector shape_x, shape_y, shape_z, shape_l;
+   mutable Vector dshape_x, dshape_y, dshape_z, dshape_l;
+   mutable DenseMatrix u;
+   mutable Vector divu;
+#endif
+   Array<int> dof2nk;
+   DenseMatrixInverse Ti;
+
+public:
+   /// Construct the RT_TetrahedronElement of order @a p
+   RT_TetrahedronElement(const int p);
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { LocalInterpolation_RT(*this, nk, dof2nk, Trans, I); }
+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { LocalRestriction_RT(nk, dof2nk, Trans, R); }
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { LocalInterpolation_RT(CheckVectorFE(fe), nk, dof2nk, Trans, I); }
+   using FiniteElement::Project;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectMatrixCoefficient(
+      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
+   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
+   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                        DenseMatrix &I) const
+   { Project_RT(nk, dof2nk, fe, Trans, I); }
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+};
+
+} // namespace mfem
+
+#endif

fem/fe/fe_ser.cpp

@@ -0,0 +1,247 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Serendipity Finite Element classes
+
+#include "fe_ser.hpp"
+#include "fe_fixed_order.hpp"
+
+namespace mfem
+{
+
+using namespace std;
+
+H1Ser_QuadrilateralElement::H1Ser_QuadrilateralElement(const int p)
+   : ScalarFiniteElement(2, Geometry::SQUARE, (p*p + 3*p +6) / 2, p,
+                         FunctionSpace::Qk)
+{
+   // Store the dof_map of the associated TensorBasisElement, which will be used
+   // to create the serendipity dof map.  Its size is larger than the size of
+   // the serendipity element.
+   TensorBasisElement tbeTemp =
+      TensorBasisElement(2, p, BasisType::GaussLobatto,
+                         TensorBasisElement::DofMapType::Sr_DOF_MAP);
+   const Array<int> tp_dof_map = tbeTemp.GetDofMap();
+
+   const double *cp = poly1d.ClosedPoints(p, BasisType::GaussLobatto);
+
+   // Fixing the Nodes is exactly the same as the H1_QuadrilateralElement
+   // constructor except we only use those values of the associated tensor
+   // product dof_map that are <= the number of serendipity Dofs e.g. only DoFs
+   // 0-7 out of the 9 tensor product dofs (at quadratic order)
+   int o = 0;
+
+   for (int j = 0; j <= p; j++)
+   {
+      for (int i = 0; i <= p; i++)
+      {
+         if (tp_dof_map[o] < Nodes.Size())
+         {
+            Nodes.IntPoint(tp_dof_map[o]).x = cp[i];
+            Nodes.IntPoint(tp_dof_map[o]).y = cp[j];
+         }
+         o++;
+      }
+   }
+}
+
+void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
+                                           Vector &shape) const
+{
+   int p = (this)->GetOrder();
+   double x = ip.x, y = ip.y;
+
+   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Vector nodalX(p+1);
+   Vector nodalY(p+1);
+
+   edgeNodalBasis.Eval(x, nodalX);
+   edgeNodalBasis.Eval(y, nodalY);
+
+   // First, fix edge-based shape functions. Use a nodal interpolant for edge
+   // points, weighted by the linear function that vanishes on opposite edge.
+   for (int i = 0; i < p-1; i++)
+   {
+      shape(4 + 0*(p-1) + i) = (nodalX(i+1))*(1.-y);         // south edge 0->1
+      shape(4 + 1*(p-1) + i) = (nodalY(i+1))*x;              // east edge  1->2
+      shape(4 + 3*(p-1) - i - 1) = (nodalX(i+1)) * y;        // north edge 3->2
+      shape(4 + 4*(p-1) - i - 1) = (nodalY(i+1)) * (1. - x); // west edge  0->3
+   }
+
+   BiLinear2DFiniteElement bilinear = BiLinear2DFiniteElement();
+   Vector bilinearsAtIP(4);
+   bilinear.CalcShape(ip, bilinearsAtIP);
+
+   const double *edgePts(poly1d.ClosedPoints(p, BasisType::GaussLobatto));
+
+   // Next, set the shape function associated with vertex V, evaluated at (x,y)
+   // to be: bilinear function associated to V, evaluated at (x,y) - sum (shape
+   // function at edge point P, weighted by bilinear function for V evaluated at
+   // P) where the sum is taken only for points P on edges incident to V.
+
+   double vtx0fix =0;
+   double vtx1fix =0;
+   double vtx2fix =0;
+   double vtx3fix =0;
+   for (int i = 0; i<p-1; i++)
+   {
+      vtx0fix += (1-edgePts[i+1])*(shape(4 + i) +
+                                   shape(4 + 4*(p-1) - i - 1)); // bot+left edge
+      vtx1fix += (1-edgePts[i+1])*(shape(4 + 1*(p-1) + i) +
+                                   shape(4 + (p-2)-i));        // right+bot edge
+      vtx2fix += (1-edgePts[i+1])*(shape(4 + 2*(p-1) + i) +
+                                   shape(1 + 2*p-i));          // top+right edge
+      vtx3fix += (1-edgePts[i+1])*(shape(4 + 3*(p-1) + i) +
+                                   shape(3*p - i));            // left+top edge
+   }
+   shape(0) = bilinearsAtIP(0) - vtx0fix;
+   shape(1) = bilinearsAtIP(1) - vtx1fix;
+   shape(2) = bilinearsAtIP(2) - vtx2fix;
+   shape(3) = bilinearsAtIP(3) - vtx3fix;
+
+   // Interior basis functions appear starting at order p=4. These are non-nodal
+   // bubble functions.
+   if (p > 3)
+   {
+      double *legX = new double[p-1];
+      double *legY = new double[p-1];
+      Poly_1D *storeLegendre = new Poly_1D();
+
+      storeLegendre->CalcLegendre(p-2, x, legX);
+      storeLegendre->CalcLegendre(p-2, y, legY);
+
+      int interior_total = 0;
+      for (int j = 4; j < p + 1; j++)
+      {
+         for (int k = 0; k < j-3; k++)
+         {
+            shape(4 + 4*(p-1) + interior_total)
+               = legX[k] * legY[j-4-k] * x * (1. - x) * y * (1. - y);
+            interior_total++;
+         }
+      }
+
+      delete[] legX;
+      delete[] legY;
+      delete storeLegendre;
+   }
+}
+
+void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
+                                            DenseMatrix &dshape) const
+{
+   int p = (this)->GetOrder();
+   double x = ip.x, y = ip.y;
+
+   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Vector nodalX(p+1);
+   Vector DnodalX(p+1);
+   Vector nodalY(p+1);
+   Vector DnodalY(p+1);
+
+   edgeNodalBasis.Eval(x, nodalX, DnodalX);
+   edgeNodalBasis.Eval(y, nodalY, DnodalY);
+
+   for (int i = 0; i < p-1; i++)
+   {
+      dshape(4 + 0*(p-1) + i,0) =  DnodalX(i+1) * (1.-y);
+      dshape(4 + 0*(p-1) + i,1) = -nodalX(i+1);
+      dshape(4 + 1*(p-1) + i,0) =  nodalY(i+1);
+      dshape(4 + 1*(p-1) + i,1) =  DnodalY(i+1)*x;
+      dshape(4 + 3*(p-1) - i - 1,0) =  DnodalX(i+1)*y;
+      dshape(4 + 3*(p-1) - i - 1,1) =  nodalX(i+1);
+      dshape(4 + 4*(p-1) - i - 1,0) = -nodalY(i+1);
+      dshape(4 + 4*(p-1) - i - 1,1) =  DnodalY(i+1) * (1.-x);
+   }
+
+   BiLinear2DFiniteElement bilinear = BiLinear2DFiniteElement();
+   DenseMatrix DbilinearsAtIP(4);
+   bilinear.CalcDShape(ip, DbilinearsAtIP);
+
+   const double *edgePts(poly1d.ClosedPoints(p, BasisType::GaussLobatto));
+
+   dshape(0,0) = DbilinearsAtIP(0,0);
+   dshape(0,1) = DbilinearsAtIP(0,1);
+   dshape(1,0) = DbilinearsAtIP(1,0);
+   dshape(1,1) = DbilinearsAtIP(1,1);
+   dshape(2,0) = DbilinearsAtIP(2,0);
+   dshape(2,1) = DbilinearsAtIP(2,1);
+   dshape(3,0) = DbilinearsAtIP(3,0);
+   dshape(3,1) = DbilinearsAtIP(3,1);
+
+   for (int i = 0; i<p-1; i++)
+   {
+      dshape(0,0) -= (1-edgePts[i+1])*(dshape(4 + 0*(p-1) + i, 0) +
+                                       dshape(4 + 4*(p-1) - i - 1,0));
+      dshape(0,1) -= (1-edgePts[i+1])*(dshape(4 + 0*(p-1) + i, 1) +
+                                       dshape(4 + 4*(p-1) - i - 1,1));
+      dshape(1,0) -= (1-edgePts[i+1])*(dshape(4 + 1*(p-1) + i, 0) +
+                                       dshape(4 + (p-2)-i, 0));
+      dshape(1,1) -= (1-edgePts[i+1])*(dshape(4 + 1*(p-1) + i, 1) +
+                                       dshape(4 + (p-2)-i, 1));
+      dshape(2,0) -= (1-edgePts[i+1])*(dshape(4 + 2*(p-1) + i, 0) +
+                                       dshape(1 + 2*p-i, 0));
+      dshape(2,1) -= (1-edgePts[i+1])*(dshape(4 + 2*(p-1) + i, 1) +
+                                       dshape(1 + 2*p-i, 1));
+      dshape(3,0) -= (1-edgePts[i+1])*(dshape(4 + 3*(p-1) + i, 0) +
+                                       dshape(3*p - i, 0));
+      dshape(3,1) -= (1-edgePts[i+1])*(dshape(4 + 3*(p-1) + i, 1) +
+                                       dshape(3*p - i, 1));
+   }
+
+   if (p > 3)
+   {
+      double *legX = new double[p-1];
+      double *legY = new double[p-1];
+      double *DlegX = new double[p-1];
+      double *DlegY = new double[p-1];
+      Poly_1D *storeLegendre = new Poly_1D();
+
+      storeLegendre->CalcLegendre(p-2, x, legX, DlegX);
+      storeLegendre->CalcLegendre(p-2, y, legY, DlegY);
+
+      int interior_total = 0;
+      for (int j = 4; j < p + 1; j++)
+      {
+         for (int k = 0; k < j-3; k++)
+         {
+            dshape(4 + 4*(p-1) + interior_total, 0) =
+               legY[j-4-k]*y*(1-y) * (DlegX[k]*x*(1-x) + legX[k]*(1-2*x));
+            dshape(4 + 4*(p-1) + interior_total, 1) =
+               legX[k]*x*(1-x) * (DlegY[j-4-k]*y*(1-y) + legY[j-4-k]*(1-2*y));
+            interior_total++;
+         }
+      }
+      delete[] legX;
+      delete[] legY;
+      delete[] DlegX;
+      delete[] DlegY;
+      delete storeLegendre;
+   }
+}
+
+void H1Ser_QuadrilateralElement::GetLocalInterpolation(ElementTransformation
+                                                       &Trans,
+                                                       DenseMatrix &I) const
+{
+   // For p<=4, the basis is nodal; for p>4, the quad-interior functions are
+   // non-nodal.
+   if (order <= 4)
+   {
+      NodalLocalInterpolation(Trans, I, *this);
+   }
+   else
+   {
+      ScalarLocalInterpolation(Trans, I, *this);
+   }
+}
+
+}

fem/fe/fe_ser.hpp

@@ -0,0 +1,37 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_SERENDIPITY
+#define MFEM_FE_SERENDIPITY
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+/// Arbitrary order H1 serendipity elements in 2D on a quad
+class H1Ser_QuadrilateralElement : public ScalarFiniteElement
+{
+public:
+   /// Construct the H1Ser_QuadrilateralElement of order @a p
+   H1Ser_QuadrilateralElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const;
+   using FiniteElement::Project;
+};
+
+
+} // namespace mfem
+
+#endif

fem/fespace.cpp

@@ -106,8 +106,11 @@ void FiniteElementSpace::CopyProlongationAndRestriction(
    SparseMatrix *perm_mat = NULL, *perm_mat_tr = NULL;
    if (perm)
    {
+      // Note: although n and fes.GetVSize() are typically equal, in
+      // variable-order spaces they may differ, since nonconforming edges/faces
+      // my have fictitious DOFs.
       int n = perm->Size();
-      perm_mat = new SparseMatrix(n, n);
+      perm_mat = new SparseMatrix(n, fes.GetVSize());
       for (int i=0; i<n; ++i)
       {
          double s;
@@ -124,11 +127,22 @@ void FiniteElementSpace::CopyProlongationAndRestriction(
       else { cP = new SparseMatrix(*fes.GetConformingProlongation()); }
       cP_is_set = true;
    }
+   else if (perm != NULL)
+   {
+      cP = perm_mat;
+      cP_is_set = true;
+      perm_mat = NULL;
+   }
    if (fes.GetConformingRestriction() != NULL)
    {
       if (perm) { cR = Mult(*fes.GetConformingRestriction(), *perm_mat_tr); }
       else { cR = new SparseMatrix(*fes.GetConformingRestriction()); }
    }
+   else if (perm != NULL)
+   {
+      cR = perm_mat_tr;
+      perm_mat_tr = NULL;
+   }
 
    delete perm_mat;
    delete perm_mat_tr;

fem/gridfunc.cpp

@@ -502,25 +502,24 @@ const
    {
       doftrans->InvTransformPrimal(loc_data);
    }
-   for (int k = 0; k < n; k++)
-      if (FElem->GetMapType() == FiniteElement::VALUE)
+   if (FElem->GetMapType() == FiniteElement::VALUE)
+   {
+      for (int k = 0; k < n; k++)
       {
-         for (int k = 0; k < n; k++)
-         {
-            FElem->CalcShape(ir.IntPoint(k), DofVal);
-            vals(k) = DofVal * loc_data;
-         }
+         FElem->CalcShape(ir.IntPoint(k), DofVal);
+         vals(k) = DofVal * loc_data;
       }
-      else
+   }
+   else
+   {
+      ElementTransformation *Tr = fes->GetElementTransformation(i);
+      for (int k = 0; k < n; k++)
       {
-         ElementTransformation *Tr = fes->GetElementTransformation(i);
-         for (int k = 0; k < n; k++)
-         {
-            Tr->SetIntPoint(&ir.IntPoint(k));
-            FElem->CalcPhysShape(*Tr, DofVal);
-            vals(k) = DofVal * loc_data;
-         }
+         Tr->SetIntPoint(&ir.IntPoint(k));
+         FElem->CalcPhysShape(*Tr, DofVal);
+         vals(k) = DofVal * loc_data;
       }
+   }
 }
 
 void GridFunction::GetValues(int i, const IntegrationRule &ir, Vector &vals,
@@ -1342,21 +1341,20 @@ void GridFunction::ProjectVectorFieldOn(GridFunction &vec_field, int comp)
    }
 }
 
-void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
+void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
+                                                      GridFunction &der,
+                                                      Array<int> &zones_per_dof)
 {
    FiniteElementSpace * der_fes = der.FESpace();
    ElementTransformation * transf;
-   Array<int> overlap(der_fes->GetVSize());
+   zones_per_dof.SetSize(der_fes->GetVSize());
    Array<int> der_dofs, vdofs;
    DenseMatrix dshape, inv_jac;
    Vector pt_grad, loc_func;
    int i, j, k, dim, dof, der_dof, ind;
    double a;
 
-   for (i = 0; i < overlap.Size(); i++)
-   {
-      overlap[i] = 0;
-   }
+   zones_per_dof = 0;
    der = 0.0;
 
    comp--;
@@ -1391,11 +1389,17 @@ void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
             a += inv_jac(j, der_comp) * pt_grad(j);
          }
          der(der_dofs[k]) += a;
-         overlap[der_dofs[k]]++;
+         zones_per_dof[der_dofs[k]]++;
       }
    }
+}
 
-   for (i = 0; i < overlap.Size(); i++)
+void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
+{
+   Array<int> overlap;
+   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
+
+   for (int i = 0; i < overlap.Size(); i++)
    {
       der(i) /= overlap[i];
    }
@@ -2257,9 +2261,10 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
       }
       fe = fes->GetBE(i);
       T = fes->GetBdrElementTransformation(i);
-      fes->GetBdrElementDofs(i, dofs);
+      DofTransformation *dof_tr = fes->GetBdrElementDofs(i, dofs);
       lvec.SetSize(fe->GetDof());
       fe->Project(vcoeff, *T, lvec);
+      if (dof_tr) { dof_tr->TransformPrimal(lvec); }
       accumulate_dofs(dofs, lvec, *this, values_counter);
    }
 

fem/gridfunc.hpp

@@ -310,6 +310,16 @@ public:
 
    void ProjectVectorFieldOn(GridFunction &vec_field, int comp = 0);
 
+   /** @brief Compute a certain derivative of a function's component.
+       Derivatives of the function are computed at the DOF locations of @a der,
+       and averaged over overlapping DOFs. Thus this function projects the
+       derivative to the FiniteElementSpace of @a der.
+       @param[in]  comp  Index of the function's component to be differentiated.
+                         The index is 1-based, i.e., use 1 for scalar functions.
+       @param[in]  der_comp  Use 0/1/2 for derivatives in x/y/z directions.
+       @param[out] der       The resulting derivative (scalar function). The
+                             FiniteElementSpace of this function must be set
+                             before the call. */
    void GetDerivative(int comp, int der_comp, GridFunction &der);
 
    double GetDivergence(ElementTransformation &tr) const;
@@ -411,6 +421,12 @@ protected:
    void AccumulateAndCountZones(VectorCoefficient &vcoeff, AvgType type,
                                 Array<int> &zones_per_vdof);
 
+   /** @brief Used for the serial and parallel implementations of the
+       GetDerivative() method; see its documentation. */
+   void AccumulateAndCountDerivativeValues(int comp, int der_comp,
+                                           GridFunction &der,
+                                           Array<int> &zones_per_dof);
+
    void AccumulateAndCountBdrValues(Coefficient *coeff[],
                                     VectorCoefficient *vcoeff, Array<int> &attr,
                                     Array<int> &values_counter);

fem/intrules.cpp

@@ -955,7 +955,7 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
             IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
             int RealOrder = Order;
             while (RealOrder+1 < ir_array->Size() &&
-            /*  */ (*ir_array)[RealOrder+1] == ir)
+                   (*ir_array)[RealOrder+1] == ir)
             {
                RealOrder++;
             }

fem/lor.cpp

@@ -34,7 +34,8 @@ void LORBase::AddIntegratorsAndMarkers(BilinearForm &a_from,
                                        BilinearForm &a_to,
                                        GetIntegratorsFn get_integrators,
                                        GetMarkersFn get_markers,
-                                       AddIntegratorMarkersFn add_integrator,
+                                       AddIntegratorMarkersFn add_integrator_marker,
+                                       AddIntegratorFn add_integrator,
                                        const IntegrationRule *ir)
 {
    Array<BilinearFormIntegrator*> *integrators = (a_from.*get_integrators)();
@@ -42,7 +43,14 @@ void LORBase::AddIntegratorsAndMarkers(BilinearForm &a_from,
 
    for (int i=0; i<integrators->Size(); ++i)
    {
-      (a_to.*add_integrator)((*integrators)[i], *(*markers[i]));
+      if (*markers[i])
+      {
+         (a_to.*add_integrator_marker)((*integrators)[i], *(*markers[i]));
+      }
+      else
+      {
+         (a_to.*add_integrator)((*integrators)[i]);
+      }
       ir_map[(*integrators)[i]] = ((*integrators)[i])->GetIntegrationRule();
       if (ir) { ((*integrators)[i])->SetIntegrationRule(*ir); }
    }
@@ -92,13 +100,29 @@ void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
    int dim = mesh_lor.Dimension();
    const CoarseFineTransformations &cf_tr = mesh_lor.GetRefinementTransforms();
 
+   using GeomRef = std::pair<Geometry::Type, int>;
+   std::map<GeomRef, int> point_matrices_offsets;
    perm_.SetSize(fes_lor.GetVSize());
 
    Array<int> vdof_ho, vdof_lor;
    for (int ilor=0; ilor<mesh_lor.GetNE(); ++ilor)
    {
       int iho = cf_tr.embeddings[ilor].parent;
+      int p = fes_ho.GetOrder(iho);
       int lor_index = cf_tr.embeddings[ilor].matrix;
+      // We use the point matrix index to identify the local LOR element index
+      // within the high-order coarse element.
+      //
+      // In variable-order spaces, the point matrices for each order are
+      // concatenated sequentially, so for the given element order, we need to
+      // find the offset that will give us the point matrix index relative to
+      // the current element order only.
+      GeomRef id(mesh_lor.GetElementBaseGeometry(ilor), p);
+      if (point_matrices_offsets.find(id) == point_matrices_offsets.end())
+      {
+         point_matrices_offsets[id] = lor_index;
+      }
+      lor_index -= point_matrices_offsets[id];
 
       fes_ho.GetElementVDofs(iho, vdof_ho);
       fes_lor.GetElementVDofs(ilor, vdof_lor);
@@ -109,7 +133,6 @@ void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
          continue;
       }
 
-      int p = fes_ho.GetOrder(iho);
       int p1 = p+1;
       int ndof_per_dim = (dim == 2) ? p*p1 : type == ND ? p*p1*p1 : p*p*p1;
 
@@ -181,7 +204,7 @@ void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
 void LORBase::ConstructDofPermutation() const
 {
    FESpaceType type = GetFESpaceType();
-   if (type == H1 || type == L2 || nonconforming)
+   if (type == H1 || type == L2)
    {
       // H1 and L2: no permutation necessary, return identity
       perm.SetSize(fes->GetTrueVSize());
@@ -226,10 +249,10 @@ const Array<int> &LORBase::GetDofPermutation() const
    return perm;
 }
 
-bool LORBase::RequiresDofPermutation() const
+bool LORBase::HasSameDofNumbering() const
 {
    FESpaceType type = GetFESpaceType();
-   return (type == H1 || type == L2 || nonconforming) ? false : true;
+   return type == H1 || type == L2;
 }
 
 const OperatorHandle &LORBase::GetAssembledSystem() const
@@ -238,7 +261,7 @@ const OperatorHandle &LORBase::GetAssembledSystem() const
    return A;
 }
 
-void LORBase::AssembleSystem(BilinearForm &a_ho, const Array<int> &ess_dofs)
+void LORBase::AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs)
 {
    a->UseExternalIntegrators();
    AddIntegrators(a_ho, *a, &BilinearForm::GetDBFI,
@@ -247,40 +270,23 @@ void LORBase::AssembleSystem(BilinearForm &a_ho, const Array<int> &ess_dofs)
                   &BilinearForm::AddInteriorFaceIntegrator, ir_face);
    AddIntegratorsAndMarkers(a_ho, *a, &BilinearForm::GetBBFI,
                             &BilinearForm::GetBBFI_Marker,
+                            &BilinearForm::AddBoundaryIntegrator,
                             &BilinearForm::AddBoundaryIntegrator, ir_face);
    AddIntegratorsAndMarkers(a_ho, *a, &BilinearForm::GetBFBFI,
                             &BilinearForm::GetBFBFI_Marker,
+                            &BilinearForm::AddBdrFaceIntegrator,
                             &BilinearForm::AddBdrFaceIntegrator, ir_face);
    a->Assemble();
-   if (RequiresDofPermutation())
-   {
-      const Array<int> &p = GetDofPermutation();
-      // Form inverse permutation: given high-order dof i, pi[i] is corresp. LO
-      Array<int> pi(p.Size());
-      for (int i=0; i<p.Size(); ++i)
-      {
-         pi[absdof(p[i])] = i;
-      }
-      Array<int> ess_dofs_perm(ess_dofs.Size());
-      for (int i=0; i<ess_dofs.Size(); ++i)
-      {
-         ess_dofs_perm[i] = pi[ess_dofs[i]];
-      }
-      a->FormSystemMatrix(ess_dofs_perm, A);
-   }
-   else
-   {
-      a->FormSystemMatrix(ess_dofs, A);
-   }
+   a->FormSystemMatrix(ess_dofs, A);
    ResetIntegrationRules(&BilinearForm::GetDBFI);
    ResetIntegrationRules(&BilinearForm::GetFBFI);
    ResetIntegrationRules(&BilinearForm::GetBBFI);
    ResetIntegrationRules(&BilinearForm::GetBFBFI);
 }
 
-void LORBase::SetupNonconforming()
+void LORBase::SetupProlongationAndRestriction()
 {
-   if (RequiresDofPermutation())
+   if (!HasSameDofNumbering())
    {
       Array<int> p;
       ConstructLocalDofPermutation(p);
@@ -290,7 +296,6 @@ void LORBase::SetupNonconforming()
    {
       fes->CopyProlongationAndRestriction(fes_ho, NULL);
    }
-   nonconforming = true;
 }
 
 template <typename FEC>
@@ -373,7 +378,6 @@ LORDiscretization::LORDiscretization(BilinearForm &a_ho_,
                                      int ref_type)
    : LORDiscretization(*a_ho_.FESpace(), ref_type)
 {
-   a = new BilinearForm(fes);
    AssembleSystem(a_ho_, ess_tdof_list);
 }
 
@@ -382,23 +386,32 @@ LORDiscretization::LORDiscretization(FiniteElementSpace &fes_ho,
 {
    CheckBasisType(fes_ho);
 
-   // TODO: support variable-order spaces
-   MFEM_VERIFY(!fes_ho.IsVariableOrder(),
-               "Cannot construct LOR operators on variable-order spaces");
-
-   int order = fes_ho.GetMaxElementOrder();
-   if (GetFESpaceType() == L2) { ++order; }
-
    Mesh &mesh_ho = *fes_ho.GetMesh();
-   mesh = new Mesh(Mesh::MakeRefined(mesh_ho, order, ref_type));
+   // For H1, ND and RT spaces, use refinement = element order, for DG spaces,
+   // use refinement = element order + 1 (since LOR is p = 0 in this case).
+   int increment = (GetFESpaceType() == L2) ? 1 : 0;
+   Array<int> refinements(mesh_ho.GetNE());
+   for (int i=0; i<refinements.Size(); ++i)
+   {
+      refinements[i] = fes_ho.GetOrder(i) + increment;
+   }
+   mesh = new Mesh(Mesh::MakeRefined(mesh_ho, refinements, ref_type));
 
    fec = fes_ho.FEColl()->Clone(GetLOROrder());
    fes = new FiniteElementSpace(mesh, fec);
-   if (fes_ho.Nonconforming()) { SetupNonconforming(); }
+   SetupProlongationAndRestriction();
 
    A.SetType(Operator::MFEM_SPARSEMAT);
 }
 
+void LORDiscretization::AssembleSystem(BilinearForm &a_ho,
+                                       const Array<int> &ess_dofs)
+{
+   delete a;
+   a = new BilinearForm(&GetFESpace());
+   AssembleSystem_(a_ho, ess_dofs);
+}
+
 SparseMatrix &LORDiscretization::GetAssembledMatrix() const
 {
    MFEM_VERIFY(a != NULL && A.Ptr() != NULL, "No LOR system assembled");
@@ -412,7 +425,6 @@ ParLORDiscretization::ParLORDiscretization(ParBilinearForm &a_ho_,
                                            int ref_type)
    : ParLORDiscretization(*a_ho_.ParFESpace(), ref_type)
 {
-   a = new ParBilinearForm(static_cast<ParFiniteElementSpace*>(fes));
    AssembleSystem(a_ho_, ess_tdof_list);
 }
 
@@ -420,7 +432,7 @@ ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
                                            int ref_type) : LORBase(fes_ho)
 {
    if (fes_ho.GetMyRank() == 0) { CheckBasisType(fes_ho); }
-   // TODO: support variable-order spaces
+   // TODO: support variable-order spaces in parallel
    MFEM_VERIFY(!fes_ho.IsVariableOrder(),
                "Cannot construct LOR operators on variable-order spaces");
 
@@ -434,11 +446,19 @@ ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
    fec = fes_ho.FEColl()->Clone(GetLOROrder());
    ParFiniteElementSpace *pfes = new ParFiniteElementSpace(pmesh, fec);
    fes = pfes;
-   if (fes_ho.Nonconforming()) { SetupNonconforming(); }
+   SetupProlongationAndRestriction();
 
    A.SetType(Operator::Hypre_ParCSR);
 }
 
+void ParLORDiscretization::AssembleSystem(ParBilinearForm &a_ho,
+                                          const Array<int> &ess_dofs)
+{
+   delete a;
+   a = new ParBilinearForm(&GetParFESpace());
+   AssembleSystem_(a_ho, ess_dofs);
+}
+
 HypreParMatrix &ParLORDiscretization::GetAssembledMatrix() const
 {
    MFEM_VERIFY(a != NULL && A.Ptr() != NULL, "No LOR system assembled");

fem/lor.hpp

@@ -35,7 +35,7 @@ private:
    /// Adds all the integrators from the BilinearForm @a a_from to @a a_to. If
    /// the mesh consists of tensor product elements, temporarily changes the
    /// integration rules of the integrators to use collocated quadrature for
-   /// better conditioning of the %LOR system.
+   /// better conditioning of the LOR system.
    void AddIntegrators(BilinearForm &a_from,
                        BilinearForm &a_to,
                        GetIntegratorsFn get_integrators,
@@ -49,11 +49,12 @@ private:
                                  BilinearForm &a_to,
                                  GetIntegratorsFn get_integrators,
                                  GetMarkersFn get_markers,
-                                 AddIntegratorMarkersFn add_integrator,
+                                 AddIntegratorMarkersFn add_integrator_marker,
+                                 AddIntegratorFn add_integrator,
                                  const IntegrationRule *ir);
 
    /// Resets the integration rules of the integrators of @a a to their original
-   /// values (after temporarily changing them for %LOR assembly).
+   /// values (after temporarily changing them for LOR assembly).
    void ResetIntegrationRules(GetIntegratorsFn get_integrators);
 
    static inline int absdof(int i) { return i < 0 ? -1-i : i; }
@@ -68,37 +69,42 @@ protected:
    BilinearForm *a;
    OperatorHandle A;
    mutable Array<int> perm;
-   bool nonconforming = false;
 
    /// Constructs the local DOF (ldof) permutation. In parallel this is used as
    /// an intermediate step in computing the DOF permutation (see
    /// ConstructDofPermutation and GetDofPermutation).
    void ConstructLocalDofPermutation(Array<int> &perm_) const;
 
-   /// Construct the permutation that maps %LOR DOFs to high-order DOFs. See
+   /// Construct the permutation that maps LOR DOFs to high-order DOFs. See
    /// GetDofPermutation.
    void ConstructDofPermutation() const;
 
-   /// Sets up the prolongation and restriction operators required for
-   /// nonconforming spaces.
-   void SetupNonconforming();
+   /// Returns true if the LOR space and HO space have the same DOF numbering
+   /// (H1 or L2 spaces), false otherwise (ND or RT spaces).
+   bool HasSameDofNumbering() const;
+
+   /// Sets up the prolongation and restriction operators required in the case
+   /// of different DOF numberings (ND or RT spaces) or nonconforming spaces.
+   void SetupProlongationAndRestriction();
 
    /// Returns the type of finite element space: H1, ND, RT or L2.
    FESpaceType GetFESpaceType() const;
 
-   /// Returns the order of the %LOR space. 1 for H1 or ND, 0 for L2 or RT.
+   /// Returns the order of the LOR space. 1 for H1 or ND, 0 for L2 or RT.
    int GetLOROrder() const;
 
+   /// Assembles the LOR system (used internally by
+   /// LORDiscretization::AssembleSystem and
+   /// ParLORDiscretization::AssembleSystem).
+   void AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs);
+
    LORBase(FiniteElementSpace &fes_ho_);
 
 public:
-   /// Returns the assembled %LOR system.
+   /// Returns the assembled LOR system.
    const OperatorHandle &GetAssembledSystem() const;
 
-   /// Assembles the %LOR system.
-   void AssembleSystem(BilinearForm &a_ho, const Array<int> &ess_dofs);
-
-   /// @brief Returns the permutation that maps %LOR DOFs to high-order DOFs.
+   /// @brief Returns the permutation that maps LOR DOFs to high-order DOFs.
    ///
    /// This permutation is constructed the first time it is requested, and then
    /// is cached. For H1 and L2 finite element spaces (or for nonconforming
@@ -108,16 +114,9 @@ public:
    ///
    /// For vector finite element spaces (ND and RT), the DOF permutation is
    /// nontrivial. Returns an array @a perm such that, given an index @a i of a
-   /// %LOR dof, @a perm[i] is the index of the corresponding HO dof.
+   /// LOR dof, @a perm[i] is the index of the corresponding HO dof.
    const Array<int> &GetDofPermutation() const;
 
-   /// Returns true if the %LOR spaces requires a DOF permutation (if the
-   /// corresponding %LOR and HO DOFs are numbered differently), false
-   /// otherwise. Note: permutations are not required in the case of
-   /// nonconforming spaces, since the DOF numbering is incorporated into the
-   /// prolongation operators.
-   bool RequiresDofPermutation() const;
-
    /// Returns the low-order refined finite element space.
    FiniteElementSpace &GetFESpace() const { return *fes; }
 
@@ -144,7 +143,10 @@ public:
    LORDiscretization(FiniteElementSpace &fes_ho,
                      int ref_type=BasisType::GaussLobatto);
 
-   /// Return the assembled %LOR operator as a SparseMatrix.
+   /// Assembles the LOR system corresponding to @a a_ho.
+   void AssembleSystem(BilinearForm &a_ho, const Array<int> &ess_dofs);
+
+   /// Return the assembled LOR operator as a SparseMatrix.
    SparseMatrix &GetAssembledMatrix() const;
 };
 
@@ -170,10 +172,13 @@ public:
    ParLORDiscretization(ParFiniteElementSpace &fes_ho,
                         int ref_type=BasisType::GaussLobatto);
 
-   /// Return the assembled %LOR operator as a HypreParMatrix.
+   /// Assembles the LOR system corresponding to @a a_ho.
+   void AssembleSystem(ParBilinearForm &a_ho, const Array<int> &ess_dofs);
+
+   /// Return the assembled LOR operator as a HypreParMatrix.
    HypreParMatrix &GetAssembledMatrix() const;
 
-   /// Return the %LOR ParFiniteElementSpace.
+   /// Return the LOR ParFiniteElementSpace.
    ParFiniteElementSpace &GetParFESpace() const;
 };
 
@@ -192,12 +197,11 @@ class LORSolver : public Solver
 protected:
    LORBase *lor;
    bool own_lor = true;
-   bool use_permutation = true;
    SolverType solver;
    mutable Vector px, py;
 public:
    /// @brief Create a solver of type @a SolverType, formed using the assembled
-   /// SparseMatrix of the %LOR version of @a a_ho. @see LORDiscretization
+   /// SparseMatrix of the LOR version of @a a_ho. @see LORDiscretization
    LORSolver(BilinearForm &a_ho, const Array<int> &ess_tdof_list,
              int ref_type=BasisType::GaussLobatto)
    {
@@ -207,7 +211,7 @@ public:
 
 #ifdef MFEM_USE_MPI
    /// @brief Create a solver of type @a SolverType, formed using the assembled
-   /// HypreParMatrix of the %LOR version of @a a_ho. @see ParLORDiscretization
+   /// HypreParMatrix of the LOR version of @a a_ho. @see ParLORDiscretization
    LORSolver(ParBilinearForm &a_ho, const Array<int> &ess_tdof_list,
              int ref_type=BasisType::GaussLobatto)
    {
@@ -218,8 +222,6 @@ public:
 
    /// @brief Create a solver of type @a SolverType using Operator @a op and
    /// arguments @a args.
-   ///
-   /// The object @a lor_ will be used for DOF permutations.
    template <typename... Args>
    LORSolver(const Operator &op, LORBase &lor_, Args&&... args) : solver(args...)
    {
@@ -228,7 +230,7 @@ public:
       SetOperator(op);
    }
 
-   /// @brief Create a solver of type @a SolverType using the assembled %LOR
+   /// @brief Create a solver of type @a SolverType using the assembled LOR
    /// operator represented by @a lor_.
    ///
    /// The given @a args will be used as arguments to the solver constructor.
@@ -243,42 +245,7 @@ public:
       height = solver.Height();
    }
 
-   void Mult(const Vector &x, Vector &y) const
-   {
-      if (use_permutation && lor->RequiresDofPermutation())
-      {
-         const Array<int> &p = lor->GetDofPermutation();
-         px.SetSize(x.Size());
-         py.SetSize(y.Size());
-         for (int i=0; i<x.Size(); ++i)
-         { px[i] = p[i] < 0 ? -x[-1-p[i]] : x[p[i]]; }
-
-         solver.Mult(px, py);
-
-         for (int i=0; i<y.Size(); ++i)
-         {
-            int pi = p[i];
-            int s = pi < 0 ? -1 : 1;
-            y[pi < 0 ? -1-pi : pi] = s*py[i];
-         }
-      }
-      else
-      {
-         solver.Mult(x, y);
-      }
-   }
-
-   /// @brief Enable or disable the DOF permutation (enabled by default).
-   ///
-   /// The corresponding %LOR and high-order DOFs may not have the same
-   /// numbering (for example, when using ND or RT spaces), and so a permutation
-   /// is required when applying the %LOR solver as a preconditioner for the
-   /// high-order problem. This permutation can be disabled (for example, in
-   /// order to precondition the low-order problem directly).
-   void UsePermutation(bool use_permutation_)
-   {
-      use_permutation = use_permutation_;
-   }
+   void Mult(const Vector &x, Vector &y) const { solver.Mult(x, y); }
 
    /// Access the underlying solver.
    SolverType &GetSolver() { return solver; }
@@ -286,6 +253,9 @@ public:
    /// Access the underlying solver.
    const SolverType &GetSolver() const { return solver; }
 
+   /// Access the LOR discretization object.
+   const LORBase &GetLOR() const { return *lor; }
+
    ~LORSolver() { if (own_lor) { delete lor; } }
 };
 

fem/pfespace.cpp

@@ -780,14 +780,14 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
    else
    {
       // Some shared dofs will be linear combinations of others
-      int ldof  = GetVSize();
-      int ltdof = TrueVSize();
+      HYPRE_BigInt ldof  = GetVSize();
+      HYPRE_BigInt ltdof = TrueVSize();
 
-      HYPRE_Int gdof  = -1;
-      HYPRE_Int gtdof = -1;
+      HYPRE_BigInt gdof  = -1;
+      HYPRE_BigInt gtdof = -1;
 
-      MPI_Allreduce(&ldof, &gdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
-      MPI_Allreduce(&ltdof, &gtdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
+      MPI_Allreduce(&ldof, &gdof, 1, HYPRE_MPI_BIG_INT, MPI_SUM, MyComm);
+      MPI_Allreduce(&ltdof, &gtdof, 1, HYPRE_MPI_BIG_INT, MPI_SUM, MyComm);
 
       // Ensure face orientations have been communicated
       pmesh->ExchangeFaceNbrData();
@@ -2294,6 +2294,12 @@ int ParFiniteElementSpace
                                        Array<int> *dof_tdof,
                                        bool partial) const
 {
+   // TODO: general face DOF transformations in NeighborRowMessage::Decode()
+   MFEM_VERIFY(!(fec->GetOrder() >= 2
+                 && pmesh->HasGeometry(Geometry::TETRAHEDRON)
+                 && fec->GetContType() == FiniteElementCollection::TANGENTIAL),
+               "Nedelec NC tets of order >= 2 are not supported yet.");
+
    bool dg = (nvdofs == 0 && nedofs == 0 && nfdofs == 0);
 
 #ifdef MFEM_PMATRIX_STATS
@@ -3189,8 +3195,11 @@ void ParFiniteElementSpace::CopyProlongationAndRestriction(
    SparseMatrix *perm_mat = NULL, *perm_mat_tr = NULL;
    if (perm)
    {
+      // Note: although n and fes.GetVSize() are typically equal, in
+      // variable-order spaces they may differ, since nonconforming edges/faces
+      // my have fictitious DOFs.
       int n = perm->Size();
-      perm_mat = new SparseMatrix(n, n);
+      perm_mat = new SparseMatrix(n, fes.GetVSize());
       for (int i=0; i<n; ++i)
       {
          double s;
@@ -3207,11 +3216,26 @@ void ParFiniteElementSpace::CopyProlongationAndRestriction(
       else { P = new HypreParMatrix(*pfes->P); }
       nonconf_P = true;
    }
+   else if (perm != NULL)
+   {
+      HYPRE_BigInt glob_nrows = GlobalVSize();
+      HYPRE_BigInt glob_ncols = GlobalTrueVSize();
+      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
+      HYPRE_BigInt *row_starts = GetDofOffsets();
+      P = new HypreParMatrix(MyComm, glob_nrows, glob_ncols, row_starts,
+                             col_starts, perm_mat);
+      nonconf_P = true;
+   }
    if (pfes->R != NULL)
    {
       if (perm) { R = Mult(*pfes->R, *perm_mat_tr); }
       else { R = new SparseMatrix(*pfes->R); }
    }
+   else if (perm != NULL)
+   {
+      R = perm_mat_tr;
+      perm_mat_tr = NULL;
+   }
 
    delete perm_mat;
    delete perm_mat_tr;

fem/pgridfunc.cpp

@@ -481,6 +481,27 @@ void ParGridFunction::GetVectorValue(ElementTransformation &T,
    }
 }
 
+void ParGridFunction::GetDerivative(int comp, int der_comp,
+                                    ParGridFunction &der)
+{
+   Array<int> overlap;
+   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
+
+   // Count the zones globally.
+   GroupCommunicator &gcomm = der.ParFESpace()->GroupComm();
+   gcomm.Reduce<int>(overlap, GroupCommunicator::Sum);
+   gcomm.Bcast(overlap);
+
+   // Accumulate for all dofs.
+   gcomm.Reduce<double>(der.HostReadWrite(), GroupCommunicator::Sum);
+   gcomm.Bcast<double>(der.HostReadWrite());
+
+   for (int i = 0; i < overlap.Size(); i++)
+   {
+      der(i) /= overlap[i];
+   }
+}
+
 void ParGridFunction::GetElementDofValues(int el, Vector &dof_vals) const
 {
    int ne = fes->GetNE();

fem/pgridfunc.hpp

@@ -226,6 +226,9 @@ public:
                                const IntegrationPoint &ip,
                                Vector &val, Vector *tr = NULL) const;
 
+   /// Parallel version of GridFunction::GetDerivative(); see its documentation.
+   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
+
    /** Sets the output vector @a dof_vals to the values of the degrees of
        freedom of element @a el. If @a el is greater than or equal to the number
        of local elements, it will be interpreted as a shifted index of a face

fem/restriction.cpp

@@ -167,8 +167,8 @@ void ElementRestriction::MultTranspose(const Vector& x, Vector& y) const
          for (int j = offset; j < nextOffset; ++j)
          {
             const int idx_j = (d_indices[j] >= 0) ? d_indices[j] : -1 - d_indices[j];
-            dofValue += (d_indices[j] >= 0) ? d_x(idx_j % nd, c,
-            idx_j / nd) : -d_x(idx_j % nd, c, idx_j / nd);
+            dofValue += ((d_indices[j] >= 0) ? d_x(idx_j % nd, c, idx_j / nd) :
+                         -d_x(idx_j % nd, c, idx_j / nd));
          }
          d_y(t?c:i,t?i:c) = dofValue;
       }

fem/tmop_amr.cpp

@@ -234,7 +234,7 @@ void TMOPRefinerEstimator::SetTriIntRules()
 
    // Reftype = 0 // original element
    const int Nvert = 3, NEsplit = 1;
-   Mesh meshsplit(2, Nvert, NEsplit, 0 ,2);
+   Mesh meshsplit(2, Nvert, NEsplit, 0, 2);
    const double tri_v[3][2] =
    {
       {0, 0}, {1, 0}, {0, 1}

general/backends.hpp

@@ -58,6 +58,23 @@
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #endif
 
+// 'double' atomicAdd implementation for previous versions of CUDA
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600
+MFEM_DEVICE double atomicAdd(double *add, double val)
+{
+   unsigned long long int *ptr = (unsigned long long int *) add;
+   unsigned long long int old = *ptr, reg;
+   do
+   {
+      reg = old;
+      old = atomicCAS(ptr, reg,
+                      __double_as_longlong(val + __longlong_as_double(reg)));
+   }
+   while (reg != old);
+   return __longlong_as_double(old);
+}
+#endif
+
 template <typename T>
 MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 {
@@ -66,6 +83,9 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
    return atomicAdd(&add,val);
 #else
    T old = add;
+#ifdef MFEM_USE_OPENMP
+   #pragma omp atomic
+#endif
    add += val;
    return old;
 #endif

general/forall.hpp

@@ -139,7 +139,7 @@ void RajaCuWrap2D(const int N, DBODY &&d_body,
    using RAJA::RangeSegment;
 
    launch<cuda_launch_policy>
-   (DEVICE, Resources(Teams(G), Threads(X, Y, BZ)),
+   (DEVICE, Grid(Teams(G), Threads(X, Y, BZ)),
     [=] RAJA_DEVICE (LaunchContext ctx)
    {
 
@@ -172,7 +172,7 @@ void RajaCuWrap3D(const int N, DBODY &&d_body,
    using RAJA::RangeSegment;
 
    launch<cuda_launch_policy>
-   (DEVICE, Resources(Teams(GRID), Threads(X, Y, Z)),
+   (DEVICE, Grid(Teams(GRID), Threads(X, Y, Z)),
     [=] RAJA_DEVICE (LaunchContext ctx)
    {
 
@@ -205,7 +205,7 @@ void RajaHipWrap2D(const int N, DBODY &&d_body,
    using RAJA::RangeSegment;
 
    launch<hip_launch_policy>
-   (DEVICE, Resources(Teams(G), Threads(X, Y, BZ)),
+   (DEVICE, Grid(Teams(G), Threads(X, Y, BZ)),
     [=] RAJA_DEVICE (LaunchContext ctx)
    {
 
@@ -238,7 +238,7 @@ void RajaHipWrap3D(const int N, DBODY &&d_body,
    using RAJA::RangeSegment;
 
    launch<hip_launch_policy>
-   (DEVICE, Resources(Teams(GRID), Threads(X, Y, Z)),
+   (DEVICE, Grid(Teams(GRID), Threads(X, Y, Z)),
     [=] RAJA_DEVICE (LaunchContext ctx)
    {
 

general/isockstream.cpp

@@ -23,7 +23,9 @@
 #include <unistd.h>
 #else
 #include <winsock.h>
+#ifdef _MSC_VER
 typedef int ssize_t;
+#endif
 typedef int socklen_t;
 #define close closesocket
 // Link with ws2_32.lib

general/mem_manager.cpp

@@ -33,7 +33,8 @@
 #endif
 
 #ifdef MFEM_USE_UMPIRE
-#include "umpire/Umpire.hpp"
+#include <umpire/Umpire.hpp>
+#include <umpire/strategy/QuickPool.hpp>
 
 // Make sure Umpire is build with CUDA support if MFEM is built with it.
 #if defined(MFEM_USE_CUDA) && !defined(UMPIRE_ENABLE_CUDA)
@@ -535,7 +536,7 @@ public:
    {
       if (!rm.isAllocator(name))
       {
-         allocator = rm.makeAllocator<umpire::strategy::DynamicPool>(
+         allocator = rm.makeAllocator<umpire::strategy::QuickPool>(
                         name, rm.getAllocator(space));
          owns_allocator = true;
       }
@@ -910,7 +911,12 @@ MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType h_mt, unsigned flags)
    MFEM_ASSERT(IsHostMemory(h_mt), "invalid h_mt = " << (int)h_mt);
    // MFEM_ASSERT(registered || IsHostMemory(h_mt),"");
    MFEM_ASSERT(!owns_device || owns_internal, "invalid Memory state");
-   MFEM_ASSERT(registered || !(owns_host || owns_device || owns_internal),
+   // If at least one of the 'own_*' flags is true then 'registered' must be
+   // true too. An acceptable exception is the special case when 'h_ptr' is
+   // NULL, and both 'own_device' and 'own_internal' are false -- this case is
+   // an exception only when 'own_host' is true and 'registered' is false.
+   MFEM_ASSERT(registered || !(owns_host || owns_device || owns_internal) ||
+               (!(owns_device || owns_internal) && h_ptr == nullptr),
                "invalid Memory state");
    if (!mm.exists || !registered) { return h_mt; }
    if (alias)

general/socketstream.cpp

@@ -28,7 +28,9 @@
 #define closesocket (::close)
 #else
 #include <winsock.h>
+#ifdef _MSC_VER
 typedef int ssize_t;
+#endif
 // Link with ws2_32.lib
 #pragma comment(lib, "ws2_32.lib")
 #endif

linalg/amgxsolver.cpp

@@ -604,11 +604,16 @@ void AmgXSolver::SetMatrix(const HypreParMatrix &A, const bool update_mat)
    mfem_error("Hypre version 2.16+ is required when using AmgX \n");
 #endif
 
+   // Ensure HypreParMatrix is on the host
+   A.HostRead();
+
    hypre_ParCSRMatrix * A_ptr =
       (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(A);
 
    hypre_CSRMatrix *A_csr = hypre_MergeDiagAndOffd(A_ptr);
 
+   A.HypreRead();
+
    Array<double> loc_A(A_csr->data, (int)A_csr->num_nonzeros);
    const Array<HYPRE_Int> loc_I(A_csr->i, (int)A_csr->num_rows+1);
 

linalg/blockmatrix.cpp

@@ -571,6 +571,8 @@ void BlockMatrix::PrintMatlab(std::ostream & os) const
          os << i+1 << " " << row_ind[j]+1 << " " << row_data[j] << std::endl;
       }
    }
+   // Write a zero entry at (m,n) to make sure MATLAB doesn't shrink the matrix
+   os << row_offsets.Last() << " " << col_offsets.Last () << " 0.0\n";
 
    os.precision(old_prec);
    os.flags(old_fmt);

linalg/blockmatrix.hpp

@@ -89,7 +89,7 @@ public:
    //! Returns a monolithic CSR matrix that represents this operator.
    SparseMatrix * CreateMonolithic() const;
    //! Export the monolithic matrix to file.
-   void PrintMatlab(std::ostream & os = mfem::out) const;
+   virtual void PrintMatlab(std::ostream & os = mfem::out) const;
 
    /// @name Matrix interface
    ///@{

linalg/hiop.cpp

@@ -22,7 +22,7 @@ using namespace hiop;
 namespace mfem
 {
 
-bool HiopOptimizationProblem::get_prob_sizes(long long &n, long long &m)
+bool HiopOptimizationProblem::get_prob_sizes(size_type &n, size_type &m)
 {
    n = ntdofs_glob;
    m = problem.GetNumConstraints();
@@ -30,7 +30,7 @@ bool HiopOptimizationProblem::get_prob_sizes(long long &n, long long &m)
    return true;
 }
 
-bool HiopOptimizationProblem::get_starting_point(const long long &n, double *x0)
+bool HiopOptimizationProblem::get_starting_point(const size_type &n, double *x0)
 {
    MFEM_ASSERT(x_start != NULL && ntdofs_loc == x_start->Size(),
                "Starting point is not set properly.");
@@ -40,7 +40,7 @@ bool HiopOptimizationProblem::get_starting_point(const long long &n, double *x0)
    return true;
 }
 
-bool HiopOptimizationProblem::get_vars_info(const long long &n,
+bool HiopOptimizationProblem::get_vars_info(const size_type &n,
                                             double *xlow, double *xupp,
                                             NonlinearityType *type)
 {
@@ -55,7 +55,7 @@ bool HiopOptimizationProblem::get_vars_info(const long long &n,
    return true;
 }
 
-bool HiopOptimizationProblem::get_cons_info(const long long &m,
+bool HiopOptimizationProblem::get_cons_info(const size_type &m,
                                             double *clow, double *cupp,
                                             NonlinearityType *type)
 {
@@ -79,7 +79,7 @@ bool HiopOptimizationProblem::get_cons_info(const long long &m,
    return true;
 }
 
-bool HiopOptimizationProblem::eval_f(const long long &n, const double *x,
+bool HiopOptimizationProblem::eval_f(const size_type &n, const double *x,
                                      bool new_x, double &obj_value)
 {
    MFEM_ASSERT(n == ntdofs_glob, "Global input mismatch.");
@@ -93,7 +93,7 @@ bool HiopOptimizationProblem::eval_f(const long long &n, const double *x,
    return true;
 }
 
-bool HiopOptimizationProblem::eval_grad_f(const long long &n, const double *x,
+bool HiopOptimizationProblem::eval_grad_f(const size_type &n, const double *x,
                                           bool new_x, double *gradf)
 {
    MFEM_ASSERT(n == ntdofs_glob, "Global input mismatch.");
@@ -108,9 +108,9 @@ bool HiopOptimizationProblem::eval_grad_f(const long long &n, const double *x,
    return true;
 }
 
-bool HiopOptimizationProblem::eval_cons(const long long &n, const long long &m,
-                                        const long long &num_cons,
-                                        const long long *idx_cons,
+bool HiopOptimizationProblem::eval_cons(const size_type &n, const size_type &m,
+                                        const size_type &num_cons,
+                                        const index_type *idx_cons,
                                         const double *x, bool new_x,
                                         double *cons)
 {
@@ -134,10 +134,10 @@ bool HiopOptimizationProblem::eval_cons(const long long &n, const long long &m,
    return true;
 }
 
-bool HiopOptimizationProblem::eval_Jac_cons(const long long &n,
-                                            const long long &m,
-                                            const long long &num_cons,
-                                            const long long *idx_cons,
+bool HiopOptimizationProblem::eval_Jac_cons(const size_type &n,
+                                            const size_type &m,
+                                            const size_type &num_cons,
+                                            const index_type *idx_cons,
                                             const double *x, bool new_x,
                                             double *Jac)
 {
@@ -165,16 +165,16 @@ bool HiopOptimizationProblem::eval_Jac_cons(const long long &n,
    return true;
 }
 
-bool HiopOptimizationProblem::get_vecdistrib_info(long long global_n,
-                                                  long long *cols)
+bool HiopOptimizationProblem::get_vecdistrib_info(size_type global_n,
+                                                  index_type *cols)
 {
 #ifdef MFEM_USE_MPI
    int nranks;
    MPI_Comm_size(comm, &nranks);
 
-   long long *sizes = new long long[nranks];
-   MPI_Allgather(&ntdofs_loc, 1, MPI_LONG_LONG_INT, sizes, 1,
-                 MPI_LONG_LONG_INT, comm);
+   size_type *sizes = new size_type[nranks];
+   MPI_Allgather(&ntdofs_loc, 1, MPI_HIOP_SIZE_TYPE, sizes, 1,
+                 MPI_HIOP_SIZE_TYPE, comm);
    cols[0] = 0;
    for (int r = 1; r <= nranks; r++)
    {
@@ -302,6 +302,8 @@ void HiopNlpOptimizer::Mult(const Vector &xt, Vector &x) const
    hiopInstance.options->SetNumericValue("fixed_var_tolerance", 1e-20);
    hiopInstance.options->SetNumericValue("fixed_var_perturb", 1e-9);
 
+   hiopInstance.options->SetNumericValue("mu0", 1e-1);
+
    // 0: no output; 3: not too much
    hiopInstance.options->SetIntegerValue("verbosity_level", print_level);