Fixed some bugs introduced by the previous commit.

ex14 and ex14p.

.github/stale.yml

@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 30
+
+# Number of days of inactivity before an Issue or Pull Request with the stale
+# label is closed.  Set to false to disable. If disabled, issues still need to
+# be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale.
+# Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set
+# to `[]` to disable
+exemptLabels:
+  - bug
+  - WIP
+  - ready-for-review
+  - in-review
+  - in-next
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking an issue as stale
+staleLabel: stale
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  :warning: This issue or PR has been automatically marked as stale because it has not
+  had any activity in the last month. *If no activity occurs in the next week, it will
+  be automatically closed.* Thank you for your contributions.
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/stale.yml

@@ -1,31 +0,0 @@
-# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
-# For more information, see: https://github.com/actions/stale
-name: Mark stale issues and pull requests
-
-on:
-  workflow_dispatch:
-  schedule:
-  - cron: '0 0 * * *'
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: ':warning: This issue has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
-        days-before-stale: 30
-        days-before-close: 7
-        stale-issue-label: 'stale'
-        stale-pr-label: 'stale'
-        operations-per-run: 500
-        exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
-        exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"

.github/workflows/trigger-pymfem-ci.yml

@@ -1,31 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Trigger PyMFEM CI"
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  trigger-pymfem:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Send POST request to trigger PyMFEM CI
-      run: |
-        curl -L \
-             -X POST \
-             -H "Accept: application/vnd.github+json" \
-             -H "Authorization: Bearer ${{ secrets.PYMFEM_CI_TOKEN }}" \
-             -H "X-GitHub-Api-Version: 2022-11-28" \
-             https://api.github.com/repos/mfem/pymfem/actions/workflows/build-and-test-dispatch.yml/dispatches \
-             -d '{"ref":"master", "inputs":{"test_options":"fast"}}'

.gitignore

@@ -8,7 +8,6 @@
 
 # Object and library files
 *.o
-*.o.tmp
 /libmfem.*
 /miniapps/common/libmfem-common.*
 
@@ -16,9 +15,6 @@
 CMakeCache.txt
 CMakeFiles/
 
-# Clangd server cache
-*.cache*
-
 # Backup files
 *~
 
@@ -276,27 +272,16 @@ miniapps/navier/*_output
 
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex3
-miniapps/nurbs/nurbs_ex5
 miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_ex24
-miniapps/nurbs/nurbs_solenoidal
 miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
-miniapps/nurbs/sol_?.gf
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
-miniapps/nurbs/Example3*
-miniapps/nurbs/Example5*
-miniapps/nurbs/Solenoidal*
-miniapps/nurbs/ParaView
 miniapps/nurbs/sin-fit.mesh
-miniapps/nurbs/ex5.mesh
-miniapps/nurbs/exsol.mesh
 miniapps/nurbs/CurveInt
 miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh

.gitlab-ci.yml

@@ -22,7 +22,7 @@ include:
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where ruby resource are allocated/released once for all.
+# - Allocate/Release is where quartz resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
@@ -53,7 +53,7 @@ variables:
   AUTOTEST_COMMIT: "YES"
 
 # Trigger subpipelines:
-ruby-build-and-test:
+quartz-build-and-test:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -61,10 +61,10 @@ ruby-build-and-test:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/ruby-build-and-test.yml
+    include: .gitlab/quartz-build-and-test.yml
     strategy: depend
 
-ruby-baseline:
+quartz-baseline:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -73,7 +73,7 @@ ruby-baseline:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/ruby-baseline.yml
+    include: .gitlab/quartz-baseline.yml
     strategy: depend
 
 lassen-build-and-test:

.gitlab/README.md

@@ -24,7 +24,7 @@ and `test type`.
 
 Machines typically include:
 
-* Ruby: 2nd Gen Intel Xeon (Cascade Lake)
+* Quartz: Intel bi-socket x86
 * Lassen: Power9 + Nvidia GPU
 * Corona: AMD GPU
 
@@ -76,13 +76,13 @@ with a spack spec of MFEM, within the limits permitted by the MFEM spack
 package.
 
 In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on ruby for example resumes to:
+spack spec to use. Adding a job on quartz for example resumes to:
 
 ```yaml
 <job_name>:
   variables:
     SPEC: "<spack_spec>"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 ```
 
 The remaining and non trivial work is to make sure this spec is working. To

.gitlab/configs/common.yml

@@ -24,7 +24,7 @@ variables:
 # TODO: add a clean-up mechanism
   BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
 
-# On LLNL's ruby, there is only one allocation shared among jobs in order to
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}

.gitlab/configs/quartz-config.yml

@@ -9,17 +9,17 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# GitLab pipelines configurations for the Ruby machine at LLNL
+# GitLab pipelines configurations for the Quartz machine at LLNL
 variables:
-  MACHINE_NAME: ruby
+  MACHINE_NAME: quartz
 
-.on_ruby:
+.on_quartz:
   tags:
     - shell
-    - ruby
+    - quartz
   rules:
-    # Don't run ruby jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"'
+    # Don't run quartz jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
       when: never
     # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
@@ -40,13 +40,13 @@ variables:
     - when: on_success
 
 # Spack helped builds
-# Generic ruby build job, extending build script
-.build_and_test_on_ruby:
-  extends: [.on_ruby]
+# Generic quartz build job, extending build script
+.build_and_test_on_quartz:
+  extends: [.on_quartz]
   stage: build_and_test
   script:
     # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=16
+    - export THREADS=12
     - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
     - echo ${JOBID}

.gitlab/configs/setup-baseline.yml

@@ -18,7 +18,7 @@
 setup_baseline:
   tags:
     - shell
-    - ruby
+    - quartz
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/configs/setup-build-and-test.yml

@@ -16,7 +16,7 @@
 setup:
   tags:
     - shell
-    - ruby
+    - quartz
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/quartz-baseline.yml

@@ -19,8 +19,8 @@ stages:
   - cleanup
   - baseline_publish
 
-baselinecheck_mfem_intel_ruby:
-  extends: [.on_ruby]
+baselinecheck_mfem_intel_quartz:
+  extends: [.on_quartz]
   stage: baseline_check
   variables:
     # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
@@ -32,7 +32,7 @@ baselinecheck_mfem_intel_ruby:
     - echo ${BUILD_ROOT}
     - echo ${TPLS_DIR}
     # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=48
+    - export MFEM_TEST_NP=32
     # The next script uses the following environment variables:
     # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
     # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
@@ -44,16 +44,18 @@ baselinecheck_mfem_intel_ruby:
   allow_failure: true
 
 cleanup:
-  extends: .on_ruby
+  extends: .on_quartz
   stage: cleanup
   variables:
     GIT_STRATEGY: none
   script:
     - echo "BUILD_ROOT=${BUILD_ROOT}"
     - rm -rf "${BUILD_ROOT}" || true
+    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
+    - make -C "${CI_PROJECT_DIR}" distclean
 
 report_baseline:
-  extends: [.on_ruby]
+  extends: [.on_quartz]
   stage: baseline_report
   script:
     - echo ${MACHINE_NAME}
@@ -113,8 +115,8 @@ report_baseline:
         exit $err
       ) 9> autotest.lock
 
-baselinepublish_mfem_ruby:
-  extends: [.on_ruby]
+baselinepublish_mfem_quartz:
+  extends: [.on_quartz]
   stage: baseline_publish
   rules:
     # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
@@ -129,5 +131,5 @@ baselinepublish_mfem_ruby:
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/quartz-config.yml
   - local: .gitlab/configs/setup-baseline.yml

.gitlab/quartz-build-and-test.yml

@@ -19,54 +19,54 @@ stages:
 allocate_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_quartz
   stage: allocate_resource
   script:
     - echo ${ALLOC_NAME}
     - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
   timeout: 6h
 
-# GitLab jobs for the Ruby machine at LLNL
+# GitLab jobs for the Quartz machine at LLNL
 debug_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 debug_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10_sundials:
   variables:
     SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10_petsc:
   variables:
     SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10_pumi:
   variables:
     SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 # Release
 release_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_quartz
   stage: release_resource_and_report
   script:
     - echo ${ALLOC_NAME}
@@ -78,17 +78,17 @@ release_resource:
 report_job_success:
   stage: release_resource_and_report
   extends:
-    - .on_ruby
+    - .on_quartz
     - .report_job_success
 
 report_job_failure:
   stage: release_resource_and_report
   extends:
-    - .on_ruby
+    - .on_quartz
     - .report_job_failure
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/quartz-config.yml
   - local: .gitlab/configs/setup-build-and-test.yml
   - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -14,7 +14,7 @@
 # locals
 glob_err=${BASELINE_TEST}.err
 base=${BASELINE_TEST}-${SYS_TYPE}
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
+if [[ "${MACHINE_NAME}" == "quartz" ]]; then
   base="${BASELINE_TEST}-${MACHINE_NAME}"
 fi
 base_diff=${base}.diff
@@ -31,8 +31,8 @@ cd tests
 mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
   salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
@@ -41,11 +41,11 @@ else
   echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
   exit 1
 fi
-status="$?"
 
 # post
 mkdir ${artifacts_path}
 
+status=0
 if [[ -f ${BASELINE_TEST}.out ]]; then
   cp ${BASELINE_TEST}.out ${artifacts_path}
 fi

.gitlab/scripts/rebaseline

@@ -11,7 +11,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and quartz baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
@@ -21,7 +21,7 @@ PATCH_FILE=${ARTIFACT_PATH}.patch
 FULL_FILE=${ARTIFACT_PATH}.out
 DIFF_FILE=${ARTIFACT_PATH}.diff
 
-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and quartz baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 SAVED_NAME=baseline-${SYS_TYPE}.saved

CHANGELOG

@@ -10,92 +10,12 @@
 
 Version 4.7.1 (development)
 ===========================
-- Refactored ALGOIM cut integration rules. The interface is unified with
-  the interface for moment based cut integration rules.
-
-Discretization improvements
----------------------------
-- Added NURBS-based H(div) and H(curl) elements in 2D and 3D. Only on single
-  patch meshes. Only implemented for serial computations.
-
-- Added support for boundary constraints to the hybridization class.
-
-- Added support for external boundary submeshes with nonconformal mesh adaptation.
-
-Meshing improvements
---------------------
-- The ExodusII reader now handles pyramid and wedge element types. Mixed meshes
-  are also supported.
-
-New and updated examples and miniapps
--------------------------------------
-- Added miniapps to demonstrate the H(div) and H(curl) NURBS elements.
 
 - Added an MFEM example for the eikonal equation. This new solver is based on
   the proximal Galerkin method introduced by Keith and Surowiec.
 
-- Added a command line option to all miniapps (`-p` or `--send-port`) for
-  specifying the GLVis server socket port (19916 by default).
-
-GPU computing
--------------
-- Added support for GPU-accelerated batched linear algebra (using cuBLAS,
-  hipBLAS, MAGMA, or native MFEM functionality) through the BatchedLinAlg class.
-
-- A new GPU kernel dispatch mechanism was introduced. Users can instantiate
-  specialized kernels for specific combinations of (for example) polynomial
-  degree and number of quadrature points using
-  `DiffusionIntegrator::AddSpecialization` and
-  `MassIntegrator::AddSpecialization` (this functionality may be added to more
-  integrators in the future).
-
-- Calls to slower fallback kernels can be reported to `mfem::err` by setting
-  the environment variable `MFEM_REPORT_KERNELS` to any value other than `NO`
-  or by explicitly calling `KernelReporter::Enable`. Users can then add
-  specializations for these kernels to achieve higher performance.
-
-- Element assembly kernels have been added for low-order refined to
-  high-order transfer operators. New kernels can be offloaded as device
-  kernels. Example usage may be found in lor-transfer.cpp under miniapps/tools.
-
-Miscellaneous
--------------
-- Added support for SUNDIALS v7. See the section "API changes" for some small
-  changes related to this new version.
-
-- Refactored the `ARKStepSolver` class (ARKODE interface) to use
-  `TimeDependentOperator::Mult` only when the associated ODE operator is
-  expressed in explicit form (i.e., `TimeDependentOperator::isExplicit()`),
-  otherwise `TimeDependentOperator::ExplicitMult` is used. A check has been
-  added to `ARKStepSolver` to verify that the associated ODE operator is not in
-  explicit form when a mass matrix solver is enabled via a call to either the
-  `UseMFEMMassLinearSolver` or `UseSundialsMassLinearSolver` methods. This is
-  because enabling a mass matrix solver assumes that F(u,k,t) = M k in the
-  associated ODE operator.
-
-- Added support for custom interpolation procedure in FindPointsGSLIB.
-
-- `FiniteElementSpace` has new methods to directly set prolongation and
-  restriction operators to arbitrary sparse matrices.
-
-- There are new convenience constructors for NURBS patches and knot vectors.
-
-API changes
------------
 - API change: in class GridFunction, 'fec' was renamed to 'fec_owned'.
 
-- API change: support for SUNDIALS v7:
-  * the SUNDIALS types `realtype` and `booleantype` are no longer defined by v7
-    and therefore MFEM now uses the new type names `sunrealtype` and
-    `sunbooleantype`, respectively, which MFEM defines when using SUNDIALS < v6
-    where these types were not defined.
-  * The SUNDIALS macro `SUNLS_SUCCESS` and some other `*_SUCCESS` macros were
-    removed and replaced by `SUN_SUCCESS` in v7, so to avoid tedious checks for
-    SUNDIALS versions, MFEM now defines and uses the constant `SUN_SUCCESS` when
-    using SUNDIALS < v7.
-  * The constants `SUN_PREC_*`, introduced by SUNDIALS v6 are now introduced by
-    MFEM when using SUNDIALS < v6 to avoid tedious version checks.
-
 
 Version 4.7, released on May 7, 2024
 ====================================
@@ -120,9 +40,6 @@ Meshing improvements
 
 - Added support for internal boundary elements in nonconforming meshes.
 
-- Added ExodusII output capability. The writer can handle first-order (Pyramid5,
-  Wedge6, Hex8, Tet4) and second-order FE types (Pyramid14, Wedge18, Hex27, Tet10).
-
 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.
 
 Discretization improvements
@@ -182,15 +99,6 @@ New and updated examples and miniapps
 - Added two new example codes: 38 and 39/39p described above. Substantially
   updated Example 18/18p.
 
-- Added ODE solvers selection routines. This creates a uniformity across examples,
-  miniapps and other executables in regard to ODE(time-integrator) selection.
-
-- Added new mechanism for retrieving and setting state vectors in ODE solvers.
-  This is relevant for AB/AM and gen-alpha solvers.
-
-- Added ODEsolver/ODEsolver2 unit tests to verify order of convergence and
-  read/write functionality.
-
 Miscellaneous
 -------------
 - Updated the Doxygen documentation style, which now requires Doxygen version

CMakeLists.txt

@@ -146,9 +146,7 @@ if (MFEM_USE_CUDA)
    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
    find_package(CUDAToolkit REQUIRED)
    set(CUSPARSE_FOUND TRUE)
-   set(CUBLAS_FOUND TRUE)
    get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-   get_target_property(CUBLAS_LIBRARIES CUDA::cublas LOCATION)
 endif()
 
 if (XSDK_ENABLE_C)
@@ -233,7 +231,6 @@ if (MFEM_USE_HIP)
     list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
   endif()
   find_package(HIP REQUIRED)
-  find_package(HIPBLAS REQUIRED)
   find_package(HIPSPARSE REQUIRED)
 endif()
 
@@ -340,10 +337,7 @@ if (MFEM_USE_SUNDIALS)
   if (MFEM_USE_HIP)
     list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
   endif()
-  # The Core component was added in SUNDIALS v7, so we treat it as optional in
-  # order to support older versions.
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS}
-    OPTIONAL_COMPONENTS Core)
+  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
 # SuperLU_DIST can only be enabled in parallel
@@ -402,10 +396,6 @@ if (MFEM_USE_AMGX)
   find_package(AMGX REQUIRED)
 endif()
 
-if (MFEM_USE_MAGMA)
-  find_package(MAGMA REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint)
 endif()
@@ -567,9 +557,8 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
-    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
-    ALGOIM ENZYME)
+    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -684,7 +673,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
 
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
       "${PROJECT_BINARY_DIR}/${Header}.tmp"
       "${PROJECT_BINARY_DIR}/${Header}"
     )
@@ -698,7 +687,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 #include \"mfem/${Header}\"
 ")
 
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
       "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
       "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
     )

INSTALL

@@ -273,13 +273,7 @@ Installation options:
    PREFIX  - Specify the installation directory. The library (libmfem.a) will be
              installed in $(PREFIX)/lib, the headers in $(PREFIX)/include, and
              the configuration makefile (config.mk) in $(PREFIX)/share/mfem.
-   INSTALL - Specify the install program, default = /usr/bin/install
-   INSTALL_DEF_PERM - Specify the default install permissions. This affects
-                      headers and configuration makefiles, default = 644
-   INSTALL_BIN_PERM - Specify the install permissions for binaries. This only
-                      affects the shared version of the library, default = 755
-   INSTALL_DIR_PERM - Specify the install permissions for directories and,
-                      on macOS/BSD, for symlinks as well, default = 755
+   INSTALL - Specify the install program, e.g /usr/bin/install
 
 MFEM library features/options (GNU make)
 ----------------------------------------
@@ -394,11 +388,6 @@ MFEM_USE_AMGX = YES/NO
    Allows the user to use SparseMatrices and HypreParMatrices to solve linear
    systems with the routines from the AmgX library.
 
-MFEM_USE_MAGMA = YES/NO
-   Enable MFEM functionality based on the MAGMA high-performance linear algebra
-   library. The MAGMA library provides a BLAS/LAPACK interface, with
-   implementations that have been optimized for Nvidia and AMD GPUs.
-
 MFEM_USE_GNUTLS = YES/NO
    Enable secure socket support in class socketstream, using the auxiliary
    GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -502,14 +491,10 @@ MFEM_USE_CODIPACK = YES/NO
 MFEM_USE_ALGOIM = YES/NO
   Enable the usage of Algoim - a collection of high-order accurate numerical
   methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods, see https://algoim.github.io. MFEM provides interface to
-  Algoim v1. To check out the specific Algoim state use:
-  https://github.com/algoim/algoim
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM
+  provides interface to Algoim v1. Thus, to check out the specific state use:
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
-  The Algoim library requires the Blitz++ library. To use the latest state of
-  Blitz++ that has been tested with MFEM, use:
-  https://github.com/blitzpp/blitz
-  git checkout f24a250a43dff88c31ad92916da828b7ea9a98b7
+  https://algoim.github.io
 
 MFEM_USE_ADFORWARD = YES/NO
   Enable forward mode for AD packages. This option is valid
@@ -714,11 +699,6 @@ The specific libraries and their options are:
   Options: AMGX_OPT, AMGX_LIB.
   Versions: AmgX >= 2.1, older versions may work too.
 
-- MAGMA (optional), used with MFEM_USE_MAGMA = YES.
-  URL: https://icl.utk.edu/magma/
-  Options: MAGMA_OPT, MAGMA_LIB
-  Versions: MAGMA >= 2.8.0
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
   GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
   one can get the library through the Homebrew package manager (http://brew.sh).

config/cmake/MFEMConfig.cmake.in

@@ -37,7 +37,6 @@ set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
-set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
 set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)

config/cmake/config.hpp.in

@@ -114,9 +114,6 @@
 // Enable MFEM functionality based on the AmgX library.
 #cmakedefine MFEM_USE_AMGX
 
-// Enable MFEM functionality based on the MAGMA library.
-#cmakedefine MFEM_USE_MAGMA
-
 // Enable secure socket streams based on the GNUTLS library.
 #cmakedefine MFEM_USE_GNUTLS
 

config/cmake/modules/FindMAGMA.cmake

@@ -1,37 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MAGMA_FOUND
-#   - MAGMA_LIBRARIES
-#   - MAGMA_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(MAGMA MAGMA MAGMA_DIR "include" "magma.h" "lib" "magma"
-  "Paths to headers required by MAGMA." "Libraries required by MAGMA.")
-
-if (MAGMA_FOUND AND MFEM_USE_CUDA)
-  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-  get_target_property(CUBLAS_LIBRARIES CUDA::cublas LOCATION)
-  list(APPEND MAGMA_LIBRARIES ${CUSPARSE_LIBRARIES} ${CUBLAS_LIBRARIES})
-  set(MAGMA_LIBRARIES ${MAGMA_LIBRARIES} CACHE STRING
-      "MAGMA libraries + dependencies." FORCE)
-  message(STATUS "Updated MAGMA_LIBRARIES: ${MAGMA_LIBRARIES}")
-endif()
-
-if (MAGMA_FOUND AND MFEM_USE_HIP)
-  find_package(HIPBLAS REQUIRED)
-  find_package(HIPSPARSE REQUIRED)
-  list(APPEND MAGMA_LIBRARIES ${HIPBLAS_LIBRARIES} ${HIPSPARSE_LIBRARIES})
-  set(MAGMA_LIBRARIES ${MAGMA_LIBRARIES} CACHE STRING
-      "MAGMA libraries + dependencies." FORCE)
-  message(STATUS "Updated MAGMA_LIBRARIES: ${MAGMA_LIBRARIES}")
-endif()

config/cmake/modules/FindSUNDIALS.cmake

@@ -31,5 +31,4 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
   ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
   ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
   ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
-  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol
-  ADD_COMPONENT Core "include" sundials/sundials_core.h "lib" sundials_core)
+  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -846,14 +846,14 @@ function(mfem_export_mk_files)
       MFEM_USE_ZLIB MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE
       MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS
       MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS
-      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_MAGMA
-      MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC
-      MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA
-      MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_TRIBOL MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS
+      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB
+      MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED
+      MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2
+      MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_ADFORWARD
+      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)

config/config.hpp.in

@@ -114,9 +114,6 @@
 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX
 
-// Enable MFEM functionality based on the MAGMA library.
-// #define MFEM_USE_MAGMA
-
 // Enable secure socket streams based on the GNUTLS library.
 // #define MFEM_USE_GNUTLS
 

config/config.mk.in

@@ -38,7 +38,6 @@ MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_AMGX          = @MFEM_USE_AMGX@
-MFEM_USE_MAGMA         = @MFEM_USE_MAGMA@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@

config/defaults.cmake

@@ -40,7 +40,6 @@ option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
-option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -184,10 +183,6 @@ set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")
 
 set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
 
-set(MAGMA_DIR "${MFEM_DIR}/../magma" CACHE PATH "Path to MAGMA")
-set(MAGMA_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
-    "Additional packages required by MAGMA.")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -264,7 +259,7 @@ set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
     "The ParELAG library.")
 
 set(TRIBOL_DIR "${MFEM_DIR}/../tribol" CACHE PATH "Path to Tribol")
-set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING
+set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING 
     "Additional packages required by Tribol")
 
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")

config/defaults.mk

@@ -95,10 +95,6 @@ else
    # Silence unused command line argument warnings when generating dependencies
    # with mpicxx and clang
    DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
-   # Silence "ignoring duplicate libraries" warnings on new (Xcode 15) linker
-   ifneq (,$(findstring PROJECT:dyld,$(shell ld -v 2>&1)))
-      LDFLAGS_INTERNAL = -Xlinker -no_warn_duplicate_libraries
-   endif
 endif
 
 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -143,7 +139,6 @@ MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
-MFEM_USE_MAGMA         = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -289,13 +284,6 @@ endif
 ifeq ($(MFEM_USE_HIP),YES)
    SUNDIALS_LIB += -lsundials_nvechip
 endif
-SUNDIALS_CORE_PAT = $(subst\
- @MFEM_DIR@,$(MFEM_DIR),$(SUNDIALS_DIR))/lib*/libsundials_core.*
-ifeq ($(MFEM_USE_SUNDIALS),YES)
-   ifneq ($(wildcard $(SUNDIALS_CORE_PAT)),)
-      SUNDIALS_LIB += -lsundials_core
-   endif
-endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
@@ -402,11 +390,6 @@ AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
 AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt
 
-# MAGMA library configuration
-MAGMA_DIR = @MFEM_DIR@/../magma
-MAGMA_OPT = -I$(MAGMA_DIR)/include
-MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a -lcublas -lcusparse $(LAPACK_LIB)
-
 # GnuTLS library configuration
 GNUTLS_OPT =
 GNUTLS_LIB = -lgnutls
@@ -514,11 +497,11 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse -lcublas
+CUDA_LIB = -lcusparse
 
 # HIP library configuration
 HIP_OPT =
-HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse -lhipblas
+HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa
@@ -540,10 +523,8 @@ ifdef GOTCHA_DIR
 endif
 
 # BLITZ library configuration
-# BLITZ_DIR must be the custom installation folder (-DCMAKE_INSTALL_PREFIX).
-BLITZ_DIR = @MFEM_DIR@/../blitz/install
+BLITZ_DIR = @MFEM_DIR@/../blitz
 BLITZ_OPT = -I$(BLITZ_DIR)/include
-# On intel machines, use /lib64 instead of /lib.
 BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz
 
 # ALGOIM library configuration

config/sample-runs.sh

@@ -32,7 +32,7 @@ groups_serial=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,[1-9]}[0-9].cpp"'
+   "ex{,1,2,3}[0-9].cpp"'
 #   "ex1.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -58,10 +58,6 @@ groups_serial=(
    "HiOp examples:"
    "examples/hiop"
    "ex9.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex1.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
@@ -70,38 +66,25 @@ groups_serial=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
-    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
     mesh-optimizer.cpp minimal-surface.cpp"'
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
    "cvsRoberts_ASAi_dns.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "seq_example.cpp seq_test.cpp"' # 'seq_test.cpp' has no sample runs
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
    "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp "'
-# todo: miniapps/mtop
 '"nurbs"
    "NURBS miniapps:"
    "miniapps/nurbs"
    "nurbs_ex1.cpp"'
-# todo: add other nurbs miniapps
-# todo: miniapps/solvers (serial)
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
    "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
     lor-transfer.cpp"'
-# todo: add other tools miniapps
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
@@ -117,7 +100,7 @@ groups_parallel=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,[1-9]}[0-9]p.cpp"'
+   "ex{,1,2,3}[0-9]p.cpp"'
 #   "ex1p.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -143,10 +126,6 @@ groups_parallel=(
    "HiOp examples:"
    "examples/hiop"
    "ex9p.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex{1,2}p.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
@@ -159,41 +138,24 @@ groups_parallel=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp
-    fit-node-position.cpp"'
+   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
    "joule.cpp"'
-#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
+#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
    "adjoint_advection_diffusion.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "par_example.cpp"'
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "p{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
    "pfindpts.cpp schwarz_ex1p.cpp"'
-'"hdiv-linear-solver"
-   "H(div) linear solver miniapps:"
-   "miniapps/hdiv-linear-solver"
-   "grad_div.cpp darcy.cpp"'
-# 'miniapps/hooke/hooke.cpp' has no sample runs
-# todo: miniapps/mtop
-# todo: miniapps/multidomain
 '"navier"
    "Navier miniapps:"
    "miniapps/navier"
    "navier_cht.cpp"'
-# todo: add other navier miniapps
 '"nurbs"
    "NURBS miniapps:"
    "miniapps/nurbs"
@@ -202,18 +164,14 @@ groups_parallel=(
    "Shifted miniapps:"
    "miniapps/shifted"
    "distance.cpp"'
-# todo: add other shifted miniapps
 '"solvers"
    "Solvers miniapps:"
    "miniapps/solvers"
    "block-solvers.cpp"'
-# todo: add other solvers miniapps
-# todo: miniapps/spde
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
-   "convert-dc.cpp get-values.cpp load-dc.cpp"'
-# todo: add other tools miniapps
+   "convert-cd.cpp get-values.cpp load-dc.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -228,7 +186,7 @@ groups_all=(
 '"examples"
    "Examples:"
    "examples"
-   "ex\"{,[1-9]}[0-9]\"{,p}.cpp"'
+   "ex\"{,1,2,3}[0-9]\"{,p}.cpp"'
 '"sundials"
    "SUNDIALS examples:"
    "examples/sundials"
@@ -257,14 +215,10 @@ groups_all=(
    "HiOp examples:"
    "examples/hiop"
    "ex9.cpp ex9p.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex1.cpp ex{1,2}p.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
-   "ex1.cpp ex2.cpp ex1p.cpp ex6p.cpp"'
+   "ex1.cpp ex1p.cpp ex2.cpp ex6p.cpp"'
 '"superlu"
    "Superlu examples:"
    "examples/superlu"
@@ -272,67 +226,43 @@ groups_all=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
-    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
-    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp
-    fit-node-position.cpp"'
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
+    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
    "joule.cpp"'
-#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
+#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
-   "cvsRoberts_ASAi_dns.cpp adjoint_advection_diffusion.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "seq_example.cpp seq_test.cpp par_example.cpp"'
-# 'seq_test.cpp' has no sample runs
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "{,p}{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
+   "adjoint_advection_diffusion.cpp cvsRoberts_ASAi_dns.cpp"'
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
    "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp
     schwarz_ex1p.cpp"'
-'"hdiv-linear-solver"
-   "H(div) linear solver miniapps:"
-   "miniapps/hdiv-linear-solver"
-   "grad_div.cpp darcy.cpp"'
-# 'miniapps/hooke/hooke.cpp' has no sample runs
-# todo: miniapps/mtop
-# todo: miniapps/multidomain
 '"navier"
    "Navier miniapps:"
    "miniapps/navier"
    "navier_cht.cpp"'
-# todo: add other navier miniapps
 '"nurbs"
    "NURBS miniapps:"
    "miniapps/nurbs"
    "nurbs_ex1.cpp nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-# todo: add other nurbs miniapps
 '"shifted"
    "Shifted miniapps:"
    "miniapps/shifted"
    "distance.cpp"'
-# todo: add other shifted miniapps
 '"solvers"
    "Solvers miniapps:"
    "miniapps/solvers"
    "block-solvers.cpp"'
-# todo: add other solvers miniapps
-# todo: miniapps/spde
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
    "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
     lor-transfer.cpp"'
-# todo: add other tools miniapps
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
@@ -456,7 +386,7 @@ function help_message()
       mfem_config [${mfem_config}]
          Set MFEM configuration options
       make [${make}], mpiexec [${mpiexec}], mpiexec_np [${mpiexec_np}]
-         Their values can also be set using the respective uppercase environment
+         Their values can also set using the respective uppercase environment
          variable
       mfem_build_dir [${mfem_build_dir}]
          Same as '-d': set this variable to something different from <mfem_dir>

data/cube-space-time.mesh

@@ -0,0 +1,102 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+24
+1 7 4 5 8 11 13
+1 7 1 4 5 7 8
+1 7 1 4 5 8 11
+1 7 1 5 8 11 13
+1 7 1 5 7 8 13
+1 7 4 5 7 8 13
+1 7 1 3 4 8 11
+1 7 1 3 4 5 11
+1 7 1 5 10 11 13
+1 7 1 8 10 11 13
+1 7 1 3 5 10 11
+1 7 1 2 3 5 10
+1 7 0 1 3 4 8
+1 7 0 1 4 7 8
+1 7 1 5 6 7 13
+1 7 1 6 7 8 13
+1 7 6 7 8 13 15
+1 7 4 7 8 13 15
+1 7 4 8 12 13 15
+1 7 4 8 11 12 13
+1 7 6 8 13 14 15
+1 7 1 6 8 13 14
+1 7 1 8 9 10 13
+1 7 1 8 9 13 14
+
+boundary
+48
+1 4 0 1 3 4
+1 4 0 1 3 8
+1 4 0 3 4 8
+1 4 0 1 4 7
+1 4 0 1 7 8
+1 4 0 4 7 8
+1 4 1 4 5 7
+1 4 1 3 8 11
+1 4 1 3 4 5
+1 4 1 5 10 13
+1 4 1 8 10 11
+1 4 1 3 10 11
+1 4 1 2 3 5
+1 4 1 2 3 10
+1 4 1 2 5 10
+1 4 1 5 6 7
+1 4 1 5 6 13
+1 4 1 6 7 8
+1 4 1 6 8 14
+1 4 1 6 13 14
+1 4 1 8 9 10
+1 4 1 9 10 13
+1 4 1 8 9 14
+1 4 1 9 13 14
+1 4 2 3 5 10
+1 4 3 4 8 11
+1 4 3 4 5 11
+1 4 3 5 10 11
+1 4 4 5 11 13
+1 4 4 5 7 13
+1 4 4 7 8 15
+1 4 4 7 13 15
+1 4 4 8 12 15
+1 4 4 12 13 15
+1 4 4 8 11 12
+1 4 4 11 12 13
+1 4 5 10 11 13
+1 4 5 6 7 13
+1 4 6 7 8 15
+1 4 6 7 13 15
+1 4 6 8 14 15
+1 4 6 13 14 15
+2 4 8 10 11 13
+2 4 8 12 13 15
+2 4 8 11 12 13
+2 4 8 13 14 15
+2 4 8 9 10 13
+2 4 8 9 13 14
+
+vertices
+16
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000

data/cube4d_24.MFEM

@@ -0,0 +1,102 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+24
+17 8 4 5 8 11 13
+11 8 1 4 5 7 8
+15 8 1 4 5 8 11
+16 8 1 5 8 11 13
+12 8 1 5 7 8 13
+13 8 4 5 7 8 13
+7  8 1 3 4 8 11
+2  8 1 3 4 5 11
+4  8 1 5 10 11 13
+10 8 1 8 10 11 13
+3  8 1 3 5 10 11
+1  8 1 2 3 5 10
+6  8 0 1 3 4 8
+8  8 0 1 4 7 8
+5  8 1 5 6 7 13
+20 8 1 6 7 8 13
+19 8 6 7 8 13 15
+14 8 4 7 8 13 15
+21 8 4 8 12 13 15
+22 8 4 8 11 12 13
+23 8 6 8 13 14 15
+24 8 1 6 8 13 14
+9  8 1 8 9 10 13
+18 8 1 8 9 13 14
+
+boundary
+48
+1 4 0 1 3 4
+3 4 0 1 3 8
+3 4 0 3 4 8
+1 4 0 1 4 7
+3 4 0 1 7 8
+3 4 0 4 7 8
+1 4 1 4 5 7
+3 4 1 3 8 11
+1 4 1 3 4 5
+3 4 1 5 10 13
+3 4 1 8 10 11
+3 4 1 3 10 11
+1 4 1 2 3 5
+3 4 1 2 3 10
+3 4 1 2 5 10
+1 4 1 5 6 7
+3 4 1 5 6 13
+3 4 1 6 7 8
+3 4 1 6 8 14
+3 4 1 6 13 14
+3 4 1 8 9 10
+3 4 1 9 10 13
+3 4 1 8 9 14
+3 4 1 9 13 14
+3 4 2 3 5 10
+3 4 3 4 8 11
+3 4 3 4 5 11
+3 4 3 5 10 11
+3 4 4 5 11 13
+3 4 4 5 7 13
+3 4 4 7 8 15
+3 4 4 7 13 15
+3 4 4 8 12 15
+3 4 4 12 13 15
+3 4 4 8 11 12
+3 4 4 11 12 13
+3 4 5 10 11 13
+3 4 5 6 7 13
+3 4 6 7 8 15
+3 4 6 7 13 15
+3 4 6 8 14 15
+3 4 6 13 14 15
+5 4 8 10 11 13
+5 4 8 12 13 15
+5 4 8 11 12 13
+5 4 8 13 14 15
+5 4 8 9 10 13
+5 4 8 9 13 14
+
+vertices
+16
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000

data/cube4d_96.MFEM

@@ -0,0 +1,231 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+96
+1 8 0 1 7 8 9
+1 8 1 6 7 8 9
+1 8 4 5 6 8 9
+1 8 4 6 7 8 9
+1 8 0 1 3 8 9
+1 8 1 2 3 8 9
+1 8 0 4 7 8 9
+1 8 0 3 4 8 9
+1 8 1 2 6 8 9
+1 8 2 5 6 8 9
+1 8 3 4 5 8 9
+1 8 2 3 5 8 9
+1 8 9 10 11 17 18
+1 8 9 11 16 17 18
+1 8 9 14 15 16 18
+1 8 9 14 16 17 18
+1 8 9 10 11 13 18
+1 8 9 11 12 13 18
+1 8 9 10 14 17 18
+1 8 9 10 13 14 18
+1 8 9 11 12 16 18
+1 8 9 12 15 16 18
+1 8 9 13 14 15 18
+1 8 9 12 13 15 18
+1 8 9 12 15 16 19
+1 8 9 11 12 16 19
+1 8 1 6 9 11 19
+1 8 6 9 11 16 19
+1 8 1 2 6 9 19
+1 8 2 5 6 9 19
+1 8 2 5 9 15 19
+1 8 2 9 12 15 19
+1 8 5 6 9 16 19
+1 8 5 9 15 16 19
+1 8 1 9 11 12 19
+1 8 1 2 9 12 19
+1 8 9 10 13 14 20
+1 8 9 10 14 17 20
+1 8 0 9 10 17 20
+1 8 0 7 9 17 20
+1 8 0 3 4 9 20
+1 8 0 4 7 9 20
+1 8 3 4 9 13 20
+1 8 4 9 13 14 20
+1 8 4 7 9 14 20
+1 8 7 9 14 17 20
+1 8 0 3 9 10 20
+1 8 3 9 10 13 20
+1 8 2 5 9 15 21
+1 8 2 9 12 15 21
+1 8 2 3 5 9 21
+1 8 3 4 5 9 21
+1 8 9 13 14 15 21
+1 8 9 12 13 15 21
+1 8 4 5 9 14 21
+1 8 5 9 14 15 21
+1 8 2 3 9 12 21
+1 8 3 9 12 13 21
+1 8 3 4 9 13 21
+1 8 4 9 13 14 21
+1 8 1 6 9 11 22
+1 8 6 9 11 16 22
+1 8 6 7 9 16 22
+1 8 7 9 16 17 22
+1 8 0 7 9 17 22
+1 8 0 9 10 17 22
+1 8 0 1 9 10 22
+1 8 1 9 10 11 22
+1 8 0 1 7 9 22
+1 8 1 6 7 9 22
+1 8 9 10 11 17 22
+1 8 9 11 16 17 22
+1 8 5 9 15 16 23
+1 8 5 6 9 16 23
+1 8 6 7 9 16 23
+1 8 7 9 16 17 23
+1 8 7 9 14 17 23
+1 8 4 7 9 14 23
+1 8 4 5 9 14 23
+1 8 5 9 14 15 23
+1 8 4 6 7 9 23
+1 8 4 5 6 9 23
+1 8 9 14 15 16 23
+1 8 9 14 16 17 23
+1 8 1 2 9 12 24
+1 8 1 9 11 12 24
+1 8 0 1 9 10 24
+1 8 1 9 10 11 24
+1 8 0 3 9 10 24
+1 8 3 9 10 13 24
+1 8 2 3 9 12 24
+1 8 3 9 12 13 24
+1 8 1 2 3 9 24
+1 8 0 1 3 9 24
+1 8 9 10 11 13 24
+1 8 9 11 12 13 24
+
+boundary
+96
+1 4 0 1 7 8
+1 4 0 1 3 8
+1 4 0 4 7 8
+1 4 0 3 4 8
+2 4 0 10 17 20
+2 4 0 7 17 20
+2 4 0 3 4 20
+2 4 0 4 7 20
+2 4 0 3 10 20
+2 4 0 7 17 22
+2 4 0 10 17 22
+2 4 0 1 10 22
+2 4 0 1 7 22
+2 4 0 1 10 24
+2 4 0 3 10 24
+2 4 0 1 3 24
+1 4 1 6 7 8
+1 4 1 2 3 8
+1 4 1 2 6 8
+2 4 1 6 11 19
+2 4 1 2 6 19
+2 4 1 11 12 19
+2 4 1 2 12 19
+2 4 1 6 11 22
+2 4 1 10 11 22
+2 4 1 6 7 22
+2 4 1 2 12 24
+2 4 1 11 12 24
+2 4 1 10 11 24
+2 4 1 2 3 24
+1 4 2 5 6 8
+1 4 2 3 5 8
+2 4 2 5 6 19
+2 4 2 5 15 19
+2 4 2 12 15 19
+2 4 2 5 15 21
+2 4 2 12 15 21
+2 4 2 3 5 21
+2 4 2 3 12 21
+2 4 2 3 12 24
+1 4 3 4 5 8
+2 4 3 4 13 20
+2 4 3 10 13 20
+2 4 3 4 5 21
+2 4 3 12 13 21
+2 4 3 4 13 21
+2 4 3 10 13 24
+2 4 3 12 13 24
+1 4 4 5 6 8
+1 4 4 6 7 8
+2 4 4 13 14 20
+2 4 4 7 14 20
+2 4 4 5 14 21
+2 4 4 13 14 21
+2 4 4 7 14 23
+2 4 4 5 14 23
+2 4 4 6 7 23
+2 4 4 5 6 23
+2 4 5 6 16 19
+2 4 5 15 16 19
+2 4 5 14 15 21
+2 4 5 15 16 23
+2 4 5 6 16 23
+2 4 5 14 15 23
+2 4 6 11 16 19
+2 4 6 11 16 22
+2 4 6 7 16 22
+2 4 6 7 16 23
+2 4 7 14 17 20
+2 4 7 16 17 22
+2 4 7 16 17 23
+2 4 7 14 17 23
+3 4 10 11 17 18
+3 4 10 11 13 18
+3 4 10 14 17 18
+3 4 10 13 14 18
+2 4 10 13 14 20
+2 4 10 14 17 20
+2 4 10 11 17 22
+2 4 10 11 13 24
+3 4 11 16 17 18
+3 4 11 12 13 18
+3 4 11 12 16 18
+2 4 11 12 16 19
+2 4 11 16 17 22
+2 4 11 12 13 24
+3 4 12 15 16 18
+3 4 12 13 15 18
+2 4 12 15 16 19
+2 4 12 13 15 21
+3 4 13 14 15 18
+2 4 13 14 15 21
+3 4 14 15 16 18
+3 4 14 16 17 18
+2 4 14 15 16 23
+2 4 14 16 17 23
+
+vertices
+25
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 0.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 1.0000000000000000
+1.0000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.0000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 1.0000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 0.0000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 0.5000000000000000 1.0000000000000000 0.5000000000000000
+0.5000000000000000 0.5000000000000000 0.0000000000000000 0.5000000000000000

data/line-space-time.mesh

@@ -0,0 +1,36 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+2
+1 2 2 0 1
+1 2 0 2 3
+
+boundary
+4
+1 1 0 1
+1 1 1 2
+2 1 2 3
+1 1 3 0
+
+vertices
+4
+2
+0 0
+1 0
+1 1
+0 1

data/pipe-nurbs-2d.mesh

@@ -18,9 +18,9 @@ elements
 boundary
 4
 1 1 0 1
-2 1 2 3
-3 1 3 0
-4 1 1 2
+1 1 2 3
+1 1 3 0
+1 1 1 2
 
 edges
 4

data/square-space-time.mesh

@@ -0,0 +1,52 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+6
+1 4 3 1 7 5
+1 4 1 6 7 4
+1 4 6 1 0 2
+1 4 1 6 4 2
+1 4 6 1 3 0
+1 4 1 6 3 7
+
+boundary
+12
+1 2 6 0 3
+1 2 0 6 2
+1 2 1 3 0
+1 2 3 1 5
+1 2 3 7 6
+1 2 7 3 5
+1 2 4 6 7
+1 2 6 4 2
+2 2 1 7 5
+2 2 7 1 4
+1 2 1 2 4
+1 2 2 1 0
+
+vertices
+8
+3
+0 0 0
+0 0 1
+1 0 0
+0 1 0
+1 0 1
+0 1 1
+1 1 0
+1 1 1

doc/CodeDocumentation.conf.in

@@ -938,7 +938,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \
-                         @MFEM_SOURCE_DIR@/linalg/batched \
                          @MFEM_SOURCE_DIR@/linalg/simd \
                          @MFEM_SOURCE_DIR@/mesh \
                          @MFEM_SOURCE_DIR@/mesh/submesh \
@@ -1050,8 +1049,7 @@ RECURSIVE              = NO
 EXCLUDE                = @MFEM_SOURCE_DIR@/config/_config.hpp \
                          @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp \
                          @MFEM_SOURCE_DIR@/general/tinyxml2.h \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp \
-                         @MFEM_SOURCE_DIR@/linalg/lapack.hpp
+                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -1210,13 +1208,13 @@ STRIP_CODE_COMMENTS    = NO
 # entity all documented functions referencing it will be listed.
 # The default value is: NO.
 
-REFERENCED_BY_RELATION = NO
+REFERENCED_BY_RELATION = YES
 
 # If the REFERENCES_RELATION tag is set to YES then for each documented function
 # all documented entities called/used by that function will be listed.
 # The default value is: NO.
 
-REFERENCES_RELATION    = NO
+REFERENCES_RELATION    = YES
 
 # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
 # to YES then the hyperlinks from functions in REFERENCES_RELATION and

doc/CodeDocumentation.dox

@@ -182,21 +182,6 @@ namespace mfem {
  *   <a class="el" href="examples_2superlu_2ex1p_8cpp_source.html">1p</a>,
  *   demonstrating the use of MFEM's \link superlu.hpp SuperLU integration\endlink.
  *
- * <H4>NURBS Examples</H4>
- * - Variants of Examples
- *  <a class="el" href="nurbs__ex1_8cpp_source.html">1</a>,
- *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
- *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
- *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
- *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
- *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
- *   demonstrating howto perform NURBS-based Isogeometric Analysis.
- * - Variant of Example <a class="el" href="nurbs__patch__ex1_8cpp_source.html">1</a>: demonstrates the use of patch integration
- * - <a class="el" href="nurbs__solenoidal_8cpp_source.html">NURBS Divergence-free</a>: solve a solenoidal vector projection with NURBS-based H(div) elements
- * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
- * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
- * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
- *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code

examples/CMakeLists.txt

@@ -73,6 +73,9 @@ if (MFEM_USE_MPI)
     ex20p.cpp
     ex21p.cpp
     ex22p.cpp
+    ex1p_4d.cpp
+    ex3p_4d.cpp
+    ex4D_DivSkew.cpp
     ex24p.cpp
     ex25p.cpp
     ex26p.cpp

examples/amgx/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/amgx/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/caliper/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/caliper,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/ex10.cpp

@@ -3,14 +3,14 @@
 // Compile with: make ex10
 //
 // Sample runs:
-//    ex10 -m ../data/beam-quad.mesh -s 23 -r 2 -o 2 -dt 3
-//    ex10 -m ../data/beam-tri.mesh -s 23 -r 2 -o 2 -dt 3
-//    ex10 -m ../data/beam-hex.mesh -s 22 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-tet.mesh -s 22 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-wedge.mesh -s 22 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-quad.mesh -s 4 -r 2 -o 2 -dt 0.03 -vs 20
-//    ex10 -m ../data/beam-hex.mesh -s 4 -r 1 -o 2 -dt 0.05 -vs 20
-//    ex10 -m ../data/beam-quad-amr.mesh -s 23 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-quad.mesh -s 3 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-tri.mesh -s 3 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-hex.mesh -s 2 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-tet.mesh -s 2 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-wedge.mesh -s 2 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-quad.mesh -s 14 -r 2 -o 2 -dt 0.03 -vs 20
+//    ex10 -m ../data/beam-hex.mesh -s 14 -r 1 -o 2 -dt 0.05 -vs 20
+//    ex10 -m ../data/beam-quad-amr.mesh -s 3 -r 2 -o 2 -dt 3
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -87,16 +87,16 @@ public:
                         real_t visc, real_t mu, real_t K);
 
    /// Compute the right-hand side of the ODE system.
-   void Mult(const Vector &vx, Vector &dvx_dt) const override;
+   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
    /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
 
    real_t ElasticEnergy(const Vector &x) const;
    real_t KineticEnergy(const Vector &v) const;
    void GetElasticEnergyDensity(const GridFunction &x, GridFunction &w) const;
 
-   ~HyperelasticOperator() override;
+   virtual ~HyperelasticOperator();
 };
 
 /** Nonlinear operator of the form:
@@ -120,12 +120,12 @@ public:
    void SetParameters(real_t dt_, const Vector *v_, const Vector *x_);
 
    /// Compute y = H(x + dt (v + dt k)) + M k + S (v + dt k).
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual void Mult(const Vector &k, Vector &y) const;
 
    /// Compute J = M + dt S + dt^2 grad_H(x + dt (v + dt k)).
-   Operator &GetGradient(const Vector &k) const override;
+   virtual Operator &GetGradient(const Vector &k) const;
 
-   ~ReducedSystemOperator() override;
+   virtual ~ReducedSystemOperator();
 };
 
 
@@ -141,8 +141,8 @@ private:
 public:
    ElasticEnergyCoefficient(HyperelasticModel &m, const GridFunction &x_)
       : model(m), x(x_) { }
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
-   ~ElasticEnergyCoefficient() override { }
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual ~ElasticEnergyCoefficient() { }
 };
 
 void InitialDeformation(const Vector &x, Vector &y);
@@ -160,7 +160,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/beam-quad.mesh";
    int ref_levels = 2;
    int order = 2;
-   int ode_solver_type = 23;
+   int ode_solver_type = 3;
    real_t t_final = 300.0;
    real_t dt = 3.0;
    real_t visc = 1e-2;
@@ -177,7 +177,11 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4."
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -209,7 +213,28 @@ int main(int argc, char *argv[])
    // 3. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1: ode_solver = new BackwardEulerSolver; break;
+      case 2: ode_solver = new SDIRK23Solver(2); break;
+      case 3: ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -346,6 +371,7 @@ int main(int argc, char *argv[])
    }
 
    // 10. Free the used memory.
+   delete ode_solver;
    delete mesh;
 
    return 0;

examples/ex10p.cpp

@@ -3,14 +3,14 @@
 // Compile with: make ex10p
 //
 // Sample runs:
-//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 23 -rs 2 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-tri.mesh -s 23 -rs 2 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 22 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-tet.mesh -s 22 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-wedge.mesh -s 22 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 4 -rs 2 -dt 0.03 -vs 20
-//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 4 -rs 1 -dt 0.05 -vs 20
-//    mpirun -np 4 ex10p -m ../data/beam-quad-amr.mesh -s 23 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 3 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-tri.mesh -s 3 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 2 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-tet.mesh -s 2 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-wedge.mesh -s 2 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 14 -rs 2 -dt 0.03 -vs 20
+//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 14 -rs 1 -dt 0.05 -vs 20
+//    mpirun -np 4 ex10p -m ../data/beam-quad-amr.mesh -s 3 -rs 2 -dt 3
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -89,17 +89,17 @@ public:
                         real_t visc, real_t mu, real_t K);
 
    /// Compute the right-hand side of the ODE system.
-   void Mult(const Vector &vx, Vector &dvx_dt) const override;
+   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
    /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
 
    real_t ElasticEnergy(const ParGridFunction &x) const;
    real_t KineticEnergy(const ParGridFunction &v) const;
    void GetElasticEnergyDensity(const ParGridFunction &x,
                                 ParGridFunction &w) const;
 
-   ~HyperelasticOperator() override;
+   virtual ~HyperelasticOperator();
 };
 
 /** Nonlinear operator of the form:
@@ -125,12 +125,12 @@ public:
    void SetParameters(real_t dt_, const Vector *v_, const Vector *x_);
 
    /// Compute y = H(x + dt (v + dt k)) + M k + S (v + dt k).
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual void Mult(const Vector &k, Vector &y) const;
 
    /// Compute J = M + dt S + dt^2 grad_H(x + dt (v + dt k)).
-   Operator &GetGradient(const Vector &k) const override;
+   virtual Operator &GetGradient(const Vector &k) const;
 
-   ~ReducedSystemOperator() override;
+   virtual ~ReducedSystemOperator();
 };
 
 
@@ -146,8 +146,8 @@ private:
 public:
    ElasticEnergyCoefficient(HyperelasticModel &m, const ParGridFunction &x_)
       : model(m), x(x_) { }
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
-   ~ElasticEnergyCoefficient() override { }
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual ~ElasticEnergyCoefficient() { }
 };
 
 void InitialDeformation(const Vector &x, Vector &y);
@@ -172,7 +172,7 @@ int main(int argc, char *argv[])
    int ser_ref_levels = 2;
    int par_ref_levels = 0;
    int order = 2;
-   int ode_solver_type = 23;
+   int ode_solver_type = 3;
    real_t t_final = 300.0;
    real_t dt = 3.0;
    real_t visc = 1e-2;
@@ -192,7 +192,11 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4."
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -234,7 +238,31 @@ int main(int argc, char *argv[])
    // 4. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         if (myid == 0)
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         return 3;
+   }
 
    // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -405,6 +433,7 @@ int main(int argc, char *argv[])
    }
 
    // 12. Free the used memory.
+   delete ode_solver;
    delete pmesh;
 
    return 0;

examples/ex14.cpp

@@ -20,6 +20,7 @@
 //               ex14 -m ../data/fichera-amr.mesh
 //               ex14 -pa -r 1 -o 3
 //               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
+//               ex14 -m ../data/inline-tet.mesh -o 0 -nt 4 -s 1
 //
 // Device sample runs:
 //               ex14 -pa -r 2 -d cuda -o 3
@@ -55,10 +56,16 @@ int main(int argc, char *argv[])
    bool pa = false;
    bool visualization = 1;
    const char *device_config = "cpu";
+   int nt = 0;
+   double st = 1.0;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&nt, "-nt", "--number-of-timeslices",
+                  "Number of slices through the hyper-prism in the 4th coordinate.");
+   args.AddOption(&st, "-st", "--size-time",
+                  "Length of hyper-prims in 4th coordinate (e.g. time).");
    args.AddOption(&ref_levels, "-r", "--refine",
                   "Number of times to refine the mesh uniformly, -1 for auto.");
    args.AddOption(&order, "-o", "--order",
@@ -97,8 +104,17 @@ int main(int argc, char *argv[])
    // 3. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
    //    NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   const int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   if (dim == 3 && nt > 0)
+   {
+      Mesh* spat_mesh = mesh;
+      mesh = new Mesh(spat_mesh, nt, Element::PENTATOPE, true, st);
+      delete spat_mesh;
+
+      ref_levels = 0;
+      dim = 4;
+   }
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
@@ -107,23 +123,23 @@ int main(int argc, char *argv[])
    {
       if (ref_levels < 0)
       {
-         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/(dim < 4 ? dim : 1.));
       }
       for (int l = 0; l < ref_levels; l++)
       {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
       }
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
 
    // 5. Define a finite element space on the mesh. Here we use discontinuous
    //    finite elements of the specified order >= 0.
    const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
    DG_FECollection fec(order, dim, bt);
-   FiniteElementSpace fespace(&mesh, &fec);
+   FiniteElementSpace fespace(mesh, &fec);
    cout << "Number of unknowns: " << fespace.GetVSize() << endl;
 
    // 6. Set up the linear form b(.) which corresponds to the right-hand side of
@@ -198,7 +214,7 @@ int main(int argc, char *argv[])
    //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
    ofstream mesh_ofs("refined.mesh");
    mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
    ofstream sol_ofs("sol.gf");
    sol_ofs.precision(8);
    x.Save(sol_ofs);
@@ -210,8 +226,10 @@ int main(int argc, char *argv[])
       int  visport   = 19916;
       socketstream sol_sock(vishost, visport);
       sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
    }
 
+   delete mesh;
+
    return 0;
 }

examples/ex14p.cpp

@@ -19,6 +19,7 @@
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
 //               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
 //               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
+//               mpirun -np 4 ex14p -m ../data/inline-tet.mesh -o 0 -nt 4 -s 1
 //
 // Device sample runs:
 //               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
@@ -53,7 +54,7 @@ public:
       pmesh(pmesh_),
       pgf(pgf_) {}
 
-   void MonitorSolution(int i, real_t norm, const Vector &x, bool final) override
+   void MonitorSolution(int i, real_t norm, const Vector &x, bool final)
    {
       char vishost[] = "localhost";
       int  visport   = 19916;
@@ -90,10 +91,16 @@ int main(int argc, char *argv[])
    bool pa = false;
    bool visualization = 1;
    const char *device_config = "cpu";
+   int nt = 0;
+   double st = 1.0;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&nt, "-nt", "--number-of-timeslices",
+                  "Number of slices through the hyper-prism in the 4th coordinate.");
+   args.AddOption(&st, "-st", "--size-time",
+                  "Length of hyper-prims in 4th coordinate (e.g. time).");
    args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
                   "Number of times to refine the mesh uniformly in serial,"
                   " -1 for auto.");
@@ -139,8 +146,17 @@ int main(int argc, char *argv[])
    // 3. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   if (dim == 3 && nt > 0)
+   {
+      Mesh* spat_mesh = mesh;
+      mesh = new Mesh(spat_mesh, nt, Element::PENTATOPE, true, st);
+      delete spat_mesh;
+      dim = 4;
+   }
+   if (dim == 4)
+      ser_ref_levels = 0;
 
    // 4. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -149,23 +165,23 @@ int main(int argc, char *argv[])
    {
       if (ser_ref_levels < 0)
       {
-         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
       }
       for (int l = 0; l < ser_ref_levels; l++)
       {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
       }
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
 
    // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
    {
       for (int l = 0; l < par_ref_levels; l++)
       {

examples/ex16.cpp

@@ -5,10 +5,10 @@
 // Sample runs:  ex16
 //               ex16 -m ../data/inline-tri.mesh
 //               ex16 -m ../data/disc-nurbs.mesh -tf 2
-//               ex16 -s 21 -a 0.0 -k 1.0
-//               ex16 -s 22 -a 1.0 -k 0.0
-//               ex16 -s 23 -a 0.5 -k 0.5 -o 4
-//               ex16 -s 4 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               ex16 -s 1 -a 0.0 -k 1.0
+//               ex16 -s 2 -a 1.0 -k 0.0
+//               ex16 -s 3 -a 0.5 -k 0.5 -o 4
+//               ex16 -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               ex16 -m ../data/fichera-q2.mesh
 //               ex16 -m ../data/fichera-mixed.mesh
 //               ex16 -m ../data/escher.mesh
@@ -76,15 +76,15 @@ public:
    ConductionOperator(FiniteElementSpace &f, real_t alpha, real_t kappa,
                       const Vector &u);
 
-   void Mult(const Vector &u, Vector &du_dt) const override;
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
    /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &u, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &u, Vector &k);
 
    /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
    void SetParameters(const Vector &u);
 
-   ~ConductionOperator() override;
+   virtual ~ConductionOperator();
 };
 
 real_t InitialTemperature(const Vector &x);
@@ -95,13 +95,11 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int ref_levels = 2;
    int order = 2;
-
-   int ode_solver_type = 23;  // SDIRK33Solver
+   int ode_solver_type = 3;
    real_t t_final = 0.5;
    real_t dt = 1.0e-2;
    real_t alpha = 1.0e-2;
    real_t kappa = 0.5;
-
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
@@ -117,7 +115,8 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -150,7 +149,28 @@ int main(int argc, char *argv[])
    // 3. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -267,6 +287,7 @@ int main(int argc, char *argv[])
    }
 
    // 10. Free the used memory.
+   delete ode_solver;
    delete mesh;
 
    return 0;

examples/ex16p.cpp

@@ -5,10 +5,10 @@
 // Sample runs:  mpirun -np 4 ex16p
 //               mpirun -np 4 ex16p -m ../data/inline-tri.mesh
 //               mpirun -np 4 ex16p -m ../data/disc-nurbs.mesh -tf 2
-//               mpirun -np 4 ex16p -s 21 -a 0.0 -k 1.0
-//               mpirun -np 4 ex16p -s 22 -a 1.0 -k 0.0
-//               mpirun -np 8 ex16p -s 23 -a 0.5 -k 0.5 -o 4
-//               mpirun -np 4 ex16p -s 4 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               mpirun -np 4 ex16p -s 1 -a 0.0 -k 1.0
+//               mpirun -np 4 ex16p -s 2 -a 1.0 -k 0.0
+//               mpirun -np 8 ex16p -s 3 -a 0.5 -k 0.5 -o 4
+//               mpirun -np 4 ex16p -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               mpirun -np 16 ex16p -m ../data/fichera-q2.mesh
 //               mpirun -np 16 ex16p -m ../data/fichera-mixed.mesh
 //               mpirun -np 16 ex16p -m ../data/escher-p2.mesh
@@ -78,15 +78,15 @@ public:
    ConductionOperator(ParFiniteElementSpace &f, real_t alpha, real_t kappa,
                       const Vector &u);
 
-   void Mult(const Vector &u, Vector &du_dt) const override;
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
    /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &u, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &u, Vector &k);
 
    /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
    void SetParameters(const Vector &u);
 
-   ~ConductionOperator() override;
+   virtual ~ConductionOperator();
 };
 
 real_t InitialTemperature(const Vector &x);
@@ -104,13 +104,11 @@ int main(int argc, char *argv[])
    int ser_ref_levels = 2;
    int par_ref_levels = 1;
    int order = 2;
-
-   int ode_solver_type = 23;  // SDIRK33Solver
+   int ode_solver_type = 3;
    real_t t_final = 0.5;
    real_t dt = 1.0e-2;
    real_t alpha = 1.0e-2;
    real_t kappa = 0.5;
-
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
@@ -129,7 +127,8 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -170,7 +169,28 @@ int main(int argc, char *argv[])
    // 4. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
 
    // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -356,6 +376,7 @@ int main(int argc, char *argv[])
    }
 
    // 12. Free the used memory.
+   delete ode_solver;
    delete pmesh;
 
    return 0;

examples/ex17.cpp

@@ -69,7 +69,7 @@ public:
    void SetDisplacement(GridFunction &u_) { u = &u_; }
    void SetComponent(int i, int j) { si = i; sj = j; }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };
 
 // Simple GLVis visualization manager.
@@ -89,7 +89,7 @@ public:
    void NewWindow();
    void CloseConnection();
    void PositionWindow();
-   ~VisMan() override;
+   virtual ~VisMan();
 };
 
 // Manipulators for the GLVis visualization manager.

examples/ex17p.cpp

@@ -69,7 +69,7 @@ public:
    void SetDisplacement(GridFunction &u_) { u = &u_; }
    void SetComponent(int i, int j) { si = i; sj = j; }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };
 
 // Simple GLVis visualization manager.
@@ -89,7 +89,7 @@ public:
    void NewWindow();
    void CloseConnection();
    void PositionWindow();
-   ~VisMan() override;
+   virtual ~VisMan();
 };
 
 // Manipulators for the GLVis visualization manager.

examples/ex18.cpp

@@ -90,7 +90,8 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::ExplicitTypes.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
    args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step. Positive number skips CFL timestep calculation.");
@@ -124,7 +125,18 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectExplicit(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         return 3;
+   }
 
    // 4. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
@@ -292,5 +304,8 @@ int main(int argc, char *argv[])
    const real_t error = sol.ComputeLpError(2, u0);
    cout << "Solution error: " << error << endl;
 
+   // Free the used memory.
+   delete ode_solver;
+
    return 0;
 }

examples/ex18p.cpp

@@ -99,7 +99,8 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::ExplicitTypes.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
    args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step. Positive number skips CFL timestep calculation.");
@@ -147,7 +148,18 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectExplicit(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         return 3;
+   }
 
    // 4. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
@@ -348,5 +360,8 @@ int main(int argc, char *argv[])
       cout << "Solution error: " << error << endl;
    }
 
+   // Free the used memory.
+   delete ode_solver;
+
    return 0;
 }

examples/ex19.cpp

@@ -48,7 +48,7 @@ public:
       print_level = print_lvl;
    }
 
-   void MonitorResidual(int it, real_t norm, const Vector &r, bool final) override;
+   virtual void MonitorResidual(int it, real_t norm, const Vector &r, bool final);
 
 private:
    const std::string prefix;
@@ -116,10 +116,10 @@ public:
    JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
                           SparseMatrix &mass, Array<int> &offsets);
 
-   void Mult(const Vector &k, Vector &y) const override;
-   void SetOperator(const Operator &op) override;
+   virtual void Mult(const Vector &k, Vector &y) const;
+   virtual void SetOperator(const Operator &op);
 
-   ~JacobianPreconditioner() override;
+   virtual ~JacobianPreconditioner();
 };
 
 // After spatial discretization, the rubber model can be written as:
@@ -161,13 +161,13 @@ public:
                   int iter, Coefficient &mu);
 
    // Required to use the native newton solver
-   Operator &GetGradient(const Vector &xp) const override;
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual Operator &GetGradient(const Vector &xp) const;
+   virtual void Mult(const Vector &k, Vector &y) const;
 
    // Driver for the newton solver
    void Solve(Vector &xp) const;
 
-   ~RubberOperator() override;
+   virtual ~RubberOperator();
 };
 
 // Visualization driver

examples/ex19p.cpp

@@ -62,7 +62,7 @@ public:
 #endif
    }
 
-   void MonitorResidual(int it, real_t norm, const Vector &r, bool final) override;
+   virtual void MonitorResidual(int it, real_t norm, const Vector &r, bool final);
 
 private:
    const std::string prefix;
@@ -130,10 +130,10 @@ public:
    JacobianPreconditioner(Array<ParFiniteElementSpace *> &fes,
                           Operator &mass, Array<int> &offsets);
 
-   void Mult(const Vector &k, Vector &y) const override;
-   void SetOperator(const Operator &op) override;
+   virtual void Mult(const Vector &k, Vector &y) const;
+   virtual void SetOperator(const Operator &op);
 
-   ~JacobianPreconditioner() override;
+   virtual ~JacobianPreconditioner();
 };
 
 // After spatial discretization, the rubber model can be written as:
@@ -175,13 +175,13 @@ public:
                   int iter, Coefficient &mu);
 
    // Required to use the native newton solver
-   Operator &GetGradient(const Vector &xp) const override;
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual Operator &GetGradient(const Vector &xp) const;
+   virtual void Mult(const Vector &k, Vector &y) const;
 
    // Driver for the newton solver
    void Solve(Vector &xp) const;
 
-   ~RubberOperator() override;
+   virtual ~RubberOperator();
 };
 
 // Visualization driver

examples/ex1p_4d.cpp

@@ -0,0 +1,412 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "./spe10_coeff.cpp"
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_grad.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_grad.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+using namespace std;
+using namespace mfem;
+
+double kappa = 1.0;
+
+double u_exact(const Vector &x)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      return cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+   }
+   else { return 0.0; }
+}
+
+double f_exact(const Vector &x)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      return (kappa + 4.0 * M_PI*M_PI) * cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(
+                                                                              2))*cos(M_PI*x(3));
+   }
+   else { return 0.0; }
+}
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   bool set_bc = true;
+   bool standardCG = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   //   if(dim !=4 || sdim != 4)
+   //   {
+   //    MPI_Finalize();
+   //    return 0;
+   //   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      if (dim==4)
+      {
+         if (order==1) { fec = new LinearFECollection; }
+         else { fec = new QuadraticFECollection; }
+      }
+      else { fec = new H1_FECollection(order, dim); }
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = set_bc ? 1 : 0;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   FunctionCoefficient uExact(u_exact);
+   ParGridFunction x(fespace);
+
+   int NExpo =8;
+   for (int expo=-NExpo; expo<=NExpo; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      x.ProjectCoefficient(uExact);
+
+      ParLinearForm *b = new ParLinearForm(fespace);
+      FunctionCoefficient ffunc(f_exact);
+      b->AddDomainIntegrator(new DomainLFIntegrator(ffunc));
+      b->Assemble();
+
+      x = 0.0;
+
+      // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+      //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+      //     domain integrator.
+
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+      //    FunctionCoefficient *cspe10 = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      Coefficient *beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DiffusionIntegrator);
+      a->AddDomainIntegrator(new MassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreSolver *amg = new HypreBoomerAMG(A);
+
+      int iter = -1;
+      if (standardCG)
+      {
+         IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+         pcg->SetOperator(A);
+         pcg->SetRelTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetPrintLevel(1);
+         pcg->SetPreconditioner(*amg);
+         pcg->Mult(B, X);
+
+         iter = pcg->GetNumIterations();
+
+         delete pcg;
+      }
+      else
+      {
+         HyprePCG *pcg = new HyprePCG(A);
+         pcg->SetTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetResidualConvergenceOptions(1,tol);
+         pcg->SetPrintLevel(2);
+         pcg->SetPreconditioner(*amg);
+         pcg->Mult(B, X);
+
+         pcg->GetNumIterations(iter);
+
+         delete pcg;
+      }
+
+
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      {
+         double err = x.ComputeL2Error(uExact);
+         if (myid == 0)
+         {
+            cout << "\n|| u - u_h ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 14. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //   {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //   }
+
+      // 15. Send the solution by socket to a GLVis server.
+      //   if (visualization)
+      //   {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //   }
+
+      delete amg;
+      delete a;
+      delete beta;
+      delete b;
+   }
+
+   // 16. Free the used memory.
+
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/ex20.cpp

@@ -79,14 +79,14 @@ class GradT : public Operator
 {
 public:
    GradT() : Operator(1) {}
-   void Mult(const Vector &x, Vector &y) const override { y.Set(1.0/m_, x); }
+   void Mult(const Vector &x, Vector &y) const { y.Set(1.0/m_, x); }
 };
 
 class NegGradV : public TimeDependentOperator
 {
 public:
    NegGradV() : TimeDependentOperator(1) {}
-   void Mult(const Vector &x, Vector &y) const override;
+   void Mult(const Vector &x, Vector &y) const;
 };
 
 int main(int argc, char *argv[])

examples/ex20p.cpp

@@ -84,14 +84,14 @@ class GradT : public Operator
 {
 public:
    GradT() : Operator(1) {}
-   void Mult(const Vector &x, Vector &y) const override { y.Set(1.0/m_, x); }
+   void Mult(const Vector &x, Vector &y) const { y.Set(1.0/m_, x); }
 };
 
 class NegGradV : public TimeDependentOperator
 {
 public:
    NegGradV() : TimeDependentOperator(1) {}
-   void Mult(const Vector &x, Vector &y) const override;
+   void Mult(const Vector &x, Vector &y) const;
 };
 
 int main(int argc, char *argv[])

examples/ex23.cpp

@@ -44,7 +44,7 @@ protected:
    BilinearForm *M;
    BilinearForm *K;
 
-   SparseMatrix Mmat, Kmat;
+   SparseMatrix Mmat, Kmat, Kmat0;
    SparseMatrix *T; // T = M + dt K
    real_t current_dt;
 
@@ -61,20 +61,20 @@ public:
    WaveOperator(FiniteElementSpace &f, Array<int> &ess_bdr, real_t speed);
 
    using SecondOrderTimeDependentOperator::Mult;
-   void Mult(const Vector &u, const Vector &du_dt,
-             Vector &d2udt2) const override;
+   virtual void Mult(const Vector &u, const Vector &du_dt,
+                     Vector &d2udt2) const;
 
    /** Solve the Backward-Euler equation:
        d2udt2 = f(u + fac0*d2udt2,dudt + fac1*d2udt2, t),
        for the unknown d2udt2. */
    using SecondOrderTimeDependentOperator::ImplicitSolve;
-   void ImplicitSolve(const real_t fac0, const real_t fac1,
-                      const Vector &u, const Vector &dudt, Vector &d2udt2) override;
+   virtual void ImplicitSolve(const real_t fac0, const real_t fac1,
+                              const Vector &u, const Vector &dudt, Vector &d2udt2);
 
    ///
    void SetParameters(const Vector &u);
 
-   ~WaveOperator() override;
+   virtual ~WaveOperator();
 };
 
 
@@ -83,24 +83,25 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
    : SecondOrderTimeDependentOperator(f.GetTrueVSize(), (real_t) 0.0),
      fespace(f), M(NULL), K(NULL), T(NULL), current_dt(0.0), z(height)
 {
-   // Assemble Laplace matrix
+   const real_t rel_tol = 1e-8;
+
+   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
    c2 = new ConstantCoefficient(speed*speed);
+
    K = new BilinearForm(&fespace);
    K->AddDomainIntegrator(new DiffusionIntegrator(*c2));
    K->Assemble();
 
-   // Assemble Mass matrix
+   Array<int> dummy;
+   K->FormSystemMatrix(dummy, Kmat0);
+   K->FormSystemMatrix(ess_tdof_list, Kmat);
+
    M = new BilinearForm(&fespace);
    M->AddDomainIntegrator(new MassIntegrator());
    M->Assemble();
-
-   // Apply Bcs
-   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   K->FormSystemMatrix(ess_tdof_list, Kmat);
    M->FormSystemMatrix(ess_tdof_list, Mmat);
 
-   // Configure preconditioner
-   const real_t rel_tol = 1e-8;
    M_solver.iterative_mode = false;
    M_solver.SetRelTol(rel_tol);
    M_solver.SetAbsTol(0.0);
@@ -109,13 +110,14 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(Mmat);
 
-   // Configure solver
    T_solver.iterative_mode = false;
    T_solver.SetRelTol(rel_tol);
    T_solver.SetAbsTol(0.0);
    T_solver.SetMaxIter(100);
    T_solver.SetPrintLevel(0);
    T_solver.SetPreconditioner(T_prec);
+
+   T = NULL;
 }
 
 void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
@@ -124,11 +126,9 @@ void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
    // Compute:
    //    d2udt2 = M^{-1}*-K(u)
    // for d2udt2
-   K->FullMult(u, z);
+   Kmat.Mult(u, z);
    z.Neg(); // z = -z
-   z.SetSubVector(ess_tdof_list, 0.0);
    M_solver.Mult(z, d2udt2);
-   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }
 
 void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
@@ -142,11 +142,14 @@ void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
       T = Add(1.0, Mmat, fac0, Kmat);
       T_solver.SetOperator(*T);
    }
-   K->FullMult(u, z);
+   Kmat0.Mult(u, z);
    z.Neg();
-   z.SetSubVector(ess_tdof_list, 0.0);
+
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      z[ess_tdof_list[i]] = 0.0;
+   }
    T_solver.Mult(z, d2udt2);
-   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }
 
 void WaveOperator::SetParameters(const Vector &u)
@@ -201,7 +204,9 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  SecondOrderODESolver::Types.c_str());
+                  "ODE solver: [0--10] - GeneralizedAlpha(0.1 * s),\n\t"
+                  "\t   11 - Average Acceleration, 12 - Linear Acceleration\n"
+                  "\t   13 - CentralDifference, 14 - FoxGoodwin");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -236,7 +241,32 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several second order
    //    time integrators are available.
-   SecondOrderODESolver *ode_solver= SecondOrderODESolver::Select(ode_solver_type);
+   SecondOrderODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit methods
+      case 0: ode_solver = new GeneralizedAlpha2Solver(0.0); break;
+      case 1: ode_solver = new GeneralizedAlpha2Solver(0.1); break;
+      case 2: ode_solver = new GeneralizedAlpha2Solver(0.2); break;
+      case 3: ode_solver = new GeneralizedAlpha2Solver(0.3); break;
+      case 4: ode_solver = new GeneralizedAlpha2Solver(0.4); break;
+      case 5: ode_solver = new GeneralizedAlpha2Solver(0.5); break;
+      case 6: ode_solver = new GeneralizedAlpha2Solver(0.6); break;
+      case 7: ode_solver = new GeneralizedAlpha2Solver(0.7); break;
+      case 8: ode_solver = new GeneralizedAlpha2Solver(0.8); break;
+      case 9: ode_solver = new GeneralizedAlpha2Solver(0.9); break;
+      case 10: ode_solver = new GeneralizedAlpha2Solver(1.0); break;
+
+      case 11: ode_solver = new AverageAccelerationSolver(); break;
+      case 12: ode_solver = new LinearAccelerationSolver(); break;
+      case 13: ode_solver = new CentralDifferenceSolver(); break;
+      case 14: ode_solver = new FoxGoodwinSolver(); break;
+
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -284,6 +314,7 @@ int main(int argc, char *argv[])
          ess_bdr = 0;
       }
    }
+
    WaveOperator oper(fespace, ess_bdr, speed);
 
    u_gf.SetFromTrueDofs(u);

examples/ex25.cpp

@@ -103,8 +103,8 @@ public:
 
    using VectorCoefficient::Eval;
 
-   void Eval(Vector &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
    {
       real_t x[3];
       Vector transip(x, 3);

examples/ex25p.cpp

@@ -102,8 +102,8 @@ public:
 
    using VectorCoefficient::Eval;
 
-   void Eval(Vector &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
    {
       real_t x[3];
       Vector transip(x, 3);

examples/ex26p.cpp

@@ -58,7 +58,7 @@ public:
       }
    }
 
-   ~DiffusionMultigrid() override
+   virtual ~DiffusionMultigrid()
    {
       delete amg;
    }

examples/ex36.cpp

@@ -53,7 +53,7 @@ public:
                                     real_t min_val_=-36)
       : u(&u_), obstacle(&obst_), min_val(min_val_) { }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };
 
 class ExponentialGridFunctionCoefficient : public Coefficient
@@ -69,7 +69,7 @@ public:
                                       real_t min_val_=0.0, real_t max_val_=1e6)
       : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };
 
 int main(int argc, char *argv[])
@@ -258,7 +258,7 @@ int main(int argc, char *argv[])
          MixedBilinearForm a10(&H1fes,&L2fes);
          a10.AddDomainIntegrator(new MixedScalarMassIntegrator());
          a10.Assemble();
-         a10.EliminateTrialEssentialBC(ess_bdr, x.GetBlock(0), rhs.GetBlock(1));
+         a10.EliminateTrialDofs(ess_bdr, x.GetBlock(0), rhs.GetBlock(1));
          a10.Finalize();
          SparseMatrix &A10 = a10.SpMat();
 

examples/ex36p.cpp

@@ -53,7 +53,7 @@ public:
                                     real_t min_val_=-36)
       : u(&u_), obstacle(&obst_), min_val(min_val_) { }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };
 
 class ExponentialGridFunctionCoefficient : public Coefficient
@@ -69,7 +69,7 @@ public:
                                       real_t min_val_=0.0, real_t max_val_=1e6)
       : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };
 
 int main(int argc, char *argv[])

examples/ex37.hpp

@@ -52,8 +52,8 @@ public:
        fun(fun_) {}
 
 
-   real_t Eval(ElementTransformation &T,
-               const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
    {
       return fun(GridFunctionCoefficient::Eval(T, ip));
    }
@@ -83,8 +83,8 @@ public:
        OtherGridF_cf(OtherGridF),
        fun(fun_) {}
 
-   real_t Eval(ElementTransformation &T,
-               const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
    {
       const real_t value1 = fun(GridFunctionCoefficient::Eval(T, ip));
       const real_t value2 = fun(OtherGridF_cf.Eval(T, ip));
@@ -108,7 +108,7 @@ public:
       : rho_filter(rho_filter_), min_val(min_val_), max_val(max_val_),
         exponent(exponent_) { }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip)
    {
       real_t val = rho_filter->GetValue(T, ip);
       real_t coeff = min_val + pow(val,exponent)*(max_val-min_val);
@@ -142,7 +142,7 @@ public:
       MFEM_ASSERT(rho_filter, "density field is not set");
    }
 
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip)
    {
       real_t L = lambda->Eval(T, ip);
       real_t M = mu->Eval(T, ip);
@@ -176,8 +176,8 @@ public:
 
    using VectorCoefficient::Eval;
 
-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip)
    {
       Vector xx; xx.SetSize(T.GetDimension());
       T.Transform(ip,xx);

examples/ex38.cpp

@@ -3,18 +3,18 @@
 // Compile with: make ex38
 //
 // Sample runs:
-// (since all sample runs require LAPACK or ALGOIM, the * symbol is used to
-//  exclude them from the automatically generated internal MFEM tests).
+// (since all sample runs require LAPACK, the * symbol is used to exclude them
+// from the automatically generated internal MFEM tests).
 //              * ex38
 //              * ex38 -i volumetric1d
 //              * ex38 -i surface2d
-//              * ex38 -i surface2d -o 4 -r 5 -m 1
+//              * ex38 -i surface2d -o 4 -r 5
 //              * ex38 -i volumetric2d
-//              * ex38 -i volumetric2d -o 4 -r 5 -m 1
+//              * ex38 -i volumetric2d -o 4 -r 5
 //              * ex38 -i surface3d
-//              * ex38 -i surface3d -o 3 -r 4 -m 1
+//              * ex38 -i surface3d -o 4 -r 5
 //              * ex38 -i volumetric3d
-//              * ex38 -i volumetric3d -o 3 -r 4 -m 1
+//              * ex38 -i volumetric3d -o 4 -r 5
 //
 // Description: This example code demonstrates the use of MFEM to integrate
 //              functions over implicit interfaces and subdomains bounded by
@@ -71,7 +71,7 @@ real_t integrand(const Vector& X)
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return pow(X(0), 2.);
+         return 1.;
       case IntegrationType::Surface2D:
          return 3. * pow(X(0), 2.) - pow(X(1), 2.);
       case IntegrationType::Volumetric2D:
@@ -91,7 +91,7 @@ real_t Surface()
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return .3025;
+         return 1.;
       case IntegrationType::Surface2D:
          return 2. * M_PI;
       case IntegrationType::Volumetric2D:
@@ -111,7 +111,7 @@ real_t Volume()
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return pow(.55, 3.) / 3.;
+         return .55;
       case IntegrationType::Surface2D:
          return NAN;
       case IntegrationType::Volumetric2D:
@@ -125,6 +125,7 @@ real_t Volume()
    }
 }
 
+#ifdef MFEM_USE_LAPACK
 /**
  @brief Class for surface IntegrationRule
 
@@ -134,14 +135,11 @@ real_t Volume()
 class SIntegrationRule : public IntegrationRule
 {
 protected:
-   /// method 0 is moments-based, 1 is Algoim.
-   int method, ir_order, ls_order;
-   Coefficient &level_set;
-   /// Space Dimension of the IntegrationRule
+   /// @brief Space Dimension of the IntegrationRule
    int dim;
-   /// Column-wise matrix of the quadtrature weights
+   /// @brief Column-wise matrix of the quadtrature weights
    DenseMatrix Weights;
-   /// Column-wise matrix of the transformation weights of the normal
+   /// @brief Column-wise matrix of the transformation weights of the normal
    DenseMatrix SurfaceWeights;
 
 public:
@@ -155,21 +153,15 @@ public:
     @param [in] lsOrder Polynomial degree for approx of level-set function
     @param [in] mesh Pointer to the mesh that is used
    */
-   SIntegrationRule(int method_, int Order,
-                    Coefficient& LvlSet, int lsOrder, Mesh* mesh)
-      : method(method_), ir_order(Order), ls_order(lsOrder),
-        level_set(LvlSet), dim(mesh->Dimension())
+   SIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
    {
-      // Nothing gets pre-computed for Algoim.
-      if (method == 1) { return; }
-
-#ifdef MFEM_USE_LAPACK
-      MomentFittingIntRules mf_ir(ir_order, level_set, ls_order);
+      dim = mesh->Dimension();
 
       IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
       mesh->GetElementTransformation(0, &Tr);
       IntegrationRule ir;
-      mf_ir.GetSurfaceIntegrationRule(Tr, ir);
+      MFIRs.GetSurfaceIntegrationRule(Tr, ir);
       if (dim >1)
       {
          Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
@@ -180,7 +172,7 @@ public:
       }
       SurfaceWeights.SetSize(ir.GetNPoints(), mesh->GetNE());
       Vector w;
-      mf_ir.GetSurfaceWeights(Tr, ir, w);
+      MFIRs.GetSurfaceWeights(Tr, ir, w);
       SurfaceWeights.SetCol(0, w);
       SetSize(ir.GetNPoints());
 
@@ -206,8 +198,8 @@ public:
       for (int elem = 1; elem < mesh->GetNE(); elem++)
       {
          mesh->GetElementTransformation(elem, &Tr);
-         mf_ir.GetSurfaceIntegrationRule(Tr, ir);
-         mf_ir.GetSurfaceWeights(Tr, ir, w);
+         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+         MFIRs.GetSurfaceWeights(Tr, ir, w);
          SurfaceWeights.SetCol(elem, w);
 
          for (int ip = 0; ip < GetNPoints(); ip++)
@@ -223,48 +215,48 @@ public:
             }
          }
       }
-#else
-      MFEM_ABORT("Moment-fitting requires MFEM to be built with LAPACK!");
-#endif
    }
 
    /**
     @brief Set the weights for the given element and multiply them with the
     transformation of the interface
     */
-   void SetElementAndSurfaceWeight(ElementTransformation &Tr)
+   void SetElementinclSurfaceWeight(int Element)
    {
-      if (method == 1)
-      {
-#ifdef MFEM_USE_ALGOIM
-         AlgoimIntegrationRules a_ir(ir_order, level_set, ls_order);
-         a_ir.GetSurfaceIntegrationRule(Tr, *this);
-         Vector w;
-         a_ir.GetSurfaceWeights(Tr, *this, w);
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntPoint(ip).weight *= w(ip);
-         }
-         return;
-#else
-         MFEM_ABORT("MFEM is not built with Algoim support!");
-#endif
-      }
-
       if (dim == 1)
       {
-         IntPoint(0).x = Weights(0, Tr.ElementNo);
-         IntPoint(0).weight = Weights(1, Tr.ElementNo);
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+         cout << intp.x << " " << Element << endl;
       }
       else
-      {
          for (int ip = 0; ip < GetNPoints(); ip++)
          {
-            IntPoint(ip).weight = Weights(ip, Tr.ElementNo) *
-                                  SurfaceWeights(ip, Tr.ElementNo);
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element) * SurfaceWeights(ip, Element);
          }
-      }
    }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of SIntegrationRule
+   ~SIntegrationRule() {}
 };
 
 /**
@@ -276,12 +268,9 @@ public:
 class CIntegrationRule : public IntegrationRule
 {
 protected:
-   /// method 0 is moments-based, 1 is Algoim.
-   int method, ir_order, ls_order;
-   Coefficient &level_set;
-   /// Space Dimension of the IntegrationRule
+   /// @brief Space Dimension of the IntegrationRule
    int dim;
-   /// Column-wise matrix of the quadtrature positions and weights.
+   /// @brief Column-wise matrix of the quadtrature weights
    DenseMatrix Weights;
 
 public:
@@ -295,21 +284,15 @@ public:
     @param [in] lsOrder Polynomial degree for approx of level-set function
     @param [in] mesh Pointer to the mesh that is used
    */
-   CIntegrationRule(int method_, int Order,
-                    Coefficient &LvlSet, int lsOrder, Mesh *mesh)
-      : method(method_), ir_order(Order), ls_order(lsOrder),
-        level_set(LvlSet), dim(mesh->Dimension())
+   CIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
    {
-      // Nothing gets pre-computed for Algoim.
-      if (method == 1) { return; }
-
-#ifdef MFEM_USE_LAPACK
-      MomentFittingIntRules mf_ir(ir_order, level_set, ls_order);
+      dim = mesh->Dimension();
 
       IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
       mesh->GetElementTransformation(0, &Tr);
       IntegrationRule ir;
-      mf_ir.GetVolumeIntegrationRule(Tr, ir);
+      MFIRs.GetVolumeIntegrationRule(Tr, ir);
       if (dim > 1)
       {
          Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
@@ -341,9 +324,9 @@ public:
       for (int elem = 1; elem < mesh->GetNE(); elem++)
       {
          mesh->GetElementTransformation(elem, &Tr);
-         mf_ir.GetVolumeIntegrationRule(Tr, ir);
+         MFIRs.GetVolumeIntegrationRule(Tr, ir);
 
-         for (int ip = 0; ip < ir.GetNPoints(); ip++)
+         for (int ip = 0; ip < GetNPoints(); ip++)
          {
             if (dim > 1)
             {
@@ -356,39 +339,29 @@ public:
             }
          }
       }
-#else
-      MFEM_ABORT("Moment-fitting requires MFEM to be built with LAPACK!");
-#endif
    }
 
    /// @brief Set the weights for the given element
-   void SetElement(ElementTransformation &Tr)
+   void SetElement(int Element)
    {
-      if (method == 1)
-      {
-#ifdef MFEM_USE_ALGOIM
-         AlgoimIntegrationRules a_ir(ir_order, level_set, ls_order);
-         a_ir.GetVolumeIntegrationRule(Tr, *this);
-         return;
-#else
-         MFEM_ABORT("MFEM is not built with Algoim support!");
-#endif
-      }
-
-      for (int ip = 0; ip < GetNPoints(); ip++)
-      {
-         IntegrationPoint &intp = IntPoint(ip);
-         if (dim == 1)
+      if (dim == 1)
+         for (int ip = 0; ip < GetNPoints(); ip++)
          {
-            intp.x = Weights(2 * ip, Tr.ElementNo);
-            intp.weight = Weights(2 * ip + 1, Tr.ElementNo);
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.x = Weights(2 * ip, Element);
+            intp.weight = Weights(2 * ip + 1, Element);
+         }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
          }
-         else { intp.weight = Weights(ip, Tr.ElementNo); }
-      }
    }
+
+   /// @brief Destructor of CIntegrationRule
+   ~CIntegrationRule() {}
 };
-
-
 /**
  @brief Class for surface linearform integrator
 
@@ -435,9 +408,9 @@ public:
     @param [in] Tr transformation of finite element
     @param [out] elvect vector containing the
    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &Tr,
-                               Vector &elvect) override
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
    {
       int dof = el.GetDof();
       shape.SetSize(dof);
@@ -445,7 +418,7 @@ public:
       elvect = 0.;
 
       // Update the surface integration rule for the current element
-      SIntRule->SetElementAndSurfaceWeight(Tr);
+      SIntRule->SetElementinclSurfaceWeight(Tr.ElementNo);
 
       for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
       {
@@ -455,8 +428,6 @@ public:
          add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
       }
    }
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
 /**
@@ -505,9 +476,9 @@ public:
     @param [in] Tr transformation of finite element
     @param [out] elvect vector containing the
    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &Tr,
-                               Vector &elvect) override
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
    {
       int dof = el.GetDof();
       shape.SetSize(dof);
@@ -515,7 +486,7 @@ public:
       elvect = 0.;
 
       // Update the subdomain integration rule
-      CIntRule->SetElement(Tr);
+      CIntRule->SetElement(Tr.ElementNo);
 
       for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
       {
@@ -526,17 +497,18 @@ public:
          add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
       }
    }
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
 };
+#endif // MFEM_USE_LAPACK
 
 int main(int argc, char *argv[])
 {
-#if defined(MFEM_USE_LAPACK) || defined(MFEM_USE_ALGOIM)
+#ifndef MFEM_USE_LAPACK
+   cout << "MFEM must be built with LAPACK for this example." << endl;
+   return MFEM_SKIP_RETURN_VALUE;
+#else
    // 1. Parse he command-line options.
    int ref_levels = 3;
    int order = 2;
-   int method = 0;
    const char *inttype = "surface2d";
    bool visualization = true;
    itype = IntegrationType::Surface2D;
@@ -544,8 +516,6 @@ int main(int argc, char *argv[])
    OptionsParser args(argc, argv);
    args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
    args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
-   args.AddOption(&method, "-m", "--method",
-                  "Cut integration method: 0 for moments-based, 1 for Algoim.");
    args.AddOption(&inttype, "-i", "--integrationtype",
                   "IntegrationType to demonstrate");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -580,7 +550,7 @@ int main(int argc, char *argv[])
    }
 
    // 2. Construct and refine the mesh.
-   Mesh *mesh = nullptr;
+   Mesh *mesh;
    if (itype == IntegrationType::Volumetric1D)
    {
       mesh = new Mesh("../data/inline-segment.mesh");
@@ -628,14 +598,13 @@ int main(int argc, char *argv[])
    // 5. Define the necessary Integration rules on element 0.
    IsoparametricTransformation Tr;
    mesh->GetElementTransformation(0, &Tr);
-   SIntegrationRule* sir = new SIntegrationRule(method, order,
-                                                levelset, 2, mesh);
+   SIntegrationRule* sir = new SIntegrationRule(order, levelset, 2, mesh);
    CIntegrationRule* cir = NULL;
    if (itype == IntegrationType::Volumetric1D
        || itype == IntegrationType::Volumetric2D
        || itype == IntegrationType::Volumetric3D)
    {
-      cir = new CIntegrationRule(method, order, levelset, 2, mesh);
+      cir = new CIntegrationRule(order, levelset, 2, mesh);
    }
 
    // 6. Define and assemble the linear forms on the finite element space.
@@ -678,11 +647,11 @@ int main(int argc, char *argv[])
       cout << "Number of div free basis functions: " << nbasis << endl;
       cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
    }
-   cout << scientific << setprecision(10);
+   cout << scientific << setprecision(2);
    cout << "============================================" << endl;
    cout << "Computed value of surface integral: " << surface.Sum() << endl;
    cout << "True value of surface integral:     " << Surface() << endl;
-   cout << "Absolute Error (Surface):           ";
+   cout << "Absolute Error (Surface):            ";
    cout << abs(surface.Sum() - Surface()) << endl;
    cout << "Relative Error (Surface):           ";
    cout << abs(surface.Sum() - Surface()) / Surface() << endl;
@@ -693,7 +662,7 @@ int main(int argc, char *argv[])
       cout << "--------------------------------------------" << endl;
       cout << "Computed value of volume integral:  " << volume.Sum() << endl;
       cout << "True value of volume integral:      " << Volume() << endl;
-      cout << "Absolute Error (Volume):            ";
+      cout << "Absolute Error (Volume):             ";
       cout << abs(volume.Sum() - Volume()) << endl;
       cout << "Relative Error (Volume):            ";
       cout << abs(volume.Sum() - Volume()) / Volume() << endl;
@@ -722,8 +691,5 @@ int main(int argc, char *argv[])
    delete fespace;
    delete mesh;
    return EXIT_SUCCESS;
-#else
-   cout << "MFEM must be built with LAPACK or ALGOIM for this example." << endl;
-   return MFEM_SKIP_RETURN_VALUE;
-#endif // MFEM_USE_LAPACK
+#endif //MFEM_USE_LAPACK
 }

examples/ex3p.cpp

@@ -58,6 +58,180 @@ void f_exact(const Vector &, Vector &);
 real_t freq = 1.0, kappa;
 int dim;
 
+class Curl4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+
+   HypreParMatrix *idMat;
+   HypreParMatrix *H1VecLaplaceMat;
+   HypreBoomerAMG *amgVecH1;
+
+
+   HypreParMatrix *gradMat;
+   HypreParMatrix *H1LaplaceMat;
+   HypreBoomerAMG *amgH1;
+
+   HypreSmoother * smoother;
+   CGSolver *pcgGrad;
+   CGSolver *pcgH1Vec;
+
+   Vector *f;
+   Vector *fGrad, *uGrad;
+   Vector *fH1Vec, *uH1Vec;
+
+   bool exactSolves;
+
+public:
+   Curl4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+              const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //vecH1  --> H(curl)
+      int orderKer=orderKernel; //grad V --> H(curl)
+
+      smoother = new HypreSmoother(*A, 16, 3);
+
+      //    //for the pure dirichlet case
+      //    Array<int> essBnd(pmesh->bdr_attributes.Max()); essBnd = 1;
+
+      Array<int> HCurl_essDof(fespace->GetVSize()); HCurl_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HCurl_essDof);
+
+      //setup the H1 FESpace
+      FiniteElementCollection* fecH1;
+      if (orderKer==1) { fecH1 = new LinearFECollection; }
+      else { fecH1 = new QuadraticFECollection; }
+
+      ParFiniteElementSpace *H1FESpace = new ParFiniteElementSpace(pmesh, fecH1);
+      Array<int> H1_essDof(H1FESpace->GetVSize()); H1_essDof = 0;
+      H1FESpace->GetEssentialVDofs(essBnd, H1_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disGrad = new ParDiscreteLinearOperator(H1FESpace,
+                                                                         fespace);
+      disGrad->AddDomainInterpolator(new GradientInterpolator);
+      disGrad->Assemble();
+      disGrad->Finalize();
+      SparseMatrix* smat = &(disGrad->SpMat());
+      smat->EliminateCols(H1_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smat->EliminateRow(dof); }
+      gradMat = disGrad->ParallelAssemble();
+      delete disGrad;
+
+      //setup the H1 preconditioner
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1FESpace);
+      H1Varf->AddDomainIntegrator(new DiffusionIntegrator);
+      H1Varf->AddDomainIntegrator(new MassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1_essDof.Size(); dof++) if (H1_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1LaplaceMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1 = new HypreBoomerAMG(*H1LaplaceMat);
+
+
+      //setup the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1VecFESpace = new ParFiniteElementSpace(pmesh, fecH1Vec,
+                                                                      dim, Ordering::byVDIM);
+      Array<int> H1Vec_essDof(H1VecFESpace->GetVSize()); H1Vec_essDof = 0;
+      H1VecFESpace->GetEssentialVDofs(essBnd, H1Vec_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1VecFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Vec_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      idMat = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the H1-vec preconditioner
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1VecFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator);
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator);
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Vec_essDof.Size(); dof++) if (H1Vec_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1VecLaplaceMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgVecH1 = new HypreBoomerAMG(*H1VecLaplaceMat);
+      amgVecH1->SetSystemsOptions(dim);
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fGrad = new Vector(H1FESpace->GetTrueVSize());
+      uGrad = new Vector(H1FESpace->GetTrueVSize());
+
+      fH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+      uH1Vec = new Vector(H1VecFESpace->GetTrueVSize());;
+
+
+      amgH1->Mult(*fGrad, *uGrad);
+      amgVecH1->Mult(*fH1Vec, *uH1Vec);
+
+      pcgGrad = new CGSolver(MPI_COMM_WORLD);
+      pcgGrad->SetOperator(*H1LaplaceMat);
+      pcgGrad->SetPreconditioner(*amgH1);
+      pcgGrad->SetRelTol(1e-16);
+      pcgGrad->SetMaxIter(100000000);
+      pcgGrad->SetPrintLevel(-2);
+
+      pcgH1Vec = new CGSolver(MPI_COMM_WORLD);
+      pcgH1Vec->SetOperator(*H1VecLaplaceMat);
+      pcgH1Vec->SetPreconditioner(*amgVecH1);
+      pcgH1Vec->SetRelTol(1e-16);
+      pcgH1Vec->SetMaxIter(100000000);
+      pcgH1Vec->SetPrintLevel(-2);
+
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smoother->Mult(x,y);
+
+      idMat->MultTranspose(x,*fH1Vec);
+      *uH1Vec = 0.0;
+      if (exactSolves) { pcgH1Vec->Mult(*fH1Vec, *uH1Vec); }
+      else { amgVecH1->Mult(*fH1Vec, *uH1Vec); }
+      idMat->Mult(1.0, *uH1Vec, 1.0, y);
+
+      gradMat->MultTranspose(x,*fGrad);
+      *uGrad = 0.0;
+      if (exactSolves) { pcgGrad->Mult(*fGrad, *uGrad); }
+      else { amgH1->Mult(*fGrad, *uGrad); }
+      gradMat->Mult(1.0, *uGrad, 1.0, y);
+
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
 int main(int argc, char *argv[])
 {
    // 1. Initialize MPI and HYPRE.
@@ -162,7 +336,13 @@ int main(int argc, char *argv[])
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementCollection *fec;
+   if (dim==4)
+   {
+      if (order==1) { fec = new ND1_4DFECollection; }
+      else { fec = new ND2_4DFECollection; }
+   }
+   else { fec = new ND_FECollection(order, dim); }
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
    HYPRE_BigInt size = fespace->GlobalTrueVSize();
    if (myid == 0)
@@ -248,15 +428,29 @@ int main(int argc, char *argv[])
 
       ParFiniteElementSpace *prec_fespace =
          (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      Solver *prec;
       HypreAMS ams(*A.As<HypreParMatrix>(), prec_fespace);
-      HyprePCG pcg(*A.As<HypreParMatrix>());
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(ams);
-      pcg.Mult(B, X);
+      if (dim <= 3)
+      {
+         prec = new HypreAMS(*A.As<HypreParMatrix>(), prec_fespace);
+      }
+      else if (dim == 4)
+      {
+         prec = new Curl4dPrec(A.As<HypreParMatrix>(), fespace, ess_bdr, order, false);
+      }
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(*A.As<HypreParMatrix>());
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete prec;
    }
 
+   // 13. Recover the parallel grid function corresponding to X. This is the
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
@@ -312,7 +506,14 @@ int main(int argc, char *argv[])
 
 void E_exact(const Vector &x, Vector &E)
 {
-   if (dim == 3)
+   if (dim==4)
+   {
+      E(0) =  sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(1) = -cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(2) =  cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(M_PI*x(3));
+      E(3) = -cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(M_PI*x(3));
+   }
+   else if (dim == 3)
    {
       E(0) = sin(kappa * x(1));
       E(1) = sin(kappa * x(2));
@@ -328,7 +529,19 @@ void E_exact(const Vector &x, Vector &E)
 
 void f_exact(const Vector &x, Vector &f)
 {
-   if (dim == 3)
+   //f_exact = E +  DivSkew P( curl E ), where P is the 4d permutation operator
+   if (dim==4)
+   {
+      f(0) =  (1.0+4.0*M_PI*M_PI)*sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(1) = -(1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                M_PI*x(3));
+      f(2) =  (1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(3) = -(1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(
+                M_PI*x(3));
+   }
+   else if (dim == 3)
    {
       f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
       f(1) = (1. + kappa * kappa) * sin(kappa * x(2));

examples/ex3p_4d.cpp

@@ -0,0 +1,650 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_curl.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_curl.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa = 1.0;
+int dim;
+
+double osziCoeff(const Vector &x)
+{
+   return 1.0001 + sin(100*x(0))*sin(200*x(1))*sin(300*x(2))*sin(400*x(3));
+}
+
+class Curl4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+   Coefficient *alpha_, *beta_, *neg_beta_;
+
+   HypreParMatrix *idMat;
+   HypreParMatrix *H1VecLaplaceMat;
+   HypreBoomerAMG *amgVecH1;
+
+
+   HypreParMatrix *gradMat;
+   HypreParMatrix *H1LaplaceMat;
+   HypreBoomerAMG *amgH1;
+
+   HypreSmoother * smoother;
+   CGSolver *pcgGrad;
+   CGSolver *pcgH1Vec;
+
+   Vector *f;
+   Vector *fGrad, *uGrad;
+   Vector *fH1Vec, *uH1Vec;
+
+   bool exactSolves;
+
+public:
+   ~Curl4dPrec()
+   {
+      delete pcgH1Vec;
+      delete pcgGrad;
+
+      delete   f, fGrad, uGrad, fH1Vec, uH1Vec;
+
+      delete smoother;
+
+      delete amgVecH1, H1VecLaplaceMat;
+      delete idMat;
+      delete amgH1, H1LaplaceMat;
+      delete gradMat;
+   }
+
+   Curl4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+              Coefficient *alpha, Coefficient *beta, Coefficient *neg_beta,
+              const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+      neg_beta_=neg_beta;
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //vecH1  --> H(curl)
+      int orderKer=orderKernel; //grad V --> H(curl)
+
+      smoother = new HypreSmoother(*A, 16, 3);
+
+      //    //for the pure dirichlet case
+      //    Array<int> essBnd(pmesh->bdr_attributes.Max()); essBnd = 1;
+
+      Array<int> HCurl_essDof(fespace->GetVSize()); HCurl_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HCurl_essDof);
+
+      //setup the H1 FESpace
+      FiniteElementCollection* fecH1;
+      if (orderKer==1) { fecH1 = new LinearFECollection; }
+      else { fecH1 = new QuadraticFECollection; }
+
+      ParFiniteElementSpace *H1FESpace = new ParFiniteElementSpace(pmesh, fecH1);
+      Array<int> H1_essDof(H1FESpace->GetVSize()); H1_essDof = 0;
+      H1FESpace->GetEssentialVDofs(essBnd, H1_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disGrad = new ParDiscreteLinearOperator(H1FESpace,
+                                                                         fespace);
+      disGrad->AddDomainInterpolator(new GradientInterpolator);
+      disGrad->Assemble();
+      disGrad->Finalize();
+      SparseMatrix* smat = &(disGrad->SpMat());
+      smat->EliminateCols(H1_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smat->EliminateRow(dof); }
+      gradMat = disGrad->ParallelAssemble();
+      delete disGrad;
+
+      //setup the H1 preconditioner
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1FESpace);
+      H1Varf->AddDomainIntegrator(new DiffusionIntegrator(*beta_));
+      //    H1Varf->AddDomainIntegrator(new MassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1_essDof.Size(); dof++) if (H1_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1LaplaceMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1 = new HypreBoomerAMG(*H1LaplaceMat);
+
+
+      //setup the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1VecFESpace = new ParFiniteElementSpace(pmesh, fecH1Vec,
+                                                                      dim, Ordering::byVDIM);
+      Array<int> H1Vec_essDof(H1VecFESpace->GetVSize()); H1Vec_essDof = 0;
+      H1VecFESpace->GetEssentialVDofs(essBnd, H1Vec_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1VecFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Vec_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      idMat = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the H1-vec preconditioner
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1VecFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_));
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator(*neg_beta_));
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Vec_essDof.Size(); dof++) if (H1Vec_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1VecLaplaceMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgVecH1 = new HypreBoomerAMG(*H1VecLaplaceMat);
+      amgVecH1->SetSystemsOptions(dim);
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fGrad = new Vector(H1FESpace->GetTrueVSize());
+      uGrad = new Vector(H1FESpace->GetTrueVSize());
+
+      fH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+      uH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+
+
+      amgH1->Mult(*fGrad, *uGrad);
+      amgVecH1->Mult(*fH1Vec, *uH1Vec);
+
+      pcgGrad = new CGSolver(MPI_COMM_WORLD);
+      pcgGrad->SetOperator(*H1LaplaceMat);
+      pcgGrad->SetPreconditioner(*amgH1);
+      pcgGrad->SetRelTol(1e-16);
+      pcgGrad->SetMaxIter(100000000);
+      pcgGrad->SetPrintLevel(-2);
+
+      pcgH1Vec = new CGSolver(MPI_COMM_WORLD);
+      pcgH1Vec->SetOperator(*H1VecLaplaceMat);
+      pcgH1Vec->SetPreconditioner(*amgVecH1);
+      pcgH1Vec->SetRelTol(1e-16);
+      pcgH1Vec->SetMaxIter(100000000);
+      pcgH1Vec->SetPrintLevel(-2);
+
+      delete H1FESpace; delete fecH1;
+      delete H1VecFESpace; delete fecH1Vec;
+
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smoother->Mult(x,y);
+
+      idMat->MultTranspose(x,*fH1Vec);
+      *uH1Vec = 0.0;
+      if (exactSolves) { pcgH1Vec->Mult(*fH1Vec, *uH1Vec); }
+      else { amgVecH1->Mult(*fH1Vec, *uH1Vec); }
+      idMat->Mult(1.0, *uH1Vec, 1.0, y);
+
+      gradMat->MultTranspose(x,*fGrad);
+      *uGrad = 0.0;
+      if (exactSolves) { pcgGrad->Mult(*fGrad, *uGrad); }
+      else { amgH1->Mult(*fGrad, *uGrad); }
+      gradMat->Mult(1.0, *uGrad, 1.0, y);
+
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool exactH1Solver = false;
+   bool spe10Coeff = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   kappa = freq * M_PI;
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+
+   if (dim !=4 || sdim != 4)
+   {
+      MPI_Finalize();
+      return 0;
+   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->ReorientTetMesh();
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (dim==4)
+   {
+      if (order==1) { fec = new ND1_4DFECollection; }
+      else { fec = new ND2_4DFECollection; }
+   }
+   else { fec = new ND_FECollection(order, dim); }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      VectorFunctionCoefficient f(sdim, f_exact);
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      x.ProjectCoefficient(E);
+
+      // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+      //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+      //     mass domain integrators.
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+      Coefficient *neg_beta = new ConstantCoefficient(-weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new CurlCurlIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      Solver *prec;
+      if (dim<=3) { prec = new HypreAMS(A, prec_fespace); }
+      else if (dim==4) { prec = new Curl4dPrec(&A, fespace, alpha, beta, neg_beta, ess_bdr, order, false); }
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(A);
+      pcg->SetRelTol(tol);
+      pcg->SetMaxIter(5000);
+      pcg->SetPrintLevel(1);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      int iter = pcg->GetNumIterations();
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double err = x.ComputeL2Error(E);
+         if (myid == 0)
+         {
+            cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 15. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //   {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //   }
+
+      //   // 16. Send the solution by socket to a GLVis server.
+      //   if (visualization)
+      //   {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //   }
+
+      delete pcg;
+      delete prec;
+      delete a;
+      delete alpha;
+      delete beta;
+      delete b;
+
+   }
+
+   // 17. Free the used memory.
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim==4)
+   {
+      E(0) =  sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(1) = -cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(2) =  cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(M_PI*x(3));
+      E(3) = -cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(M_PI*x(3));
+   }
+   else if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   //f_exact = E +  DivSkew P( curl E ), where P is the 4d permutation operator
+   if (dim==4)
+   {
+      f(0) =  (kappa+4.0*M_PI*M_PI)*sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(1) = -(kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                M_PI*x(3));
+      f(2) =  (kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(3) = -(kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(
+                M_PI*x(3));
+   }
+   else if (dim == 3)
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ex40.cpp

@@ -67,10 +67,8 @@ public:
    ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
       : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
 
-   using VectorCoefficient::Eval;
-
-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
    void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };
 
@@ -84,8 +82,8 @@ public:
    DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
       : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
 
-   void Eval(DenseMatrix &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
    void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };
 

examples/ex40p.cpp

@@ -67,10 +67,8 @@ public:
    ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
       : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
 
-   using VectorCoefficient::Eval;
-
-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
    void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };
 
@@ -84,8 +82,8 @@ public:
    DZCoefficient(int height, ParGridFunction &psi_, real_t alpha_ = 1.0)
       : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
 
-   void Eval(DenseMatrix &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
    void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };
 

examples/ex4D_DivSkew.cpp

@@ -0,0 +1,782 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_DivSkew.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_DivSkew.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact_vec(const Vector &x, Vector &E);
+void E_exact(const Vector &, DenseMatrix &);
+void f_exact(const Vector &, DenseMatrix &);
+
+
+class DivSkew4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+   Coefficient *alpha_, *beta_;
+
+   //kernel operators
+   HypreParMatrix *P_d_HCurl_HDivSkew;
+
+
+   HypreParMatrix *P_H1_HCurl;
+   HypreParMatrix *H1_KernelMat;
+   HypreBoomerAMG *amgH1_Kernel;
+
+   //"image" operators
+   HypreParMatrix *P_H1_HDivSkew;
+   HypreParMatrix *H1_ImageMat;
+   HypreBoomerAMG *amgH1_Image;
+
+
+   HypreParMatrix *HCurlMat;
+   HypreSmoother * smootherDivSkew;
+   HypreSmoother * smootherCurl;
+
+   CGSolver *pcgKernel;
+   CGSolver *pcgImage;
+
+   Vector *f;
+   Vector *fKernel, *uKernel;
+   Vector *fImage, *uImage;
+   Vector *fCurl, *uCurl;
+
+   bool exactSolves;
+
+   FiniteElementCollection* fecHCurlKernel;
+   ParFiniteElementSpace *HCurlKernelFESpace;
+
+
+public:
+   ~DivSkew4dPrec()
+   {
+      delete pcgImage, pcgKernel;
+
+      delete f, fKernel, uKernel, fImage, uImage, fCurl, uCurl;
+
+      delete smootherCurl, HCurlMat;
+
+      delete P_d_HCurl_HDivSkew, P_H1_HDivSkew, P_H1_HCurl;
+
+      delete amgH1_Image, H1_ImageMat;
+      delete amgH1_Kernel, H1_KernelMat;
+
+      delete smootherDivSkew;
+
+      delete HCurlKernelFESpace, fecHCurlKernel;
+   }
+   DivSkew4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+                 Coefficient *alpha, Coefficient *beta,
+                 const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //H1  --> H(divSkew)
+      int orderKer=orderKernel; //curl V --> H(divSkew)
+
+      smootherDivSkew = new HypreSmoother(*A, 16, 3);
+
+      Array<int> HDivSkew_essDof(fespace->GetVSize()); HDivSkew_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HDivSkew_essDof);
+
+      //setup the H1 FESpace for the kernel
+      FiniteElementCollection* fecH1Kernel = new H1_FECollection(orderKer, 4);
+
+      ParFiniteElementSpace *H1KernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Kernel, dim, Ordering::byVDIM);
+      Array<int> H1Kernel_essDof(H1KernelFESpace->GetVSize()); H1Kernel_essDof = 0;
+      H1KernelFESpace->GetEssentialVDofs(essBnd, H1Kernel_essDof);
+
+
+      //setup the H(curl) FESpace for the kernel
+      if (orderKer==1) { fecHCurlKernel = new ND1_4DFECollection; }
+      else { fecHCurlKernel = new ND2_4DFECollection; }
+
+      HCurlKernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                     fecHCurlKernel);
+      Array<int> HCurlKernel_essDof(HCurlKernelFESpace->GetVSize());
+      HCurlKernel_essDof = 0;
+      HCurlKernelFESpace->GetEssentialVDofs(essBnd, HCurlKernel_essDof);
+
+
+      //setup the FESpace for the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1_ImageFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Vec, 6, Ordering::byVDIM);
+      Array<int> H1Image_essDof(H1_ImageFESpace->GetVSize()); H1Image_essDof = 0;
+      H1_ImageFESpace->GetEssentialVDofs(essBnd, H1Image_essDof);
+
+
+
+      //setup the H1 preconditioner for the kernel
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1KernelFESpace);
+      H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*beta_));
+      //      H1Varf->AddDomainIntegrator(new VectorMassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof = 0; dof < H1Kernel_essDof.Size(); dof++)
+         if (H1Kernel_essDof[dof] < 0)
+         {
+            matH1.EliminateRowCol(dof);
+         }
+      H1_KernelMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1_Kernel = new HypreBoomerAMG(*H1_KernelMat);
+      amgH1_Kernel->SetSystemsOptions(dim);
+      amgH1_Kernel->SetPrintLevel(0);
+
+      //setup the H1 preconditioner for the image
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1_ImageFESpace);
+      VectorDiffusionIntegrator *alpha_integ = new VectorDiffusionIntegrator(*alpha_);
+      alpha_integ->SetVDim(6);
+      H1VecVarf->AddDomainIntegrator(alpha_integ);
+      VectorMassIntegrator *beta_integ = new VectorMassIntegrator(*beta);
+      beta_integ->SetVDim(6);
+      H1VecVarf->AddDomainIntegrator(beta_integ);
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Image_essDof.Size(); dof++) if (H1Image_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1_ImageMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgH1_Image = new HypreBoomerAMG(*H1_ImageMat);
+      amgH1_Image->SetSystemsOptions(6);
+      amgH1_Image->SetPrintLevel(0);
+
+
+      //setup the injection of H1 into H(curl)
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1KernelFESpace, HCurlKernelFESpace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Kernel_essDof);
+      for (int dof=0; dof<HCurlKernel_essDof.Size();
+           dof++) if (HCurlKernel_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      P_H1_HCurl = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the injection of H1 into H(DivSkew)
+      ParDiscreteLinearOperator *disInterpolIm = new ParDiscreteLinearOperator(
+         H1_ImageFESpace, fespace);
+      disInterpolIm->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpolIm->Assemble();
+      disInterpolIm->Finalize();
+      SparseMatrix* smatIDIm = &(disInterpolIm->SpMat());
+      smatIDIm->EliminateCols(H1Image_essDof);
+      for (int dof=0; dof<HDivSkew_essDof.Size(); dof++) if (HDivSkew_essDof[dof]<0) { smatIDIm->EliminateRow(dof); }
+      P_H1_HDivSkew = disInterpolIm->ParallelAssemble();
+      delete disInterpolIm;
+
+
+      //setup the injection of the curl(H(curl)) into H(DivSkew)
+      ParDiscreteLinearOperator *disCurl = new ParDiscreteLinearOperator(
+         HCurlKernelFESpace, fespace);
+      disCurl->AddDomainInterpolator(new CurlInterpolator);
+      disCurl->Assemble();
+      disCurl->Finalize();
+      SparseMatrix* smatCurl = &(disCurl->SpMat());
+      smatCurl->EliminateCols(HCurlKernel_essDof);
+      for (int dof=0; dof<HDivSkew_essDof.Size(); dof++) if (HDivSkew_essDof[dof]<0) { smatCurl->EliminateRow(dof); }
+      P_d_HCurl_HDivSkew = disCurl->ParallelAssemble();
+      delete disCurl;
+
+      //setup the smoother for H(curl)
+      //      Coefficient *massC = new ConstantCoefficient(1.0);
+      //      Coefficient *CurlCurlC = new ConstantCoefficient(1.0);
+      ParBilinearForm *a_HCurl = new ParBilinearForm(HCurlKernelFESpace);
+      a_HCurl->AddDomainIntegrator(new CurlCurlIntegrator(*beta_));
+      //      a_HCurl->AddDomainIntegrator(new CurlCurlIntegrator(*CurlCurlC));
+      //      a_HCurl->AddDomainIntegrator(new VectorFEMassIntegrator(*massC));
+      a_HCurl->Assemble();
+      a_HCurl->Finalize();
+      SparseMatrix &matHCurl(a_HCurl->SpMat());
+      for (int dof=0; dof<HCurlKernel_essDof.Size();
+           dof++) if (HCurlKernel_essDof[dof]<0) { matHCurl.EliminateRowCol(dof); }
+      HCurlMat = a_HCurl->ParallelAssemble();
+      delete a_HCurl;
+      smootherCurl = new HypreSmoother(*HCurlMat, 16, 3);
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+      uKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+
+      fImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+      uImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+
+      fCurl = new Vector(HCurlKernelFESpace->GetTrueVSize());
+      uCurl = new Vector(HCurlKernelFESpace->GetTrueVSize());
+
+
+      amgH1_Kernel->Mult(*fKernel, *uKernel);
+      amgH1_Image->Mult(*fImage, *uImage);
+
+      pcgKernel = new CGSolver(MPI_COMM_WORLD);
+      pcgKernel->SetOperator(*H1_KernelMat);
+      pcgKernel->SetPreconditioner(*amgH1_Kernel);
+      pcgKernel->SetRelTol(1e-16);
+      pcgKernel->SetMaxIter(100000000);
+      pcgKernel->SetPrintLevel(-2);
+
+      pcgImage = new CGSolver(MPI_COMM_WORLD);
+      pcgImage->SetOperator(*H1_ImageMat);
+      pcgImage->SetPreconditioner(*amgH1_Image);
+      pcgImage->SetRelTol(1e-16);
+      pcgImage->SetMaxIter(100000000);
+      pcgImage->SetPrintLevel(-2);
+
+      delete H1KernelFESpace, fecH1Kernel;
+      delete H1_ImageFESpace, fecH1Vec;
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smootherDivSkew->Mult(x,y);
+
+      P_H1_HDivSkew->MultTranspose(x,*fImage);
+      *uImage = 0.0;
+      if (exactSolves) { pcgImage->Mult(*fImage, *uImage); }
+      else { amgH1_Image->Mult(*fImage, *uImage); }
+      P_H1_HDivSkew->Mult(1.0, *uImage, 1.0, y);
+
+      *uCurl = 0.0;
+      P_d_HCurl_HDivSkew->MultTranspose(x,*fCurl);
+
+      smootherCurl->Mult(*fCurl, *uCurl);
+
+      P_H1_HCurl->MultTranspose(*fCurl,*fKernel);
+      *uKernel = 0.0;
+      if (exactSolves) { pcgKernel->Mult(*fKernel, *uKernel); }
+      else { amgH1_Kernel->Mult(*fKernel, *uKernel); }
+      P_H1_HCurl->Mult(1.0, *uKernel, 1.0, *uCurl);
+
+      P_d_HCurl_HDivSkew->Mult(1.0, *uCurl, 1.0, y);
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool exactH1Solver = false;
+   bool spe10Coeff = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   if (dim !=4 || sdim != 4)
+   {
+      MPI_Finalize();
+      return 0;
+   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (order==1) { fec = new DivSkew1_4DFECollection; }
+   //    else fec = new F2K1_4DFECollection;
+
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   fespace->SetUpdateOperatorType(Operator::Hypre_ParCSR);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   MatrixFunctionCoefficient f(sdim, f_exact);
+   MatrixFunctionCoefficient solMat(sdim, E_exact);
+   VectorFunctionCoefficient solVec(6, E_exact_vec);
+
+
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+
+      x.ProjectCoefficient(solVec);
+
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new MatFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      //   cout << x << endl;
+      //   x = 0.0;
+
+      // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+      //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+      //     mass domain integrators.
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFE_DivSkewMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      //Define the preconditioner
+
+      if (myid == 0) { cout << "Set up the preconditioner" << endl; }
+      Solver *prec;
+      if (dim==4) { prec = new DivSkew4dPrec(&A, fespace, alpha, beta, ess_bdr, order, exactH1Solver); }
+
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(A);
+      pcg->SetRelTol(tol);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(1);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      delete prec;
+
+      int iter = pcg->GetNumIterations();
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double error = 0.0;
+         for (int i = 0; i < fespace->GetNE(); i++)
+         {
+            const FiniteElement* fe = fespace->GetFE(i);
+            int fdof = fe->GetDof();
+            ElementTransformation* transf = fespace->GetElementTransformation(i);
+            DenseMatrix shape(fdof,dim*dim);
+
+            int intorder = 2*fe->GetOrder() + 1; // <----------
+            const IntegrationRule *ir;
+            ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+
+            Vector elSol(dim*dim);
+            DenseMatrix elSolMat(dim,dim);
+            DenseMatrix exactSol(dim,dim);
+            Vector exactSolVec(dim*dim);
+
+
+
+            Array<int> vdofs;
+            fespace->GetElementVDofs(i, vdofs);
+            for (int j = 0; j < ir->GetNPoints(); j++)
+            {
+               const IntegrationPoint &ip = ir->IntPoint(j);
+               transf->SetIntPoint(&ip);
+
+               fe->CalcVShape(*transf, shape);
+
+               elSol = 0.0;
+               for (int k = 0; k < fdof; k++)
+               {
+                  if (vdofs[k] >= 0)
+                  {
+                     for (int l=0; l<dim*dim; l++) { elSol(l) += shape(k,l)*x(vdofs[k]); }
+                  }
+                  else
+                  {
+                     for (int l=0; l<dim*dim; l++) { elSol(l) -= shape(k,l)*x(-1-vdofs[k]); }
+                  }
+               }
+               for (int k=0; k<dim; k++)
+                  for (int l=0; l<dim; l++)
+                  {
+                     elSolMat(k,l) = elSol(dim*k+l);
+                  }
+
+
+               solMat.Eval(exactSol,*transf, ip);
+               for (int k=0; k<dim; k++)
+                  for (int l=0; l<dim; l++)
+                  {
+                     exactSolVec(dim*k+l) = exactSol(k,l);
+                  }
+               elSol.Add(-1.0, exactSolVec);
+
+               error += ip.weight * fabs(transf->Weight()) * (elSol * elSol);
+            }
+         }
+         double globalError = 0.0;
+         MPI_Allreduce(&error, &globalError, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+         if (myid==0) { std::cout << "L2 error: " << sqrt(globalError) << std::endl; }
+
+
+      }
+
+      delete pcg;
+      delete a;
+      delete alpha;
+      delete beta;
+
+      delete b;
+   }
+
+   // 17. Free the used memory.
+
+
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+void E_exact_vec(const Vector &x, Vector &E)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      E.SetSize(6);
+
+      double s0 = sin(M_PI*x(0)), s1 = sin(M_PI*x(1)), s2 = sin(M_PI*x(2)),
+             s3 = sin(M_PI*x(3));
+      double c0 = cos(M_PI*x(0)), c1 = cos(M_PI*x(1)), c2 = cos(M_PI*x(2)),
+             c3 = cos(M_PI*x(3));
+
+      E(0) =  c0*c1*s2*s3;
+      E(1) = -c0*s1*c2*s3;
+      E(2) =  c0*s1*s2*c3;
+      E(3) =  s0*c1*c2*s3;
+      E(4) = -s0*c1*s2*c3;
+      E(5) =  s0*s1*c2*c3;
+   }
+}
+
+void E_exact(const Vector &x, DenseMatrix &E)
+{
+   int dim = x.Size();
+
+   E.SetSize(dim*dim);
+
+   if (dim==4)
+   {
+      Vector vecE; E_exact_vec(x, vecE);
+
+      E = 0.0;
+
+      E(0,1) = vecE(0);
+      E(0,2) = vecE(1);
+      E(0,3) = vecE(2);
+      E(1,2) = vecE(3);
+      E(1,3) = vecE(4);
+      E(2,3) = vecE(5);
+
+      E(1,0) =  -E(0,1);
+      E(2,0) =  -E(0,2);
+      E(3,0) =  -E(0,3);
+      E(2,1) =  -E(1,2);
+      E(3,1) =  -E(1,3);
+      E(3,2) =  -E(2,3);
+   }
+}
+
+
+
+//f_exact = E + 0.5 * P( curl DivSkew E ), where P is the 4d permutation operator
+void f_exact(const Vector &x, DenseMatrix &f)
+{
+   int dim = x.Size();
+
+   f.SetSize(dim,dim);
+
+   if (dim==4)
+   {
+      f = 0.0;
+
+      double s0 = sin(M_PI*x(0)), s1 = sin(M_PI*x(1)), s2 = sin(M_PI*x(2)),
+             s3 = sin(M_PI*x(3));
+      double c0 = cos(M_PI*x(0)), c1 = cos(M_PI*x(1)), c2 = cos(M_PI*x(2)),
+             c3 = cos(M_PI*x(3));
+
+      f(0,1) =  (1.0 + 1.0  * M_PI*M_PI)*c0*c1*s2*s3;
+      f(0,2) = -(1.0 + 0.0  * M_PI*M_PI)*c0*s1*c2*s3;
+      f(0,3) =  (1.0 + 1.0  * M_PI*M_PI)*c0*s1*s2*c3;
+      f(1,2) =  (1.0 - 1.0  * M_PI*M_PI)*s0*c1*c2*s3;
+      f(1,3) = -(1.0 + 0.0  * M_PI*M_PI)*s0*c1*s2*c3;
+      f(2,3) =  (1.0 + 1.0  * M_PI*M_PI)*s0*s1*c2*c3;
+
+      f(1,0) =  -f(0,1);
+      f(2,0) =  -f(0,2);
+      f(3,0) =  -f(0,3);
+      f(2,1) =  -f(1,2);
+      f(3,1) =  -f(1,3);
+      f(3,2) =  -f(2,3);
+   }
+}

examples/ex4p_4d.cpp

@@ -0,0 +1,800 @@
+//                       MFEM Example 4 - Parallel Version
+//
+// Compile with: make ex4p
+//
+// Sample runs:  mpirun -np 4 ex4p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex4p -m ../data/star.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/beam-hex-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
+//
+// Description:  This example code solves a simple 2D/3D H(div) diffusion
+//               problem corresponding to the second order definite equation
+//               -grad(alpha div F) + beta F = f with boundary condition F dot n
+//               = <given normal field>. Here, we use a given exact solution F
+//               and compute the corresponding r.h.s. f.  We discretize with
+//               Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of H(div) finite element
+//               spaces with the grad-div and H(div) vector finite element mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Bilinear form
+//               hybridization and static condensation are also illustrated.
+//
+//               We recommend viewing examples 1-3 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_div.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_div.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+void F_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+
+
+
+class div4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+
+   Coefficient *alpha_, *beta_;
+
+   //kernel operators
+   HypreParMatrix *P_d_HSkewDiv_Hdiv;
+
+   HypreParMatrix *P_H1_HDivSkew;
+   HypreParMatrix *H1_KernelMat;
+   HypreBoomerAMG *amgH1_Kernel;
+
+   //"image" operators
+   HypreParMatrix *P_H1_Hdiv;
+   HypreParMatrix *H1_ImageMat;
+   HypreBoomerAMG *amgH1_Image;
+
+   HypreParMatrix *HDivSkewMat;
+   HypreSmoother * smootherdiv;
+   HypreSmoother * smootherDivSkew;
+
+   CGSolver *pcgKernel;
+   CGSolver *pcgImage;
+
+   Vector *f;
+   Vector *fKernel, *uKernel;
+   Vector *fImage, *uImage;
+   Vector *fDivSkew, *uDivSkew;
+
+   FiniteElementCollection* fecHDivSkewKernel;
+   ParFiniteElementSpace *HDivSkewKernelFESpace;
+
+   bool exactSolves;
+
+public:
+   ~div4dPrec()
+   {
+      delete pcgImage;
+      delete pcgKernel;
+
+      delete uDivSkew, fDivSkew, uImage, fImage, uKernel, fKernel, f;
+
+      delete smootherDivSkew;
+      delete HDivSkewMat;
+
+      delete P_d_HSkewDiv_Hdiv;
+      delete P_H1_Hdiv;
+      delete P_H1_HDivSkew;
+
+      delete amgH1_Image, H1_ImageMat;
+      delete amgH1_Kernel, H1_KernelMat;
+
+      delete smootherdiv;
+
+      delete HDivSkewKernelFESpace;
+      delete fecHDivSkewKernel;
+   }
+   div4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+             Coefficient *alpha, Coefficient *beta, const Array<int> &essBnd,
+             int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+
+
+
+      int orderIm=1;  //H1  --> H(div)
+      int orderKer=orderKernel; //DivSkew V --> H(div)
+
+
+
+      smootherdiv = new HypreSmoother(*A, 16, 3);
+
+      Array<int> Hdiv_essDof(fespace->GetVSize()); Hdiv_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, Hdiv_essDof);
+
+
+
+
+      //setup the H1 FESpace for the kernel
+      FiniteElementCollection* fecH1Kernel;
+      if (orderKer==1) { fecH1Kernel = new LinearFECollection; }
+      else { fecH1Kernel = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1KernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Kernel, 6, Ordering::byVDIM);
+      Array<int> H1Kernel_essDof(H1KernelFESpace->GetVSize()); H1Kernel_essDof = 0;
+      H1KernelFESpace->GetEssentialVDofs(essBnd, H1Kernel_essDof);
+
+
+      //setup the H(DivSkew) FESpace for the kernel
+      if (orderKer==1) { fecHDivSkewKernel = new DivSkew1_4DFECollection; }
+      //       else fecHDivSkewKernel = new DivSkewFull1_4DFECollection;
+      HDivSkewKernelFESpace = new ParFiniteElementSpace(pmesh, fecHDivSkewKernel);
+      Array<int> HDivSkewKernel_essDof(HDivSkewKernelFESpace->GetVSize());
+      HDivSkewKernel_essDof = 0;
+      HDivSkewKernelFESpace->GetEssentialVDofs(essBnd, HDivSkewKernel_essDof);
+
+
+      //setup the FESpace for the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1_ImageFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Vec, dim, Ordering::byVDIM);
+      Array<int> H1Image_essDof(H1_ImageFESpace->GetVSize()); H1Image_essDof = 0;
+      H1_ImageFESpace->GetEssentialVDofs(essBnd, H1Image_essDof);
+
+
+
+      //setup the H1 preconditioner for the kernel
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1KernelFESpace);
+      //    H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_, 6));
+      //    H1Varf->AddDomainIntegrator(new VectorMassIntegrator(6, beta_));
+
+      H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*beta_, 6));
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1Kernel_essDof.Size(); dof++) if (H1Kernel_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1_KernelMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1_Kernel = new HypreBoomerAMG(*H1_KernelMat);
+      amgH1_Kernel->SetSystemsOptions(6);
+
+      //setup the H1 preconditioner for the image
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1_ImageFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_));
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator(-1, beta_));
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Image_essDof.Size(); dof++) if (H1Image_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1_ImageMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgH1_Image = new HypreBoomerAMG(*H1_ImageMat);
+      amgH1_Image->SetSystemsOptions(dim);
+
+
+      //setup the injection of H1 into H(DivSkew)
+      ParDiscreteLinearOperator *disInterpolIm = new ParDiscreteLinearOperator(
+         H1KernelFESpace, HDivSkewKernelFESpace);
+      disInterpolIm->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpolIm->Assemble();
+      disInterpolIm->Finalize();
+      SparseMatrix* smatIDIm = &(disInterpolIm->SpMat());
+      smatIDIm->EliminateCols(H1Kernel_essDof);
+      for (int dof=0; dof<HDivSkewKernel_essDof.Size();
+           dof++) if (HDivSkewKernel_essDof[dof]<0) { smatIDIm->EliminateRow(dof); }
+      P_H1_HDivSkew = disInterpolIm->ParallelAssemble();
+      delete disInterpolIm;
+
+      //setup the injection of H1 into H(div)
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1_ImageFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Image_essDof);
+      for (int dof=0; dof<Hdiv_essDof.Size(); dof++) if (Hdiv_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      P_H1_Hdiv = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+
+
+
+      //setup the injection of the DivSkew(H(DivSkew)) into H(div)
+      ParDiscreteLinearOperator *disDivSkew = new ParDiscreteLinearOperator(
+         HDivSkewKernelFESpace, fespace);
+      disDivSkew->AddDomainInterpolator(new DivSkewInterpolator);
+      disDivSkew->Assemble();
+      disDivSkew->Finalize();
+      SparseMatrix* smatDivSkew= &(disDivSkew->SpMat());
+      smatDivSkew->EliminateCols(HDivSkewKernel_essDof);
+      for (int dof=0; dof<Hdiv_essDof.Size(); dof++) if (Hdiv_essDof[dof]<0) { smatDivSkew->EliminateRow(dof); }
+      P_d_HSkewDiv_Hdiv = disDivSkew->ParallelAssemble();
+      delete disDivSkew;
+
+
+      //setup the smoother for H(DivSkew)
+      ParBilinearForm *a_HDivSkew = new ParBilinearForm(HDivSkewKernelFESpace);
+      //    a_HDivSkew->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*alpha_));
+      //    a_HDivSkew->AddDomainIntegrator(new VectorFE_DivSkewMassIntegrator(*beta_));
+
+      a_HDivSkew->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*beta_));
+
+      a_HDivSkew->Assemble();
+      a_HDivSkew->Finalize();
+      SparseMatrix &matHDivSkew(a_HDivSkew->SpMat());
+      for (int dof=0; dof<HDivSkewKernel_essDof.Size();
+           dof++) if (HDivSkewKernel_essDof[dof]<0) { matHDivSkew.EliminateRowCol(dof); }
+      HDivSkewMat = a_HDivSkew->ParallelAssemble();
+      delete a_HDivSkew;
+      smootherDivSkew = new HypreSmoother(*HDivSkewMat, 16, 3);
+
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+      uKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+
+      fImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+      uImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+
+      fDivSkew = new Vector(HDivSkewKernelFESpace->GetTrueVSize());
+      uDivSkew = new Vector(HDivSkewKernelFESpace->GetTrueVSize());
+
+      amgH1_Kernel->Mult(*fKernel, *uKernel);
+      amgH1_Image->Mult(*fImage, *uImage);
+
+      pcgKernel = new CGSolver(MPI_COMM_WORLD);
+      pcgKernel->SetOperator(*H1_KernelMat);
+      pcgKernel->SetPreconditioner(*amgH1_Kernel);
+      pcgKernel->SetRelTol(1e-16);
+      pcgKernel->SetMaxIter(100000000);
+      pcgKernel->SetPrintLevel(-2);
+
+      pcgImage = new CGSolver(MPI_COMM_WORLD);
+      pcgImage->SetOperator(*H1_ImageMat);
+      pcgImage->SetPreconditioner(*amgH1_Image);
+      pcgImage->SetRelTol(1e-16);
+      pcgImage->SetMaxIter(100000000);
+      pcgImage->SetPrintLevel(-2);
+
+      delete H1_ImageFESpace;
+      delete H1KernelFESpace;
+      delete fecH1Kernel;
+      delete fecH1Vec;
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smootherdiv->Mult(x,y);
+
+      P_H1_Hdiv->MultTranspose(x,*fImage);
+      *uImage = 0.0;
+      if (exactSolves) { pcgImage->Mult(*fImage, *uImage); }
+      else { amgH1_Image->Mult(*fImage, *uImage); }
+      P_H1_Hdiv->Mult(1.0, *uImage, 1.0, y);
+
+
+      *uDivSkew = 0.0;
+      P_d_HSkewDiv_Hdiv->MultTranspose(x,*fDivSkew);
+
+      smootherDivSkew->Mult(*fDivSkew, *uDivSkew);
+
+      P_H1_HDivSkew->MultTranspose(*fDivSkew,*fKernel);
+      *uKernel = 0.0;
+      if (exactSolves) { pcgKernel->Mult(*fKernel, *uKernel); }
+      else { amgH1_Kernel->Mult(*fKernel, *uKernel); }
+      P_H1_HDivSkew->Mult(1.0, *uKernel, 1.0, *uDivSkew);
+
+      P_d_HSkewDiv_Hdiv->Mult(1.0, *uDivSkew, 1.0, y);
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool hybridization = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool spe10Coeff = false;
+   bool exactH1Solver = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
+                  "--no-hybridization", "Enable hybridization.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume, as well as periodic meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      for (int l = 0; l < sequ_ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Raviart-Thomas finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (dim==4) { fec = new RT0_4DFECollection; }
+   else { fec = new RT_FECollection(order-1, dim); }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary faces will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient F(sdim, F_exact);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      x.ProjectCoefficient(F);
+
+      VectorFunctionCoefficient f(sdim, f_exact);
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      // 10. Set up the parallel bilinear form corresponding to the H(div)
+      //     diffusion operator grad alpha div + beta I, by adding the div-div and
+      //     the mass domain integrators.
+
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation,
+      //     hybridization, etc.
+      FiniteElementCollection *hfec = NULL;
+      ParFiniteElementSpace *hfes = NULL;
+      if (static_cond)
+      {
+         a->EnableStaticCondensation();
+      }
+      else if (hybridization)
+      {
+         hfec = new DG_Interface_FECollection(order-1, dim);
+         hfes = new ParFiniteElementSpace(pmesh, hfec);
+         a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
+                                ess_tdof_list);
+      }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      HYPRE_Int glob_size = A.GetGlobalNumRows();
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << glob_size << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
+      //     the 3D ADS preconditioners from hypre. If using hybridization, the
+      //     system is preconditioned with hypre's BoomerAMG.
+      Solver *prec = NULL;
+      if (hybridization) { prec = new HypreBoomerAMG(A); }
+      else
+      {
+         ParFiniteElementSpace *prec_fespace =
+            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+         if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
+         else if (dim==3) { prec = new HypreADS(A, prec_fespace); }
+         else if (dim==4) { prec = new div4dPrec(&A, fespace, alpha, beta, ess_bdr, order, exactH1Solver); }
+         else { prec = NULL; }
+      }
+
+      int iter = -1;
+      if (standardCG)
+      {
+         IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+         pcg->SetOperator(A);
+         pcg->SetRelTol(tol);
+         pcg->SetMaxIter(500);
+         pcg->SetPrintLevel(1);
+         pcg->SetPreconditioner(*prec);
+         pcg->Mult(B, X);
+
+         iter = pcg->GetNumIterations();
+
+         delete pcg;
+      }
+      else
+      {
+         HyprePCG *pcg = new HyprePCG(A);
+         pcg->SetTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetResidualConvergenceOptions(1,tol);
+         pcg->SetPrintLevel(2);
+         //          pcg->SetPreconditioner(*prec);
+         pcg->Mult(B, X);
+
+         pcg->GetNumIterations(iter);
+
+         delete pcg;
+      }
+
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double err = x.ComputeL2Error(F);
+         if (myid == 0)
+         {
+            cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 15. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //    {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //    }
+
+      // 16. Send the solution by socket to a GLVis server.
+      //    if (visualization)
+      //    {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //    }
+
+      if (prec!=NULL) { delete prec; }
+      delete hfes;
+      delete hfec;
+      delete a;
+      delete alpha;
+      delete beta;
+
+      delete b;
+   }
+
+   // 17. Free the used memory.
+
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   if (dim==4)
+   {
+      double s0 = sin(M_PI*p(0)), s1 = sin(M_PI*p(1)), s2 = sin(M_PI*p(2)),
+             s3 = sin(M_PI*p(3));
+      double c0 = cos(M_PI*p(0)), c1 = cos(M_PI*p(1)), c2 = cos(M_PI*p(2)),
+             c3 = cos(M_PI*p(3));
+
+      F(0) = c0 * s1 * s2 * s3;
+      F(1) = s0 * c1 * s2 * s3;
+      F(2) = s0 * s1 * c2 * s3;
+      F(3) = s0 * s1 * s2 * c3;
+   }
+   else
+   {
+      double x = p(0);
+      double y = p(1);
+      // double z = (dim == 3) ? p(2) : 0.0;
+
+      F(0) = cos(kappa*x)*sin(kappa*y);
+      F(1) = cos(kappa*y)*sin(kappa*x);
+      if (dim == 3)
+      {
+         F(2) = 0.0;
+      }
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+   if (dim==4)
+   {
+      double s0 = sin(M_PI*p(0)), s1 = sin(M_PI*p(1)), s2 = sin(M_PI*p(2)),
+             s3 = sin(M_PI*p(3));
+      double c0 = cos(M_PI*p(0)), c1 = cos(M_PI*p(1)), c2 = cos(M_PI*p(2)),
+             c3 = cos(M_PI*p(3));
+
+      f(0) = c0 * s1 * s2 * s3;
+      f(1) = s0 * c1 * s2 * s3;
+      f(2) = s0 * s1 * c2 * s3;
+      f(3) = s0 * s1 * s2 * c3;
+
+      f *= (kappa + 4.0 * M_PI*M_PI);
+   }
+   else
+   {
+      double x = p(0);
+      double y = p(1);
+      // double z = (dim == 3) ? p(2) : 0.0;
+
+      double temp = 1 + 2*kappa*kappa;
+
+      f(0) = temp*cos(kappa*x)*sin(kappa*y);
+      f(1) = temp*cos(kappa*y)*sin(kappa*x);
+      if (dim == 3)
+      {
+         f(2) = 0;
+      }
+   }
+}

examples/ex8.cpp

@@ -157,7 +157,7 @@ int main(int argc, char *argv[])
    MixedBilinearForm *B0 = new MixedBilinearForm(x0_space,test_space);
    B0->AddDomainIntegrator(new DiffusionIntegrator(one));
    B0->Assemble();
-   B0->EliminateTrialEssentialBC(ess_bdr, x.GetBlock(x0_var), F);
+   B0->EliminateTrialDofs(ess_bdr, x.GetBlock(x0_var), F);
    B0->Finalize();
 
    MixedBilinearForm *Bhat = new MixedBilinearForm(xhat_space,test_space);

examples/ex9.cpp

@@ -9,7 +9,7 @@
 //    ex9 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 23 -tf 9
+//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 13 -tf 9
 //    ex9 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
@@ -104,12 +104,12 @@ public:
       }
    }
 
-   void SetOperator(const Operator &op) override
+   void SetOperator(const Operator &op)
    {
       linear_solver.SetOperator(op);
    }
 
-   void Mult(const Vector &x, Vector &y) const override
+   virtual void Mult(const Vector &x, Vector &y) const
    {
       linear_solver.Mult(x, y);
    }
@@ -134,10 +134,10 @@ private:
 public:
    FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);
 
-   void Mult(const Vector &x, Vector &y) const override;
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
 
-   ~FE_Evolution() override;
+   virtual ~FE_Evolution();
 };
 
 
@@ -182,7 +182,12 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - Backward Euler,\n\t"
+                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -219,7 +224,28 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // Explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      // Implicit (L-stable) methods
+      case 11: ode_solver = new BackwardEulerSolver; break;
+      case 12: ode_solver = new SDIRK23Solver(2); break;
+      case 13: ode_solver = new SDIRK33Solver; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         return 3;
+   }
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -414,6 +440,7 @@ int main(int argc, char *argv[])
    }
 
    // 10. Free the used memory.
+   delete ode_solver;
    delete pd;
    delete dc;
 

examples/ex9p.cpp

@@ -9,7 +9,7 @@
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-hexagon.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9
-//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 23 -tf 9
+//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 13 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-mixed.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
@@ -92,7 +92,7 @@ private:
 public:
    AIR_prec(int blocksize_) : AIR_solver(NULL), blocksize(blocksize_) { }
 
-   void SetOperator(const Operator &op) override
+   void SetOperator(const Operator &op)
    {
       width = op.Width();
       height = op.Height();
@@ -110,7 +110,7 @@ public:
       AIR_solver->SetMaxLevels(50);
    }
 
-   void Mult(const Vector &x, Vector &y) const override
+   virtual void Mult(const Vector &x, Vector &y) const
    {
       // Scale the rhs by block inverse and solve system
       HypreParVector z_s;
@@ -119,7 +119,7 @@ public:
       AIR_solver->Mult(z_s, y);
    }
 
-   ~AIR_prec() override
+   ~AIR_prec()
    {
       delete AIR_solver;
    }
@@ -185,17 +185,17 @@ public:
       }
    }
 
-   void SetOperator(const Operator &op) override
+   void SetOperator(const Operator &op)
    {
       linear_solver.SetOperator(op);
    }
 
-   void Mult(const Vector &x, Vector &y) const override
+   virtual void Mult(const Vector &x, Vector &y) const
    {
       linear_solver.Mult(x, y);
    }
 
-   ~DG_Solver() override
+   ~DG_Solver()
    {
       delete prec;
       delete A;
@@ -223,10 +223,10 @@ public:
    FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_,
                 PrecType prec_type);
 
-   void Mult(const Vector &x, Vector &y) const override;
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
 
-   ~FE_Evolution() override;
+   virtual ~FE_Evolution();
 };
 
 
@@ -285,7 +285,12 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - Backward Euler,\n\t"
+                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -333,7 +338,31 @@ int main(int argc, char *argv[])
 
    // 4. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // Explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      // Implicit (L-stable) methods
+      case 11: ode_solver = new BackwardEulerSolver; break;
+      case 12: ode_solver = new SDIRK23Solver(2); break;
+      case 13: ode_solver = new SDIRK33Solver; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         if (Mpi::Root())
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         return 3;
+   }
 
    // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -613,6 +642,7 @@ int main(int argc, char *argv[])
    delete m;
    delete fes;
    delete pmesh;
+   delete ode_solver;
    delete pd;
 #ifdef MFEM_USE_ADIOS2
    if (adios2)

examples/ginkgo/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ginkgo/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/hiop/ex9p.cpp

@@ -96,7 +96,6 @@ public:
    {
       Vector w_glob(width);
       pfes.Dof_TrueDof_Matrix()->MultTranspose(w, w_glob);
-      w_glob.HostReadWrite(); // read+write -> can use w_glob(i) (non-const)
       for (int i = 0; i < width; i++) { grad(0, i) = w_glob(i); }
    }
 

examples/hiop/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/hiop/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ..
 MFEM_BUILD_DIR ?= ..
-MFEM_INSTALL_DIR ?= ../mfem
 SRC = $(if $(MFEM_DIR:..=),$(MFEM_DIR)/examples/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -26,7 +27,8 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
    ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
    ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p ex40p
+   ex37p ex39p ex40p \
+   ex1p_4d ex3p_4d ex4D_DivSkew
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
    ex22p ex24p ex25p ex26p ex34p ex35p

examples/moonolith/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/moonolith/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/petsc/ex10p.cpp

@@ -206,7 +206,6 @@ int main(int argc, char *argv[])
    bool use_petsc = true;
    const char *petscrc_file = "";
    bool petsc_use_jfnk = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -244,8 +243,6 @@ int main(int argc, char *argv[])
    args.AddOption(&petsc_use_jfnk, "-jfnk", "--jfnk", "-no-jfnk",
                   "--no-jfnk",
                   "Use JFNK with user-defined preconditioner factory.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -260,12 +257,7 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
    if (use_petsc)
    {
       MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL);

examples/petsc/ex2p.cpp

@@ -67,7 +67,6 @@ int main(int argc, char *argv[])
    bool use_petsc = true;
    const char *petscrc_file = "";
    bool use_nonoverlapping = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -96,8 +95,6 @@ int main(int argc, char *argv[])
                   "-no-nonoverlapping", "--no-nonoverlapping",
                   "Use or not the block diagonal PETSc's matrix format "
                   "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -112,12 +109,7 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
    if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
 
    // 3. Read the (serial) mesh from the given mesh file on all processors.  We

examples/petsc/ex3p.cpp

@@ -61,7 +61,6 @@ int main(int argc, char *argv[])
    bool use_petsc = true;
    const char *petscrc_file = "";
    bool use_nonoverlapping = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -88,8 +87,6 @@ int main(int argc, char *argv[])
                   "-no-nonoverlapping", "--no-nonoverlapping",
                   "Use or not the block diagonal PETSc's matrix format "
                   "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -103,15 +100,9 @@ int main(int argc, char *argv[])
    {
       args.PrintOptions(cout);
    }
-   kappa = freq * M_PI;
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
    if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
+   kappa = freq * M_PI;
 
    // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface

examples/petsc/ex4p.cpp

@@ -58,7 +58,6 @@ int main(int argc, char *argv[])
    bool use_petsc = true;
    const char *petscrc_file = "";
    bool use_nonoverlapping = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -89,8 +88,6 @@ int main(int argc, char *argv[])
                   "-no-nonoverlapping", "--no-nonoverlapping",
                   "Use or not the block diagonal PETSc's matrix format "
                   "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -104,15 +101,9 @@ int main(int argc, char *argv[])
    {
       args.PrintOptions(cout);
    }
-   kappa = freq * M_PI;
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
    if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
+   kappa = freq * M_PI;
 
    // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface

examples/petsc/ex5p.cpp

@@ -59,8 +59,6 @@ int main(int argc, char *argv[])
 
    // 2. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = -1;
-   int par_ref_levels = 2;
    int order = 1;
    bool par_format = false;
    bool visualization = 1;
@@ -68,22 +66,15 @@ int main(int argc, char *argv[])
    bool use_nonoverlapping = false;
    bool local_bdr_spec = false;
    const char *petscrc_file = "";
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
    args.AddOption(&par_format, "-pf", "--parallel-format", "-sf",
                   "--serial-format",
                   "Format to use when saving the results for VisIt.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -112,13 +103,7 @@ int main(int argc, char *argv[])
    {
       args.PrintOptions(cout);
    }
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
    if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
 
    // 3. Read the (serial) mesh from the given mesh file on all processors.  We
@@ -132,11 +117,9 @@ int main(int argc, char *argv[])
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 10,000 elements.
    {
-      if (ser_ref_levels < 0)
-      {
-         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      }
-      for (int l = 0; l < ser_ref_levels; l++)
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
       }
@@ -148,6 +131,7 @@ int main(int argc, char *argv[])
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
+      int par_ref_levels = 2;
       for (int l = 0; l < par_ref_levels; l++)
       {
          pmesh->UniformRefinement();
@@ -203,26 +187,21 @@ int main(int argc, char *argv[])
 
    // 9. Define the parallel grid function and parallel linear forms, solution
    //    vector and rhs.
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
-   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
 
    ParLinearForm *fform(new ParLinearForm);
    fform->Update(R_space, rhs.GetBlock(0), 0);
    fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
    fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
    fform->Assemble();
-   fform->SyncAliasMemory(rhs);
    fform->ParallelAssemble(trueRhs.GetBlock(0));
-   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);
 
    ParLinearForm *gform(new ParLinearForm);
    gform->Update(W_space, rhs.GetBlock(1), 0);
    gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
    gform->Assemble();
-   gform->SyncAliasMemory(rhs);
    gform->ParallelAssemble(trueRhs.GetBlock(1));
-   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);
 
    // 10. Assemble the finite element matrices for the Darcy operator
    //

examples/petsc/ex6p.cpp

@@ -53,7 +53,6 @@ int main(int argc, char *argv[])
    bool use_petsc = true;
    const char *petscrc_file = "";
    bool use_nonoverlapping = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -74,8 +73,6 @@ int main(int argc, char *argv[])
                   "-no-nonoverlapping", "--no-nonoverlapping",
                   "Use or not the block diagonal PETSc's matrix format "
                   "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -89,13 +86,7 @@ int main(int argc, char *argv[])
    {
       args.PrintOptions(cout);
    }
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
    if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
 
    // 3. Read the (serial) mesh from the given mesh file on all processors.  We

examples/petsc/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/petsc/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/pumi/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/pumi/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/spe10_coeff.cpp

@@ -0,0 +1,352 @@
+/*
+ * spe10_coeff.cpp
+ *
+ *  Created on: Aug 23, 2017
+ *      Author: neumueller
+ */
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class InversePermeabilityFunction
+{
+public:
+
+   enum SliceOrientation {NONE, XY, XZ, YZ};
+
+   static void SetNumberCells(int Nx_, int Ny_, int Nz_);
+   static void SetMeshSizes(double hx, double hy, double hz);
+   static void Set2DSlice(SliceOrientation o, int npos );
+
+   static void ReadPermeabilityFile(const std::string fileName);
+#ifdef MFEM_USE_MPI
+   static void ReadPermeabilityFile(const std::string fileName, MPI_Comm comm);
+#endif
+   static void SetConstantInversePermeability(double ipx, double ipy, double ipz);
+
+   template<class F>
+   static void Transform(const F & f)
+   {
+      for (int i = 0; i < 3*Nx*Ny*Nz; ++i)
+      {
+         inversePermeability[i] = f(inversePermeability[i]);
+      }
+   }
+
+   static void InversePermeability(const Vector & x, Vector & val);
+   static double PermeabilityXY(Vector &x);
+   static void NegativeInversePermeability(const Vector & x, Vector & val);
+   static void Permeability(const Vector & x, Vector & val);
+
+   static double Norm2Permeability(const Vector & x);
+
+   static double Norm2InversePermeability(const Vector & x);
+   static double Norm1InversePermeability(const Vector & x);
+   static double NormInfInversePermeability(const Vector & x);
+
+   static double InvNorm2(const Vector & x);
+   static double InvNorm1(const Vector & x);
+   static double InvNormInf(const Vector & x);
+
+
+   static void ClearMemory();
+
+private:
+   static int Nx;
+   static int Ny;
+   static int Nz;
+   static double hx;
+   static double hy;
+   static double hz;
+   static double * inversePermeability;
+
+   static SliceOrientation orientation;
+   static int npos;
+};
+
+
+void InversePermeabilityFunction::SetNumberCells(int Nx_, int Ny_, int Nz_)
+{
+   Nx = Nx_;
+   Ny = Ny_;
+   Nz = Nz_;
+}
+
+void InversePermeabilityFunction::SetMeshSizes(double hx_, double hy_,
+                                               double hz_)
+{
+   hx = hx_;
+   hy = hy_;
+   hz = hz_;
+}
+
+void InversePermeabilityFunction::Set2DSlice(SliceOrientation o, int npos_ )
+{
+   orientation = o;
+   npos = npos_;
+}
+
+void InversePermeabilityFunction::SetConstantInversePermeability(double ipx,
+                                                                 double ipy, double ipz)
+{
+   int compSize = Nx*Ny*Nz;
+   int size = 3*compSize;
+   inversePermeability = new double [size];
+   double *ip = inversePermeability;
+
+   for (int i(0); i < compSize; ++i)
+   {
+      ip[i] = ipx;
+      ip[i+compSize] = ipy;
+      ip[i+2*compSize] = ipz;
+   }
+
+}
+
+#ifdef MFEM_USE_MPI
+void InversePermeabilityFunction::ReadPermeabilityFile(const std::string
+                                                       fileName, MPI_Comm comm)
+{
+   int num_procs, myid;
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   StopWatch chrono;
+
+   chrono.Start();
+   if (myid == 0)
+   {
+      ReadPermeabilityFile(fileName);
+   }
+   else
+   {
+      inversePermeability = new double [3*Nx*Ny*Nz];
+   }
+   chrono.Stop();
+
+   if (myid==0)
+   {
+      std::cout<<"Permeability file read in " << chrono.RealTime() << ".s \n";
+   }
+
+   chrono.Clear();
+
+   chrono.Start();
+   MPI_Bcast(inversePermeability, 3*Nx*Ny*Nz, MPI_DOUBLE, 0, comm);
+   chrono.Stop();
+
+   if (myid==0)
+   {
+      std::cout<<"Permeability field distributed in " << chrono.RealTime() <<
+               ".s \n";
+   }
+
+}
+#endif
+
+void InversePermeabilityFunction::ReadPermeabilityFile(const std::string
+                                                       fileName)
+{
+   std::ifstream permfile(fileName.c_str());
+
+   if (!permfile.is_open())
+   {
+      std::cout << "Error in opening file " << fileName << "\n";
+      mfem_error("File do not exists");
+   }
+
+   inversePermeability = new double [3*Nx*Ny*Nz];
+   double *ip = inversePermeability;
+   double tmp;
+   for (int l = 0; l < 3; l++)
+   {
+      for (int k = 0; k < Nz; k++)
+      {
+         for (int j = 0; j < Ny; j++)
+         {
+            for (int i = 0; i < Nx; i++)
+            {
+               permfile >> *ip;
+               *ip = 1./(*ip);
+               ip++;
+            }
+            for (int i = 0; i < 60-Nx; i++)
+            {
+               permfile >> tmp;   // skip unneeded part
+            }
+         }
+         for (int j = 0; j < 220-Ny; j++)
+            for (int i = 0; i < 60; i++)
+            {
+               permfile >> tmp;   // skip unneeded part
+            }
+      }
+
+      if (l < 2) // if not processing Kz, skip unneeded part
+         for (int k = 0; k < 85-Nz; k++)
+            for (int j = 0; j < 220; j++)
+               for (int i = 0; i < 60; i++)
+               {
+                  permfile >> tmp;
+               }
+   }
+
+}
+
+void InversePermeabilityFunction::InversePermeability(const Vector & x,
+                                                      Vector & val)
+{
+   val.SetSize(3);
+
+   unsigned int i,j,k;
+
+   switch (orientation)
+   {
+      case NONE:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      case XY:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = npos;
+         break;
+      case XZ:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = npos;
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      case YZ:
+         i = npos;
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      default:
+      {
+         mfem_error("InversePermeabilityFunction::InversePermeability");
+      }
+   }
+
+
+   int NMax = 3*Nx*Ny*Nz-1;
+   if (Ny*Nx*k + Nx*j + i>NMax || Ny*Nx*k + Nx*j + i + Nx*Ny*Nz>NMax ||
+       Ny*Nx*k + Nx*j + i + 2*Nx*Ny*Nz>NMax)
+   {
+      cout << " the indicies are wrong!" << endl;
+      cout << i << " " << j << " " << k << endl;
+   }
+
+   val[0] = inversePermeability[Ny*Nx*k + Nx*j + i];
+   val[1] = inversePermeability[Ny*Nx*k + Nx*j + i + Nx*Ny*Nz];
+
+   if (orientation == NONE)
+   {
+      val[2] = inversePermeability[Ny*Nx*k + Nx*j + i + 2*Nx*Ny*Nz];
+   }
+
+}
+
+double InversePermeabilityFunction::PermeabilityXY(Vector &x)
+{
+   unsigned int i,j,k;
+
+   i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+   j = (int)floor(x[1]/hy/(1.+3e-16));
+   k = npos;
+
+   return 1./inversePermeability[Ny*Nx*k + Nx*j + i];
+}
+
+void InversePermeabilityFunction::NegativeInversePermeability(const Vector & x,
+                                                              Vector & val)
+{
+   InversePermeability(x,val);
+   val *= -1.;
+}
+
+
+void InversePermeabilityFunction::Permeability(const Vector & x, Vector & val)
+{
+   InversePermeability(x,val);
+
+   for (double * it = val.GetData(), *end = val.GetData()+val.Size(); it != end;
+        ++it )
+   {
+      (*it) = 1./ (*it);
+   }
+}
+
+double InversePermeabilityFunction::Norm2Permeability(const Vector & x)
+{
+   Vector val(3);
+   Permeability(x,val);
+   return val.Norml2();
+}
+
+
+double InversePermeabilityFunction::Norm2InversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Norml2();
+}
+
+double InversePermeabilityFunction::Norm1InversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Norml1();
+}
+
+double InversePermeabilityFunction::NormInfInversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Normlinf();
+}
+
+double InversePermeabilityFunction::InvNorm2(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Norml2();
+}
+
+double InversePermeabilityFunction::InvNorm1(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Norml1();
+}
+
+double InversePermeabilityFunction::InvNormInf(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Normlinf();
+}
+
+
+void InversePermeabilityFunction::ClearMemory()
+{
+   delete[] inversePermeability;
+}
+
+int InversePermeabilityFunction::Nx(60);
+int InversePermeabilityFunction::Ny(220);
+int InversePermeabilityFunction::Nz(85);
+double InversePermeabilityFunction::hx(20);
+double InversePermeabilityFunction::hy(10);
+double InversePermeabilityFunction::hz(2);
+double * InversePermeabilityFunction::inversePermeability(NULL);
+InversePermeabilityFunction::SliceOrientation
+InversePermeabilityFunction::orientation( InversePermeabilityFunction::NONE );
+int InversePermeabilityFunction::npos(-1);
+
+
+

examples/sundials/CMakeLists.txt

@@ -31,21 +31,11 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 add_custom_target(test_sundials
   ${CMAKE_CTEST_COMMAND} -R sundials USES_TERMINAL)
 
-# Add one executable per cpp file, adding "sundials_" as prefix so the CMake
-# target is unique from those in the non-SUNDIALS examples. Also sets
-# "test_sundials" as a target that depends on the given SUNDIALS examples.
+# Add one executable per cpp file, adding "sundials_" as prefix. Sets
+# "test_sundials" as a target that depends on the given examples.
 set(PFX sundials_)
 add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)
 
-# Remove "sundials_" prefix from exectuable name for consistency with GNU build
-# system.
-foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TARGET_NAME "${PFX}${SRC_FILENAME}")
-  string(REPLACE ${PFX} "" EXE_NAME ${TARGET_NAME})
-  set_target_properties(${TARGET_NAME} PROPERTIES OUTPUT_NAME ${EXE_NAME})
-endforeach()
-
 # Testing.
 # The SUNDIALS tests can be run separately using the target "test_sundials"
 # which builds the examples and runs:
@@ -61,10 +51,7 @@ if (MFEM_ENABLE_TESTING)
   set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
   set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
   set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
-  # Example 16: test ARKODE with implicit time stepping using mass form
-  set(EX16_COMMON_OPTS -s 15)
-  set(EX16_TEST_OPTS ${EX16_COMMON_OPTS})
-  set(EX16P_TEST_OPTS ${EX16_COMMON_OPTS})
+  # Example 16: use the default options
 
   # Add the tests: one test per source file.
   foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})

examples/sundials/ex10.cpp

@@ -1,17 +1,15 @@
 //                                MFEM Example 10
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex10             (GNU make)
-//    make sundials_ex10    (CMake)
+// Compile with: make ex10
 //
 // Sample runs:
 //    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 12 -dt 0.15 -vs 10
 //    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 16 -dt 0.3  -vs 5
 //    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 12 -dt 0.2  -vs 5
-//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  2 -dt 3 -nls 1
-//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  2 -dt 3 -nls 2
-//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  2 -dt 3 -nls 4
+//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  2 -dt 3 -nls kinsol
+//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  2 -dt 3 -nls kinsol
+//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  2 -dt 3 -nls kinsol
 //    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 14 -dt 0.15 -vs 10
 //    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 17 -dt 0.01 -vs 30
 //    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 14 -dt 0.15 -vs 10
@@ -99,11 +97,16 @@ protected:
    double saved_gamma; // saved gamma value from implicit setup
 
 public:
+   /// Solver type to use in the ImplicitSolve() method, used by SDIRK methods.
+   enum NonlinearSolverType
+   {
+      NEWTON = 0, ///< Use MFEM's plain NewtonSolver
+      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KINSolver)
+   };
 
    HyperelasticOperator(FiniteElementSpace &f, Array<int> &ess_bdr,
                         double visc, double mu, double K,
-                        int kinsol_nls_type = -1, double kinsol_damping = 0.0,
-                        int kinsol_aa_n = 0);
+                        NonlinearSolverType nls_type);
 
    /// Compute the right-hand side of the ODE system.
    virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
@@ -221,10 +224,8 @@ int main(int argc, char *argv[])
    double mu = 0.25;
    double K = 5.0;
    bool visualization = true;
-   int nonlinear_solver_type = 0;
+   const char *nls = "newton";
    int vis_steps = 1;
-   double kinsol_damping = 0.0;
-   int kinsol_aa_n = -1;
 
    // Relative and absolute tolerances for CVODE and ARKODE.
    const double reltol = 1e-1, abstol = 1e-1;
@@ -261,18 +262,9 @@ int main(int argc, char *argv[])
                   "15 - ARKODE implicit, approximate Jacobian,\n\t"
                   "16 - ARKODE implicit, specified Jacobian,\n\t"
                   "17 - ARKODE explicit, 4th order.");
-   args.AddOption(&nonlinear_solver_type, "-nls", "--nonlinear-solver",
-                  "Nonlinear system solver:\n\t"
-                  "0  - MFEM Newton method,\n\t"
-                  "1  - KINSOL Newton method,\n\t"
-                  "2  - KINSOL Newton method with globalization,\n\t"
-                  "3  - KINSOL fixed-point method (with or without AA),\n\t"
-                  "4  - KINSOL Picard method (with or without AA).");
-   args.AddOption(&kinsol_damping, "-damp", "--kinsol-damping",
-                  "Picard or Fixed-Point damping parameter (only valid with KINSOL): "
-                  "0 < d <= 1.0");
-   args.AddOption(&kinsol_aa_n, "-aan", "--anderson-subspace",
-                  "Anderson Acceleration subspace size (only valid with KINSOL)");
+   args.AddOption(&nls, "-nls", "--nonlinear-solver",
+                  "Nonlinear systems solver: "
+                  "\"newton\" (plain Newton) or \"kinsol\" (KINSOL).");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -303,32 +295,22 @@ int main(int argc, char *argv[])
       return 1;
    }
 
-   // check for valid nonlinear solver options
-   if (nonlinear_solver_type < 0 || nonlinear_solver_type > 4)
-   {
-      cout << "Unknown nonlinear solver type: " << nonlinear_solver_type << "\n";
-      return 1;
-   }
-   if (kinsol_damping > 0.0 &&
-       !(nonlinear_solver_type == 3 || nonlinear_solver_type == 4))
-   {
-      cout << "Only KINSOL fixed-point and Picard methods can use damping\n";
-      return 1;
-   }
-   if (kinsol_aa_n > 0 &&
-       !(nonlinear_solver_type == 3 || nonlinear_solver_type == 4))
-   {
-      cout << "Only KINSOL fixed-point and Picard methods can use AA\n";
-      return 1;
-   }
-
-
    // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 3. Setup the nonlinear solver
+   map<string,HyperelasticOperator::NonlinearSolverType> nls_map;
+   nls_map["newton"] = HyperelasticOperator::NEWTON;
+   nls_map["kinsol"] = HyperelasticOperator::KINSOL;
+   if (nls_map.find(nls) == nls_map.end())
+   {
+      cout << "Unknown type of nonlinear solver: " << nls << endl;
+      return 4;
+   }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
    //    command-line parameter.
    for (int lev = 0; lev < ref_levels; lev++)
@@ -336,7 +318,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 4. Define the vector finite element spaces representing the mesh
+   // 5. Define the vector finite element spaces representing the mesh
    //    deformation x, the velocity v, and the initial configuration, x_ref.
    //    Define also the elastic energy density, w, which is in a discontinuous
    //    higher-order space. Since x and v are integrated in time as a system,
@@ -364,7 +346,7 @@ int main(int argc, char *argv[])
    FiniteElementSpace w_fespace(mesh, &w_fec);
    GridFunction w(&w_fespace);
 
-   // 5. Set the initial conditions for v and x, and the boundary conditions on
+   // 6. Set the initial conditions for v and x, and the boundary conditions on
    //    a beam-like mesh (see description above).
    VectorFunctionCoefficient velo(dim, InitialVelocity);
    v.ProjectCoefficient(velo);
@@ -377,34 +359,9 @@ int main(int argc, char *argv[])
    ess_bdr = 0;
    ess_bdr[0] = 1; // boundary attribute 1 (index 0) is fixed
 
-   // 6. Initialize the hyperelastic operator, the GLVis visualization and print
+   // 7. Initialize the hyperelastic operator, the GLVis visualization and print
    //    the initial energies.
-   std::unique_ptr<HyperelasticOperator> oper;
-   if (nonlinear_solver_type == 0)
-      oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr, visc, mu,
-                                                    K);
-   else
-   {
-      switch (nonlinear_solver_type)
-      {
-         case 1:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_NONE);
-            break;
-         case 2:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_LINESEARCH);
-            break;
-         case 3:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_FP, kinsol_damping, kinsol_aa_n);
-            break;
-         case 4:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_PICARD, kinsol_damping, kinsol_aa_n);
-            break;
-      }
-   }
+   HyperelasticOperator oper(fespace, ess_bdr, visc, mu, K, nls_map[nls]);
 
    socketstream vis_v, vis_w;
    if (visualization)
@@ -418,23 +375,23 @@ int main(int argc, char *argv[])
       vis_w.open(vishost, visport);
       if (vis_w)
       {
-         oper->GetElasticEnergyDensity(x, w);
+         oper.GetElasticEnergyDensity(x, w);
          vis_w.precision(8);
          visualize(vis_w, mesh, &x, &w, "Elastic energy density", true);
       }
    }
 
-   double ee0 = oper->ElasticEnergy(x.GetTrueVector());
-   double ke0 = oper->KineticEnergy(v.GetTrueVector());
+   double ee0 = oper.ElasticEnergy(x.GetTrueVector());
+   double ke0 = oper.KineticEnergy(v.GetTrueVector());
    cout << "initial elastic energy (EE) = " << ee0 << endl;
    cout << "initial kinetic energy (KE) = " << ke0 << endl;
    cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;
 
-   // 7. Define the ODE solver used for time integration. Several implicit
+   // 8. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
    double t = 0.0;
-   oper->SetTime(t);
+   oper.SetTime(t);
 
    ODESolver *ode_solver = NULL;
    CVODESolver *cvode = NULL;
@@ -458,7 +415,7 @@ int main(int argc, char *argv[])
       case 11:
       case 12:
          cvode = new CVODESolver(CV_BDF);
-         cvode->Init(*oper);
+         cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
          cvode->SetMaxStep(dt);
@@ -471,7 +428,7 @@ int main(int argc, char *argv[])
       case 13:
       case 14:
          cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(*oper);
+         cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
          cvode->SetMaxStep(dt);
@@ -484,13 +441,9 @@ int main(int argc, char *argv[])
       case 15:
       case 16:
          arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
-         arkode->Init(*oper);
+         arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
-#if MFEM_SUNDIALS_VERSION < 70100
          ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-#else
-         ARKodeSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-#endif
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 15)
          {
@@ -500,16 +453,16 @@ int main(int argc, char *argv[])
       // ARKStep Explicit methods
       case 17:
          arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(*oper);
+         arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
          ode_solver = arkode; break;
    }
 
    // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 11) { ode_solver->Init(*oper); }
+   if (ode_solver_type < 11) { ode_solver->Init(oper); }
 
-   // 8. Perform time-integration (looping over the time iterations, ti, with a
+   // 9. Perform time-integration (looping over the time iterations, ti, with a
    //    time-step dt).
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
@@ -522,8 +475,8 @@ int main(int argc, char *argv[])
 
       if (last_step || (ti % vis_steps) == 0)
       {
-         double ee = oper->ElasticEnergy(x.GetTrueVector());
-         double ke = oper->KineticEnergy(v.GetTrueVector());
+         double ee = oper.ElasticEnergy(x.GetTrueVector());
+         double ke = oper.KineticEnergy(v.GetTrueVector());
 
          cout << "step " << ti << ", t = " << t << ", EE = " << ee << ", KE = "
               << ke << ", ΔTE = " << (ee+ke)-(ee0+ke0) << endl;
@@ -537,14 +490,14 @@ int main(int argc, char *argv[])
             visualize(vis_v, mesh, &x, &v);
             if (vis_w)
             {
-               oper->GetElasticEnergyDensity(x, w);
+               oper.GetElasticEnergyDensity(x, w);
                visualize(vis_w, mesh, &x, &w);
             }
          }
       }
    }
 
-   // 9. Save the displaced mesh, the velocity and elastic energy.
+   // 10. Save the displaced mesh, the velocity and elastic energy.
    {
       v.SetFromTrueVector(); x.SetFromTrueVector();
       GridFunction *nodes = &x;
@@ -559,11 +512,11 @@ int main(int argc, char *argv[])
       v.Save(velo_ofs);
       ofstream ee_ofs("elastic_energy.sol");
       ee_ofs.precision(8);
-      oper->GetElasticEnergyDensity(x, w);
+      oper.GetElasticEnergyDensity(x, w);
       w.Save(ee_ofs);
    }
 
-   // 10. Free the used memory.
+   // 11. Free the used memory.
    delete ode_solver;
    delete mesh;
 
@@ -647,9 +600,7 @@ ReducedSystemOperator::~ReducedSystemOperator()
 HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
                                            Array<int> &ess_bdr, double visc,
                                            double mu, double K,
-                                           int kinsol_nls_type,
-                                           double kinsol_damping,
-                                           int kinsol_aa_n)
+                                           NonlinearSolverType nls_type)
    : TimeDependentOperator(2*f.GetTrueVSize(), 0.0), fespace(f),
      M(&fespace), S(&fespace), H(&fespace),
      viscosity(visc), z(height/2),
@@ -700,28 +651,15 @@ HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
    J_prec = NULL;
 #endif
 
-   if (kinsol_nls_type > 0)
+   if (nls_type == KINSOL)
    {
-      KINSolver *kinsolver = new KINSolver(kinsol_nls_type, true);
-      if (kinsol_nls_type != KIN_PICARD)
-      {
-         kinsolver->SetJFNK(true);
-         kinsolver->SetLSMaxIter(100);
-      }
-      if (kinsol_aa_n > 0)
-      {
-         kinsolver->EnableAndersonAcc(kinsol_aa_n);
-      }
+      KINSolver *kinsolver = new KINSolver(KIN_NONE, true);
       newton_solver = kinsolver;
       newton_solver->SetOperator(*reduced_oper);
       newton_solver->SetMaxIter(200);
       newton_solver->SetRelTol(rel_tol);
       newton_solver->SetPrintLevel(0);
       kinsolver->SetMaxSetupCalls(4);
-      if (kinsol_damping > 0.0)
-      {
-         kinsolver->SetDamping(kinsol_damping);
-      }
    }
    else
    {

examples/sundials/ex10p.cpp

@@ -1,17 +1,15 @@
 //                       MFEM Example 10 - Parallel Version
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex10p            (GNU make)
-//    make sundials_ex10p   (CMake)
+// Compile with: make ex10p
 //
 // Sample runs:
 //    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 12 -dt 0.15 -vs 10
 //    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 16 -dt 0.25 -vs 10
 //    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 12 -dt 0.15 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s  2 -dt 3 -nls 1
-//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s  2 -dt 3 -nls 2
-//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rs 1 -o 2 -s  2 -dt 3 -nls 4
+//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s  2 -dt 3 -nls kinsol
+//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s  2 -dt 3 -nls kinsol
+//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rs 1 -o 2 -s  2 -dt 3 -nls kinsol
 //    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 14 -dt 0.15 -vs 10
 //    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 17 -dt 5e-3 -vs 60
 //    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 14 -dt 0.15 -vs 10
@@ -101,11 +99,16 @@ protected:
    double saved_gamma; // saved gamma value from implicit setup
 
 public:
+   /// Solver type to use in the ImplicitSolve() method, used by SDIRK methods.
+   enum NonlinearSolverType
+   {
+      NEWTON = 0, ///< Use MFEM's plain NewtonSolver
+      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KINSolver)
+   };
 
    HyperelasticOperator(ParFiniteElementSpace &f, Array<int> &ess_bdr,
                         double visc, double mu, double K,
-                        int kinsol_nls_type = -1, double kinsol_damping = 0.0,
-                        int kinsol_aa_n = 0);
+                        NonlinearSolverType nls_type);
 
    /// Compute the right-hand side of the ODE system.
    virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
@@ -230,10 +233,8 @@ int main(int argc, char *argv[])
    double mu = 0.25;
    double K = 5.0;
    bool visualization = true;
-   int nonlinear_solver_type = 0;
+   const char *nls = "newton";
    int vis_steps = 1;
-   double kinsol_damping = 0.0;
-   int kinsol_aa_n = -1;
 
    // Relative and absolute tolerances for CVODE and ARKODE.
    const double reltol = 1e-1, abstol = 1e-1;
@@ -272,18 +273,9 @@ int main(int argc, char *argv[])
                   "15 - ARKODE implicit, approximate Jacobian,\n\t"
                   "16 - ARKODE implicit, specified Jacobian,\n\t"
                   "17 - ARKODE explicit, 4th order.");
-   args.AddOption(&nonlinear_solver_type, "-nls", "--nonlinear-solver",
-                  "Nonlinear system solver:\n\t"
-                  "0  - MFEM Newton method,\n\t"
-                  "1  - KINSOL Newton method,\n\t"
-                  "2  - KINSOL Newton method with globalization,\n\t"
-                  "3  - KINSOL fixed-point method (with or without AA),\n\t"
-                  "4  - KINSOL Picard method (with or without AA).");
-   args.AddOption(&kinsol_damping, "-damp", "--kinsol-damping",
-                  "Picard or Fixed-Point damping parameter (only valid with KINSOL): "
-                  "0 < d <= 1.0");
-   args.AddOption(&kinsol_aa_n, "-aan", "--anderson-subspace",
-                  "Anderson Acceleration subspace size (only valid with KINSOL)");
+   args.AddOption(&nls, "-nls", "--nonlinear-solver",
+                  "Nonlinear systems solver: "
+                  "\"newton\" (plain Newton) or \"kinsol\" (KINSOL).");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -323,42 +315,27 @@ int main(int argc, char *argv[])
       return 1;
    }
 
-   // check for valid nonlinear solver options
-   if (nonlinear_solver_type < 0 || nonlinear_solver_type > 4)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown nonlinear solver type: " << nonlinear_solver_type
-              << "\n";
-      }
-      return 1;
-   }
-   if (kinsol_damping > 0.0 &&
-       !(nonlinear_solver_type == 3 || nonlinear_solver_type == 4))
-   {
-      if (myid == 0)
-      {
-         cout << "Only KINSOL fixed-point and Picard methods can use damping\n";
-      }
-      return 1;
-   }
-   if (kinsol_aa_n > 0 &&
-       !(nonlinear_solver_type == 3 || nonlinear_solver_type == 4))
-   {
-      if (myid == 0)
-      {
-         cout << "Only KINSOL fixed-point and Picard methods can use AA\n";
-      }
-      return 1;
-   }
-
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the mesh in serial to increase the resolution. In this example
+   // 4. Nonlinear solver
+   map<string,HyperelasticOperator::NonlinearSolverType> nls_map;
+   nls_map["newton"] = HyperelasticOperator::NEWTON;
+   nls_map["kinsol"] = HyperelasticOperator::KINSOL;
+   if (nls_map.find(nls) == nls_map.end())
+   {
+      if (myid == 0)
+      {
+         cout << "Unknown type of nonlinear solver: " << nls << endl;
+      }
+      delete mesh;
+      return 4;
+   }
+
+   // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
    //    a command-line parameter.
    for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -366,7 +343,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -376,7 +353,7 @@ int main(int argc, char *argv[])
       pmesh->UniformRefinement();
    }
 
-   // 6. Define the parallel vector finite element spaces representing the mesh
+   // 7. Define the parallel vector finite element spaces representing the mesh
    //    deformation x_gf, the velocity v_gf, and the initial configuration,
    //    x_ref. Define also the elastic energy density, w_gf, which is in a
    //    discontinuous higher-order space. Since x and v are integrated in time
@@ -408,7 +385,7 @@ int main(int argc, char *argv[])
    ParFiniteElementSpace w_fespace(pmesh, &w_fec);
    ParGridFunction w_gf(&w_fespace);
 
-   // 7. Set the initial conditions for v_gf, x_gf and vx, and define the
+   // 8. Set the initial conditions for v_gf, x_gf and vx, and define the
    //    boundary conditions on a beam-like mesh (see description above).
    VectorFunctionCoefficient velo(dim, InitialVelocity);
    v_gf.ProjectCoefficient(velo);
@@ -423,38 +400,9 @@ int main(int argc, char *argv[])
    ess_bdr = 0;
    ess_bdr[0] = 1; // boundary attribute 1 (index 0) is fixed
 
-   // 8. Initialize the hyperelastic operator, the GLVis visualization and print
+   // 9. Initialize the hyperelastic operator, the GLVis visualization and print
    //    the initial energies.
-   std::unique_ptr<HyperelasticOperator> oper;
-   if (nonlinear_solver_type == 0)
-      oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr, visc, mu,
-                                                    K);
-   else
-   {
-      switch (nonlinear_solver_type)
-      {
-         case 1:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_NONE);
-            break;
-         case 2:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_LINESEARCH);
-            break;
-         case 3:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_FP, kinsol_damping, kinsol_aa_n);
-            break;
-         case 4:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_PICARD, kinsol_damping, kinsol_aa_n);
-            break;
-         default:
-            cout << "Unknown type of nonlinear solver: "
-                 << nonlinear_solver_type << endl;
-            return 4;
-      }
-   }
+   HyperelasticOperator oper(fespace, ess_bdr, visc, mu, K, nls_map[nls]);
 
    socketstream vis_v, vis_w;
    if (visualization)
@@ -470,14 +418,14 @@ int main(int argc, char *argv[])
       vis_w.open(vishost, visport);
       if (vis_w)
       {
-         oper->GetElasticEnergyDensity(x_gf, w_gf);
+         oper.GetElasticEnergyDensity(x_gf, w_gf);
          vis_w.precision(8);
          visualize(vis_w, pmesh, &x_gf, &w_gf, "Elastic energy density", true);
       }
    }
 
-   double ee0 = oper->ElasticEnergy(x_gf);
-   double ke0 = oper->KineticEnergy(v_gf);
+   double ee0 = oper.ElasticEnergy(x_gf);
+   double ke0 = oper.KineticEnergy(v_gf);
    if (myid == 0)
    {
       cout << "initial elastic energy (EE) = " << ee0 << endl;
@@ -485,11 +433,11 @@ int main(int argc, char *argv[])
       cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;
    }
 
-   // 9. Define the ODE solver used for time integration. Several implicit
-   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //    explicit Runge-Kutta methods are available.
+   // 10. Define the ODE solver used for time integration. Several implicit
+   //     singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //     explicit Runge-Kutta methods are available.
    double t = 0.0;
-   oper->SetTime(t);
+   oper.SetTime(t);
 
    ODESolver *ode_solver = NULL;
    CVODESolver *cvode = NULL;
@@ -513,7 +461,7 @@ int main(int argc, char *argv[])
       case 11:
       case 12:
          cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF);
-         cvode->Init(*oper);
+         cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
          cvode->SetMaxStep(dt);
@@ -526,7 +474,7 @@ int main(int argc, char *argv[])
       case 13:
       case 14:
          cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
-         cvode->Init(*oper);
+         cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
          cvode->SetMaxStep(dt);
@@ -539,13 +487,9 @@ int main(int argc, char *argv[])
       case 15:
       case 16:
          arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
-         arkode->Init(*oper);
+         arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
-#if MFEM_SUNDIALS_VERSION < 70100
          ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-#else
-         ARKodeSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-#endif
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 15)
          {
@@ -555,16 +499,16 @@ int main(int argc, char *argv[])
       // ARKStep Explicit methods
       case 17:
          arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(*oper);
+         arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
          ode_solver = arkode; break;
    }
 
    // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 11) { ode_solver->Init(*oper); }
+   if (ode_solver_type < 11) { ode_solver->Init(oper); }
 
-   // 10. Perform time-integration
+   // 11. Perform time-integration
    //     (looping over the time iterations, ti, with a time-step dt).
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
@@ -579,8 +523,8 @@ int main(int argc, char *argv[])
       {
          v_gf.SetFromTrueVector(); x_gf.SetFromTrueVector();
 
-         double ee = oper->ElasticEnergy(x_gf);
-         double ke = oper->KineticEnergy(v_gf);
+         double ee = oper.ElasticEnergy(x_gf);
+         double ke = oper.KineticEnergy(v_gf);
 
          if (myid == 0)
          {
@@ -596,14 +540,14 @@ int main(int argc, char *argv[])
             visualize(vis_v, pmesh, &x_gf, &v_gf);
             if (vis_w)
             {
-               oper->GetElasticEnergyDensity(x_gf, w_gf);
+               oper.GetElasticEnergyDensity(x_gf, w_gf);
                visualize(vis_w, pmesh, &x_gf, &w_gf);
             }
          }
       }
    }
 
-   // 11. Save the displaced mesh, the velocity and elastic energy.
+   // 12. Save the displaced mesh, the velocity and elastic energy.
    {
       v_gf.SetFromTrueVector(); x_gf.SetFromTrueVector();
       GridFunction *nodes = &x_gf;
@@ -624,11 +568,11 @@ int main(int argc, char *argv[])
       v_gf.Save(velo_ofs);
       ofstream ee_ofs(ee_name.str().c_str());
       ee_ofs.precision(8);
-      oper->GetElasticEnergyDensity(x_gf, w_gf);
+      oper.GetElasticEnergyDensity(x_gf, w_gf);
       w_gf.Save(ee_ofs);
    }
 
-   // 12. Free the used memory.
+   // 13. Free the used memory.
    delete ode_solver;
    delete pmesh;
 
@@ -718,10 +662,7 @@ ReducedSystemOperator::~ReducedSystemOperator()
 HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
                                            Array<int> &ess_bdr, double visc,
                                            double mu, double K,
-                                           int kinsol_nls_type,
-                                           double kinsol_damping,
-                                           int kinsol_aa_n)
-
+                                           NonlinearSolverType nls_type)
    : TimeDependentOperator(2*f.TrueVSize(), 0.0), fespace(f),
      M(&fespace), S(&fespace), H(&fespace),
      viscosity(visc), M_solver(f.GetComm()), z(height/2),
@@ -773,28 +714,17 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
    J_minres->SetPreconditioner(*J_prec);
    J_solver = J_minres;
 
-   if (kinsol_nls_type > 0)
+   if (nls_type == KINSOL)
    {
-      KINSolver *kinsolver = new KINSolver(f.GetComm(), kinsol_nls_type, true);
-      if (kinsol_nls_type != KIN_PICARD)
-      {
-         kinsolver->SetJFNK(true);
-         kinsolver->SetLSMaxIter(100);
-      }
-      if (kinsol_aa_n > 0)
-      {
-         kinsolver->EnableAndersonAcc(kinsol_aa_n);
-      }
+      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_LINESEARCH, true);
+      kinsolver->SetJFNK(true);
+      kinsolver->SetLSMaxIter(100);
       newton_solver = kinsolver;
       newton_solver->SetOperator(*reduced_oper);
       newton_solver->SetMaxIter(200);
       newton_solver->SetRelTol(rel_tol);
-      newton_solver->SetPrintLevel(0);
+      newton_solver->SetPrintLevel(1);
       kinsolver->SetMaxSetupCalls(4);
-      if (kinsol_damping > 0.0)
-      {
-         kinsolver->SetDamping(kinsol_damping);
-      }
    }
    else
    {

examples/sundials/ex16.cpp

@@ -1,21 +1,15 @@
 //                                MFEM Example 16
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex16             (GNU make)
-//    make sundials_ex16    (CMake)
+// Compile with: make ex16
 //
 // Sample runs:  ex16
 //               ex16 -m ../../data/inline-tri.mesh
 //               ex16 -m ../../data/disc-nurbs.mesh -tf 2
 //               ex16 -s 12 -a 0.0 -k 1.0
-//               ex16 -s 15 -a 0.0 -k 1.0
 //               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
-//               ex16 -s 11 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
 //               ex16 -s 9 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
-//               ex16 -s 12 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
 //               ex16 -s 10 -dt 1.0e-4 -tf 4.0e-2 -vs 40
-//               ex16 -s 13 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               ex16 -m ../../data/fichera-q2.mesh
 //               ex16 -m ../../data/escher.mesh
 //               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -43,102 +37,75 @@
 using namespace std;
 using namespace mfem;
 
-/** After spatial discretization, the conduction model is expressed as
+/** After spatial discretization, the conduction model can be written as:
  *
- *   M du/dt = - K(u) u
+ *     du/dt = M^{-1}(-Ku)
  *
  *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K(u) is the diffusion operator with diffusivity depending on u:
+ *  and K is the diffusion operator with diffusivity depending on u:
  *  (\kappa + \alpha u).
  *
- *  Class ConductionOperatorOperator represents the above ODE operator in the
- *  general form F(u, k, t) = G(u, t) where
- *
- *    1. F(u, du/dt, t) = du/dt            (ODE is expressed in EXPLICIT form)
- *       G(u, t) = - inv(M) K(u) u
- *    2. F(u, du/dt, t) = M du/dt          (ODE is expressed in IMPLICIT form)
- *       G(u, t) = - K(u) u
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
  */
 class ConductionOperator : public TimeDependentOperator
 {
+protected:
    FiniteElementSpace &fespace;
    Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
 
-   BilinearForm M;
-   SparseMatrix Mmat;
+   BilinearForm *M;
+   BilinearForm *K;
 
-   const real_t alpha, kappa;
-   std::unique_ptr<BilinearForm> K;
-   SparseMatrix Kmat;
-
-   std::unique_ptr<SparseMatrix> T; // T = M + gam K(u)
+   SparseMatrix Mmat, Kmat;
+   SparseMatrix *T; // T = M + dt K
 
    CGSolver M_solver; // Krylov solver for inverting the mass matrix M
    DSmoother M_prec;  // Preconditioner for the mass matrix M
 
-   CGSolver T_solver; // Implicit solver for T = M + gam K(u)
+   CGSolver T_solver; // Implicit solver for T = M + dt K
    DSmoother T_prec;  // Preconditioner for the implicit solver
 
+   double alpha, kappa;
+
    mutable Vector z; // auxiliary vector
 
 public:
+   ConductionOperator(FiniteElementSpace &f, double alpha, double kappa,
+                      const Vector &u);
 
-   ConductionOperator(FiniteElementSpace &f, const real_t alpha,
-                      const real_t kappa, const Vector &u,
-                      const Type &ode_expression_type);
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
 
-   // Compute K(u_n) for use as an approximation in - K(u) u
-   void SetConductionTensor(const Vector &u);
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
 
-   /** Compute G(u, t) as defined in the IMPLICIT expression form of the ODE
-    operator, i.e., @a v = - K(u_n) @a u. Note that K(u_n) is an
-      approximation to K(u). */
-   void ExplicitMult(const Vector &u, Vector &v) const override;
+   /// Custom Jacobian system solver for the SUNDIALS time integrators.
+   /** For the ODE system represented by ConductionOperator
 
-   /** Solve for k in F(u, k, t) = G(u, t) for either EXPLICIT or IMPLICIT
-       expression forms of the ODE operator, i.e., @a k = - inv(M) K(u_n) @a u.
-       Note that K(u_n) is an approximation to K(u). */
-   void Mult(const Vector &u, Vector &k) const override;
+           M du/dt = -K(u),
 
-   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t) for either EXPLICIT
-       or IMPLICIT expression forms of the ODE operator, i.e.,
-       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u . Note that K(u_n) is an
-       approximation to K(u). */
-   void ImplicitSolve(const real_t gam, const Vector &u, Vector &k) override;
+       this class facilitates the solution of linear systems of the form
 
-   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F for
-       either EXPLICIT or IMPLICIT expression forms of the ODE operator, i.e.,
-       [M - @a gam Jf(u)] dk = G - F, where Jf(u) is an approximation of the
-       Jacobian of -K(u) u. The approximation chosen here is Jf(u) = -K(u_n). */
-   int SUNImplicitSetup(const Vector &u, const Vector &fu, int jok, int *jcur,
-                        real_t gam) override;
+           (M + γK) y = M b,
 
-   /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing either
-        1. @a r = G - F = inv(M) f(u) - k       (EXPLICIT expression form)
-        1. @a r = G - F = f(u) - M k            (IMPLICIT expression form)
-       */
-   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
+       for given b, u (not used), and γ = GetTimeStep(). */
 
-   int SUNMassSetup() override;
+   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+                                int jok, int *jcur, double gamma);
 
-   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override;
+   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
 
-   int SUNMassMult(const Vector &x, Vector &v) override;
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
+
+   virtual ~ConductionOperator();
 };
 
-real_t InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 2.0;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
+double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
@@ -150,16 +117,16 @@ int main(int argc, char *argv[])
    int ref_levels = 2;
    int order = 2;
    int ode_solver_type = 9; // CVODE implicit BDF
-   real_t t_final = 0.5;
-   real_t dt = 1.0e-2;
-   real_t alpha = 1.0e-2;
-   real_t kappa = 0.5;
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
 
    // Relative and absolute tolerances for CVODE and ARKODE.
-   const real_t reltol = 1e-4, abstol = 1e-4;
+   const double reltol = 1e-4, abstol = 1e-4;
 
    int precision = 8;
    cout.precision(precision);
@@ -184,10 +151,7 @@ int main(int argc, char *argv[])
                   "9  - CVODE (implicit BDF),\n\t"
                   "10 - ARKODE (default explicit),\n\t"
                   "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default implicit),\n\t"
-                  "13 - ARKODE (default explicit with MFEM mass solve),\n\t"
-                  "14 - ARKODE (explicit Fehlberg-6-4-5 with MFEM mass solve),\n\t"
-                  "15 - ARKODE (default implicit with MFEM mass solve).");
+                  "12 - ARKODE (default impicit).");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -210,13 +174,16 @@ int main(int argc, char *argv[])
       args.PrintUsage(cout);
       return 1;
    }
+   if (ode_solver_type < 1 || ode_solver_type > 12)
+   {
+      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      return 3;
+   }
    args.PrintOptions(cout);
 
-   bool use_mass_solver = ode_solver_type >= 13;
-
    // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
-   std::unique_ptr<Mesh> mesh(new Mesh(mesh_file, 1, 1));
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
    // 3. Refine the mesh to increase the resolution. In this example we do
@@ -230,7 +197,7 @@ int main(int argc, char *argv[])
    // 4. Define the vector finite element space representing the current and the
    //    initial temperature, u_ref.
    H1_FECollection fe_coll(order, dim);
-   FiniteElementSpace fespace(mesh.get(), &fe_coll);
+   FiniteElementSpace fespace(mesh, &fe_coll);
 
    int fe_size = fespace.GetTrueVSize();
    cout << "Number of temperature unknowns: " << fe_size << endl;
@@ -244,17 +211,8 @@ int main(int argc, char *argv[])
    Vector u;
    u_gf.GetTrueDofs(u);
 
-   // 6. Initialize the conduction ODE operator and the visualization.
-   ConductionOperator::Type ode_expression_type;
-   if (use_mass_solver)
-   {
-      ode_expression_type = ConductionOperator::Type::IMPLICIT;
-   }
-   else
-   {
-      ode_expression_type = ConductionOperator::Type::EXPLICIT;
-   }
-   ConductionOperator oper(fespace, alpha, kappa, u, ode_expression_type);
+   // 6. Initialize the conduction operator and the visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -266,7 +224,7 @@ int main(int argc, char *argv[])
       u_gf.Save(osol);
    }
 
-   VisItDataCollection visit_dc("Example16", mesh.get());
+   VisItDataCollection visit_dc("Example16", mesh);
    visit_dc.RegisterField("temperature", &u_gf);
    if (visit)
    {
@@ -300,75 +258,52 @@ int main(int argc, char *argv[])
    }
 
    // 7. Define the ODE solver used for time integration.
-   real_t t = 0.0;
-   std::unique_ptr<ODESolver> ode_solver;
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKStepSolver *arkode = NULL;
    switch (ode_solver_type)
    {
       // MFEM explicit methods
-      case 1: ode_solver = std::make_unique<ForwardEulerSolver>(); break;
-      case 2: ode_solver = std::make_unique<RK2Solver>(0.5); break; // midpoint method
-      case 3: ode_solver = std::make_unique<RK3SSPSolver>(); break;
-      case 4: ode_solver = std::make_unique<RK4Solver>(); break;
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
       // MFEM implicit L-stable methods
-      case 5: ode_solver = std::make_unique<BackwardEulerSolver>(); break;
-      case 6: ode_solver = std::make_unique<SDIRK23Solver>(2); break;
-      case 7: ode_solver = std::make_unique<SDIRK33Solver>(); break;
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
       // CVODE
       case 8:
-      case 9:
-      {
-         int cvode_solver_type;
-         if (ode_solver_type == 8)
-         {
-            cvode_solver_type = CV_ADAMS;
-         }
-         else
-         {
-            cvode_solver_type = CV_BDF;
-         }
-         std::unique_ptr<CVODESolver> cvode(new CVODESolver(cvode_solver_type));
+         cvode = new CVODESolver(CV_ADAMS);
          cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          cvode->SetMaxStep(dt);
-         ode_solver = std::move(cvode);
-         break;
-      }
+         ode_solver = cvode; break;
+      case 9:
+         cvode = new CVODESolver(CV_BDF);
+         cvode->Init(oper);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
       // ARKODE
       case 10:
       case 11:
-      case 12:
-      case 13:
-      case 14:
-      case 15:
-      {
-         ARKStepSolver::Type arkode_solver_type;
-         if (ode_solver_type == 12 || ode_solver_type == 15)
-         {
-            arkode_solver_type = ARKStepSolver::IMPLICIT;
-         }
-         else
-         {
-            arkode_solver_type = ARKStepSolver::EXPLICIT;
-         }
-         std::unique_ptr<ARKStepSolver> arkode(
-            new ARKStepSolver(arkode_solver_type));
+         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11 || ode_solver_type == 14)
+         if (ode_solver_type == 11)
          {
             arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
          }
-         if (use_mass_solver)
-         {
-            arkode->UseMFEMMassLinearSolver(SUNFALSE);
-         }
-         ode_solver = std::move(arkode);
-         break;
-      }
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
+         arkode->Init(oper);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
    }
 
    // Initialize MFEM integrators, SUNDIALS integrators are initialized above
@@ -376,14 +311,8 @@ int main(int argc, char *argv[])
 
    // Since we want to update the diffusion coefficient after every time step,
    // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-   {
-      cvode->SetStepMode(CV_ONE_STEP);
-   }
-   else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-   {
-      arkode->SetStepMode(ARK_ONE_STEP);
-   }
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
 
    // 8. Perform time-integration (looping over the time iterations, ti, with a
    //    time-step dt).
@@ -394,7 +323,7 @@ int main(int argc, char *argv[])
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
    {
-      real_t dt_real = min(dt, t_final - t);
+      double dt_real = min(dt, t_final - t);
 
       // Note that since we are using the "one-step" mode of the SUNDIALS
       // solvers, they will, generally, step over the final time and will not
@@ -408,14 +337,8 @@ int main(int argc, char *argv[])
       if (last_step || (ti % vis_steps) == 0)
       {
          cout << "step " << ti << ", t = " << t << endl;
-         if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-         {
-            cvode->PrintInfo();
-         }
-         else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-         {
-            arkode->PrintInfo();
-         }
+         if (cvode) { cvode->PrintInfo(); }
+         if (arkode) { arkode->PrintInfo(); }
 
          u_gf.SetFromTrueDofs(u);
          if (visualization)
@@ -430,153 +353,137 @@ int main(int argc, char *argv[])
             visit_dc.Save();
          }
       }
-      oper.SetConductionTensor(u);
+      oper.SetParameters(u);
    }
    tic_toc.Stop();
    cout << "Done, " << tic_toc.RealTime() << "s." << endl;
 
    // 9. Save the final solution. This output can be viewed later using GLVis:
    //    "glvis -m ex16.mesh -g ex16-final.gf".
-   u_gf.Save("ex16-final.gf", precision);
+   {
+      ofstream osol("ex16-final.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete mesh;
 
    return 0;
 }
 
-ConductionOperator::ConductionOperator(FiniteElementSpace &fes,
-                                       const real_t alpha, const real_t kappa,
-                                       const Vector &u,
-                                       const Type &ode_expression_type)
-   : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa), z(height)
+ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
+     T(NULL), z(height)
 {
-   // specify a relative tolerance for all solves with MFEM integrators
-   const real_t rel_tol = 1e-8;
+   const double rel_tol = 1e-8;
 
-   M.AddDomainIntegrator(new MassIntegrator());
-   M.Assemble();
-   M.FormSystemMatrix(ess_tdof_list, Mmat);
+   M = new BilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble();
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
 
    M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   M_solver.SetRelTol(rel_tol);
    M_solver.SetAbsTol(0.0);
    M_solver.SetMaxIter(50);
    M_solver.SetPrintLevel(0);
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(Mmat);
 
+   alpha = al;
+   kappa = kap;
+
    T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   T_solver.SetRelTol(rel_tol);
    T_solver.SetAbsTol(0.0);
    T_solver.SetMaxIter(100);
    T_solver.SetPrintLevel(0);
    T_solver.SetPreconditioner(T_prec);
 
-   SetConductionTensor(u);
+   SetParameters(u);
 }
 
-void ConductionOperator::SetConductionTensor(const Vector &u)
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Kmat.Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, dt, Kmat);
+   T_solver.SetOperator(*T);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
 {
-   // Compute K(u_n).
    GridFunction u_alpha_gf(&fespace);
    u_alpha_gf.SetFromTrueDofs(u);
    for (int i = 0; i < u_alpha_gf.Size(); i++)
    {
       u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
    }
+
+   delete K;
+   K = new BilinearForm(&fespace);
+
    GridFunctionCoefficient u_coeff(&u_alpha_gf);
 
-   K = std::make_unique<BilinearForm>(&fespace);
    K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
    K->Assemble();
    K->FormSystemMatrix(ess_tdof_list, Kmat);
 }
 
-void ConductionOperator::ExplicitMult(const Vector &u, Vector &v) const
+int ConductionOperator::SUNImplicitSetup(const Vector &x,
+                                         const Vector &fx, int jok, int *jcur,
+                                         double gamma)
 {
-   // Compute - K(u_n) u.
-   Kmat.Mult(u, v);
-   v.Neg();
-}
-
-void ConductionOperator::Mult(const Vector &u, Vector &k) const
-{
-   // Compute - inv(M) K(u_n) u.
-   ExplicitMult(u, z);
-   M_solver.Mult(z, k);
-}
-
-void ConductionOperator::ImplicitSolve(const real_t gam, const Vector &u,
-                                       Vector &k)
-{
-   // Solve for k in M k = - K(u_n) [u + gam*k].
-   ExplicitMult(u, z);
-   T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
+   // Setup the ODE Jacobian T = M + gamma K.
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, gamma, Kmat);
    T_solver.SetOperator(*T);
-   T_solver.Mult(z, k);
+   *jcur = 1;
+   return (0);
 }
 
-int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
-                                         int jok, int *jcur, real_t gam)
+int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
 {
-   // Compute T = M + gamma K(u_n).
-   T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
-   T_solver.SetOperator(*T);
-   *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUN_SUCCESS;
+   // Solve the system A x = z => (M - gamma K) x = M b.
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, x);
+   return (0);
 }
 
-int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
-                                         real_t tol)
+ConductionOperator::~ConductionOperator()
 {
-   // Solve the system [M + gamma K(u_n)] dk = - K(u_n) u - M k.
-   // What value r is providing depends on the ODE expression form:
-   //   EXPLICIT form: r = -inv(M) K(u_n) u - k
-   //   IMPLICIT form: r = -K(u_n) u - M k
-   T_solver.SetRelTol(tol);
-   if (isExplicit())
+   delete T;
+   delete M;
+   delete K;
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5)
    {
-      Mmat.Mult(r, z);
-      T_solver.Mult(z, dk);
+      return 2.0;
    }
    else
    {
-      T_solver.Mult(r, dk);
-   }
-   if (T_solver.GetConverged())
-   {
-      return SUN_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
+      return 1.0;
    }
 }
-
-int ConductionOperator::SUNMassSetup()
-{
-   // Do nothing b/c mass solver was setup in constructor.
-   return SUN_SUCCESS;
-}
-
-int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
-{
-   // Solve the system M x = b.
-   M_solver.SetRelTol(tol);
-   M_solver.Mult(b, x);
-   if (M_solver.GetConverged())
-   {
-      return SUN_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
-   }
-}
-
-int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
-{
-   // Compute M x.
-   Mmat.Mult(x, v);
-   return SUN_SUCCESS;
-}
-

examples/sundials/ex16p.cpp

@@ -1,22 +1,16 @@
 //                         MFEM Example 16 - Parallel Version
 //                              SUNDIALS Modification
 //
-// Compile with:
-//    make ex16p            (GNU make)
-//    make sundials_ex16p   (CMake)
+// Compile with: make ex16p
 //
 // Sample runs:
 //     mpirun -np 4 ex16p
 //     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
 //     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
 //     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
-//     mpirun -np 4 ex16p -s 15 -a 0.0 -k 1.0
 //     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
-//     mpirun -np 4 ex16p -s 11 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
 //     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
-//     mpirun -np 8 ex16p -s 12 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
 //     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
-//     mpirun -np 4 ex16p -s 13 -dt 2.0e-4 -tf 4.0e-2
 //     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
 //     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
 //     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -44,102 +38,66 @@
 using namespace std;
 using namespace mfem;
 
-/** After spatial discretization, the conduction model is expressed as
+/** After spatial discretization, the conduction model can be written as:
  *
- *   M du/dt = - K(u) u
+ *     du/dt = M^{-1}(-Ku)
  *
  *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K(u) is the diffusion operator with diffusivity depending on u:
+ *  and K is the diffusion operator with diffusivity depending on u:
  *  (\kappa + \alpha u).
  *
- *  Class ConductionOperatorOperator represents the above ODE operator in the
- *  general form F(u, k, t) = G(u, t) where either
- *
- *    1. F(u, du/dt, t) = du/dt            (ODE is expressed in EXPLICIT form)
- *       G(u, t) = - inv(M) K(u) u
- *    2. F(u, du/dt, t) = M du/dt          (ODE is expressed in IMPLICIT form)
- *       G(u, t) = - K(u) u
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
  */
 class ConductionOperator : public TimeDependentOperator
 {
+protected:
    ParFiniteElementSpace &fespace;
    Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
 
-   ParBilinearForm M;
+   ParBilinearForm *M;
+   ParBilinearForm *K;
+
    HypreParMatrix Mmat;
-
-   const real_t alpha, kappa;
-   std::unique_ptr<BilinearForm> K;
    HypreParMatrix Kmat;
+   HypreParMatrix *T; // T = M + dt K
+   double current_dt;
 
-   std::unique_ptr<HypreParMatrix> T; // T = M + gam K(u)
+   CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
+   HypreSmoother M_prec; // Preconditioner for the mass matrix M
 
-   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
-   HypreSmoother M_prec;  // Preconditioner for the mass matrix M
+   CGSolver T_solver;    // Implicit solver for T = M + dt K
+   HypreSmoother T_prec; // Preconditioner for the implicit solver
 
-   CGSolver T_solver; // Implicit solver for T = M + gam K(u)
-   HypreSmoother T_prec;  // Preconditioner for the implicit solver
+   double alpha, kappa;
 
    mutable Vector z; // auxiliary vector
 
 public:
+   ConductionOperator(ParFiniteElementSpace &f, double alpha, double kappa,
+                      const Vector &u);
 
-   ConductionOperator(ParFiniteElementSpace &f, const real_t alpha,
-                      const real_t kappa, const Vector &u,
-                      const Type &ode_expression_type);
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
 
-   // Compute K(u_n) for use as an approximation in - K(u) u
-   void SetConductionTensor(const Vector &u);
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
 
-   /** Compute G(u, t) as defined in the IMPLICIT expression form of the ODE
-       operator, i.e., @a v = - K(u_n) @a u. Note that K(u_n) is an
-       approximation to K(u). */
-   void ExplicitMult(const Vector &u, Vector &v) const override;
+   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+                                int jok, int *jcur, double gamma);
 
-   /** Solve for k in F(u, k, t) = G(u, t) for either EXPLICIT or IMPLICIT
-       expression forms of the ODE operator, i.e., @a k = - inv(M) K(u_n) @a u.
-       Note that K(u_n) is an approximation to K(u). */
-   void Mult(const Vector &u, Vector &k) const override;
+   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
 
-   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t) for either EXPLICIT
-       or IMPLICIT expression forms of the ODE operator, i.e.,
-       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u . Note that K(u_n) is an
-       approximation to K(u). */
-   void ImplicitSolve(const real_t gam, const Vector &u, Vector &k) override;
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
 
-   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F for
-       either EXPLICIT or IMPLICIT expression forms of the ODE operator, i.e.,
-       [M - @a gam Jf(u)] dk = G - F, where Jf(u) is an approximation of the
-       Jacobian of -K(u) u. The approximation chosen here is Jf(u) = -K(u_n). */
-   int SUNImplicitSetup(const Vector &u, const Vector &fu, int jok, int *jcur,
-                        real_t gam) override;
-
-   /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing either
-        1. @a r = G - F = inv(M) f(u) - k       (EXPLICIT expression form)
-        1. @a r = G - F = f(u) - M k            (IMPLICIT expression form)
-       */
-   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
-
-   int SUNMassSetup() override;
-
-   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override;
-
-   int SUNMassMult(const Vector &x, Vector &v) override;
+   virtual ~ConductionOperator();
 };
 
-real_t InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 2.0;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
+double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
@@ -156,16 +114,16 @@ int main(int argc, char *argv[])
    int par_ref_levels = 1;
    int order = 2;
    int ode_solver_type = 9; // CVODE implicit BDF
-   real_t t_final = 0.5;
-   real_t dt = 1.0e-2;
-   real_t alpha = 1.0e-2;
-   real_t kappa = 0.5;
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
 
    // Relative and absolute tolerances for CVODE and ARKODE.
-   const real_t reltol = 1e-4, abstol = 1e-4;
+   const double reltol = 1e-4, abstol = 1e-4;
 
    int precision = 8;
    cout.precision(precision);
@@ -192,10 +150,7 @@ int main(int argc, char *argv[])
                   "9  - CVODE (implicit BDF),\n\t"
                   "10 - ARKODE (default explicit),\n\t"
                   "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default implicit),\n\t"
-                  "13 - ARKODE (default explicit with MFEM mass solve),\n\t"
-                  "14 - ARKODE (explicit Fehlberg-6-4-5 with MFEM mass solve),\n\t"
-                  "15 - ARKODE (default implicit with MFEM mass solve).");
+                  "12 - ARKODE (default impicit).");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -219,33 +174,40 @@ int main(int argc, char *argv[])
       return 1;
    }
 
-   if (Mpi::Root())
+   if (myid == 0)
    {
       args.PrintOptions(cout);
    }
 
-   bool use_mass_solver = ode_solver_type >= 13;
+   // check for valid ODE solver option
+   if (ode_solver_type < 1 || ode_solver_type > 12)
+   {
+      if (myid == 0)
+      {
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      }
+      return 1;
+   }
 
-   // 3. Define a parallel mesh by a partitioning of a serial mesh. Read the
-   //    serial mesh from the given mesh file on all processors. We can
+   // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code.
-   std::unique_ptr<ParMesh> pmesh;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the mesh in serial to increase the resolution. In this example
+   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
+   //    a command-line parameter.
+   for (int lev = 0; lev < ser_ref_levels; lev++)
    {
-      std::unique_ptr<Mesh> mesh(new Mesh(mesh_file, 1, 1));
-
-      // 4. Refine the mesh in serial to increase the resolution. In this example
-      //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-      //    a command-line parameter.
-      for (int lev = 0; lev < ser_ref_levels; lev++)
-      {
-         mesh->UniformRefinement();
-      }
-
-      // 5. Refine this mesh further in parallel to increase the resolution.
-      //    Once the parallel mesh is defined, the serial mesh can be deleted.
-      pmesh = std::make_unique<ParMesh>(MPI_COMM_WORLD, *mesh);
+      mesh->UniformRefinement();
    }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
    for (int lev = 0; lev < par_ref_levels; lev++)
    {
       pmesh->UniformRefinement();
@@ -253,9 +215,8 @@ int main(int argc, char *argv[])
 
    // 6. Define the vector finite element space representing the current and the
    //    initial temperature, u_ref.
-   int dim = pmesh->Dimension();
    H1_FECollection fe_coll(order, dim);
-   ParFiniteElementSpace fespace(pmesh.get(), &fe_coll);
+   ParFiniteElementSpace fespace(pmesh, &fe_coll);
 
    int fe_size = fespace.GlobalTrueVSize();
    if (myid == 0)
@@ -272,17 +233,8 @@ int main(int argc, char *argv[])
    Vector u;
    u_gf.GetTrueDofs(u);
 
-   // 8. Initialize the conduction ODE operator and the visualization.
-   ConductionOperator::Type ode_expression_type;
-   if (use_mass_solver)
-   {
-      ode_expression_type = ConductionOperator::Type::IMPLICIT;
-   }
-   else
-   {
-      ode_expression_type = ConductionOperator::Type::EXPLICIT;
-   }
-   ConductionOperator oper(fespace, alpha, kappa, u, ode_expression_type);
+   // 8. Initialize the conduction operator and the VisIt visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -297,7 +249,7 @@ int main(int argc, char *argv[])
       u_gf.Save(osol);
    }
 
-   VisItDataCollection visit_dc("Example16-Parallel", pmesh.get());
+   VisItDataCollection visit_dc("Example16-Parallel", pmesh);
    visit_dc.RegisterField("temperature", &u_gf);
    if (visit)
    {
@@ -341,76 +293,52 @@ int main(int argc, char *argv[])
    }
 
    // 9. Define the ODE solver used for time integration.
-   real_t t = 0.0;
-   std::unique_ptr<ODESolver> ode_solver;
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKStepSolver *arkode = NULL;
    switch (ode_solver_type)
    {
       // MFEM explicit methods
-      case 1: ode_solver = std::make_unique<ForwardEulerSolver>(); break;
-      case 2: ode_solver = std::make_unique<RK2Solver>(0.5); break; // midpoint method
-      case 3: ode_solver = std::make_unique<RK3SSPSolver>(); break;
-      case 4: ode_solver = std::make_unique<RK4Solver>(); break;
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
       // MFEM implicit L-stable methods
-      case 5: ode_solver = std::make_unique<BackwardEulerSolver>(); break;
-      case 6: ode_solver = std::make_unique<SDIRK23Solver>(2); break;
-      case 7: ode_solver = std::make_unique<SDIRK33Solver>(); break;
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
       // CVODE
       case 8:
-      case 9:
-      {
-         int cvode_solver_type;
-         if (ode_solver_type == 8)
-         {
-            cvode_solver_type = CV_ADAMS;
-         }
-         else
-         {
-            cvode_solver_type = CV_BDF;
-         }
-         std::unique_ptr<CVODESolver> cvode(
-            new CVODESolver(MPI_COMM_WORLD, cvode_solver_type));
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
          cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          cvode->SetMaxStep(dt);
-         ode_solver = std::move(cvode);
-         break;
-      }
+         ode_solver = cvode; break;
+      case 9:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF);
+         cvode->Init(oper);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
       // ARKODE
       case 10:
       case 11:
-      case 12:
-      case 13:
-      case 14:
-      case 15:
-      {
-         ARKStepSolver::Type arkode_solver_type;
-         if (ode_solver_type == 12 || ode_solver_type == 15)
-         {
-            arkode_solver_type = ARKStepSolver::IMPLICIT;
-         }
-         else
-         {
-            arkode_solver_type = ARKStepSolver::EXPLICIT;
-         }
-         std::unique_ptr<ARKStepSolver> arkode(
-            new ARKStepSolver(MPI_COMM_WORLD, arkode_solver_type));
+         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11 || ode_solver_type == 14)
+         if (ode_solver_type == 11)
          {
             arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
          }
-         if (use_mass_solver)
-         {
-            arkode->UseMFEMMassLinearSolver(SUNFALSE);
-         }
-         ode_solver = std::move(arkode);
-         break;
-      }
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
+         arkode->Init(oper);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
    }
 
    // Initialize MFEM integrators, SUNDIALS integrators are initialized above
@@ -418,18 +346,12 @@ int main(int argc, char *argv[])
 
    // Since we want to update the diffusion coefficient after every time step,
    // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-   {
-      cvode->SetStepMode(CV_ONE_STEP);
-   }
-   else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-   {
-      arkode->SetStepMode(ARK_ONE_STEP);
-   }
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
 
    // 10. Perform time-integration (looping over the time iterations, ti, with a
    //     time-step dt).
-   if (Mpi::Root())
+   if (myid == 0)
    {
       cout << "Integrating the ODE ..." << endl;
    }
@@ -439,7 +361,7 @@ int main(int argc, char *argv[])
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
    {
-      real_t dt_real = min(dt, t_final - t);
+      double dt_real = min(dt, t_final - t);
 
       // Note that since we are using the "one-step" mode of the SUNDIALS
       // solvers, they will, generally, step over the final time and will not
@@ -455,14 +377,8 @@ int main(int argc, char *argv[])
          if (myid == 0)
          {
             cout << "step " << ti << ", t = " << t << endl;
-            if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-            {
-               cvode->PrintInfo();
-            }
-            else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-            {
-               arkode->PrintInfo();
-            }
+            if (cvode) { cvode->PrintInfo(); }
+            if (arkode) { arkode->PrintInfo(); }
          }
 
          u_gf.SetFromTrueDofs(u);
@@ -479,38 +395,46 @@ int main(int argc, char *argv[])
             visit_dc.Save();
          }
       }
-      oper.SetConductionTensor(u);
+      oper.SetParameters(u);
    }
    tic_toc.Stop();
-   if (Mpi::Root())
+   if (myid == 0)
    {
       cout << "Done, " << tic_toc.RealTime() << "s." << endl;
    }
 
    // 11. Save the final solution in parallel. This output can be viewed later
    //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
-   u_gf.Save("ex16-final", precision);
+   {
+      ostringstream sol_name;
+      sol_name << "ex16-final." << setfill('0') << setw(6) << myid;
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   // 12. Free the used memory.
+   delete ode_solver;
+   delete pmesh;
 
    return 0;
 }
 
-ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
-                                       const real_t alpha, const real_t kappa,
-                                       const Vector &u,
-                                       const Type &ode_expression_type)
-   : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa),
-     M_solver(fes.GetComm()), T_solver(fes.GetComm()), z(height)
+ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
+     T(NULL),
+     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
 {
-   // specify a relative tolerance for all solves with MFEM integrators
-   const real_t rel_tol = 1e-8;
+   const double rel_tol = 1e-8;
 
-   M.AddDomainIntegrator(new MassIntegrator());
-   M.Assemble(0); // keep zeros to keep sparsity pattern of M and K the same
-   M.FormSystemMatrix(ess_tdof_list, Mmat);
+   M = new ParBilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble(0); // keep sparsity pattern of M and K the same
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
 
    M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   M_solver.SetRelTol(rel_tol);
    M_solver.SetAbsTol(0.0);
    M_solver.SetMaxIter(100);
    M_solver.SetPrintLevel(0);
@@ -518,118 +442,97 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(Mmat);
 
+   alpha = al;
+   kappa = kap;
+
    T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   T_solver.SetRelTol(rel_tol);
    T_solver.SetAbsTol(0.0);
    T_solver.SetMaxIter(100);
    T_solver.SetPrintLevel(0);
    T_solver.SetPreconditioner(T_prec);
 
-   SetConductionTensor(u);
+   SetParameters(u);
 }
 
-void ConductionOperator::SetConductionTensor(const Vector &u)
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Kmat.Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, dt, Kmat);
+   T_solver.SetOperator(*T);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
+}
+
+int ConductionOperator::SUNImplicitSetup(const Vector &x,
+                                         const Vector &fx, int jok, int *jcur,
+                                         double gamma)
+{
+   // Setup the ODE Jacobian T = M + gamma K.
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, gamma, Kmat);
+   T_solver.SetOperator(*T);
+   *jcur = 1;
+   return (0);
+}
+
+int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
+{
+   // Solve the system A x = z => (M - gamma K) x = M b.
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, x);
+   return (0);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
 {
-   // Compute K(u_n).
    ParGridFunction u_alpha_gf(&fespace);
    u_alpha_gf.SetFromTrueDofs(u);
    for (int i = 0; i < u_alpha_gf.Size(); i++)
    {
       u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
    }
+
+   delete K;
+   K = new ParBilinearForm(&fespace);
+
    GridFunctionCoefficient u_coeff(&u_alpha_gf);
 
-   K = std::make_unique<ParBilinearForm>(&fespace);
    K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
-   K->Assemble(0); // keep zeros to keep sparsity pattern of M and K the same
+   K->Assemble(0); // keep sparsity pattern of M and K the same
    K->FormSystemMatrix(ess_tdof_list, Kmat);
 }
 
-void ConductionOperator::ExplicitMult(const Vector &u, Vector &v) const
+ConductionOperator::~ConductionOperator()
 {
-   // Compute - K(u_n) u.
-   Kmat.Mult(u, v);
-   v.Neg();
+   delete T;
+   delete M;
+   delete K;
 }
 
-void ConductionOperator::Mult(const Vector &u, Vector &k) const
+double InitialTemperature(const Vector &x)
 {
-   // Compute - inv(M) K(u_n) u.
-   ExplicitMult(u, z);
-   M_solver.Mult(z, k);
-}
-
-void ConductionOperator::ImplicitSolve(const real_t gam, const Vector &u,
-                                       Vector &k)
-{
-   // Solve for k in M k = - K(u_n) [u + gam*k].
-   ExplicitMult(u, z);
-   T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
-   T_solver.SetOperator(*T);
-   T_solver.Mult(z, k);
-}
-
-int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
-                                         int jok, int *jcur, real_t gam)
-{
-   // Compute T = M + gamma K(u_n).
-   T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
-   T_solver.SetOperator(*T);
-   *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUN_SUCCESS;
-}
-
-int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
-                                         real_t tol)
-{
-   // Solve the system [M + gamma K(u_n)] dk = - K(u_n) u - M k.
-   // What value r is providing depends on the ODE expression form:
-   //   EXPLICIT form: r = -inv(M) K(u_n) u - k
-   //   IMPLICIT form: r = -K(u_n) u - M k
-   T_solver.SetRelTol(tol);
-   if (isExplicit())
+   if (x.Norml2() < 0.5)
    {
-      Mmat.Mult(r, z);
-      T_solver.Mult(z, dk);
+      return 2.0;
    }
    else
    {
-      T_solver.Mult(r, dk);
-   }
-   if (T_solver.GetConverged())
-   {
-      return SUN_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
+      return 1.0;
    }
 }
-
-int ConductionOperator::SUNMassSetup()
-{
-   // Do nothing b/c mass solver was setup in constructor.
-   return SUN_SUCCESS;
-}
-
-int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
-{
-   // Solve the system M x = b.
-   M_solver.SetRelTol(tol);
-   M_solver.Mult(b, x);
-   if (M_solver.GetConverged())
-   {
-      return SUN_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
-   }
-}
-
-int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
-{
-   // Compute M x.
-   Mmat.Mult(x, v);
-   return SUN_SUCCESS;
-}

examples/sundials/ex9.cpp

@@ -1,9 +1,7 @@
 //                                MFEM Example 9
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex9              (GNU make)
-//    make sundials_ex9     (CMake)
+// Compile with: make ex9
 //
 // Sample runs:
 //    ex9 -m ../../data/periodic-segment.mesh -p 0 -r 2 -s 7 -dt 0.005

examples/sundials/ex9p.cpp

@@ -1,9 +1,7 @@
 //                       MFEM Example 9 - Parallel Version
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex9p             (GNU make)
-//    make sundials_ex9p    (CMake)
+// Compile with: make ex9p
 //
 // Sample runs:
 //    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -p 1 -rp 1 -s 7 -dt 0.0025

examples/sundials/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/sundials/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -99,12 +100,6 @@ ex10-test-seq: ex10
 	@$(call mfem-test,$<,, $(SERIAL_NAME),$(EX10_ARGS))
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX10P_ARGS))
-# Example 16: test ARKODE with implicit time stepping using mass form
-EX16_COMMON_ARGS := -s 15
-ex16-test-seq: ex16
-	@$(call mfem-test,$<,, $(SERIAL_NAME),$(EX16_COMMON_ARGS))
-ex16p-test-par: ex16p
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX16_COMMON_ARGS))
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/superlu/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/superlu/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

fem/CMakeLists.txt

@@ -112,6 +112,8 @@ set(SRCS
   qinterp/eval_by_vdim.cpp
   qinterp/grad_by_nodes.cpp
   qinterp/grad_by_vdim.cpp
+  qinterp/grad_phys_by_nodes.cpp
+  qinterp/grad_phys_by_vdim.cpp
   qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
@@ -190,9 +192,6 @@ set(HDRS
   hybridization.hpp
   intrules.hpp
   intrules_cut.hpp
-  kernel_dispatch.hpp
-  kernel_reporter.hpp
-  kernels.hpp
   ceed/interface/basis.hpp
   ceed/interface/integrator.hpp
   ceed/interface/interface.hpp
@@ -224,6 +223,7 @@ set(HDRS
   nonlinearform_ext.hpp
   nonlininteg.hpp
   qfunction.hpp
+  qinterp/dispatch.hpp
   qinterp/eval.hpp
   qinterp/grad.hpp
   qspace.hpp

fem/bilinearform.cpp

@@ -280,7 +280,7 @@ void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
    boundary_face_integs_marker.Append(&bdr_marker);
 }
 
-void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat) const
+void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
    if (element_matrices)
    {
@@ -289,10 +289,9 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat) const
       return;
    }
 
-   const FiniteElement &fe = *fes->GetFE(i);
-
    if (domain_integs.Size())
    {
+      const FiniteElement &fe = *fes->GetFE(i);
       ElementTransformation *eltrans = fes->GetElementTransformation(i);
       domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
       for (int k = 1; k < domain_integs.Size(); k++)
@@ -303,18 +302,17 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat) const
    }
    else
    {
-      const int ndof = fe.GetDof() * fes->GetVDim();
-      elmat.SetSize(ndof);
+      fes->GetElementVDofs(i, vdofs);
+      elmat.SetSize(vdofs.Size());
       elmat = 0.0;
    }
 }
 
-void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat) const
+void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   const FiniteElement &be = *fes->GetBE(i);
-
    if (boundary_integs.Size())
    {
+      const FiniteElement &be = *fes->GetBE(i);
       ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
       boundary_integs[0]->AssembleElementMatrix(be, *eltrans, elmat);
       for (int k = 1; k < boundary_integs.Size(); k++)
@@ -325,81 +323,8 @@ void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat) const
    }
    else
    {
-      const int ndof = be.GetDof() * fes->GetVDim();
-      elmat.SetSize(ndof);
-      elmat = 0.0;
-   }
-}
-
-void BilinearForm::ComputeFaceMatrix(int i, DenseMatrix &elmat) const
-{
-   FaceElementTransformations *tr;
-   Mesh *mesh = fes -> GetMesh();
-   tr = mesh -> GetFaceElementTransformations (i);
-
-   const FiniteElement *fe1, *fe2;
-   fe1 = fes->GetFE(tr->Elem1No);
-   if (tr->Elem2No >= 0)
-   {
-      fe2 = fes->GetFE(tr->Elem2No);
-   }
-   else
-   {
-      // The fe2 object is really a dummy and not used on the
-      // boundaries, but we can't dereference a NULL pointer, and we don't
-      // want to actually make a fake element.
-      fe2 = fe1;
-   }
-
-   if (interior_face_integs.Size())
-   {
-      interior_face_integs[0] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elmat);
-      for (int k = 1; k < interior_face_integs.Size(); k++)
-      {
-         interior_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      int ndof = fe1->GetDof() * fes->GetVDim();
-      if (tr->Elem2No >= 0)
-      {
-         ndof += fe2->GetDof() * fes->GetVDim();
-      }
-
-      elmat.SetSize(ndof);
-      elmat = 0.0;
-   }
-}
-
-void BilinearForm::ComputeBdrFaceMatrix(int i, DenseMatrix &elmat) const
-{
-   FaceElementTransformations *tr;
-   Mesh *mesh = fes -> GetMesh();
-   tr = mesh -> GetBdrFaceTransformations (i);
-
-   const FiniteElement *fe1, *fe2;
-
-   fe1 = fes -> GetFE (tr -> Elem1No);
-   // The fe2 object is really a dummy and not used on the boundaries,
-   // but we can't dereference a NULL pointer, and we don't want to
-   // actually make a fake element.
-   fe2 = fe1;
-
-   if (boundary_face_integs.Size())
-   {
-      boundary_face_integs[0] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elmat);
-      for (int k = 1; k < boundary_face_integs.Size(); k++)
-      {
-         boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      int ndof = fe1->GetDof() * fes->GetVDim();
-      elmat.SetSize(ndof);
+      fes->GetBdrElementVDofs(i, vdofs);
+      elmat.SetSize(vdofs.Size());
       elmat = 0.0;
    }
 }
@@ -1431,50 +1356,32 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
    mat->GetBlocks(blocks);
 }
 
-void MixedBilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
+void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   domain_integs.Append(bfi);
+   domain_integs.Append (bfi);
    domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }
 
-void MixedBilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi,
-                                            Array<int> &elem_marker)
+void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi,
+                                             Array<int> &elem_marker)
 {
-   domain_integs.Append(bfi);
+   domain_integs.Append (bfi);
    domain_integs_marker.Append(&elem_marker);
 }
 
-void MixedBilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi)
+void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   boundary_integs.Append(bfi);
+   boundary_integs.Append (bfi);
    boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }
 
-void MixedBilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi,
-                                              Array<int> &bdr_marker)
+void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
+                                               Array<int> &bdr_marker)
 {
-   boundary_integs.Append(bfi);
+   boundary_integs.Append (bfi);
    boundary_integs_marker.Append(&bdr_marker);
 }
 
-void MixedBilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *bfi)
-{
-   interior_face_integs.Append(bfi);
-}
-
-void MixedBilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi)
-{
-   boundary_face_integs.Append(bfi);
-   boundary_face_integs_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
-                                             Array<int> &bdr_marker)
-{
-   boundary_face_integs.Append(bfi);
-   boundary_face_integs_marker.Append(&bdr_marker);
-}
-
 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
    trace_face_integs.Append (bfi);
@@ -1607,108 +1514,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
       }
    }
 
-   if (interior_face_integs.Size())
-   {
-      FaceElementTransformations *ftr;
-      Array<int> trial_vdofs2, test_vdofs2;
-      const FiniteElement *trial_fe1, *trial_fe2, *test_fe1, *test_fe2;
-
-      int nfaces = mesh->GetNumFaces();
-      for (int i = 0; i < nfaces; i++)
-      {
-         ftr = mesh->GetInteriorFaceTransformations(i);
-         if (ftr != NULL)
-         {
-            trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
-            trial_fe1 = trial_fes->GetFE(ftr->Elem1No);
-            test_fe1 = test_fes->GetFE(ftr->Elem1No);
-            if (ftr->Elem2No >= 0)
-            {
-               trial_fes->GetElementVDofs(ftr->Elem2No, trial_vdofs2);
-               test_fes->GetElementVDofs(ftr->Elem2No, test_vdofs2);
-               trial_vdofs.Append(trial_vdofs2);
-               test_vdofs.Append(test_vdofs2);
-               trial_fe2 = trial_fes->GetFE(ftr->Elem2No);
-               test_fe2 = test_fes->GetFE(ftr->Elem2No);
-            }
-            else
-            {
-               // The test_fe2 object is really a dummy and not used on the
-               // boundaries, but we can't dereference a NULL pointer, and we don't
-               // want to actually make a fake element.
-               trial_fe2 = trial_fe1;
-               test_fe2 = test_fe1;
-            }
-            for (int k = 0; k < interior_face_integs.Size(); k++)
-            {
-               interior_face_integs[k]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
-                                                           *test_fe2,
-                                                           *ftr, elemmat);
-               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
-            }
-         }
-      }
-   }
-
-   if (boundary_face_integs.Size())
-   {
-      FaceElementTransformations *ftr;
-      Array<int> tr_vdofs2, te_vdofs2;
-      const FiniteElement *trial_fe1, *trial_fe2, *test_fe1, *test_fe2;
-
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < boundary_face_integs.Size(); k++)
-      {
-         if (boundary_face_integs_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *boundary_face_integs_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary face integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < trial_fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         ftr = mesh -> GetBdrFaceTransformations (i);
-         if (ftr != NULL)
-         {
-            trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
-            trial_fe1 = trial_fes->GetFE(ftr->Elem1No);
-            test_fe1 = test_fes->GetFE(ftr->Elem1No);
-            // The test_fe2 object is really a dummy and not used on the
-            // boundaries, but we can't dereference a NULL pointer, and we don't
-            // want to actually make a fake element.
-            trial_fe2 = trial_fe1;
-            test_fe2 = test_fe1;
-            for (int k = 0; k < boundary_face_integs.Size(); k++)
-            {
-               if (boundary_face_integs_marker[k] &&
-                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0) { continue; }
-
-               boundary_face_integs[k]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
-                                                           *test_fe2,
-                                                           *ftr, elemmat);
-               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
-            }
-         }
-      }
-   }
-
    if (trace_face_integs.Size())
    {
       FaceElementTransformations *ftr;
@@ -1887,13 +1692,12 @@ void MixedBilinearForm::ConformingAssemble()
 }
 
 
-void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat) const
+void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
-   const FiniteElement &trial_fe = *trial_fes->GetFE(i);
-   const FiniteElement &test_fe = *test_fes->GetFE(i);
-
    if (domain_integs.Size())
    {
+      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
+      const FiniteElement &test_fe = *test_fes->GetFE(i);
       ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
       domain_integs[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
                                                elmat);
@@ -1906,21 +1710,19 @@ void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat) const
    }
    else
    {
-      const int tr_dofs = trial_fe.GetDof() * trial_fes->GetVDim();
-      const int te_dofs = test_fe.GetDof() * test_fes->GetVDim();
-
-      elmat.SetSize(te_dofs, tr_dofs);
+      trial_fes->GetElementVDofs(i, trial_vdofs);
+      test_fes->GetElementVDofs(i, test_vdofs);
+      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
       elmat = 0.0;
    }
 }
 
-void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat) const
+void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   const FiniteElement &trial_be = *trial_fes->GetBE(i);
-   const FiniteElement &test_be = *test_fes->GetBE(i);
-
    if (boundary_integs.Size())
    {
+      const FiniteElement &trial_be = *trial_fes->GetBE(i);
+      const FiniteElement &test_be = *test_fes->GetBE(i);
       ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
       boundary_integs[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
                                                  elmat);
@@ -1933,191 +1735,9 @@ void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat) const
    }
    else
    {
-      const int tr_dofs = trial_be.GetDof() * trial_fes->GetVDim();
-      const int te_dofs = test_be.GetDof() * test_fes->GetVDim();
-
-      elmat.SetSize(te_dofs, tr_dofs);
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::ComputeFaceMatrix(int i, DenseMatrix &elmat) const
-{
-   FaceElementTransformations *ftr;
-   Mesh *mesh = test_fes -> GetMesh();
-   ftr = mesh->GetFaceElementTransformations(i);
-   MFEM_ASSERT(ftr, "No associated face transformations.");
-
-   const FiniteElement *trial_fe1, *trial_fe2, *test_fe1, *test_fe2;
-
-   trial_fe1 = trial_fes->GetFE(ftr->Elem1No);
-   test_fe1 = test_fes->GetFE(ftr->Elem1No);
-   if (ftr->Elem2No >= 0)
-   {
-      trial_fe2 = trial_fes->GetFE(ftr->Elem2No);
-      test_fe2 = test_fes->GetFE(ftr->Elem2No);
-   }
-   else
-   {
-      // The test_fe2 object is really a dummy and not used on the
-      // boundaries, but we can't dereference a NULL pointer, and we don't
-      // want to actually make a fake element.
-      trial_fe2 = trial_fe1;
-      test_fe2 = test_fe1;
-   }
-
-   if (interior_face_integs.Size())
-   {
-      interior_face_integs[0]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
-                                                  *test_fe2,
-                                                  *ftr, elmat);
-      for (int k = 1; k < interior_face_integs.Size(); k++)
-      {
-         interior_face_integs[k]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
-                                                     *test_fe2,
-                                                     *ftr, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      int tr_dofs = trial_fe1->GetDof() * trial_fes->GetVDim();
-      int te_dofs = test_fe1->GetDof() * test_fes->GetVDim();
-      if (ftr->Elem2No >= 0)
-      {
-         tr_dofs += trial_fe2->GetDof() * trial_fes->GetVDim();
-         te_dofs += test_fe2->GetDof() * test_fes->GetVDim();
-      }
-
-      elmat.SetSize(te_dofs, tr_dofs);
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::ComputeBdrFaceMatrix(int i, DenseMatrix &elmat) const
-{
-   FaceElementTransformations *ftr;
-   Mesh *mesh = test_fes -> GetMesh();
-   ftr = mesh->GetBdrFaceTransformations(i);
-   MFEM_ASSERT(ftr, "No associated boundary face.");
-
-   const FiniteElement *trial_fe1, *trial_fe2, *test_fe1, *test_fe2;
-
-   trial_fe1 = trial_fes->GetFE(ftr->Elem1No);
-   test_fe1 = test_fes->GetFE(ftr->Elem1No);
-   // The test_fe2 object is really a dummy and not used on the
-   // boundaries, but we can't dereference a NULL pointer, and we don't
-   // want to actually make a fake element.
-   trial_fe2 = trial_fe1;
-   test_fe2 = test_fe1;
-
-   if (boundary_face_integs.Size())
-   {
-      boundary_face_integs[0]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
-                                                  *test_fe2,
-                                                  *ftr, elmat);
-      for (int k = 1; k < boundary_face_integs.Size(); k++)
-      {
-         boundary_face_integs[k]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
-                                                     *test_fe2,
-                                                     *ftr, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      const int tr_dofs = trial_fe1->GetDof() * trial_fes->GetVDim();
-      const int te_dofs = test_fe1->GetDof() * test_fes->GetVDim();
-
-      elmat.SetSize(te_dofs, tr_dofs);
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::ComputeTraceFaceMatrix(int i, DenseMatrix &elmat) const
-{
-   FaceElementTransformations *ftr;
-   Mesh *mesh = test_fes -> GetMesh();
-   ftr = mesh->GetFaceElementTransformations(i);
-   MFEM_ASSERT(ftr, "No associated face transformation.");
-
-   const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
-
-   trial_face_fe = trial_fes->GetFaceElement(i);
-   test_fe1 = test_fes->GetFE(ftr->Elem1No);
-   if (ftr->Elem2No >= 0)
-   {
-      test_fe2 = test_fes->GetFE(ftr->Elem2No);
-   }
-   else
-   {
-      // The test_fe2 object is really a dummy and not used on the
-      // boundaries, but we can't dereference a NULL pointer, and we don't
-      // want to actually make a fake element.
-      test_fe2 = test_fe1;
-   }
-
-   if (trace_face_integs.Size())
-   {
-      trace_face_integs[0]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                               *ftr, elmat);
-      for (int k = 1; k < trace_face_integs.Size(); k++)
-      {
-         trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                                  *ftr, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      const int tr_face_dofs = trial_face_fe->GetDof() * trial_fes->GetVDim();
-      int te_dofs = test_fe1->GetDof() * test_fes->GetVDim();
-      if (ftr->Elem2No >= 0)
-      {
-         te_dofs += test_fe2->GetDof() * test_fes->GetVDim();
-      }
-
-      elmat.SetSize(te_dofs, tr_face_dofs);
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::ComputeBdrTraceFaceMatrix(int i,
-                                                  DenseMatrix &elmat) const
-{
-   FaceElementTransformations *ftr;
-   Mesh *mesh = test_fes -> GetMesh();
-   ftr = mesh->GetBdrFaceTransformations(i);
-   MFEM_ASSERT(ftr, "No associated boundary face.");
-
-   const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
-   int iface = mesh->GetBdrElementFaceIndex(i);
-   trial_face_fe = trial_fes->GetFaceElement(iface);
-   test_fe1 = test_fes->GetFE(ftr->Elem1No);
-   // The test_fe2 object is really a dummy and not used on the
-   // boundaries, but we can't dereference a NULL pointer, and we don't
-   // want to actually make a fake element.
-   test_fe2 = test_fe1;
-
-   if (boundary_trace_face_integs.Size())
-   {
-      boundary_trace_face_integs[0]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
-                                                        *test_fe2,
-                                                        *ftr, elmat);
-      for (int k = 1; k < boundary_trace_face_integs.Size(); k++)
-      {
-         boundary_trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
-                                                           *test_fe2,
-                                                           *ftr, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      const int tr_face_dofs = trial_face_fe->GetDof() * trial_fes->GetVDim();
-      int te_dofs = test_fe1->GetDof() * test_fes->GetVDim();
-
-      elmat.SetSize(te_dofs, tr_face_dofs);
+      trial_fes->GetBdrElementVDofs(i, trial_vdofs);
+      test_fes->GetBdrElementVDofs(i, test_vdofs);
+      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
       elmat = 0.0;
    }
 }
@@ -2160,59 +1780,36 @@ void MixedBilinearForm::AssembleBdrElementMatrix(
    mat->AddSubMatrix(test_vdofs_, trial_vdofs_, elmat, skip_zeros);
 }
 
-void MixedBilinearForm::EliminateTrialEssentialBC(
+void MixedBilinearForm::EliminateTrialDofs (
    const Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
 {
-   Array<int> trial_ess_dofs;
-   trial_fes->GetEssentialVDofs(bdr_attr_is_ess, trial_ess_dofs);
-   mat->EliminateCols(trial_ess_dofs, &sol, &rhs);
+   int i, j, k;
+   Array<int> tr_vdofs, cols_marker (trial_fes -> GetVSize());
+
+   cols_marker = 0;
+   for (i = 0; i < trial_fes -> GetNBE(); i++)
+      if (bdr_attr_is_ess[trial_fes -> GetBdrAttribute (i)-1])
+      {
+         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
+         for (j = 0; j < tr_vdofs.Size(); j++)
+         {
+            if ( (k = tr_vdofs[j]) < 0 )
+            {
+               k = -1-k;
+            }
+            cols_marker[k] = 1;
+         }
+      }
+   mat -> EliminateCols (cols_marker, &sol, &rhs);
 }
 
-void MixedBilinearForm::EliminateTrialEssentialBC(const Array<int>
-                                                  &bdr_attr_is_ess)
-{
-   Array<int> trial_ess_dofs;
-   trial_fes->GetEssentialVDofs(bdr_attr_is_ess, trial_ess_dofs);
-   mat->EliminateCols(trial_ess_dofs);
-}
-
-void MixedBilinearForm::EliminateTrialVDofs(const Array<int> &trial_vdofs_,
-                                            const Vector &sol, Vector &rhs)
-{
-   Array<int> trial_vdofs_marker;
-   FiniteElementSpace::ListToMarker(trial_vdofs_, mat->Width(),
-                                    trial_vdofs_marker);
-   mat->EliminateCols(trial_vdofs_marker, &sol, &rhs);
-}
-
-void MixedBilinearForm::EliminateTrialVDofs(const Array<int> &trial_vdofs_)
-{
-   if (mat_e == NULL)
-   {
-      mat_e = new SparseMatrix(mat->Height(), mat->Width());
-   }
-
-   Array<int> trial_vdofs_marker;
-   FiniteElementSpace::ListToMarker(trial_vdofs_, mat->Width(),
-                                    trial_vdofs_marker);
-   mat->EliminateCols(trial_vdofs_marker, *mat_e);
-   mat_e->Finalize();
-}
-
-void MixedBilinearForm::EliminateTrialVDofsInRHS(const Array<int> &trial_vdofs_,
-                                                 const Vector &x, Vector &b)
-{
-   mat_e->AddMult(x, b, -1.);
-}
-
-void MixedBilinearForm::EliminateEssentialBCFromTrialDofs(
+void MixedBilinearForm::EliminateEssentialBCFromTrialDofs (
    const Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
 {
-   mat->EliminateCols(marked_vdofs, &sol, &rhs);
+   mat -> EliminateCols (marked_vdofs, &sol, &rhs);
 }
 
-void MixedBilinearForm::EliminateTestEssentialBC(const Array<int>
-                                                 &bdr_attr_is_ess)
+void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
 {
    int i, j, k;
    Array<int> te_vdofs;
@@ -2232,19 +1829,10 @@ void MixedBilinearForm::EliminateTestEssentialBC(const Array<int>
       }
 }
 
-void MixedBilinearForm::EliminateTestVDofs(const Array<int> &test_vdofs_)
-{
-   for (int i=0; i<test_vdofs_.Size(); ++i)
-   {
-      mat->EliminateRow(test_vdofs_[i]);
-   }
-}
-
 void MixedBilinearForm::FormRectangularSystemMatrix(
    const Array<int> &trial_tdof_list,
    const Array<int> &test_tdof_list,
    OperatorHandle &A)
-
 {
    if (ext)
    {
@@ -2276,9 +1864,20 @@ void MixedBilinearForm::FormRectangularSystemMatrix(
       mat = m;
    }
 
-   EliminateTrialVDofs(trial_tdof_list);
-   EliminateTestVDofs(test_tdof_list);
+   Array<int> ess_trial_tdof_marker, ess_test_tdof_marker;
+   FiniteElementSpace::ListToMarker(trial_tdof_list, trial_fes->GetTrueVSize(),
+                                    ess_trial_tdof_marker);
+   FiniteElementSpace::ListToMarker(test_tdof_list, test_fes->GetTrueVSize(),
+                                    ess_test_tdof_marker);
 
+   mat_e = new SparseMatrix(mat->Height(), mat->Width());
+   mat->EliminateCols(ess_trial_tdof_marker, *mat_e);
+
+   for (int i=0; i<test_tdof_list.Size(); ++i)
+   {
+      mat->EliminateRow(test_tdof_list[i]);
+   }
+   mat_e->Finalize();
    A.Reset(mat, false);
 }
 
@@ -2307,13 +1906,19 @@ void MixedBilinearForm::FormRectangularLinearSystem(
                                   A); // Set A = mat_e
    }
    // Eliminate essential BCs with B -= Ab xb
-   EliminateTrialVDofsInRHS(trial_tdof_list, X, B);
+   mat_e->AddMult(X, B, -1.0);
 
    B.SetSubVector(test_tdof_list, 0.0);
 }
 
-void MixedBilinearForm::Update()
+void MixedBilinearForm::Update(FiniteElementSpace *ntr_fes,
+      FiniteElementSpace *nte_fes)
 {
+   if ((ntr_fes && nte_fes) && (ntr_fes != trial_fes || nte_fes != test_fes))
+   {
+      trial_fes = ntr_fes;
+      test_fes = nte_fes;
+   }
    delete mat;
    mat = NULL;
    delete mat_e;
@@ -2333,10 +1938,6 @@ MixedBilinearForm::~MixedBilinearForm()
       for (i = 0; i < domain_integs.Size(); i++) { delete domain_integs[i]; }
       for (i = 0; i < boundary_integs.Size(); i++)
       { delete boundary_integs[i]; }
-      for (i = 0; i < interior_face_integs.Size(); i++)
-      { delete interior_face_integs[i]; }
-      for (i = 0; i < boundary_face_integs.Size(); i++)
-      { delete boundary_face_integs[i]; }
       for (i = 0; i < trace_face_integs.Size(); i++)
       { delete trace_face_integs[i]; }
       for (i = 0; i < boundary_trace_face_integs.Size(); i++)

fem/bilinearform.hpp

@@ -119,8 +119,8 @@ protected:
    Array<BilinearFormIntegrator*> boundary_face_integs;
    Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.
 
-   mutable DenseMatrix elemmat;
-   mutable Array<int>  vdofs;
+   DenseMatrix elemmat;
+   Array<int>  vdofs;
 
    DenseTensor *element_matrices; ///< Owned.
 
@@ -294,13 +294,13 @@ public:
    const real_t &operator()(int i, int j) { return (*mat)(i,j); }
 
    /// Returns a reference to: $ M_{ij} $
-   real_t &Elem(int i, int j) override;
+   virtual real_t &Elem(int i, int j);
 
    /// Returns constant reference to: $ M_{ij} $
-   const real_t &Elem(int i, int j) const override;
+   virtual const real_t &Elem(int i, int j) const;
 
    /// Matrix vector multiplication:  $ y = M x $
-   void Mult(const Vector &x, Vector &y) const override;
+   virtual void Mult(const Vector &x, Vector &y) const;
 
    /** @brief Matrix vector multiplication with the original uneliminated
        matrix.  The original matrix is $ M + M_e $ so we have:
@@ -309,7 +309,7 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
+   virtual void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const
    { mat -> AddMult (x, y, a); }
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
@@ -319,8 +319,8 @@ public:
    { mat->AddMult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
-   void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override
+   virtual void AddMultTranspose(const Vector & x, Vector & y,
+                                 const real_t a = 1.0) const
    { mat->AddMultTranspose(x, y, a); }
 
    /** @brief Add the original uneliminated matrix transpose vector
@@ -330,7 +330,7 @@ public:
    { mat->AddMultTranspose(x, y); mat_e->AddMultTranspose(x, y); }
 
    /// Matrix transpose vector multiplication:  $ y = M^T x $
-   void MultTranspose(const Vector & x, Vector & y) const override;
+   virtual void MultTranspose(const Vector & x, Vector & y) const;
 
    /// Compute $ y^T M x $
    real_t InnerProduct(const Vector &x, const Vector &y) const
@@ -338,13 +338,13 @@ public:
 
    /** @brief Returns a pointer to (approximation) of the matrix inverse:
        $ M^{-1} $ (currently returns NULL) */
-   MatrixInverse *Inverse() const override;
+   virtual MatrixInverse *Inverse() const;
 
    /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
        AssemblyLevel::LEGACY.
        The matrix that gets finalized is different if you are using static
        condensation or hybridization.*/
-   void Finalize(int skip_zeros = 1) override;
+   virtual void Finalize(int skip_zeros = 1);
 
    /** @brief Returns a const reference to the sparse matrix:  $ M $
     *
@@ -458,18 +458,18 @@ public:
        conforming prolongation, and |.| denotes the entry-wise absolute value.
        In general, this is just an approximation of the exact diagonal for this
        case. */
-   void AssembleDiagonal(Vector &diag) const override;
+   virtual void AssembleDiagonal(Vector &diag) const;
 
    /// Get the finite element space prolongation operator.
-   const Operator *GetProlongation() const override
+   virtual const Operator *GetProlongation() const
    { return fes->GetConformingProlongation(); }
 
    /// Get the finite element space restriction operator
-   const Operator *GetRestriction() const override
+   virtual const Operator *GetRestriction() const
    { return fes->GetConformingRestriction(); }
 
    /// Get the output finite element space prolongation matrix
-   const Operator *GetOutputProlongation() const override
+   virtual const Operator *GetOutputProlongation() const
    { return GetProlongation(); }
 
    /** @brief Returns the output fe space restriction matrix, transposed
@@ -477,11 +477,11 @@ public:
        Logically, this is the transpose of GetOutputRestriction, but in
        practice it is convenient to have it in transposed form for
        construction of RAP operators in matrix-free methods. */
-   const Operator *GetOutputRestrictionTranspose() const override
+   virtual const Operator *GetOutputRestrictionTranspose() const
    { return fes->GetRestrictionTransposeOperator(); }
 
    /// Get the output finite element space restriction matrix
-   const Operator *GetOutputRestriction() const override
+   virtual const Operator *GetOutputRestriction() const
    { return GetRestriction(); }
 
    /// Compute serial RAP operator and store it in @a A as a SparseMatrix.
@@ -566,8 +566,7 @@ public:
        FormLinearSystem() method to recover the solution as a GridFunction-size
        vector in @a x. Use the same arguments as in the FormLinearSystem() call.
    */
-   void RecoverFEMSolution(const Vector &X, const Vector &b,
-                           Vector &x) override;
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
 
    /// Compute and store internally all element matrices.
    void ComputeElementMatrices();
@@ -581,18 +580,10 @@ public:
        or the one stored internally by a prior call of ComputeElementMatrices()
        is returned when available.
    */
-   void ComputeElementMatrix(int i, DenseMatrix &elmat) const;
+   void ComputeElementMatrix(int i, DenseMatrix &elmat);
 
    /// Compute the boundary element matrix of the given boundary element
-   /** @note The boundary attribute markers of the integrators are ignored. */
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat) const;
-
-   /// Compute the face matrix of the given face element
-   void ComputeFaceMatrix(int i, DenseMatrix &elmat) const;
-
-   /// Compute the boundary face matrix of the given boundary element
-   /** @note The boundary attribute markers of the integrators are ignored. */
-   void ComputeBdrFaceMatrix(int i, DenseMatrix &elmat) const;
+   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
 
    /// Assemble the given element matrix
    /** The element matrix @a elmat is assembled for the element @a i, i.e.
@@ -715,6 +706,10 @@ public:
    */
    void SetDiagonalPolicy(DiagonalPolicy policy);
 
+   void SetIntegratorOwnership(int _extern_bfs)
+   {
+      extern_bfs = _extern_bfs;
+   }
    /// Indicate that integrators are not owned by the BilinearForm
    void UseExternalIntegrators() { extern_bfs = 1; }
 
@@ -772,14 +767,6 @@ protected:
    /// Entries are not owned.
    Array<Array<int>*> boundary_integs_marker;
 
-   /// Interior face integrators.
-   Array<BilinearFormIntegrator*> interior_face_integs;
-
-   /// Boundary face integrators.
-   Array<BilinearFormIntegrator*> boundary_face_integs;
-   /// Entries are not owned.
-   Array<Array<int>*> boundary_face_integs_marker;
-
    /// Trace face (skeleton) integrators.
    Array<BilinearFormIntegrator*> trace_face_integs;
 
@@ -788,8 +775,8 @@ protected:
    /// Entries are not owned.
    Array<Array<int>*> boundary_trace_face_integs_marker;
 
-   mutable DenseMatrix elemmat;
-   mutable Array<int>  trial_vdofs, test_vdofs;
+   DenseMatrix elemmat;
+   Array<int>  trial_vdofs, test_vdofs;
 
 private:
    /// Copy construction is not supported; body is undefined.
@@ -820,32 +807,32 @@ public:
                      MixedBilinearForm *mbf);
 
    /// Returns a reference to: $ M_{ij} $
-   real_t &Elem(int i, int j) override;
+   virtual real_t &Elem(int i, int j);
 
    /// Returns a reference to: $ M_{ij} $
-   const real_t &Elem(int i, int j) const override;
+   virtual const real_t &Elem(int i, int j) const;
 
    /// Matrix multiplication: $ y = M x $
-   void Mult(const Vector & x, Vector & y) const override;
+   virtual void Mult(const Vector & x, Vector & y) const;
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector & x, Vector & y,
-                const real_t a = 1.0) const override;
+   virtual void AddMult(const Vector & x, Vector & y,
+                        const real_t a = 1.0) const;
 
    /// Matrix transpose vector multiplication:  $ y = M^T x $
-   void MultTranspose(const Vector & x, Vector & y) const override;
+   virtual void MultTranspose(const Vector & x, Vector & y) const;
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
-   void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override;
+   virtual void AddMultTranspose(const Vector & x, Vector & y,
+                                 const real_t a = 1.0) const;
 
    /** @brief Returns a pointer to (approximation) of the matrix inverse:
        $ M^{-1} $ (currently unimplemented and returns NULL)*/
-   MatrixInverse *Inverse() const override;
+   virtual MatrixInverse *Inverse() const;
 
    /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
        AssemblyLevel::LEGACY.*/
-   void Finalize(int skip_zeros = 1) override;
+   virtual void Finalize(int skip_zeros = 1);
 
    /** @brief Extract the associated matrix as SparseMatrix blocks. The number
        of block rows and columns is given by the vector dimensions (vdim) of the
@@ -856,37 +843,15 @@ public:
    /** This will segfault if the usual sparse mat is not defined
        like when static condensation is being used or AllocMat() has
        not yet been called. */
-   const SparseMatrix &SpMat() const
-   {
-      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
-      return *mat;
-   }
+   const SparseMatrix &SpMat() const { return *mat; }
 
    /// Returns a reference to the sparse matrix:  $ M $
-   SparseMatrix &SpMat()
-   {
-      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
-      return *mat;
-   }
+   SparseMatrix &SpMat() { return *mat; }
 
    /**  @brief Nullifies the internal matrix $ M $ and returns a pointer
         to it.  Used for transferring ownership. */
    SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }
 
-   /// Returns a const reference to the sparse matrix of eliminated b.c.:  $ M_e $
-   const SparseMatrix &SpMatElim() const
-   {
-      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
-      return *mat_e;
-   }
-
-   /// Returns a reference to the sparse matrix of eliminated b.c.:  $ M_e $
-   SparseMatrix &SpMatElim()
-   {
-      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
-      return *mat_e;
-   }
-
    /// Adds a domain integrator. Assumes ownership of @a bfi.
    void AddDomainIntegrator(BilinearFormIntegrator *bfi);
 
@@ -901,16 +866,6 @@ public:
    void AddBoundaryIntegrator(BilinearFormIntegrator * bfi,
                               Array<int> &bdr_marker);
 
-   /// Adds an interior face integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator *bfi);
-
-   /// Adds a boundary face integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi);
-
-   /// Adds a boundary face integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
-                             Array<int> &bdr_marker);
-
    /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
 
        This type of integrator assembles terms over all faces of the mesh using
@@ -941,16 +896,6 @@ public:
        corresponding pointer (to Array<int>) will be NULL. */
    Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }
 
-   /// Access all integrators added with AddInteriorFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetFBFI() { return &interior_face_integs; }
-
-   /// Access all integrators added with AddBdrFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBFBFI() { return &boundary_face_integs; }
-   /** @brief Access all boundary markers added with AddBdrFaceIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBFBFI_Marker() { return &boundary_face_integs_marker; }
-
    /// Access all integrators added with AddTraceFaceIntegrator().
    Array<BilinearFormIntegrator*> *GetTFBFI() { return &trace_face_integs; }
 
@@ -979,19 +924,19 @@ public:
    void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const;
 
    /// Get the input finite element space prolongation matrix
-   const Operator *GetProlongation() const override
+   virtual const Operator *GetProlongation() const
    { return trial_fes->GetProlongationMatrix(); }
 
    /// Get the input finite element space restriction matrix
-   const Operator *GetRestriction() const override
+   virtual const Operator *GetRestriction() const
    { return trial_fes->GetRestrictionMatrix(); }
 
    /// Get the test finite element space prolongation matrix
-   const Operator *GetOutputProlongation() const override
+   virtual const Operator *GetOutputProlongation() const
    { return test_fes->GetProlongationMatrix(); }
 
    /// Get the test finite element space restriction matrix
-   const Operator *GetOutputRestriction() const override
+   virtual const Operator *GetOutputRestriction() const
    { return test_fes->GetRestrictionMatrix(); }
 
    /** @brief For partially conforming trial and/or test FE spaces, complete the
@@ -1003,25 +948,10 @@ public:
    void ConformingAssemble();
 
    /// Compute the element matrix of the given element
-   void ComputeElementMatrix(int i, DenseMatrix &elmat) const;
+   void ComputeElementMatrix(int i, DenseMatrix &elmat);
 
    /// Compute the boundary element matrix of the given boundary element
-   /** @note The boundary attribute markers of the integrators are ignored. */
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat) const;
-
-   /// Compute the trace face matrix of the given face element
-   void ComputeTraceFaceMatrix(int i, DenseMatrix &elmat) const;
-
-   /// Compute the boundary trace face matrix of the given boundary element
-   /** @note The boundary attribute markers of the integrators are ignored. */
-   void ComputeBdrTraceFaceMatrix(int i, DenseMatrix &elmat) const;
-
-   /// Compute the face matrix of the given face element
-   void ComputeFaceMatrix(int i, DenseMatrix &elmat) const;
-
-   /// Compute the boundary face matrix of the given boundary element
-   /** @note The boundary attribute markers of the integrators are ignored. */
-   void ComputeBdrFaceMatrix(int i, DenseMatrix &elmat) const;
+   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
 
    /// Assemble the given element matrix
    /** The element matrix @a elmat is assembled for the element @a i, i.e.
@@ -1063,61 +993,24 @@ public:
                                  Array<int> &test_vdofs,
                                  int skip_zeros = 1);
 
-   /// Eliminate essential boundary trial DOFs from the system.
+   /// Eliminate essential boundary DOFs from the columns of the system.
    /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. */
-   void EliminateTrialEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                  const Vector &sol, Vector &rhs);
+       the essential part of the boundary.  All entries in the columns will be
+       set to 0.0 through elimination.*/
+   void EliminateTrialDofs(const Array<int> &bdr_attr_is_ess,
+                           const Vector &sol, Vector &rhs);
 
-   /// Eliminate essential boundary trial DOFs from the system matrix.
-   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. */
-   void EliminateTrialEssentialBC(const Array<int> &bdr_attr_is_ess);
-
-   /// (DEPRECATED) Eliminate essential boundary trial DOFs from the system.
-   /** @see EliminateTrialEssentialBC() */
-   MFEM_DEPRECATED void EliminateTrialDofs(const Array<int> &bdr_attr_is_ess,
-                                           const Vector &sol, Vector &rhs)
-   { EliminateTrialEssentialBC(bdr_attr_is_ess, sol, rhs); }
-
-   /// Eliminate the given trial @a vdofs. NOTE: here, @a vdofs is a list of DOFs.
-   /** In this case the eliminations are applied to the internal $ M $
-       and @a rhs without storing the elimination matrix $ M_e $. */
-   void EliminateTrialVDofs(const Array<int> &vdofs, const Vector &sol,
-                            Vector &rhs);
-
-   /// Eliminate the given trial @a vdofs, storing the eliminated part internally in $ M_e $.
-   /** This method works in conjunction with EliminateTrialVDofsInRHS() and allows
-       elimination of boundary conditions in multiple right-hand sides. In this
-       method, @a vdofs is a list of DOFs. */
-   void EliminateTrialVDofs(const Array<int> &vdofs);
-
-   /** @brief Use the stored eliminated part of the matrix (see
-       EliminateTrialVDofs(const Array<int> &)) to modify the r.h.s.
-       @a b; @a vdofs is a list of DOFs (non-directional, i.e. >= 0). */
-   void EliminateTrialVDofsInRHS(const Array<int> &vdofs, const Vector &x,
-                                 Vector &b);
-
-   /** @brief Similar to
-      EliminateTrialVDofs(const Array<int> &, const Vector &, Vector &)
-      but here @a ess_dofs is a marker (boolean) array on all vector-dofs
-      (@a ess_dofs[i] < 0 is true). */
+   /// Eliminate the list of DOFs from the columns of the system.
+   /** @a marked_vdofs is the of colunm numbers that will be eliminated.  All
+       entries in the columns will be set to 0.0 through elimination.*/
    void EliminateEssentialBCFromTrialDofs(const Array<int> &marked_vdofs,
                                           const Vector &sol, Vector &rhs);
 
-   /// Eliminate essential boundary test DOFs from the system matrix.
+   /// Eliminate essential boundary DOFs from the rows of the system.
    /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. */
-   void EliminateTestEssentialBC(const Array<int> &bdr_attr_is_ess);
-
-   /// (DEPRECATED) Eliminate essential boundary test DOFs from the system.
-   /** @see EliminateTestEssentialBC() */
-   MFEM_DEPRECATED virtual void EliminateTestDofs(const Array<int>
-                                                  &bdr_attr_is_ess)
-   { EliminateTestEssentialBC(bdr_attr_is_ess); }
-
-   /// Eliminate the given test @a vdofs. NOTE: here, @a vdofs is a list of DOFs.
-   void EliminateTestVDofs(const Array<int> &vdofs);
+       the essential part of the boundary.  All entries in the rows will be
+       set to 0.0 through elimination.*/
+   virtual void EliminateTestDofs(const Array<int> &bdr_attr_is_ess);
 
    /** @brief Return in @a A that is column-constrained.
 
@@ -1179,8 +1072,13 @@ public:
       A.MakeRef(*A_ptr);
    }
 
-   /// Must be called after making changes to #trial_fes or #test_fes.
-   void Update();
+   virtual void Update(FiniteElementSpace *ntr_fes = NULL,
+                       FiniteElementSpace *nte_fes = NULL);
+
+   void SetIntegratorOwnership(int _extern_bfs)
+   {
+      extern_bfs = _extern_bfs;
+   }
 
    /// Return the trial FE space associated with the BilinearForm.
    FiniteElementSpace *TrialFESpace() { return trial_fes; }
@@ -1273,7 +1171,7 @@ public:
 
    /** @brief Get the output finite element space restriction matrix in
        transposed form. */
-   const Operator *GetOutputRestrictionTranspose() const override
+   virtual const Operator *GetOutputRestrictionTranspose() const
    { return test_fes->GetRestrictionTransposeOperator(); }
 };
 

fem/bilinearform_ext.hpp

@@ -37,19 +37,19 @@ protected:
 public:
    BilinearFormExtension(BilinearForm *form);
 
-   MemoryClass GetMemoryClass() const override
+   virtual MemoryClass GetMemoryClass() const
    { return Device::GetDeviceMemoryClass(); }
 
    /// Get the finite element space prolongation matrix
-   const Operator *GetProlongation() const override;
+   virtual const Operator *GetProlongation() const;
 
    /// Get the finite element space restriction matrix
-   const Operator *GetRestriction() const override;
+   virtual const Operator *GetRestriction() const;
 
    /// Assemble at the level given for the BilinearFormExtension subclass
    virtual void Assemble() = 0;
 
-   void AssembleDiagonal(Vector &diag) const override
+   virtual void AssembleDiagonal(Vector &diag) const
    {
       MFEM_ABORT("AssembleDiagonal not implemented for this assembly level!");
    }
@@ -83,17 +83,16 @@ protected:
 public:
    PABilinearFormExtension(BilinearForm*);
 
-   void Assemble() override;
-   void AssembleDiagonal(Vector &diag) const override;
-   void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                         OperatorHandle &A) override;
+   void Assemble();
+   void AssembleDiagonal(Vector &diag) const;
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
    void FormLinearSystem(const Array<int> &ess_tdof_list,
                          Vector &x, Vector &b,
                          OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) override;
-   void Mult(const Vector &x, Vector &y) const override;
-   void MultTranspose(const Vector &x, Vector &y) const override;
-   void Update() override;
+                         int copy_interior = 0);
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
+   void Update();
 
 protected:
    void SetupRestrictionOperators(const L2FaceValues m);
@@ -151,9 +150,9 @@ protected:
 public:
    EABilinearFormExtension(BilinearForm *form);
 
-   void Assemble() override;
-   void Mult(const Vector &x, Vector &y) const override;
-   void MultTranspose(const Vector &x, Vector &y) const override;
+   void Assemble();
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
 };
 
 /// Data and methods for fully-assembled bilinear forms
@@ -166,19 +165,18 @@ private:
 public:
    FABilinearFormExtension(BilinearForm *form);
 
-   void Assemble() override;
+   void Assemble();
    void RAP(OperatorHandle &A);
    /** @note Always does `DIAG_ONE` policy to be consistent with
        `Operator::FormConstrainedSystemOperator`. */
    void EliminateBC(const Array<int> &ess_dofs, OperatorHandle &A);
-   void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                         OperatorHandle &A) override;
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
    void FormLinearSystem(const Array<int> &ess_tdof_list,
                          Vector &x, Vector &b,
                          OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) override;
-   void Mult(const Vector &x, Vector &y) const override;
-   void MultTranspose(const Vector &x, Vector &y) const override;
+                         int copy_interior = 0);
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
 
    /** DGMult and DGMultTranspose use the extended L-vector to perform the
        computation. */
@@ -201,17 +199,16 @@ protected:
 public:
    MFBilinearFormExtension(BilinearForm *form);
 
-   void Assemble() override;
-   void AssembleDiagonal(Vector &diag) const override;
-   void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                         OperatorHandle &A) override;
+   void Assemble();
+   void AssembleDiagonal(Vector &diag) const;
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
    void FormLinearSystem(const Array<int> &ess_tdof_list,
                          Vector &x, Vector &b,
                          OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) override;
-   void Mult(const Vector &x, Vector &y) const override;
-   void MultTranspose(const Vector &x, Vector &y) const override;
-   void Update() override;
+                         int copy_interior = 0);
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
+   void Update();
 };
 
 /// Class extending the MixedBilinearForm class to support different AssemblyLevels.
@@ -228,20 +225,20 @@ protected:
 public:
    MixedBilinearFormExtension(MixedBilinearForm *form);
 
-   MemoryClass GetMemoryClass() const override
+   virtual MemoryClass GetMemoryClass() const
    { return Device::GetMemoryClass(); }
 
    /// Get the finite element space prolongation matrix
-   const Operator *GetProlongation() const override;
+   virtual const Operator *GetProlongation() const;
 
    /// Get the finite element space restriction matrix
-   const Operator *GetRestriction() const override;
+   virtual const Operator *GetRestriction() const;
 
    /// Get the output finite element space restriction matrix
-   const Operator *GetOutputProlongation() const override;
+   virtual const Operator *GetOutputProlongation() const;
 
    /// Get the output finite element space restriction matrix
-   const Operator *GetOutputRestriction() const override;
+   virtual const Operator *GetOutputRestriction() const;
 
    virtual void Assemble() = 0;
    virtual void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
@@ -276,7 +273,7 @@ public:
    PAMixedBilinearFormExtension(MixedBilinearForm *form);
 
    /// Partial assembly of all internal integrators
-   void Assemble() override;
+   void Assemble();
    /**
       @brief Setup OperatorHandle A to contain constrained linear operator
 
@@ -286,7 +283,7 @@ public:
    */
    void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
                                       const Array<int> &test_tdof_list,
-                                      OperatorHandle &A) override;
+                                      OperatorHandle &A);
    /**
       Setup OperatorHandle A to contain constrained linear operator and
       eliminate columns corresponding to essential dofs from system,
@@ -295,21 +292,20 @@ public:
    void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
                                     const Array<int> &test_tdof_list,
                                     Vector &x, Vector &b,
-                                    OperatorHandle &A, Vector &X, Vector &B) override;
+                                    OperatorHandle &A, Vector &X, Vector &B);
    /// y = A*x
-   void Mult(const Vector &x, Vector &y) const override;
+   void Mult(const Vector &x, Vector &y) const;
    /// y += c*A*x
-   void AddMult(const Vector &x, Vector &y, const real_t c=1.0) const override;
+   void AddMult(const Vector &x, Vector &y, const real_t c=1.0) const;
    /// y = A^T*x
-   void MultTranspose(const Vector &x, Vector &y) const override;
+   void MultTranspose(const Vector &x, Vector &y) const;
    /// y += c*A^T*x
-   void AddMultTranspose(const Vector &x, Vector &y,
-                         const real_t c=1.0) const override;
+   void AddMultTranspose(const Vector &x, Vector &y, const real_t c=1.0) const;
    /// Assemble the diagonal of ADA^T for a diagonal vector D.
-   void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const override;
+   void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const;
 
    /// Update internals for when a new MixedBilinearForm is given to this class
-   void Update() override;
+   void Update();
 };
 
 
@@ -326,17 +322,16 @@ public:
    PADiscreteLinearOperatorExtension(DiscreteLinearOperator *linop);
 
    /// Partial assembly of all internal integrators
-   void Assemble() override;
+   void Assemble();
 
-   void AddMult(const Vector &x, Vector &y, const real_t c=1.0) const override;
+   void AddMult(const Vector &x, Vector &y, const real_t c=1.0) const;
 
-   void AddMultTranspose(const Vector &x, Vector &y,
-                         const real_t c=1.0) const override;
+   void AddMultTranspose(const Vector &x, Vector &y, const real_t c=1.0) const;
 
    void FormRectangularSystemOperator(const Array<int>&, const Array<int>&,
-                                      OperatorHandle& A) override;
+                                      OperatorHandle& A);
 
-   const Operator * GetOutputRestrictionTranspose() const override;
+   const Operator * GetOutputRestrictionTranspose() const;
 
 private:
    Vector test_multiplicity;

fem/bilininteg.cpp

@@ -170,16 +170,6 @@ void BilinearFormIntegrator::AssembleFaceMatrix(
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssembleFaceMatrix(
-   const FiniteElement &trial_fe1, const FiniteElement &test_fe1,
-   const FiniteElement &trial_fe2, const FiniteElement &test_fe2,
-   FaceElementTransformations &Trans,
-   DenseMatrix &elmat)
-{
-   MFEM_ABORT("AssembleFaceMatrix (mixed form) is not implemented for this"
-              " Integrator class.");
-}
-
 void BilinearFormIntegrator::AssembleFaceMatrix(
    const FiniteElement &trial_face_fe, const FiniteElement &test_fe1,
    const FiniteElement &test_fe2, FaceElementTransformations &Trans,
@@ -233,38 +223,28 @@ void TransposeIntegrator::SetIntRule(const IntegrationRule *ir)
    bfi->SetIntRule(ir);
 }
 
-void TransposeIntegrator::AssembleElementMatrix(
+void TransposeIntegrator::AssembleElementMatrix (
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
-   bfi->AssembleElementMatrix(el, Trans, bfi_elmat);
+   bfi -> AssembleElementMatrix (el, Trans, bfi_elmat);
    // elmat = bfi_elmat^t
    elmat.Transpose (bfi_elmat);
 }
 
-void TransposeIntegrator::AssembleElementMatrix2(
+void TransposeIntegrator::AssembleElementMatrix2 (
    const FiniteElement &trial_fe, const FiniteElement &test_fe,
    ElementTransformation &Trans, DenseMatrix &elmat)
 {
-   bfi->AssembleElementMatrix2(test_fe, trial_fe, Trans, bfi_elmat);
+   bfi -> AssembleElementMatrix2 (test_fe, trial_fe, Trans, bfi_elmat);
    // elmat = bfi_elmat^t
    elmat.Transpose (bfi_elmat);
 }
 
-void TransposeIntegrator::AssembleFaceMatrix(
+void TransposeIntegrator::AssembleFaceMatrix (
    const FiniteElement &el1, const FiniteElement &el2,
    FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   bfi->AssembleFaceMatrix(el1, el2, Trans, bfi_elmat);
-   // elmat = bfi_elmat^t
-   elmat.Transpose (bfi_elmat);
-}
-
-void TransposeIntegrator::AssembleFaceMatrix(
-   const FiniteElement &tr_el1, const FiniteElement &te_el1,
-   const FiniteElement &tr_el2, const FiniteElement &te_el2,
-   FaceElementTransformations &Trans, DenseMatrix &elmat)
-{
-   bfi->AssembleFaceMatrix(te_el1, tr_el1, te_el2, tr_el2, Trans, bfi_elmat);
+   bfi -> AssembleFaceMatrix (el1, el2, Trans, bfi_elmat);
    // elmat = bfi_elmat^t
    elmat.Transpose (bfi_elmat);
 }
@@ -855,34 +835,6 @@ const IntegrationRule &GradientIntegrator::GetRule(const FiniteElement
 }
 
 
-DiffusionIntegrator::DiffusionIntegrator(const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir),
-     Q(nullptr), VQ(nullptr), MQ(nullptr), maps(nullptr), geom(nullptr)
-{
-   static Kernels kernels;
-}
-
-DiffusionIntegrator::DiffusionIntegrator(Coefficient &q,
-                                         const IntegrationRule *ir)
-   : DiffusionIntegrator(ir)
-{
-   Q = &q;
-}
-
-DiffusionIntegrator::DiffusionIntegrator(VectorCoefficient &q,
-                                         const IntegrationRule *ir)
-   : DiffusionIntegrator(ir)
-{
-   VQ = &q;
-}
-
-DiffusionIntegrator::DiffusionIntegrator(MatrixCoefficient &q,
-                                         const IntegrationRule *ir)
-   : DiffusionIntegrator(ir)
-{
-   MQ = &q;
-}
-
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -1270,8 +1222,7 @@ real_t DiffusionIntegrator::ComputeFluxEnergy
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint(&ip);
-      fluxelem.CalcPhysShape(Trans, shape);
+      fluxelem.CalcShape(ip, shape);
 
       pointflux = 0.0;
       for (int k = 0; k < spaceDim; k++)
@@ -1282,6 +1233,7 @@ real_t DiffusionIntegrator::ComputeFluxEnergy
          }
       }
 
+      Trans.SetIntPoint(&ip);
       real_t w = Trans.Weight() * ip.weight;
 
       if (MQ)
@@ -1338,17 +1290,6 @@ const IntegrationRule &DiffusionIntegrator::GetRule(
    return IntRules.Get(trial_fe.GetGeomType(), order);
 }
 
-MassIntegrator::MassIntegrator(const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir), Q(nullptr), maps(nullptr), geom(nullptr)
-{
-   static Kernels kernels;
-}
-
-MassIntegrator::MassIntegrator(Coefficient &q, const IntegrationRule *ir)
-   : MassIntegrator(ir)
-{
-   Q = &q;
-}
 
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
@@ -1469,7 +1410,9 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
       // Set the integration point in the face and the neighboring element
       Trans.SetAllIntPoints(&ip);
 
-      el1.CalcPhysShape(*Trans.Elem1, shape);
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
+      el1.CalcShape(eip, shape);
 
       w = Trans.Weight() * ip.weight;
       if (Q)
@@ -1639,9 +1582,9 @@ void VectorMassIntegrator::AssembleElementMatrix
    for (int s = 0; s < ir->GetNPoints(); s++)
    {
       const IntegrationPoint &ip = ir->IntPoint(s);
-      Trans.SetIntPoint (&ip);
-      el.CalcPhysShape(Trans, shape);
+      el.CalcShape(ip, shape);
 
+      Trans.SetIntPoint (&ip);
       norm = ip.weight * Trans.Weight();
 
       MultVVt(shape, partelmat);
@@ -1723,10 +1666,10 @@ void VectorMassIntegrator::AssembleElementMatrix2(
    for (int s = 0; s < ir->GetNPoints(); s++)
    {
       const IntegrationPoint &ip = ir->IntPoint(s);
-      Trans.SetIntPoint(&ip);
-      trial_fe.CalcPhysShape(Trans, shape);
-      test_fe.CalcPhysShape(Trans, te_shape);
+      trial_fe.CalcShape(ip, shape);
+      test_fe.CalcShape(ip, te_shape);
 
+      Trans.SetIntPoint(&ip);
       norm = ip.weight * Trans.Weight();
 
       MultVWt(te_shape, shape, partelmat);
@@ -1954,12 +1897,12 @@ void VectorFECurlIntegrator::AssembleElementMatrix2(
          if ( trial_fe.GetMapType() == mfem::FiniteElement::H_CURL )
          {
             trial_fe.CalcCurlShape(ip, curlshapeTrial_dFT);
-            test_fe.CalcPhysShape(Trans, shapeTest);
+            test_fe.CalcShape(ip, shapeTest);
          }
          else
          {
             test_fe.CalcCurlShape(ip, curlshapeTrial_dFT);
-            trial_fe.CalcPhysShape(Trans, shapeTest);
+            trial_fe.CalcShape(ip, shapeTest);
          }
       }
 
@@ -1982,89 +1925,6 @@ void VectorFECurlIntegrator::AssembleElementMatrix2(
    }
 }
 
-void VectorFEBoundaryFluxIntegrator::AssembleElementMatrix(
-   const FiniteElement &el, ElementTransformation &Tr,
-   DenseMatrix &elmat)
-{
-   int nd = el.GetDof();
-   real_t w;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector shape;
-#endif
-   elmat.SetSize(nd);
-   shape.SetSize(nd);
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int intorder = 2*el.GetOrder() + Tr.OrderW();  // <----------
-      ir = &IntRules.Get(el.GetGeomType(), intorder);
-   }
-
-   elmat = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-      el.CalcShape(ip, shape);
-
-      Tr.SetIntPoint (&ip);
-      w = ip.weight / Tr.Weight();
-
-      if (Q)
-      {
-         w *= Q->Eval(Tr, ip);
-      }
-
-      AddMult_a_VVt(w, shape, elmat);
-   }
-}
-
-void VectorFEBoundaryFluxIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe,
-   const FiniteElement &test_fe,
-   ElementTransformation &Tr,
-   DenseMatrix &elmat)
-{
-   int tr_nd = trial_fe.GetDof();
-   int te_nd = test_fe.GetDof();
-   real_t w;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector shape, te_shape;
-#endif
-   elmat.SetSize(te_nd, tr_nd);
-   shape.SetSize(tr_nd);
-   te_shape.SetSize(te_nd);
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Tr.OrderW();
-
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-   }
-
-   elmat = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-      trial_fe.CalcShape(ip, shape);
-      test_fe.CalcShape(ip, te_shape);
-
-      Tr.SetIntPoint (&ip);
-      w = ip.weight / Tr.Weight();
-
-      if (Q)
-      {
-         w *= Q->Eval(Tr, ip);
-      }
-
-      te_shape *= w;
-      AddMultVWt(te_shape, shape, elmat);
-   }
-}
-
 void DerivativeIntegrator::AssembleElementMatrix2 (
    const FiniteElement &trial_fe,
    const FiniteElement &test_fe,
@@ -2121,7 +1981,7 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
       det = Trans.Weight();
       Mult (dshape, invdfdx, dshapedxt);
 
-      test_fe.CalcPhysShape(Trans, shape);
+      test_fe.CalcShape(ip, shape);
 
       for (l = 0; l < trial_nd; l++)
       {
@@ -2139,7 +1999,11 @@ void CurlCurlIntegrator::AssembleElementMatrix
 {
    int nd = el.GetDof();
    dim = el.GetDim();
-   int dimc = el.GetCurlDim();
+   // in main
+   // int dimc = el.GetCurlDim();
+   // Taken from 4d_dev:
+   int dimc = (dim == 3) ? 3 : 1;
+   if (dim==4) { dimc = 6; }
    real_t w;
 
 #ifdef MFEM_THREAD_SAFE
@@ -2176,8 +2040,43 @@ void CurlCurlIntegrator::AssembleElementMatrix
 
       Trans.SetIntPoint (&ip);
 
+      if (dim ==4)
+      {
+         DenseMatrix tSh(4,4);
+         DenseMatrix trShTemp(4,4);
+
+         DenseMatrix J = Trans.Jacobian();
+         DenseMatrix invJ(4,4); CalcInverse(J, invJ);
+         DenseMatrix invJtr(invJ); invJtr.Transpose();
+
+         el.CalcCurlShape(ip, curlshape);
+         for (int dof=0; dof<nd; dof++)
+         {
+            tSh = 0.; trShTemp = 0.;
+            tSh(0,1) =  curlshape(dof,0); tSh(0,2) =  curlshape(dof,1);
+            tSh(0,3) =  curlshape(dof,2);
+            tSh(1,0) = -curlshape(dof,0);
+            tSh(1,2) =  curlshape(dof,3); tSh(1,3) =  curlshape(dof,4);
+            tSh(2,0) = -curlshape(dof,1); tSh(2,1) = -curlshape(dof,3);
+            tSh(2,3) =  curlshape(dof,5);
+            tSh(3,0) = -curlshape(dof,2); tSh(3,1) = -curlshape(dof,4);
+            tSh(3,2) = -curlshape(dof,5);
+
+            Mult(tSh, invJ, trShTemp);
+            Mult(invJtr, trShTemp, tSh);
+
+            curlshape_dFt(dof,0) = tSh(0,1);
+            curlshape_dFt(dof,1) = tSh(0,2);
+            curlshape_dFt(dof,2) = tSh(0,3);
+            curlshape_dFt(dof,3) = tSh(1,2);
+            curlshape_dFt(dof,4) = tSh(1,3);
+            curlshape_dFt(dof,5) = tSh(2,3);
+         }
+      }
+      else
+            el.CalcPhysCurlShape(Trans, curlshape_dFt);
+
       w = ip.weight * Trans.Weight();
-      el.CalcPhysCurlShape(Trans, curlshape_dFt);
 
       if (MQ)
       {
@@ -2706,7 +2605,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
          Trans.SetIntPoint (&ip);
 
          trial_fe.CalcVShape(Trans, trial_vshape);
-         test_fe.CalcPhysShape(Trans, shape);
+         test_fe.CalcShape(ip, shape);
 
          w = ip.weight * Trans.Weight();
          if (DQ)
@@ -2866,11 +2765,11 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(
    for (int i = 0; i < ir -> GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint (&ip);
 
       trial_fe.CalcDShape (ip, dshape);
-      test_fe.CalcPhysShape (Trans, shape);
+      test_fe.CalcShape (ip, shape);
 
+      Trans.SetIntPoint (&ip);
       CalcAdjugate(Trans.Jacobian(), Jadj);
 
       Mult (dshape, Jadj, gshape);
@@ -3371,11 +3270,11 @@ real_t ElasticityIntegrator::ComputeFluxEnergy(const FiniteElement &fluxelem,
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint(&ip);
-      fluxelem.CalcPhysShape(Trans, shape);
+      fluxelem.CalcShape(ip, shape);
 
       flux_mat.MultTranspose(shape, pointstress);
 
+      Trans.SetIntPoint(&ip);
       real_t w = Trans.Weight() * ip.weight;
 
       M = mu->Eval(Trans, ip);
@@ -3482,7 +3381,7 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
       const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
       const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
 
-      el1.CalcPhysShape(*Trans.Elem1, shape1);
+      el1.CalcShape(eip1, shape1);
 
       u->Eval(vu, *Trans.Elem1, eip1);
 
@@ -3529,7 +3428,7 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
 
       if (ndof2)
       {
-         el2.CalcPhysShape(*Trans.Elem2, shape2);
+         el2.CalcShape(eip2, shape2);
 
          if (w != 0.0)
             for (int i = 0; i < ndof2; i++)
@@ -3555,152 +3454,9 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
          }
       }
    }
+   //    elmat.PrintMatlab(std::cout);
 }
 
-void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &trial_fe1,
-                                           const FiniteElement &test_fe1,
-                                           const FiniteElement &trial_fe2,
-                                           const FiniteElement &test_fe2,
-                                           FaceElementTransformations &Trans,
-                                           DenseMatrix &elmat)
-{
-   int tr_ndof1, te_ndof1, tr_ndof2, te_ndof2;
-
-   real_t un, a, b, w;
-
-   dim = test_fe1.GetDim();
-   tr_ndof1 = trial_fe1.GetDof();
-   te_ndof1 = test_fe1.GetDof();
-   Vector vu(dim), nor(dim);
-
-   if (Trans.Elem2No >= 0)
-   {
-      tr_ndof2 = trial_fe2.GetDof();
-      te_ndof2 = test_fe2.GetDof();
-   }
-   else
-   {
-      tr_ndof2 = 0;
-      te_ndof2 = 0;
-   }
-
-   tr_shape1.SetSize(tr_ndof1);
-   te_shape1.SetSize(te_ndof1);
-   tr_shape2.SetSize(tr_ndof2);
-   te_shape2.SetSize(te_ndof2);
-   elmat.SetSize(te_ndof1 + te_ndof2, tr_ndof1 + tr_ndof2);
-   elmat = 0.0;
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order;
-      // Assuming order(u)==order(mesh)
-      if (Trans.Elem2No >= 0)
-         order = (min(Trans.Elem1->OrderW(), Trans.Elem2->OrderW()) +
-                  max(trial_fe1.GetOrder(), trial_fe2.GetOrder()) +
-                  max(test_fe1.GetOrder(), test_fe2.GetOrder()));
-      else
-      {
-         order = Trans.Elem1->OrderW() + trial_fe1.GetOrder() + test_fe1.GetOrder();
-      }
-      if (trial_fe1.Space() == FunctionSpace::Pk)
-      {
-         order++;
-      }
-      ir = &IntRules.Get(Trans.FaceGeom, order);
-   }
-
-   for (int p = 0; p < ir->GetNPoints(); p++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
-      Trans.Loc1.Transform(ip, eip1);
-      Trans.Elem1->SetIntPoint(&eip1);
-      if (tr_ndof2 && te_ndof2)
-      {
-         Trans.Loc2.Transform(ip, eip2);
-         Trans.Elem2->SetIntPoint(&eip2);
-      }
-      trial_fe1.CalcPhysShape(*Trans.Elem1, tr_shape1);
-      test_fe1.CalcPhysShape(*Trans.Elem1, te_shape1);
-
-      Trans.Face->SetIntPoint(&ip);
-
-      u->Eval(vu, *Trans.Elem1, eip1);
-
-      if (dim == 1)
-      {
-         nor(0) = 2*eip1.x - 1.0;
-      }
-      else
-      {
-         CalcOrtho(Trans.Face->Jacobian(), nor);
-      }
-
-      un = vu * nor;
-      a = 0.5 * alpha * un;
-      b = beta * fabs(un);
-      // note: if |alpha/2|==|beta| then |a|==|b|, i.e. (a==b) or (a==-b)
-      //       and therefore two blocks in the element matrix contribution
-      //       (from the current quadrature point) are 0
-
-      if (rho)
-      {
-         real_t rho_p;
-         if (un >= 0.0 && tr_ndof2 && te_ndof2)
-         {
-            Trans.Elem2->SetIntPoint(&eip2);
-            rho_p = rho->Eval(*Trans.Elem2, eip2);
-         }
-         else
-         {
-            rho_p = rho->Eval(*Trans.Elem1, eip1);
-         }
-         a *= rho_p;
-         b *= rho_p;
-      }
-
-      w = ip.weight * (a+b);
-      if (w != 0.0)
-      {
-         for (int i = 0; i < te_ndof1; i++)
-            for (int j = 0; j < tr_ndof1; j++)
-            {
-               elmat(i, j) += w * te_shape1(i) * tr_shape1(j);
-            }
-      }
-
-      if (tr_ndof2 && te_ndof2)
-      {
-         trial_fe2.CalcPhysShape(*Trans.Elem2, tr_shape2);
-         test_fe2.CalcPhysShape(*Trans.Elem2, te_shape2);
-
-         if (w != 0.0)
-            for (int i = 0; i < te_ndof2; i++)
-               for (int j = 0; j < tr_ndof1; j++)
-               {
-                  elmat(te_ndof1+i, j) -= w * te_shape2(i) * tr_shape1(j);
-               }
-
-         w = ip.weight * (b-a);
-         if (w != 0.0)
-         {
-            for (int i = 0; i < te_ndof2; i++)
-               for (int j = 0; j < tr_ndof2; j++)
-               {
-                  elmat(te_ndof1+i, tr_ndof1+j) += w * te_shape2(i) * tr_shape2(j);
-               }
-
-            for (int i = 0; i < te_ndof1; i++)
-               for (int j = 0; j < tr_ndof2; j++)
-               {
-                  elmat(i, tr_ndof1+j) -= w * te_shape1(i) * tr_shape2(j);
-               }
-         }
-      }
-   }
-}
 
 const IntegrationRule &DGTraceIntegrator::GetRule(
    Geometry::Type geom, int order, FaceElementTransformations &T)
@@ -4223,14 +3979,19 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
       // Set the integration point in the face and the neighboring elements
       Trans.SetAllIntPoints(&ip);
 
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
       // Trace finite element shape function
       trial_face_fe.CalcShape(ip, face_shape);
       // Side 1 finite element shape function
-      test_fe1.CalcPhysShape(*Trans.Elem1, shape1);
+      test_fe1.CalcShape(eip1, shape1);
       if (ndof2)
       {
          // Side 2 finite element shape function
-         test_fe2.CalcPhysShape(*Trans.Elem2, shape2);
+         test_fe2.CalcShape(eip2, shape2);
       }
       w = ip.weight;
       if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
@@ -4593,8 +4354,8 @@ struct ShapeCoefficient : public VectorCoefficient
       : VectorCoefficient(fe_.GetDof()), Q(q), fe(fe_) { }
 
    using VectorCoefficient::Eval;
-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip)
    {
       V.SetSize(vdim);
       fe.CalcPhysShape(T, V);
@@ -4636,8 +4397,8 @@ ScalarVectorProductInterpolator::AssembleElementMatrix2(
       VShapeCoefficient(Coefficient &q, const FiniteElement &fe_, int sdim)
          : MatrixCoefficient(fe_.GetDof(), sdim), Q(q), fe(fe_) { }
 
-      void Eval(DenseMatrix &M, ElementTransformation &T,
-                const IntegrationPoint &ip) override
+      virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+                        const IntegrationPoint &ip)
       {
          M.SetSize(height, width);
          fe.CalcPhysVShape(T, M);
@@ -4673,8 +4434,8 @@ VectorScalarProductInterpolator::AssembleElementMatrix2(
          : MatrixCoefficient(fe_.GetDof(), vq.GetVDim()), VQ(vq), fe(fe_),
            vc(width), shape(height) { }
 
-      void Eval(DenseMatrix &M, ElementTransformation &T,
-                const IntegrationPoint &ip) override
+      virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+                        const IntegrationPoint &ip)
       {
          M.SetSize(height, width);
          VQ.Eval(vc, T, ip);
@@ -4713,8 +4474,8 @@ ScalarCrossProductInterpolator::AssembleElementMatrix2(
            vshape(vdim, vq.GetVDim()), vc(vq.GetVDim()) { }
 
       using VectorCoefficient::Eval;
-      void Eval(Vector &V, ElementTransformation &T,
-                const IntegrationPoint &ip) override
+      virtual void Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
       {
          V.SetSize(vdim);
          VQ.Eval(vc, T, ip);
@@ -4757,8 +4518,8 @@ VectorCrossProductInterpolator::AssembleElementMatrix2(
          MFEM_ASSERT(width == 3, "");
       }
 
-      void Eval(DenseMatrix &M, ElementTransformation &T,
-                const IntegrationPoint &ip) override
+      virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+                        const IntegrationPoint &ip)
       {
          M.SetSize(height, width);
          VQ.Eval(vc, T, ip);
@@ -4806,8 +4567,8 @@ struct VDotVShapeCoefficient : public VectorCoefficient
         vshape(vdim, vq.GetVDim()), vc(vq.GetVDim()) { }
 
    using VectorCoefficient::Eval;
-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip)
    {
       V.SetSize(vdim);
       VQ.Eval(vc, T, ip);
@@ -4834,4 +4595,85 @@ VectorInnerProductInterpolator::AssembleElementMatrix2(
    ran_fe.Project(dom_shape_coeff, Trans, elmat_as_vec);
 }
 
+void HeatEquationIntegrator::AssembleElementMatrix
+( const FiniteElement &el, ElementTransformation &Trans,
+  DenseMatrix &elmat )
+{
+   int nd = el.GetDof();
+   int dim = el.GetDim();
+   int spaceDim = Trans.GetSpaceDim();
+   double w;
+
+#ifdef MFEM_THREAD_SAFE
+   DenseMatrix dshape(nd,dim), dshapedxt(nd,spaceDim), invdfdx(dim,spaceDim);
+   Vector shape(nd), vec(nd);
+#else
+   dshape.SetSize(nd,dim);
+   dshapedxt.SetSize(nd,spaceDim);
+   invdfdx.SetSize(dim,spaceDim);
+   shape.SetSize(nd);
+   dtshape.SetSize(nd);
+#endif
+   elmat.SetSize(nd);
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      el.CalcShape(ip,shape);
+      el.CalcDShape(ip, dshape);
+
+      Trans.SetIntPoint(&ip);
+      w = Trans.Weight();
+      w *= ip.weight;
+      CalcInverse(Trans.Jacobian(), invdfdx);
+      Mult(dshape, invdfdx, dshapedxt);
+
+      dshapedxt.GetColumn(spaceDim - 1, dtshape); // d_t u
+      dshapedxt.SetCol(spaceDim - 1, 0.);
+
+      AddMult_a_VWt(w,shape,dtshape,elmat); // d_t u * v
+      if (!MQ)
+      {
+         if (Q)
+         {
+            w *= Q->Eval(Trans, ip);
+         }
+         AddMult_a_AAt(w, dshapedxt, elmat);
+      }
+      else
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(dshapedxt, invdfdx, dshape);
+         AddMultABt(dshape, dshapedxt, elmat);
+      }
+   }
+}
+
+
 }

fem/coefficient.cpp

@@ -129,8 +129,10 @@ real_t PWCoefficient::Eval(ElementTransformation &T,
 real_t FunctionCoefficient::Eval(ElementTransformation & T,
                                  const IntegrationPoint & ip)
 {
-   real_t x[3];
-   Vector transip(x, 3);
+   // real_t x[Geometry::MaxDim];
+   // Vector transip(x, Geometry::MaxDim);
+   real_t x[4];
+   Vector transip(x, 4);
 
    T.Transform(ip, transip);
 
@@ -366,8 +368,10 @@ void PositionVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
 void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
-   real_t x[3];
-   Vector transip(x, 3);
+   // real_t x[Geometry::MaxDim];
+   // Vector transip(x, Geometry::MaxDim);
+   real_t x[4];
+   Vector transip(x, 4);
 
    T.Transform(ip, transip);