Fixed some bugs introduced by the previous commit.

WIP: updating the branch
Added more pentatope quadrature rules.
2026-08-10 11:36:14 +02:00 · 2025-11-28 20:16:14 -05:00 · 2024-09-17 18:32:36 +02:00 · 2024-05-29 21:16:47 +02:00 · 2021-04-12 09:33:58 +02:00 · 2020-12-04 16:41:50 +01:00
562 changed files with 26878 additions and 36588 deletions
@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 30
+
+# Number of days of inactivity before an Issue or Pull Request with the stale
+# label is closed.  Set to false to disable. If disabled, issues still need to
+# be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale.
+# Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set
+# to `[]` to disable
+exemptLabels:
+  - bug
+  - WIP
+  - ready-for-review
+  - in-review
+  - in-next
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking an issue as stale
+staleLabel: stale
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  :warning: This issue or PR has been automatically marked as stale because it has not
+  had any activity in the last month. *If no activity occurs in the next week, it will
+  be automatically closed.* Thank you for your contributions.
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed
@@ -29,10 +29,6 @@ on:
  pull_request:
  workflow_dispatch:

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
  HYPRE_TOP_DIR: hypre-2.19.0
@@ -96,7 +92,6 @@ jobs:
            build-system: cmake
            hypre-target: int32
            precision: fp64
-            config-opts: '-DCMAKE_INSTALL_PREFIX=../cmake-install'
            # This option can be set to pass additional configuration options to
            # the MFEM configuration command.
            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
@@ -119,6 +114,13 @@ jobs:
    runs-on: ${{ matrix.os }}

    steps:
+      # This external action allows to interrupt a workflow already running on
+      # the same branch to save resources.
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.12.1
+        with:
+          access_token: ${{ github.token }}
+
      # Fix 'No space left on device' errors for Ubuntu builds.
      - name: Run Actions Cleaner
        if: matrix.os == 'ubuntu-latest'
@@ -304,22 +306,6 @@ jobs:
          ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
        shell: bash

-      - name: make install
-        if: matrix.build-system == 'make'
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make install
-
-      - name: cmake install
-        if: matrix.build-system == 'cmake'
-        run: |
-          CONFIG="Release"
-          [[ ${{ matrix.target }} == 'dbg' ]] && CONFIG="Debug"
-          TARGET="install"
-          [[ ${{ matrix.os }} == 'windows-latest' ]] && TARGET="INSTALL"
-          cd ${{ env.MFEM_TOP_DIR }} && \
-          cmake --build build --target ${TARGET} --config ${CONFIG}
-        shell: bash
-
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
@@ -18,10 +18,6 @@ on:
    # The branches below must be a subset of the branches above
    branches: ["master"]

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
  analyze:
    name: Analyze
@@ -22,10 +22,6 @@ on:
  pull_request:
  workflow_dispatch:

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
  HYPRE_TOP_DIR: hypre-2.19.0
@@ -38,6 +34,11 @@ jobs:
    runs-on: ubuntu-latest

    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.12.1
+        with:
+          access_token: ${{ github.token }}
+
      - name: checkout MFEM
        uses: actions/checkout@v4
        with:
@@ -22,15 +22,22 @@ on:
  pull_request:
  workflow_dispatch:

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
  Serial:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest

    steps:
+      - name: Temporary workaround for sanitizer crashes
+        # See https://github.com/actions/runner-images/issues/9491
+        # The issue should be fixed in the next runner image for Ubuntu 22.04,
+        # see https://github.com/actions/runner-images/pull/9513
+        run: sudo sysctl vm.mmap_rnd_bits=28
+
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.12.1
+        with:
+          access_token: ${{ github.token }}
+
      - name: MFEM Checkout
        uses: actions/checkout@v4
        with:
@@ -48,7 +55,7 @@ jobs:
          build-system: make
          library-only: false
          config-options:
-            CXX="clang++-18"
+            CXX="clang++-14"
            CXXFLAGS="-g -O1 -std=c++11
                      -fsanitize=address
                      -fno-omit-frame-pointer
@@ -19,10 +19,6 @@ on:
  pull_request:
  workflow_dispatch:

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 # This workflow is run on pushes to any branch in the MFEM repo (with or without
 # PRs), as well as on updates to PRs from forks. In particular, we do not
 # duplicate work by running on both pushes and updates to local PRs. We do that
@@ -37,6 +33,11 @@ jobs:
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.12.1
+        with:
+          access_token: ${{ github.token }}
+
      - name: checkout mfem
        uses: actions/checkout@v4

@@ -1,31 +0,0 @@
-# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
-# For more information, see: https://github.com/actions/stale
-name: Mark stale issues and pull requests
-
-on:
-  workflow_dispatch:
-  schedule:
-  - cron: '0 0 * * *'
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: ':warning: This issue has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
-        days-before-stale: 30
-        days-before-close: 7
-        stale-issue-label: 'stale'
-        stale-pr-label: 'stale'
-        operations-per-run: 500
-        exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
-        exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"
@@ -1,31 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Trigger PyMFEM CI"
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  trigger-pymfem:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Send POST request to trigger PyMFEM CI
-      run: |
-        curl -L \
-             -X POST \
-             -H "Accept: application/vnd.github+json" \
-             -H "Authorization: Bearer ${{ secrets.PYMFEM_CI_TOKEN }}" \
-             -H "X-GitHub-Api-Version: 2022-11-28" \
-             https://api.github.com/repos/mfem/pymfem/actions/workflows/build-and-test-dispatch.yml/dispatches \
-             -d '{"ref":"master", "inputs":{"test_options":"fast"}}'
@@ -8,7 +8,6 @@

 # Object and library files
 *.o
-*.o.tmp
 /libmfem.*
 /miniapps/common/libmfem-common.*

@@ -16,9 +15,6 @@
 CMakeCache.txt
 CMakeFiles/

-# Clangd server cache
-*.cache*
-
 # Backup files
 *~

@@ -227,7 +223,6 @@ miniapps/meshing/extruder
 miniapps/meshing/fit-node-position
 miniapps/meshing/trimmer
 miniapps/meshing/reflector
-miniapps/meshing/ref321
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/pmesh-fitting
@@ -247,8 +242,6 @@ miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
 miniapps/meshing/reflected.mesh
-miniapps/meshing/ref321.mesh
-miniapps/meshing/sol.gf
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh
@@ -279,27 +272,16 @@ miniapps/navier/*_output

 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex3
-miniapps/nurbs/nurbs_ex5
 miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_ex24
-miniapps/nurbs/nurbs_solenoidal
 miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
-miniapps/nurbs/sol_?.gf
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
-miniapps/nurbs/Example3*
-miniapps/nurbs/Example5*
-miniapps/nurbs/Solenoidal*
-miniapps/nurbs/ParaView
 miniapps/nurbs/sin-fit.mesh
-miniapps/nurbs/ex5.mesh
-miniapps/nurbs/exsol.mesh
 miniapps/nurbs/CurveInt
 miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
@@ -343,7 +325,6 @@ miniapps/toys/rubik
 miniapps/toys/snake
 miniapps/toys/lissajous
 miniapps/toys/mondrian
-miniapps/toys/spiral
 miniapps/toys/snake-init.mesh
 miniapps/toys/snake-user.mesh
 miniapps/toys/snake-joined.mesh
@@ -432,9 +413,3 @@ pkg.gitcommit

 # Jupyter Notebook Checkpoints
 .ipynb_checkpoints
-
-# emacs tag file
-TAGS
-
-# vs code
-.vscode
@@ -22,7 +22,7 @@ include:
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where ruby resource are allocated/released once for all.
+# - Allocate/Release is where quartz resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
@@ -53,7 +53,7 @@ variables:
  AUTOTEST_COMMIT: "YES"

 # Trigger subpipelines:
-ruby-build-and-test:
+quartz-build-and-test:
  stage: sub-pipelines
  variables:
    # Explicitly pass down values that we want to be able to set when triggering
@@ -61,10 +61,10 @@ ruby-build-and-test:
    AUTOTEST: "${AUTOTEST}"
    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
-    include: .gitlab/ruby-build-and-test.yml
+    include: .gitlab/quartz-build-and-test.yml
    strategy: depend

-ruby-baseline:
+quartz-baseline:
  stage: sub-pipelines
  variables:
    # Explicitly pass down values that we want to be able to set when triggering
@@ -73,7 +73,7 @@ ruby-baseline:
    AUTOTEST: "${AUTOTEST}"
    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
-    include: .gitlab/ruby-baseline.yml
+    include: .gitlab/quartz-baseline.yml
    strategy: depend

 lassen-build-and-test:
@@ -24,7 +24,7 @@ and `test type`.

 Machines typically include:

-* Ruby: 2nd Gen Intel Xeon (Cascade Lake)
+* Quartz: Intel bi-socket x86
 * Lassen: Power9 + Nvidia GPU
 * Corona: AMD GPU

@@ -76,13 +76,13 @@ with a spack spec of MFEM, within the limits permitted by the MFEM spack
 package.

 In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on ruby for example resumes to:
+spack spec to use. Adding a job on quartz for example resumes to:

 ```yaml
 <job_name>:
  variables:
    SPEC: "<spack_spec>"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 ```

 The remaining and non trivial work is to make sure this spec is working. To
@@ -24,7 +24,7 @@ variables:
 # TODO: add a clean-up mechanism
  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}

-# On LLNL's ruby, there is only one allocation shared among jobs in order to
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
@@ -9,17 +9,17 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# GitLab pipelines configurations for the Ruby machine at LLNL
+# GitLab pipelines configurations for the Quartz machine at LLNL
 variables:
-  MACHINE_NAME: ruby
+  MACHINE_NAME: quartz

-.on_ruby:
+.on_quartz:
  tags:
    - shell
-    - ruby
+    - quartz
  rules:
-    # Don't run ruby jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"'
+    # Don't run quartz jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
      when: never
    # Don't run autotest update if...
    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
@@ -40,13 +40,13 @@ variables:
    - when: on_success

 # Spack helped builds
-# Generic ruby build job, extending build script
-.build_and_test_on_ruby:
-  extends: [.on_ruby]
+# Generic quartz build job, extending build script
+.build_and_test_on_quartz:
+  extends: [.on_quartz]
  stage: build_and_test
  script:
    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=16
+    - export THREADS=12
    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - echo ${JOBID}
@@ -18,7 +18,7 @@
 setup_baseline:
  tags:
    - shell
-    - ruby
+    - quartz
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -16,7 +16,7 @@
 setup:
  tags:
    - shell
-    - ruby
+    - quartz
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -19,8 +19,8 @@ stages:
  - cleanup
  - baseline_publish

-baselinecheck_mfem_intel_ruby:
-  extends: [.on_ruby]
+baselinecheck_mfem_intel_quartz:
+  extends: [.on_quartz]
  stage: baseline_check
  variables:
    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
@@ -32,7 +32,7 @@ baselinecheck_mfem_intel_ruby:
    - echo ${BUILD_ROOT}
    - echo ${TPLS_DIR}
    # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=48
+    - export MFEM_TEST_NP=32
    # The next script uses the following environment variables:
    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
@@ -44,16 +44,18 @@ baselinecheck_mfem_intel_ruby:
  allow_failure: true

 cleanup:
-  extends: .on_ruby
+  extends: .on_quartz
  stage: cleanup
  variables:
    GIT_STRATEGY: none
  script:
    - echo "BUILD_ROOT=${BUILD_ROOT}"
    - rm -rf "${BUILD_ROOT}" || true
+    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
+    - make -C "${CI_PROJECT_DIR}" distclean

 report_baseline:
-  extends: [.on_ruby]
+  extends: [.on_quartz]
  stage: baseline_report
  script:
    - echo ${MACHINE_NAME}
@@ -113,8 +115,8 @@ report_baseline:
        exit $err
      ) 9> autotest.lock

-baselinepublish_mfem_ruby:
-  extends: [.on_ruby]
+baselinepublish_mfem_quartz:
+  extends: [.on_quartz]
  stage: baseline_publish
  rules:
    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
@@ -129,5 +131,5 @@ baselinepublish_mfem_ruby:

 include:
  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/quartz-config.yml
  - local: .gitlab/configs/setup-baseline.yml
@@ -19,54 +19,54 @@ stages:
 allocate_resource:
  variables:
    GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_quartz
  stage: allocate_resource
  script:
    - echo ${ALLOC_NAME}
    - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h

-# GitLab jobs for the Ruby machine at LLNL
+# GitLab jobs for the Quartz machine at LLNL
 debug_ser_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz

 debug_par_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz

 opt_ser_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz

 opt_par_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz

 opt_par_gcc_10_sundials:
  variables:
    SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz

 opt_par_gcc_10_petsc:
  variables:
    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz

 opt_par_gcc_10_pumi:
  variables:
    SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz

 # Release
 release_resource:
  variables:
    GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_quartz
  stage: release_resource_and_report
  script:
    - echo ${ALLOC_NAME}
@@ -78,17 +78,17 @@ release_resource:
 report_job_success:
  stage: release_resource_and_report
  extends:
-    - .on_ruby
+    - .on_quartz
    - .report_job_success

 report_job_failure:
  stage: release_resource_and_report
  extends:
-    - .on_ruby
+    - .on_quartz
    - .report_job_failure

 include:
  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/quartz-config.yml
  - local: .gitlab/configs/setup-build-and-test.yml
  - local: .gitlab/configs/report-build-and-test.yml
@@ -14,7 +14,7 @@
 # locals
 glob_err=${BASELINE_TEST}.err
 base=${BASELINE_TEST}-${SYS_TYPE}
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
+if [[ "${MACHINE_NAME}" == "quartz" ]]; then
  base="${BASELINE_TEST}-${MACHINE_NAME}"
 fi
 base_diff=${base}.diff
@@ -31,8 +31,8 @@ cd tests
 mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
@@ -41,11 +41,11 @@ else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
 fi
-status="$?"

 # post
 mkdir ${artifacts_path}

+status=0
 if [[ -f ${BASELINE_TEST}.out ]]; then
  cp ${BASELINE_TEST}.out ${artifacts_path}
 fi
@@ -11,7 +11,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and quartz baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
@@ -21,7 +21,7 @@ PATCH_FILE=${ARTIFACT_PATH}.patch
 FULL_FILE=${ARTIFACT_PATH}.out
 DIFF_FILE=${ARTIFACT_PATH}.diff

-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and quartz baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 SAVED_NAME=baseline-${SYS_TYPE}.saved
@@ -10,139 +10,12 @@

 Version 4.7.1 (development)
 ===========================
- Refactored ALGOIM cut integration rules. The interface is unified with
-  the interface for moment based cut integration rules.
- Altered (Par)GridFunction::Compute*Error functions to ensure they return
-  non-negative values and therefore behave as "norms".
-
-Discretization improvements
---------------------------
- Added NURBS-based H(div) and H(curl) elements in 2D and 3D. Only on single
-  patch meshes. Only implemented for serial computations.
-
- Added support for boundary constraints to the hybridization class.
-
- Added support for external boundary submeshes with nonconformal mesh adaptation.
-
- Added assembly of Jacobians to `HyperbolicFormIntegrator`.
-
- Added average fluxes to `NumericalFlux` (formerly `RiemannSolver`)
-  and `FluxFunction`.
-
- Added component-wise upwinded flux (`ComponentwiseUpwindFlux`).
-
-Meshing improvements
--------------------
- Added native AD support for numerous TMOP metrics that didn't have first or
-  second derivative implementations.
-
- The ExodusII reader now handles pyramid and wedge element types. Mixed meshes
-  are also supported.
-
- Added support for nonuniform anisotropic (nonconforming) mesh refinement with
-  arbitrary spacing in each direction, for quadrilateral (2D) and hexahedral
-  (3D) meshes. This enables in particular 3:1 refinement, as demonstrated in the
-  new meshing miniapp ref321.
-
-New and updated examples and miniapps
-------------------------------------
- Added miniapps to demonstrate the H(div) and H(curl) NURBS elements.

 - Added an MFEM example for the eikonal equation. This new solver is based on
  the proximal Galerkin method introduced by Keith and Surowiec.

- Added a new toy miniapp that animates an interesting fidget spiral cone toy.
-  See miniapps/toys/spiral.cpp.
-
- Added a command line option to all miniapps (`-p` or `--send-port`) for
-  specifying the GLVis server socket port (19916 by default).
-
-GPU computing
-------------
- Added support for GPU-accelerated batched linear algebra (using cuBLAS,
-  hipBLAS, MAGMA, or native MFEM functionality) through the BatchedLinAlg class.
-
- A new GPU kernel dispatch mechanism was introduced. Users can instantiate
-  specialized kernels for specific combinations of (for example) polynomial
-  degree and number of quadrature points using
-  `DiffusionIntegrator::AddSpecialization` and
-  `MassIntegrator::AddSpecialization` (this functionality may be added to more
-  integrators in the future).
-
- Calls to slower fallback kernels can be reported to `mfem::err` by setting
-  the environment variable `MFEM_REPORT_KERNELS` to any value other than `NO`
-  or by explicitly calling `KernelReporter::Enable`. Users can then add
-  specializations for these kernels to achieve higher performance.
-
- Element assembly kernels have been added for low-order refined to
-  high-order transfer operators. New kernels can be offloaded as device
-  kernels. Example usage may be found in lor-transfer.cpp under miniapps/tools.
-
- Added support for GPU accelerated FindPointsGSLIB. Note that this will require
-  the users to switch from gslib v1.0.7 to v1.0.9.
-
-Miscellaneous
-------------
- Added support for SUNDIALS v7. See the section "API changes" for some small
-  changes related to this new version.
-
- Refactored the `ARKStepSolver` class (ARKODE interface) to use
-  `TimeDependentOperator::Mult` only when the associated ODE operator is
-  expressed in explicit form (i.e., `TimeDependentOperator::isExplicit()`),
-  otherwise `TimeDependentOperator::ExplicitMult` is used. A check has been
-  added to `ARKStepSolver` to verify that the associated ODE operator is not in
-  explicit form when a mass matrix solver is enabled via a call to either the
-  `UseMFEMMassLinearSolver` or `UseSundialsMassLinearSolver` methods. This is
-  because enabling a mass matrix solver assumes that F(u,k,t) = M k in the
-  associated ODE operator.
-
- Added support for custom interpolation procedure in FindPointsGSLIB.
-
- `FiniteElementSpace` has new methods to directly set prolongation and
-  restriction operators to arbitrary sparse matrices.
-
- There are new convenience constructors for NURBS patches and knot vectors.
-
- Added convenience methods for manipulating boundary attribute marker arrays;
-  `(Par)Mesh::MarkExternalBoundaries`, `(Par)Mesh::UnmarkInternalBoundaries`,
-  `(Par)Mesh::MarkNamedBoundaries`, and `(Par)Mesh::UnmarkNamedBoundaries`.
-  See examples `ex1.cpp`, `ex1p.cpp`, and `ex11p.cpp` for basic usage.
-
- Added `(Par)Mesh::GetExteriorFaceMarker` for identifying faces on the
-  exterior of the mesh irrespective of their presence in the list of "boundary
-  elements".
-
- Added methods to `(Par)FiniteElementSpace` to identify all degrees of freedom
-  located on the exterior of the domain without reference to the list of
-  "boundary elements"; `GetExteriorVDofs` and `GetExteriorTrueDofs`.
-
- `LinearFormIntegrator` and `NonlinearFormIntegrator` (including
-  `BilinearFormIntegrator`) now all inherit from a base class `Integrator`
-  that combines some logic related to selecting quadrature rules. This includes
-  a virtual method `Integrator::GetDefaultIntegrationRule`, which should be
-  favored over directly defining a default integration rule in the element-level
-  assembly routines (although the latter is still possible, by leaving the new
-  virtual method as its default base implementation of returning `NULL`).
-
-API changes
-----------
 - API change: in class GridFunction, 'fec' was renamed to 'fec_owned'.

- API change: `RiemannSolver` was renamed to `NumericalFlux` (the old name has
-  been been depracated through typedef)
-
- API change: support for SUNDIALS v7:
-  * the SUNDIALS types `realtype` and `booleantype` are no longer defined by v7
-    and therefore MFEM now uses the new type names `sunrealtype` and
-    `sunbooleantype`, respectively, which MFEM defines when using SUNDIALS < v6
-    where these types were not defined.
-  * The SUNDIALS macro `SUNLS_SUCCESS` and some other `*_SUCCESS` macros were
-    removed and replaced by `SUN_SUCCESS` in v7, so to avoid tedious checks for
-    SUNDIALS versions, MFEM now defines and uses the constant `SUN_SUCCESS` when
-    using SUNDIALS < v7.
-  * The constants `SUN_PREC_*`, introduced by SUNDIALS v6 are now introduced by
-    MFEM when using SUNDIALS < v6 to avoid tedious version checks.
-

 Version 4.7, released on May 7, 2024
 ====================================
@@ -167,9 +40,6 @@ Meshing improvements

 - Added support for internal boundary elements in nonconforming meshes.

- Added ExodusII output capability. The writer can handle first-order (Pyramid5,
-  Wedge6, Hex8, Tet4) and second-order FE types (Pyramid14, Wedge18, Hex27, Tet10).
-
 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.

 Discretization improvements
@@ -229,15 +99,6 @@ New and updated examples and miniapps
 - Added two new example codes: 38 and 39/39p described above. Substantially
  updated Example 18/18p.

- Added ODE solvers selection routines. This creates a uniformity across examples,
-  miniapps and other executables in regard to ODE(time-integrator) selection.
-
- Added new mechanism for retrieving and setting state vectors in ODE solvers.
-  This is relevant for AB/AM and gen-alpha solvers.
-
- Added ODEsolver/ODEsolver2 unit tests to verify order of convergence and
-  read/write functionality.
-
 Miscellaneous
 -------------
 - Updated the Doxygen documentation style, which now requires Doxygen version
@@ -146,9 +146,7 @@ if (MFEM_USE_CUDA)
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
   find_package(CUDAToolkit REQUIRED)
   set(CUSPARSE_FOUND TRUE)
-   set(CUBLAS_FOUND TRUE)
   get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-   get_target_property(CUBLAS_LIBRARIES CUDA::cublas LOCATION)
 endif()

 if (XSDK_ENABLE_C)
@@ -233,7 +231,6 @@ if (MFEM_USE_HIP)
    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
  endif()
  find_package(HIP REQUIRED)
-  find_package(HIPBLAS REQUIRED)
  find_package(HIPSPARSE REQUIRED)
 endif()

@@ -340,10 +337,7 @@ if (MFEM_USE_SUNDIALS)
  if (MFEM_USE_HIP)
    list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
  endif()
-  # The Core component was added in SUNDIALS v7, so we treat it as optional in
-  # order to support older versions.
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS}
-    OPTIONAL_COMPONENTS Core)
+  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

 # SuperLU_DIST can only be enabled in parallel
@@ -402,10 +396,6 @@ if (MFEM_USE_AMGX)
  find_package(AMGX REQUIRED)
 endif()

-if (MFEM_USE_MAGMA)
-  find_package(MAGMA REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint)
 endif()
@@ -567,9 +557,8 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
-    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
-    ALGOIM ENZYME)
+    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -618,37 +607,15 @@ set(MASTER_HEADERS
  ${PROJECT_SOURCE_DIR}/mfem.hpp
  ${PROJECT_SOURCE_DIR}/mfem-performance.hpp)

-# Installation options (we use GNUInstallDirs but prefer lib by default)
-set(MFEM_USE_GNUINSTALLDIRS OFF CACHE BOOL
-  "Use CMAKE_INSTALL_LIBDIR as defined by the GNUInstallDirs CMake module.")
-if (NOT MFEM_USE_GNUINSTALLDIRS)
-  mfem_cache_path(CMAKE_INSTALL_LIBDIR "lib" "Object code libraries (lib)")
-endif()
-include(GNUInstallDirs)
-mfem_cache_path(INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}"
-  "Relative or absolute path for installing header files.")
-mfem_cache_path(INSTALL_BIN_DIR "${CMAKE_INSTALL_BINDIR}"
-  "Relative or absolute path for installing the binaries.")
-mfem_cache_path(INSTALL_LIB_DIR "${CMAKE_INSTALL_LIBDIR}"
-  "Relative or absolute path for installing the library.")
-mfem_cache_path(INSTALL_SHARE_DIR "${CMAKE_INSTALL_DATAROOTDIR}"
-  "Relative or absolute path for installing shared data.")
-# other options: "share/mfem/cmake", "lib/mfem/cmake"
-mfem_cache_path(INSTALL_CMAKE_DIR "${INSTALL_LIB_DIR}/cmake/mfem"
-  "Relative or absolute path for installing cmake config files.")
-
-mfem_path_to_fullpath("${INSTALL_LIB_DIR}" "${CMAKE_INSTALL_PREFIX}" _lib_path)
+set(_lib_path "${CMAKE_INSTALL_PREFIX}/lib")
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON CACHE BOOL "")
-if(NOT DEFINED CMAKE_INSTALL_RPATH)
-  set(CMAKE_INSTALL_RPATH "${_lib_path}")
-endif()
-if(NOT DEFINED CMAKE_INSTALL_NAME_DIR)
-  set(CMAKE_INSTALL_NAME_DIR "${_lib_path}")
-endif()
+set(CMAKE_INSTALL_RPATH "${_lib_path}" CACHE PATH "")
+set(CMAKE_INSTALL_NAME_DIR "${_lib_path}" CACHE PATH "")

-# Variables used when generating _config.hpp, and config.mk
-set(MFEM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
+set(MFEM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH
+    "The MFEM source directory" FORCE)
+set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
+    "The MFEM install directory" FORCE)

 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
@@ -706,7 +673,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")

-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
      "${PROJECT_BINARY_DIR}/${Header}.tmp"
      "${PROJECT_BINARY_DIR}/${Header}"
    )
@@ -720,7 +687,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 #include \"mfem/${Header}\"
 ")

-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
    )
@@ -767,11 +734,7 @@ endif()
 # Create a target for all miniapps and, optionally, enable it.
 set(MFEM_ALL_MINIAPPS_TARGET_NAME miniapps)
 add_mfem_target(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_ENABLE_MINIAPPS})
-if (MFEM_ENABLE_MINIAPPS)
-  add_subdirectory(miniapps) #install miniapps if enabled
-else()
-  add_subdirectory(miniapps EXCLUDE_FROM_ALL)
-endif()
+add_subdirectory(miniapps EXCLUDE_FROM_ALL)

 # Target to build all executables, i.e. everything.
 add_custom_target(exec)
@@ -821,6 +784,15 @@ add_subdirectory(doc)
 #-------------------------------------------------------------------------------

 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
+set(INSTALL_INCLUDE_DIR include
+  CACHE PATH "Relative path for installing header files.")
+set(INSTALL_BIN_DIR bin
+  CACHE PATH "Relative path for installing the binaries.")
+set(INSTALL_LIB_DIR lib
+  CACHE PATH "Relative path for installing the library.")
+# other options: "share/mfem/cmake", "lib/mfem/cmake"
+set(INSTALL_CMAKE_DIR lib/cmake/mfem
+  CACHE PATH "Relative path for installing cmake config files.")

 target_include_directories(mfem BEFORE
  PUBLIC
@@ -845,7 +817,7 @@ foreach(Header mfem.hpp mfem-performance.hpp)
 endforeach()
 install(FILES ${MASTER_HEADERS} DESTINATION ${INSTALL_INCLUDE_DIR}/mfem)

-# Install the headers (except common miniapp which is installed from its subdir)
+# Install the headers; currently, the miniapps headers are excluded
 install(DIRECTORY ${MFEM_SOURCE_DIRS}
  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
  FILES_MATCHING PATTERN "*.hpp")
@@ -220,10 +220,9 @@ An optional installation of the library and the headers can be performed with

   make install [PREFIX=<dir>]

-The library will be installed in ${PREFIX}/lib, the headers in
-${PREFIX}/include, and the configuration and testing makefiles (config.mk and
-test.mk) in ${PREFIX}/share/mfem. The PREFIX option can also be set during
-configuration.
+The library will be installed in $(PREFIX)/lib, the headers in
+$(PREFIX)/include, and the configuration makefile (config.mk) in
+$(PREFIX)/share/mfem. The PREFIX option can also be set during configuration.

 Information about the current build configuration can be viewed using

@@ -272,16 +271,9 @@ Build options:

 Installation options:
   PREFIX  - Specify the installation directory. The library (libmfem.a) will be
-             installed in ${PREFIX}/lib, the headers in ${PREFIX}/include, and
-             the configuration and testing makefiles (config.mk and test.mk) in
-             ${PREFIX}/share/mfem.
-   INSTALL - Specify the install program, default = /usr/bin/install
-   INSTALL_DEF_PERM - Specify the default install permissions. This affects
-                      headers and configuration makefiles, default = 644
-   INSTALL_BIN_PERM - Specify the install permissions for binaries. This only
-                      affects the shared version of the library, default = 755
-   INSTALL_DIR_PERM - Specify the install permissions for directories and,
-                      on macOS/BSD, for symlinks as well, default = 755
+             installed in $(PREFIX)/lib, the headers in $(PREFIX)/include, and
+             the configuration makefile (config.mk) in $(PREFIX)/share/mfem.
+   INSTALL - Specify the install program, e.g /usr/bin/install

 MFEM library features/options (GNU make)
 ----------------------------------------
@@ -396,11 +388,6 @@ MFEM_USE_AMGX = YES/NO
   Allows the user to use SparseMatrices and HypreParMatrices to solve linear
   systems with the routines from the AmgX library.

-MFEM_USE_MAGMA = YES/NO
-   Enable MFEM functionality based on the MAGMA high-performance linear algebra
-   library. The MAGMA library provides a BLAS/LAPACK interface, with
-   implementations that have been optimized for Nvidia and AMD GPUs.
-
 MFEM_USE_GNUTLS = YES/NO
   Enable secure socket support in class socketstream, using the auxiliary
   GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -504,14 +491,10 @@ MFEM_USE_CODIPACK = YES/NO
 MFEM_USE_ALGOIM = YES/NO
  Enable the usage of Algoim - a collection of high-order accurate numerical
  methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods, see https://algoim.github.io. MFEM provides interface to
-  Algoim v1. To check out the specific Algoim state use:
-  https://github.com/algoim/algoim
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM
+  provides interface to Algoim v1. Thus, to check out the specific state use:
  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
-  The Algoim library requires the Blitz++ library. To use the latest state of
-  Blitz++ that has been tested with MFEM, use:
-  https://github.com/blitzpp/blitz
-  git checkout f24a250a43dff88c31ad92916da828b7ea9a98b7
+  https://algoim.github.io

 MFEM_USE_ADFORWARD = YES/NO
  Enable forward mode for AD packages. This option is valid
@@ -716,11 +699,6 @@ The specific libraries and their options are:
  Options: AMGX_OPT, AMGX_LIB.
  Versions: AmgX >= 2.1, older versions may work too.

- MAGMA (optional), used with MFEM_USE_MAGMA = YES.
-  URL: https://icl.utk.edu/magma/
-  Options: MAGMA_OPT, MAGMA_LIB
-  Versions: MAGMA >= 2.8.0
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
  GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
  one can get the library through the Homebrew package manager (http://brew.sh).
@@ -793,14 +771,14 @@ The specific libraries and their options are:
  Versions: 1.9.3

 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
-  built prior to the MFEM build, as follows: download gslib-1.0.9, untar it at
-  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.9 gslib".
+  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
+  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
  Build gslib in parallel or in serial based on the desired MFEM build: "make
  clean; make CC=mpicc" or "make clean; make CC=gcc MPI=0". Build MFEM with
  MFEM_USE_GSLIB=YES.
-  URL: https://github.com/gslib/gslib/archive/v1.0.9.tar.gz
+  URL: https://github.com/gslib/gslib/archive/v1.0.7.tar.gz
  Options: GSLIB_OPT, GSLIB_LIB.
-  Versions: GSLIB >= 1.0.9.
+  Versions: GSLIB >= 1.0.7.

 - ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
  headers so it just needs to be downloaded at the same level as MFEM. Download
@@ -982,20 +960,8 @@ or
   cmake --build . --config Release --target install  [Xcode]
   cmake --build . --config Release --target INSTALL  [Visual Studio]

-By default, the library will be installed in ${CMAKE_INSTALL_PREFIX}/lib, the
-headers in ${CMAKE_INSTALL_PREFIX}/include, the configuration CMake files in
-${CMAKE_INSTALL_PREFIX}/lib/cmake/mfem, and the configuration and testing GNU
-make files in ${CMAKE_INSTALL_PREFIX}/share/mfem. For fine-tuning the
-installation directories the following variables can be used:
-INSTALL_INCLUDE_DIR, INSTALL_LIB_DIR, INSTALL_BIN_DIR (e.g. for dll files on
-Windows), INSTALL_SHARE_DIR, and INSTALL_CMAKE_DIR; alternatively, with lower
-precedence, the CMake GNUInstallDirs variables can also be used:
-CMAKE_INSTALL_INCLUDEDIR, CMAKE_INSTALL_LIBDIR, CMAKE_INSTALL_BINDIR and
-CMAKE_INSTALL_DATAROOTDIR. Note that the default for INSTALL_LIB_DIR and
-CMAKE_INSTALL_LIBDIR is lib. This is in contrast to the default behavior for
-GNUInstallDirs, which defines a platform-dependent default value for
-CMAKE_INSTALL_LIBDIR (lib or lib64 or lib/<multiarch-tuple> on Debian). To
-restore this behavior, the user can set MFEM_USE_GNUINSTALLDIRS=YES.
+The library will be installed in <PREFIX>/lib, the headers in <PREFIX>/include,
+and the configuration CMake files in <PREFIX>/lib/cmake/mfem.


 Configuration variables (CMake)
@@ -37,7 +37,6 @@ set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
-set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
 set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
@@ -114,9 +114,6 @@
 // Enable MFEM functionality based on the AmgX library.
 #cmakedefine MFEM_USE_AMGX

-// Enable MFEM functionality based on the MAGMA library.
-#cmakedefine MFEM_USE_MAGMA
-
 // Enable secure socket streams based on the GNUTLS library.
 #cmakedefine MFEM_USE_GNUTLS

@@ -1,37 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MAGMA_FOUND
-#   - MAGMA_LIBRARIES
-#   - MAGMA_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(MAGMA MAGMA MAGMA_DIR "include" "magma.h" "lib" "magma"
-  "Paths to headers required by MAGMA." "Libraries required by MAGMA.")
-
-if (MAGMA_FOUND AND MFEM_USE_CUDA)
-  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-  get_target_property(CUBLAS_LIBRARIES CUDA::cublas LOCATION)
-  list(APPEND MAGMA_LIBRARIES ${CUSPARSE_LIBRARIES} ${CUBLAS_LIBRARIES})
-  set(MAGMA_LIBRARIES ${MAGMA_LIBRARIES} CACHE STRING
-      "MAGMA libraries + dependencies." FORCE)
-  message(STATUS "Updated MAGMA_LIBRARIES: ${MAGMA_LIBRARIES}")
-endif()
-
-if (MAGMA_FOUND AND MFEM_USE_HIP)
-  find_package(HIPBLAS REQUIRED)
-  find_package(HIPSPARSE REQUIRED)
-  list(APPEND MAGMA_LIBRARIES ${HIPBLAS_LIBRARIES} ${HIPSPARSE_LIBRARIES})
-  set(MAGMA_LIBRARIES ${MAGMA_LIBRARIES} CACHE STRING
-      "MAGMA libraries + dependencies." FORCE)
-  message(STATUS "Updated MAGMA_LIBRARIES: ${MAGMA_LIBRARIES}")
-endif()
@@ -117,8 +117,8 @@ macro (MULTIPASS_SOURCE_RUNS includes libraries source runs language)
  math (EXPR _tmp "${MULTIPASS_TEST_COUNT} + 1") # Why can't I add to a cache variable?
  set (MULTIPASS_TEST_COUNT ${_tmp} CACHE INTERNAL "Unique test ID")
  set (testname MULTIPASS_TEST_${MULTIPASS_TEST_COUNT}_${runs})
-  set (CMAKE_REQUIRED_INCLUDES ${includes} ${MPI_${language}_INCLUDE_PATH})
-  set (CMAKE_REQUIRED_LIBRARIES ${libraries} ${MPI_${language}_LIBRARIES})
+  set (CMAKE_REQUIRED_INCLUDES ${includes})
+  set (CMAKE_REQUIRED_LIBRARIES ${libraries})
  if(${language} STREQUAL "C")
    check_c_source_runs ("${source}" ${testname})
  elseif(${language} STREQUAL "CXX")
@@ -14,7 +14,7 @@
 #   - SLEPC_INCLUDE_DIRS
 #   - SLEPC_LIBRARIES

-set(SLEPc_REQUIRED_PACKAGES "PETSC" "MPI" CACHE STRING
+set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
  "Additional packages required by SLEPc")

 include(MfemCmakeUtilities)
@@ -31,5 +31,4 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
-  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol
-  ADD_COMPONENT Core "include" sundials/sundials_core.h "lib" sundials_core)
+  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)
@@ -138,8 +138,6 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
  # Actually add the executable
  mfem_add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
      ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
-  install(TARGETS ${MFEM_EXE_NAME}
-          RUNTIME DESTINATION miniapps)
  add_dependencies(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_EXE_NAME})
  add_dependencies(${MFEM_EXE_NAME} ${MFEM_EXEC_PREREQUISITES_TARGET_NAME})

@@ -827,20 +825,6 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
 endfunction(mfem_get_target_options)


-#
-# If ${Path} is not an absolute path, assign ${Prefix}/${Path} to the variable
-# ${OutVar}. If ${Path} is an absolute path, assign ${Path} to the variable
-# ${OutVar}.
-#
-function(mfem_path_to_fullpath Path Prefix OutVar)
-  if(IS_ABSOLUTE "${Path}")
-    set(${OutVar} "${Path}" PARENT_SCOPE)
-  else()
-    set(${OutVar} "${Prefix}/${Path}" PARENT_SCOPE)
-  endif()
-endfunction()
-
-
 #
 #   Function that creates 'config.mk' from 'config.mk.in' for the both the
 #   build- and the install-locations and define install rules for 'config.mk'
@@ -862,14 +846,14 @@ function(mfem_export_mk_files)
      MFEM_USE_ZLIB MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE
      MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS
      MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS
-      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_MAGMA
-      MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC
-      MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA
-      MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_TRIBOL MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS
+      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB
+      MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED
+      MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2
+      MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_ADFORWARD
+      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -1007,12 +991,9 @@ function(mfem_export_mk_files)
    "${PROJECT_BINARY_DIR}/config/test.mk" COPYONLY)

  # Update variables for the install-tree version of 'config.mk'
-  mfem_path_to_fullpath(
-    "${INSTALL_INCLUDE_DIR}" "${CMAKE_INSTALL_PREFIX}" MFEM_INC_DIR)
-  mfem_path_to_fullpath(
-    "${INSTALL_LIB_DIR}" "${CMAKE_INSTALL_PREFIX}" MFEM_LIB_DIR)
-  mfem_path_to_fullpath(
-    "${INSTALL_SHARE_DIR}/mfem/test.mk" "${CMAKE_INSTALL_PREFIX}" MFEM_TEST_MK)
+  set(MFEM_INC_DIR "${CMAKE_INSTALL_PREFIX}/include")
+  set(MFEM_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib")
+  set(MFEM_TEST_MK "${CMAKE_INSTALL_PREFIX}/share/mfem/test.mk")
  set(MFEM_CONFIG_EXTRA "")

  # Create the install-tree version of 'config.mk'
@@ -1022,30 +1003,8 @@ function(mfem_export_mk_files)

  # Install rules for 'config.mk' and 'test.mk'
  install(FILES ${PROJECT_SOURCE_DIR}/config/test.mk
-    DESTINATION ${INSTALL_SHARE_DIR}/mfem/)
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/share/mfem/)
  install(FILES ${PROJECT_BINARY_DIR}/config/config-install.mk
-    DESTINATION ${INSTALL_SHARE_DIR}/mfem/
-    RENAME config.mk)
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/share/mfem/ RENAME config.mk)

 endfunction()
-
-
-#
-# Function similar to the macro _GNUInstallDirs_cache_path from the module
-# GNUInstallDirs. Used to process variables like INSTALL_LIB_DIR if they are
-# set on the cmake command line without specifying type: -DINSTALL_LIB_DIR=lib.
-# Without this special treatment, relative paths are expanded to full paths
-# and we want to avoid that.
-#
-function(mfem_cache_path PathVar DefaultPath HelpStr)
-  if(NOT DEFINED ${PathVar})
-    set(${PathVar} "${DefaultPath}" CACHE PATH "${HelpStr}")
-  endif()
-  get_property(cache_type CACHE ${PathVar} PROPERTY TYPE)
-  if(cache_type STREQUAL "UNINITIALIZED")
-    file(TO_CMAKE_PATH "${${PathVar}}" cmakepath)
-    set_property(CACHE ${PathVar} PROPERTY TYPE PATH)
-    set_property(CACHE ${PathVar} PROPERTY VALUE "${cmakepath}")
-    set_property(CACHE ${PathVar} PROPERTY HELPSTRING "${HelpStr}")
-  endif()
-endfunction()
@@ -114,9 +114,6 @@
 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX

-// Enable MFEM functionality based on the MAGMA library.
-// #define MFEM_USE_MAGMA
-
 // Enable secure socket streams based on the GNUTLS library.
 // #define MFEM_USE_GNUTLS

@@ -38,7 +38,6 @@ MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_AMGX          = @MFEM_USE_AMGX@
-MFEM_USE_MAGMA         = @MFEM_USE_MAGMA@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -40,7 +40,6 @@ option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
-option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -184,10 +183,6 @@ set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")

 set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")

-set(MAGMA_DIR "${MFEM_DIR}/../magma" CACHE PATH "Path to MAGMA")
-set(MAGMA_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
-    "Additional packages required by MAGMA.")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -264,7 +259,7 @@ set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
    "The ParELAG library.")

 set(TRIBOL_DIR "${MFEM_DIR}/../tribol" CACHE PATH "Path to Tribol")
-set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING
+set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING 
    "Additional packages required by Tribol")

 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
@@ -95,10 +95,6 @@ else
   # Silence unused command line argument warnings when generating dependencies
   # with mpicxx and clang
   DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
-   # Silence "ignoring duplicate libraries" warnings on new (Xcode 15) linker
-   ifneq (,$(findstring PROJECT:dyld,$(shell ld -v 2>&1)))
-      LDFLAGS_INTERNAL = -Xlinker -no_warn_duplicate_libraries
-   endif
 endif

 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -143,7 +139,6 @@ MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
-MFEM_USE_MAGMA         = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -289,13 +284,6 @@ endif
 ifeq ($(MFEM_USE_HIP),YES)
   SUNDIALS_LIB += -lsundials_nvechip
 endif
-SUNDIALS_CORE_PAT = $(subst\
- @MFEM_DIR@,$(MFEM_DIR),$(SUNDIALS_DIR))/lib*/libsundials_core.*
-ifeq ($(MFEM_USE_SUNDIALS),YES)
-   ifneq ($(wildcard $(SUNDIALS_CORE_PAT)),)
-      SUNDIALS_LIB += -lsundials_core
-   endif
-endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES

@@ -402,11 +390,6 @@ AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
 AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt

-# MAGMA library configuration
-MAGMA_DIR = @MFEM_DIR@/../magma
-MAGMA_OPT = -I$(MAGMA_DIR)/include
-MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a -lcublas -lcusparse $(LAPACK_LIB)
-
 # GnuTLS library configuration
 GNUTLS_OPT =
 GNUTLS_LIB = -lgnutls
@@ -514,11 +497,11 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs

 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse -lcublas
+CUDA_LIB = -lcusparse

 # HIP library configuration
 HIP_OPT =
-HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse -lhipblas
+HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse

 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa
@@ -540,10 +523,8 @@ ifdef GOTCHA_DIR
 endif

 # BLITZ library configuration
-# BLITZ_DIR must be the custom installation folder (-DCMAKE_INSTALL_PREFIX).
-BLITZ_DIR = @MFEM_DIR@/../blitz/install
+BLITZ_DIR = @MFEM_DIR@/../blitz
 BLITZ_OPT = -I$(BLITZ_DIR)/include
-# On intel machines, use /lib64 instead of /lib.
 BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz

 # ALGOIM library configuration
@@ -32,7 +32,7 @@ groups_serial=(
 '"examples"
   "Examples:"
   "examples"
-   "ex{,[1-9]}[0-9].cpp"'
+   "ex{,1,2,3}[0-9].cpp"'
 #   "ex1.cpp"'
 '"sundials"
   "SUNDIALS examples:"
@@ -58,10 +58,6 @@ groups_serial=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex1.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -70,38 +66,25 @@ groups_serial=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
-    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
    mesh-optimizer.cpp minimal-surface.cpp"'
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
   "cvsRoberts_ASAi_dns.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "seq_example.cpp seq_test.cpp"' # 'seq_test.cpp' has no sample runs
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
   "GSLIB miniapps:"
   "miniapps/gslib"
   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp "'
-# todo: miniapps/mtop
 '"nurbs"
   "NURBS miniapps:"
   "miniapps/nurbs"
   "nurbs_ex1.cpp"'
-# todo: add other nurbs miniapps
-# todo: miniapps/solvers (serial)
 '"tools"
   "Tools miniapps:"
   "miniapps/tools"
   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
    lor-transfer.cpp"'
-# todo: add other tools miniapps
 '"toys"
   "Toys miniapps:"
   "miniapps/toys"
@@ -117,7 +100,7 @@ groups_parallel=(
 '"examples"
   "Examples:"
   "examples"
-   "ex{,[1-9]}[0-9]p.cpp"'
+   "ex{,1,2,3}[0-9]p.cpp"'
 #   "ex1p.cpp"'
 '"sundials"
   "SUNDIALS examples:"
@@ -143,10 +126,6 @@ groups_parallel=(
   "HiOp examples:"
   "examples/hiop"
   "ex9p.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex{1,2}p.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -159,41 +138,24 @@ groups_parallel=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp
-    fit-node-position.cpp"'
+   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
   "joule.cpp"'
-#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
+#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
   "adjoint_advection_diffusion.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "par_example.cpp"'
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "p{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
   "GSLIB miniapps:"
   "miniapps/gslib"
   "pfindpts.cpp schwarz_ex1p.cpp"'
-'"hdiv-linear-solver"
-   "H(div) linear solver miniapps:"
-   "miniapps/hdiv-linear-solver"
-   "grad_div.cpp darcy.cpp"'
-# 'miniapps/hooke/hooke.cpp' has no sample runs
-# todo: miniapps/mtop
-# todo: miniapps/multidomain
 '"navier"
   "Navier miniapps:"
   "miniapps/navier"
   "navier_cht.cpp"'
-# todo: add other navier miniapps
 '"nurbs"
   "NURBS miniapps:"
   "miniapps/nurbs"
@@ -202,18 +164,14 @@ groups_parallel=(
   "Shifted miniapps:"
   "miniapps/shifted"
   "distance.cpp"'
-# todo: add other shifted miniapps
 '"solvers"
   "Solvers miniapps:"
   "miniapps/solvers"
   "block-solvers.cpp"'
-# todo: add other solvers miniapps
-# todo: miniapps/spde
 '"tools"
   "Tools miniapps:"
   "miniapps/tools"
-   "convert-dc.cpp get-values.cpp load-dc.cpp"'
-# todo: add other tools miniapps
+   "convert-cd.cpp get-values.cpp load-dc.cpp"'
 '"convergence"
   "Convergence tests:"
   "tests/convergence"
@@ -228,7 +186,7 @@ groups_all=(
 '"examples"
   "Examples:"
   "examples"
-   "ex\"{,[1-9]}[0-9]\"{,p}.cpp"'
+   "ex\"{,1,2,3}[0-9]\"{,p}.cpp"'
 '"sundials"
   "SUNDIALS examples:"
   "examples/sundials"
@@ -257,14 +215,10 @@ groups_all=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp ex9p.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex1.cpp ex{1,2}p.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
-   "ex1.cpp ex2.cpp ex1p.cpp ex6p.cpp"'
+   "ex1.cpp ex1p.cpp ex2.cpp ex6p.cpp"'
 '"superlu"
   "Superlu examples:"
   "examples/superlu"
@@ -272,67 +226,43 @@ groups_all=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
-    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
-    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp
-    fit-node-position.cpp"'
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
+    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
   "joule.cpp"'
-#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
+#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
-   "cvsRoberts_ASAi_dns.cpp adjoint_advection_diffusion.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "seq_example.cpp seq_test.cpp par_example.cpp"'
-# 'seq_test.cpp' has no sample runs
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "{,p}{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
+   "adjoint_advection_diffusion.cpp cvsRoberts_ASAi_dns.cpp"'
 '"gslib"
   "GSLIB miniapps:"
   "miniapps/gslib"
   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp
    schwarz_ex1p.cpp"'
-'"hdiv-linear-solver"
-   "H(div) linear solver miniapps:"
-   "miniapps/hdiv-linear-solver"
-   "grad_div.cpp darcy.cpp"'
-# 'miniapps/hooke/hooke.cpp' has no sample runs
-# todo: miniapps/mtop
-# todo: miniapps/multidomain
 '"navier"
   "Navier miniapps:"
   "miniapps/navier"
   "navier_cht.cpp"'
-# todo: add other navier miniapps
 '"nurbs"
   "NURBS miniapps:"
   "miniapps/nurbs"
   "nurbs_ex1.cpp nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-# todo: add other nurbs miniapps
 '"shifted"
   "Shifted miniapps:"
   "miniapps/shifted"
   "distance.cpp"'
-# todo: add other shifted miniapps
 '"solvers"
   "Solvers miniapps:"
   "miniapps/solvers"
   "block-solvers.cpp"'
-# todo: add other solvers miniapps
-# todo: miniapps/spde
 '"tools"
   "Tools miniapps:"
   "miniapps/tools"
   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
    lor-transfer.cpp"'
-# todo: add other tools miniapps
 '"toys"
   "Toys miniapps:"
   "miniapps/toys"
@@ -456,7 +386,7 @@ function help_message()
      mfem_config [${mfem_config}]
         Set MFEM configuration options
      make [${make}], mpiexec [${mpiexec}], mpiexec_np [${mpiexec_np}]
-         Their values can also be set using the respective uppercase environment
+         Their values can also set using the respective uppercase environment
         variable
      mfem_build_dir [${mfem_build_dir}]
         Same as '-d': set this variable to something different from <mfem_dir>
@@ -0,0 +1,102 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+24
+1 7 4 5 8 11 13
+1 7 1 4 5 7 8
+1 7 1 4 5 8 11
+1 7 1 5 8 11 13
+1 7 1 5 7 8 13
+1 7 4 5 7 8 13
+1 7 1 3 4 8 11
+1 7 1 3 4 5 11
+1 7 1 5 10 11 13
+1 7 1 8 10 11 13
+1 7 1 3 5 10 11
+1 7 1 2 3 5 10
+1 7 0 1 3 4 8
+1 7 0 1 4 7 8
+1 7 1 5 6 7 13
+1 7 1 6 7 8 13
+1 7 6 7 8 13 15
+1 7 4 7 8 13 15
+1 7 4 8 12 13 15
+1 7 4 8 11 12 13
+1 7 6 8 13 14 15
+1 7 1 6 8 13 14
+1 7 1 8 9 10 13
+1 7 1 8 9 13 14
+
+boundary
+48
+1 4 0 1 3 4
+1 4 0 1 3 8
+1 4 0 3 4 8
+1 4 0 1 4 7
+1 4 0 1 7 8
+1 4 0 4 7 8
+1 4 1 4 5 7
+1 4 1 3 8 11
+1 4 1 3 4 5
+1 4 1 5 10 13
+1 4 1 8 10 11
+1 4 1 3 10 11
+1 4 1 2 3 5
+1 4 1 2 3 10
+1 4 1 2 5 10
+1 4 1 5 6 7
+1 4 1 5 6 13
+1 4 1 6 7 8
+1 4 1 6 8 14
+1 4 1 6 13 14
+1 4 1 8 9 10
+1 4 1 9 10 13
+1 4 1 8 9 14
+1 4 1 9 13 14
+1 4 2 3 5 10
+1 4 3 4 8 11
+1 4 3 4 5 11
+1 4 3 5 10 11
+1 4 4 5 11 13
+1 4 4 5 7 13
+1 4 4 7 8 15
+1 4 4 7 13 15
+1 4 4 8 12 15
+1 4 4 12 13 15
+1 4 4 8 11 12
+1 4 4 11 12 13
+1 4 5 10 11 13
+1 4 5 6 7 13
+1 4 6 7 8 15
+1 4 6 7 13 15
+1 4 6 8 14 15
+1 4 6 13 14 15
+2 4 8 10 11 13
+2 4 8 12 13 15
+2 4 8 11 12 13
+2 4 8 13 14 15
+2 4 8 9 10 13
+2 4 8 9 13 14
+
+vertices
+16
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
@@ -0,0 +1,102 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+24
+17 8 4 5 8 11 13
+11 8 1 4 5 7 8
+15 8 1 4 5 8 11
+16 8 1 5 8 11 13
+12 8 1 5 7 8 13
+13 8 4 5 7 8 13
+7  8 1 3 4 8 11
+2  8 1 3 4 5 11
+4  8 1 5 10 11 13
+10 8 1 8 10 11 13
+3  8 1 3 5 10 11
+1  8 1 2 3 5 10
+6  8 0 1 3 4 8
+8  8 0 1 4 7 8
+5  8 1 5 6 7 13
+20 8 1 6 7 8 13
+19 8 6 7 8 13 15
+14 8 4 7 8 13 15
+21 8 4 8 12 13 15
+22 8 4 8 11 12 13
+23 8 6 8 13 14 15
+24 8 1 6 8 13 14
+9  8 1 8 9 10 13
+18 8 1 8 9 13 14
+
+boundary
+48
+1 4 0 1 3 4
+3 4 0 1 3 8
+3 4 0 3 4 8
+1 4 0 1 4 7
+3 4 0 1 7 8
+3 4 0 4 7 8
+1 4 1 4 5 7
+3 4 1 3 8 11
+1 4 1 3 4 5
+3 4 1 5 10 13
+3 4 1 8 10 11
+3 4 1 3 10 11
+1 4 1 2 3 5
+3 4 1 2 3 10
+3 4 1 2 5 10
+1 4 1 5 6 7
+3 4 1 5 6 13
+3 4 1 6 7 8
+3 4 1 6 8 14
+3 4 1 6 13 14
+3 4 1 8 9 10
+3 4 1 9 10 13
+3 4 1 8 9 14
+3 4 1 9 13 14
+3 4 2 3 5 10
+3 4 3 4 8 11
+3 4 3 4 5 11
+3 4 3 5 10 11
+3 4 4 5 11 13
+3 4 4 5 7 13
+3 4 4 7 8 15
+3 4 4 7 13 15
+3 4 4 8 12 15
+3 4 4 12 13 15
+3 4 4 8 11 12
+3 4 4 11 12 13
+3 4 5 10 11 13
+3 4 5 6 7 13
+3 4 6 7 8 15
+3 4 6 7 13 15
+3 4 6 8 14 15
+3 4 6 13 14 15
+5 4 8 10 11 13
+5 4 8 12 13 15
+5 4 8 11 12 13
+5 4 8 13 14 15
+5 4 8 9 10 13
+5 4 8 9 13 14
+
+vertices
+16
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
@@ -0,0 +1,231 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+96
+1 8 0 1 7 8 9
+1 8 1 6 7 8 9
+1 8 4 5 6 8 9
+1 8 4 6 7 8 9
+1 8 0 1 3 8 9
+1 8 1 2 3 8 9
+1 8 0 4 7 8 9
+1 8 0 3 4 8 9
+1 8 1 2 6 8 9
+1 8 2 5 6 8 9
+1 8 3 4 5 8 9
+1 8 2 3 5 8 9
+1 8 9 10 11 17 18
+1 8 9 11 16 17 18
+1 8 9 14 15 16 18
+1 8 9 14 16 17 18
+1 8 9 10 11 13 18
+1 8 9 11 12 13 18
+1 8 9 10 14 17 18
+1 8 9 10 13 14 18
+1 8 9 11 12 16 18
+1 8 9 12 15 16 18
+1 8 9 13 14 15 18
+1 8 9 12 13 15 18
+1 8 9 12 15 16 19
+1 8 9 11 12 16 19
+1 8 1 6 9 11 19
+1 8 6 9 11 16 19
+1 8 1 2 6 9 19
+1 8 2 5 6 9 19
+1 8 2 5 9 15 19
+1 8 2 9 12 15 19
+1 8 5 6 9 16 19
+1 8 5 9 15 16 19
+1 8 1 9 11 12 19
+1 8 1 2 9 12 19
+1 8 9 10 13 14 20
+1 8 9 10 14 17 20
+1 8 0 9 10 17 20
+1 8 0 7 9 17 20
+1 8 0 3 4 9 20
+1 8 0 4 7 9 20
+1 8 3 4 9 13 20
+1 8 4 9 13 14 20
+1 8 4 7 9 14 20
+1 8 7 9 14 17 20
+1 8 0 3 9 10 20
+1 8 3 9 10 13 20
+1 8 2 5 9 15 21
+1 8 2 9 12 15 21
+1 8 2 3 5 9 21
+1 8 3 4 5 9 21
+1 8 9 13 14 15 21
+1 8 9 12 13 15 21
+1 8 4 5 9 14 21
+1 8 5 9 14 15 21
+1 8 2 3 9 12 21
+1 8 3 9 12 13 21
+1 8 3 4 9 13 21
+1 8 4 9 13 14 21
+1 8 1 6 9 11 22
+1 8 6 9 11 16 22
+1 8 6 7 9 16 22
+1 8 7 9 16 17 22
+1 8 0 7 9 17 22
+1 8 0 9 10 17 22
+1 8 0 1 9 10 22
+1 8 1 9 10 11 22
+1 8 0 1 7 9 22
+1 8 1 6 7 9 22
+1 8 9 10 11 17 22
+1 8 9 11 16 17 22
+1 8 5 9 15 16 23
+1 8 5 6 9 16 23
+1 8 6 7 9 16 23
+1 8 7 9 16 17 23
+1 8 7 9 14 17 23
+1 8 4 7 9 14 23
+1 8 4 5 9 14 23
+1 8 5 9 14 15 23
+1 8 4 6 7 9 23
+1 8 4 5 6 9 23
+1 8 9 14 15 16 23
+1 8 9 14 16 17 23
+1 8 1 2 9 12 24
+1 8 1 9 11 12 24
+1 8 0 1 9 10 24
+1 8 1 9 10 11 24
+1 8 0 3 9 10 24
+1 8 3 9 10 13 24
+1 8 2 3 9 12 24
+1 8 3 9 12 13 24
+1 8 1 2 3 9 24
+1 8 0 1 3 9 24
+1 8 9 10 11 13 24
+1 8 9 11 12 13 24
+
+boundary
+96
+1 4 0 1 7 8
+1 4 0 1 3 8
+1 4 0 4 7 8
+1 4 0 3 4 8
+2 4 0 10 17 20
+2 4 0 7 17 20
+2 4 0 3 4 20
+2 4 0 4 7 20
+2 4 0 3 10 20
+2 4 0 7 17 22
+2 4 0 10 17 22
+2 4 0 1 10 22
+2 4 0 1 7 22
+2 4 0 1 10 24
+2 4 0 3 10 24
+2 4 0 1 3 24
+1 4 1 6 7 8
+1 4 1 2 3 8
+1 4 1 2 6 8
+2 4 1 6 11 19
+2 4 1 2 6 19
+2 4 1 11 12 19
+2 4 1 2 12 19
+2 4 1 6 11 22
+2 4 1 10 11 22
+2 4 1 6 7 22
+2 4 1 2 12 24
+2 4 1 11 12 24
+2 4 1 10 11 24
+2 4 1 2 3 24
+1 4 2 5 6 8
+1 4 2 3 5 8
+2 4 2 5 6 19
+2 4 2 5 15 19
+2 4 2 12 15 19
+2 4 2 5 15 21
+2 4 2 12 15 21
+2 4 2 3 5 21
+2 4 2 3 12 21
+2 4 2 3 12 24
+1 4 3 4 5 8
+2 4 3 4 13 20
+2 4 3 10 13 20
+2 4 3 4 5 21
+2 4 3 12 13 21
+2 4 3 4 13 21
+2 4 3 10 13 24
+2 4 3 12 13 24
+1 4 4 5 6 8
+1 4 4 6 7 8
+2 4 4 13 14 20
+2 4 4 7 14 20
+2 4 4 5 14 21
+2 4 4 13 14 21
+2 4 4 7 14 23
+2 4 4 5 14 23
+2 4 4 6 7 23
+2 4 4 5 6 23
+2 4 5 6 16 19
+2 4 5 15 16 19
+2 4 5 14 15 21
+2 4 5 15 16 23
+2 4 5 6 16 23
+2 4 5 14 15 23
+2 4 6 11 16 19
+2 4 6 11 16 22
+2 4 6 7 16 22
+2 4 6 7 16 23
+2 4 7 14 17 20
+2 4 7 16 17 22
+2 4 7 16 17 23
+2 4 7 14 17 23
+3 4 10 11 17 18
+3 4 10 11 13 18
+3 4 10 14 17 18
+3 4 10 13 14 18
+2 4 10 13 14 20
+2 4 10 14 17 20
+2 4 10 11 17 22
+2 4 10 11 13 24
+3 4 11 16 17 18
+3 4 11 12 13 18
+3 4 11 12 16 18
+2 4 11 12 16 19
+2 4 11 16 17 22
+2 4 11 12 13 24
+3 4 12 15 16 18
+3 4 12 13 15 18
+2 4 12 15 16 19
+2 4 12 13 15 21
+3 4 13 14 15 18
+2 4 13 14 15 21
+3 4 14 15 16 18
+3 4 14 16 17 18
+2 4 14 15 16 23
+2 4 14 16 17 23
+
+vertices
+25
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 0.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 1.0000000000000000
+1.0000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.0000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 1.0000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 0.0000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 0.5000000000000000 1.0000000000000000 0.5000000000000000
+0.5000000000000000 0.5000000000000000 0.0000000000000000 0.5000000000000000
@@ -0,0 +1,36 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+2
+1 2 2 0 1
+1 2 0 2 3
+
+boundary
+4
+1 1 0 1
+1 1 1 2
+2 1 2 3
+1 1 3 0
+
+vertices
+4
+2
+0 0
+1 0
+1 1
+0 1
@@ -18,9 +18,9 @@ elements
 boundary
 4
 1 1 0 1
-2 1 2 3
-3 1 3 0
-4 1 1 2
+1 1 2 3
+1 1 3 0
+1 1 1 2

 edges
 4
@@ -0,0 +1,52 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+6
+1 4 3 1 7 5
+1 4 1 6 7 4
+1 4 6 1 0 2
+1 4 1 6 4 2
+1 4 6 1 3 0
+1 4 1 6 3 7
+
+boundary
+12
+1 2 6 0 3
+1 2 0 6 2
+1 2 1 3 0
+1 2 3 1 5
+1 2 3 7 6
+1 2 7 3 5
+1 2 4 6 7
+1 2 6 4 2
+2 2 1 7 5
+2 2 7 1 4
+1 2 1 2 4
+1 2 2 1 0
+
+vertices
+8
+3
+0 0 0
+0 0 1
+1 0 0
+0 1 0
+1 0 1
+0 1 1
+1 1 0
+1 1 1
@@ -938,7 +938,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/config \
                         @MFEM_SOURCE_DIR@/general \
                         @MFEM_SOURCE_DIR@/linalg \
-                         @MFEM_SOURCE_DIR@/linalg/batched \
                         @MFEM_SOURCE_DIR@/linalg/simd \
                         @MFEM_SOURCE_DIR@/mesh \
                         @MFEM_SOURCE_DIR@/mesh/submesh \
@@ -1050,8 +1049,7 @@ RECURSIVE              = NO
 EXCLUDE                = @MFEM_SOURCE_DIR@/config/_config.hpp \
                         @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp \
                         @MFEM_SOURCE_DIR@/general/tinyxml2.h \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp \
-                         @MFEM_SOURCE_DIR@/linalg/lapack.hpp
+                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp

 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -1210,13 +1208,13 @@ STRIP_CODE_COMMENTS    = NO
 # entity all documented functions referencing it will be listed.
 # The default value is: NO.

-REFERENCED_BY_RELATION = NO
+REFERENCED_BY_RELATION = YES

 # If the REFERENCES_RELATION tag is set to YES then for each documented function
 # all documented entities called/used by that function will be listed.
 # The default value is: NO.

-REFERENCES_RELATION    = NO
+REFERENCES_RELATION    = YES

 # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
 # to YES then the hyperlinks from functions in REFERENCES_RELATION and
@@ -182,21 +182,6 @@ namespace mfem {
 *   <a class="el" href="examples_2superlu_2ex1p_8cpp_source.html">1p</a>,
 *   demonstrating the use of MFEM's \link superlu.hpp SuperLU integration\endlink.
 *
- * <H4>NURBS Examples</H4>
- * - Variants of Examples
- *  <a class="el" href="nurbs__ex1_8cpp_source.html">1</a>,
- *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
- *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
- *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
- *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
- *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
- *   demonstrating howto perform NURBS-based Isogeometric Analysis.
- * - Variant of Example <a class="el" href="nurbs__patch__ex1_8cpp_source.html">1</a>: demonstrates the use of patch integration
- * - <a class="el" href="nurbs__solenoidal_8cpp_source.html">NURBS Divergence-free</a>: solve a solenoidal vector projection with NURBS-based H(div) elements
- * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
- * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
- * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
- *
 * <H3>Miniapps</H3>
 * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
@@ -73,6 +73,9 @@ if (MFEM_USE_MPI)
    ex20p.cpp
    ex21p.cpp
    ex22p.cpp
+    ex1p_4d.cpp
+    ex3p_4d.cpp
+    ex4D_DivSkew.cpp
    ex24p.cpp
    ex25p.cpp
    ex26p.cpp
@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/amgx/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/caliper,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -163,18 +163,13 @@ int main(int argc, char *argv[])

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
-   //    the external boundary attributes from the mesh as essential (Dirichlet)
-   //    and converting them to a list of true dofs.
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
   Array<int> ess_tdof_list;
   if (mesh.bdr_attributes.Size())
   {
      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 0;
-      // Apply boundary conditions on all external boundaries:
-      mesh.MarkExternalBoundaries(ess_bdr);
-      // Boundary conditions can also be applied based on named attributes:
-      // mesh.MarkNamedBoundaries(set_name, ess_bdr)
-
+      ess_bdr = 1;
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

@@ -3,14 +3,14 @@
 // Compile with: make ex10
 //
 // Sample runs:
-//    ex10 -m ../data/beam-quad.mesh -s 23 -r 2 -o 2 -dt 3
-//    ex10 -m ../data/beam-tri.mesh -s 23 -r 2 -o 2 -dt 3
-//    ex10 -m ../data/beam-hex.mesh -s 22 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-tet.mesh -s 22 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-wedge.mesh -s 22 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-quad.mesh -s 4 -r 2 -o 2 -dt 0.03 -vs 20
-//    ex10 -m ../data/beam-hex.mesh -s 4 -r 1 -o 2 -dt 0.05 -vs 20
-//    ex10 -m ../data/beam-quad-amr.mesh -s 23 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-quad.mesh -s 3 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-tri.mesh -s 3 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-hex.mesh -s 2 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-tet.mesh -s 2 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-wedge.mesh -s 2 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-quad.mesh -s 14 -r 2 -o 2 -dt 0.03 -vs 20
+//    ex10 -m ../data/beam-hex.mesh -s 14 -r 1 -o 2 -dt 0.05 -vs 20
+//    ex10 -m ../data/beam-quad-amr.mesh -s 3 -r 2 -o 2 -dt 3
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -87,16 +87,16 @@ public:
                        real_t visc, real_t mu, real_t K);

   /// Compute the right-hand side of the ODE system.
-   void Mult(const Vector &vx, Vector &dvx_dt) const override;
+   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
   /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);

   real_t ElasticEnergy(const Vector &x) const;
   real_t KineticEnergy(const Vector &v) const;
   void GetElasticEnergyDensity(const GridFunction &x, GridFunction &w) const;

-   ~HyperelasticOperator() override;
+   virtual ~HyperelasticOperator();
 };

 /** Nonlinear operator of the form:
@@ -120,12 +120,12 @@ public:
   void SetParameters(real_t dt_, const Vector *v_, const Vector *x_);

   /// Compute y = H(x + dt (v + dt k)) + M k + S (v + dt k).
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual void Mult(const Vector &k, Vector &y) const;

   /// Compute J = M + dt S + dt^2 grad_H(x + dt (v + dt k)).
-   Operator &GetGradient(const Vector &k) const override;
+   virtual Operator &GetGradient(const Vector &k) const;

-   ~ReducedSystemOperator() override;
+   virtual ~ReducedSystemOperator();
 };


@@ -141,8 +141,8 @@ private:
 public:
   ElasticEnergyCoefficient(HyperelasticModel &m, const GridFunction &x_)
      : model(m), x(x_) { }
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
-   ~ElasticEnergyCoefficient() override { }
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual ~ElasticEnergyCoefficient() { }
 };

 void InitialDeformation(const Vector &x, Vector &y);
@@ -160,7 +160,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/beam-quad.mesh";
   int ref_levels = 2;
   int order = 2;
-   int ode_solver_type = 23;
+   int ode_solver_type = 3;
   real_t t_final = 300.0;
   real_t dt = 3.0;
   real_t visc = 1e-2;
@@ -177,7 +177,11 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4."
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -209,7 +213,28 @@ int main(int argc, char *argv[])
   // 3. Define the ODE solver used for time integration. Several implicit
   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
   //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1: ode_solver = new BackwardEulerSolver; break;
+      case 2: ode_solver = new SDIRK23Solver(2); break;
+      case 3: ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -346,6 +371,7 @@ int main(int argc, char *argv[])
   }

   // 10. Free the used memory.
+   delete ode_solver;
   delete mesh;

   return 0;
@@ -3,14 +3,14 @@
 // Compile with: make ex10p
 //
 // Sample runs:
-//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 23 -rs 2 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-tri.mesh -s 23 -rs 2 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 22 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-tet.mesh -s 22 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-wedge.mesh -s 22 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 4 -rs 2 -dt 0.03 -vs 20
-//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 4 -rs 1 -dt 0.05 -vs 20
-//    mpirun -np 4 ex10p -m ../data/beam-quad-amr.mesh -s 23 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 3 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-tri.mesh -s 3 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 2 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-tet.mesh -s 2 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-wedge.mesh -s 2 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 14 -rs 2 -dt 0.03 -vs 20
+//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 14 -rs 1 -dt 0.05 -vs 20
+//    mpirun -np 4 ex10p -m ../data/beam-quad-amr.mesh -s 3 -rs 2 -dt 3
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -89,17 +89,17 @@ public:
                        real_t visc, real_t mu, real_t K);

   /// Compute the right-hand side of the ODE system.
-   void Mult(const Vector &vx, Vector &dvx_dt) const override;
+   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
   /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);

   real_t ElasticEnergy(const ParGridFunction &x) const;
   real_t KineticEnergy(const ParGridFunction &v) const;
   void GetElasticEnergyDensity(const ParGridFunction &x,
                                ParGridFunction &w) const;

-   ~HyperelasticOperator() override;
+   virtual ~HyperelasticOperator();
 };

 /** Nonlinear operator of the form:
@@ -125,12 +125,12 @@ public:
   void SetParameters(real_t dt_, const Vector *v_, const Vector *x_);

   /// Compute y = H(x + dt (v + dt k)) + M k + S (v + dt k).
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual void Mult(const Vector &k, Vector &y) const;

   /// Compute J = M + dt S + dt^2 grad_H(x + dt (v + dt k)).
-   Operator &GetGradient(const Vector &k) const override;
+   virtual Operator &GetGradient(const Vector &k) const;

-   ~ReducedSystemOperator() override;
+   virtual ~ReducedSystemOperator();
 };


@@ -146,8 +146,8 @@ private:
 public:
   ElasticEnergyCoefficient(HyperelasticModel &m, const ParGridFunction &x_)
      : model(m), x(x_) { }
-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
-   ~ElasticEnergyCoefficient() override { }
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   virtual ~ElasticEnergyCoefficient() { }
 };

 void InitialDeformation(const Vector &x, Vector &y);
@@ -172,7 +172,7 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 0;
   int order = 2;
-   int ode_solver_type = 23;
+   int ode_solver_type = 3;
   real_t t_final = 300.0;
   real_t dt = 3.0;
   real_t visc = 1e-2;
@@ -192,7 +192,11 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4."
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -234,7 +238,31 @@ int main(int argc, char *argv[])
   // 4. Define the ODE solver used for time integration. Several implicit
   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
   //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         if (myid == 0)
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         return 3;
+   }

   // 5. Refine the mesh in serial to increase the resolution. In this example
   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -405,6 +433,7 @@ int main(int argc, char *argv[])
   }

   // 12. Free the used memory.
+   delete ode_solver;
   delete pmesh;

   return 0;
@@ -192,11 +192,7 @@ int main(int argc, char *argv[])
   if (pmesh->bdr_attributes.Size())
   {
      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 0;
-      // Apply boundary conditions on all external boundaries:
-      pmesh->MarkExternalBoundaries(ess_bdr);
-      // Boundary conditions can also be applied based on named attributes:
-      // pmesh->MarkNamedBoundaries(set_name, ess_bdr)
+      ess_bdr = 1;
   }

   ParBilinearForm *a = new ParBilinearForm(fespace);
@@ -20,6 +20,7 @@
 //               ex14 -m ../data/fichera-amr.mesh
 //               ex14 -pa -r 1 -o 3
 //               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
+//               ex14 -m ../data/inline-tet.mesh -o 0 -nt 4 -s 1
 //
 // Device sample runs:
 //               ex14 -pa -r 2 -d cuda -o 3
@@ -55,10 +56,16 @@ int main(int argc, char *argv[])
   bool pa = false;
   bool visualization = 1;
   const char *device_config = "cpu";
+   int nt = 0;
+   double st = 1.0;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
                  "Mesh file to use.");
+   args.AddOption(&nt, "-nt", "--number-of-timeslices",
+                  "Number of slices through the hyper-prism in the 4th coordinate.");
+   args.AddOption(&st, "-st", "--size-time",
+                  "Length of hyper-prims in 4th coordinate (e.g. time).");
   args.AddOption(&ref_levels, "-r", "--refine",
                  "Number of times to refine the mesh uniformly, -1 for auto.");
   args.AddOption(&order, "-o", "--order",
@@ -97,8 +104,17 @@ int main(int argc, char *argv[])
   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
   //    NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   const int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   if (dim == 3 && nt > 0)
+   {
+      Mesh* spat_mesh = mesh;
+      mesh = new Mesh(spat_mesh, nt, Element::PENTATOPE, true, st);
+      delete spat_mesh;
+
+      ref_levels = 0;
+      dim = 4;
+   }

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
@@ -107,23 +123,23 @@ int main(int argc, char *argv[])
   {
      if (ref_levels < 0)
      {
-         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/(dim < 4 ? dim : 1.));
      }
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
   {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
   }

   // 5. Define a finite element space on the mesh. Here we use discontinuous
   //    finite elements of the specified order >= 0.
   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
   DG_FECollection fec(order, dim, bt);
-   FiniteElementSpace fespace(&mesh, &fec);
+   FiniteElementSpace fespace(mesh, &fec);
   cout << "Number of unknowns: " << fespace.GetVSize() << endl;

   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
@@ -198,7 +214,7 @@ int main(int argc, char *argv[])
   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);
@@ -210,8 +226,10 @@ int main(int argc, char *argv[])
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
   }

+   delete mesh;
+
   return 0;
 }
@@ -19,6 +19,7 @@
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
 //               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
 //               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
+//               mpirun -np 4 ex14p -m ../data/inline-tet.mesh -o 0 -nt 4 -s 1
 //
 // Device sample runs:
 //               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
@@ -53,7 +54,7 @@ public:
      pmesh(pmesh_),
      pgf(pgf_) {}

-   void MonitorSolution(int i, real_t norm, const Vector &x, bool final) override
+   void MonitorSolution(int i, real_t norm, const Vector &x, bool final)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
@@ -90,10 +91,16 @@ int main(int argc, char *argv[])
   bool pa = false;
   bool visualization = 1;
   const char *device_config = "cpu";
+   int nt = 0;
+   double st = 1.0;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
                  "Mesh file to use.");
+   args.AddOption(&nt, "-nt", "--number-of-timeslices",
+                  "Number of slices through the hyper-prism in the 4th coordinate.");
+   args.AddOption(&st, "-st", "--size-time",
+                  "Length of hyper-prims in 4th coordinate (e.g. time).");
   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
                  "Number of times to refine the mesh uniformly in serial,"
                  " -1 for auto.");
@@ -139,8 +146,17 @@ int main(int argc, char *argv[])
   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
   //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   if (dim == 3 && nt > 0)
+   {
+      Mesh* spat_mesh = mesh;
+      mesh = new Mesh(spat_mesh, nt, Element::PENTATOPE, true, st);
+      delete spat_mesh;
+      dim = 4;
+   }
+   if (dim == 4)
+      ser_ref_levels = 0;

   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -149,23 +165,23 @@ int main(int argc, char *argv[])
   {
      if (ser_ref_levels < 0)
      {
-         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ser_ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
   {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
   }

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
   {
      for (int l = 0; l < par_ref_levels; l++)
      {
@@ -5,10 +5,10 @@
 // Sample runs:  ex16
 //               ex16 -m ../data/inline-tri.mesh
 //               ex16 -m ../data/disc-nurbs.mesh -tf 2
-//               ex16 -s 21 -a 0.0 -k 1.0
-//               ex16 -s 22 -a 1.0 -k 0.0
-//               ex16 -s 23 -a 0.5 -k 0.5 -o 4
-//               ex16 -s 4 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               ex16 -s 1 -a 0.0 -k 1.0
+//               ex16 -s 2 -a 1.0 -k 0.0
+//               ex16 -s 3 -a 0.5 -k 0.5 -o 4
+//               ex16 -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               ex16 -m ../data/fichera-q2.mesh
 //               ex16 -m ../data/fichera-mixed.mesh
 //               ex16 -m ../data/escher.mesh
@@ -76,15 +76,15 @@ public:
   ConductionOperator(FiniteElementSpace &f, real_t alpha, real_t kappa,
                      const Vector &u);

-   void Mult(const Vector &u, Vector &du_dt) const override;
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &u, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &u, Vector &k);

   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
   void SetParameters(const Vector &u);

-   ~ConductionOperator() override;
+   virtual ~ConductionOperator();
 };

 real_t InitialTemperature(const Vector &x);
@@ -95,13 +95,11 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int ref_levels = 2;
   int order = 2;
-
-   int ode_solver_type = 23;  // SDIRK33Solver
+   int ode_solver_type = 3;
   real_t t_final = 0.5;
   real_t dt = 1.0e-2;
   real_t alpha = 1.0e-2;
   real_t kappa = 0.5;
-
   bool visualization = true;
   bool visit = false;
   int vis_steps = 5;
@@ -117,7 +115,8 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -150,7 +149,28 @@ int main(int argc, char *argv[])
   // 3. Define the ODE solver used for time integration. Several implicit
   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
   //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -267,6 +287,7 @@ int main(int argc, char *argv[])
   }

   // 10. Free the used memory.
+   delete ode_solver;
   delete mesh;

   return 0;
@@ -5,10 +5,10 @@
 // Sample runs:  mpirun -np 4 ex16p
 //               mpirun -np 4 ex16p -m ../data/inline-tri.mesh
 //               mpirun -np 4 ex16p -m ../data/disc-nurbs.mesh -tf 2
-//               mpirun -np 4 ex16p -s 21 -a 0.0 -k 1.0
-//               mpirun -np 4 ex16p -s 22 -a 1.0 -k 0.0
-//               mpirun -np 8 ex16p -s 23 -a 0.5 -k 0.5 -o 4
-//               mpirun -np 4 ex16p -s 4 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               mpirun -np 4 ex16p -s 1 -a 0.0 -k 1.0
+//               mpirun -np 4 ex16p -s 2 -a 1.0 -k 0.0
+//               mpirun -np 8 ex16p -s 3 -a 0.5 -k 0.5 -o 4
+//               mpirun -np 4 ex16p -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               mpirun -np 16 ex16p -m ../data/fichera-q2.mesh
 //               mpirun -np 16 ex16p -m ../data/fichera-mixed.mesh
 //               mpirun -np 16 ex16p -m ../data/escher-p2.mesh
@@ -78,15 +78,15 @@ public:
   ConductionOperator(ParFiniteElementSpace &f, real_t alpha, real_t kappa,
                      const Vector &u);

-   void Mult(const Vector &u, Vector &du_dt) const override;
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
-   void ImplicitSolve(const real_t dt, const Vector &u, Vector &k) override;
+   virtual void ImplicitSolve(const real_t dt, const Vector &u, Vector &k);

   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
   void SetParameters(const Vector &u);

-   ~ConductionOperator() override;
+   virtual ~ConductionOperator();
 };

 real_t InitialTemperature(const Vector &x);
@@ -104,13 +104,11 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 1;
   int order = 2;
-
-   int ode_solver_type = 23;  // SDIRK33Solver
+   int ode_solver_type = 3;
   real_t t_final = 0.5;
   real_t dt = 1.0e-2;
   real_t alpha = 1.0e-2;
   real_t kappa = 0.5;
-
   bool visualization = true;
   bool visit = false;
   int vis_steps = 5;
@@ -129,7 +127,8 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -170,7 +169,28 @@ int main(int argc, char *argv[])
   // 4. Define the ODE solver used for time integration. Several implicit
   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
   //    explicit Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }

   // 5. Refine the mesh in serial to increase the resolution. In this example
   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -356,6 +376,7 @@ int main(int argc, char *argv[])
   }

   // 12. Free the used memory.
+   delete ode_solver;
   delete pmesh;

   return 0;
@@ -69,7 +69,7 @@ public:
   void SetDisplacement(GridFunction &u_) { u = &u_; }
   void SetComponent(int i, int j) { si = i; sj = j; }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 // Simple GLVis visualization manager.
@@ -89,7 +89,7 @@ public:
   void NewWindow();
   void CloseConnection();
   void PositionWindow();
-   ~VisMan() override;
+   virtual ~VisMan();
 };

 // Manipulators for the GLVis visualization manager.
@@ -69,7 +69,7 @@ public:
   void SetDisplacement(GridFunction &u_) { u = &u_; }
   void SetComponent(int i, int j) { si = i; sj = j; }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 // Simple GLVis visualization manager.
@@ -89,7 +89,7 @@ public:
   void NewWindow();
   void CloseConnection();
   void PositionWindow();
-   ~VisMan() override;
+   virtual ~VisMan();
 };

 // Manipulators for the GLVis visualization manager.
@@ -90,7 +90,8 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::ExplicitTypes.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
   args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step. Positive number skips CFL timestep calculation.");
@@ -124,7 +125,18 @@ int main(int argc, char *argv[])

   // 3. Define the ODE solver used for time integration. Several explicit
   //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectExplicit(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         return 3;
+   }

   // 4. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
@@ -292,5 +304,8 @@ int main(int argc, char *argv[])
   const real_t error = sol.ComputeLpError(2, u0);
   cout << "Solution error: " << error << endl;

+   // Free the used memory.
+   delete ode_solver;
+
   return 0;
 }
@@ -99,7 +99,8 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::ExplicitTypes.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
   args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step. Positive number skips CFL timestep calculation.");
@@ -147,7 +148,18 @@ int main(int argc, char *argv[])

   // 3. Define the ODE solver used for time integration. Several explicit
   //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectExplicit(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         return 3;
+   }

   // 4. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
@@ -348,5 +360,8 @@ int main(int argc, char *argv[])
      cout << "Solution error: " << error << endl;
   }

+   // Free the used memory.
+   delete ode_solver;
+
   return 0;
 }
@@ -48,7 +48,7 @@ public:
      print_level = print_lvl;
   }

-   void MonitorResidual(int it, real_t norm, const Vector &r, bool final) override;
+   virtual void MonitorResidual(int it, real_t norm, const Vector &r, bool final);

 private:
   const std::string prefix;
@@ -116,10 +116,10 @@ public:
   JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
                          SparseMatrix &mass, Array<int> &offsets);

-   void Mult(const Vector &k, Vector &y) const override;
-   void SetOperator(const Operator &op) override;
+   virtual void Mult(const Vector &k, Vector &y) const;
+   virtual void SetOperator(const Operator &op);

-   ~JacobianPreconditioner() override;
+   virtual ~JacobianPreconditioner();
 };

 // After spatial discretization, the rubber model can be written as:
@@ -161,13 +161,13 @@ public:
                  int iter, Coefficient &mu);

   // Required to use the native newton solver
-   Operator &GetGradient(const Vector &xp) const override;
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual Operator &GetGradient(const Vector &xp) const;
+   virtual void Mult(const Vector &k, Vector &y) const;

   // Driver for the newton solver
   void Solve(Vector &xp) const;

-   ~RubberOperator() override;
+   virtual ~RubberOperator();
 };

 // Visualization driver
@@ -62,7 +62,7 @@ public:
 #endif
   }

-   void MonitorResidual(int it, real_t norm, const Vector &r, bool final) override;
+   virtual void MonitorResidual(int it, real_t norm, const Vector &r, bool final);

 private:
   const std::string prefix;
@@ -130,10 +130,10 @@ public:
   JacobianPreconditioner(Array<ParFiniteElementSpace *> &fes,
                          Operator &mass, Array<int> &offsets);

-   void Mult(const Vector &k, Vector &y) const override;
-   void SetOperator(const Operator &op) override;
+   virtual void Mult(const Vector &k, Vector &y) const;
+   virtual void SetOperator(const Operator &op);

-   ~JacobianPreconditioner() override;
+   virtual ~JacobianPreconditioner();
 };

 // After spatial discretization, the rubber model can be written as:
@@ -175,13 +175,13 @@ public:
                  int iter, Coefficient &mu);

   // Required to use the native newton solver
-   Operator &GetGradient(const Vector &xp) const override;
-   void Mult(const Vector &k, Vector &y) const override;
+   virtual Operator &GetGradient(const Vector &xp) const;
+   virtual void Mult(const Vector &k, Vector &y) const;

   // Driver for the newton solver
   void Solve(Vector &xp) const;

-   ~RubberOperator() override;
+   virtual ~RubberOperator();
 };

 // Visualization driver
@@ -190,18 +190,13 @@ int main(int argc, char *argv[])

   // 8. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the external boundary attributes from the mesh as
-   //    essential (Dirichlet) and converting them to a list of true dofs.
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
   if (pmesh.bdr_attributes.Size())
   {
      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 0;
-      // Apply boundary conditions on all external boundaries:
-      pmesh.MarkExternalBoundaries(ess_bdr);
-      // Boundary conditions can also be applied based on named attributes:
-      // pmesh.MarkNamedBoundaries(set_name, ess_bdr)
-
+      ess_bdr = 1;
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

@@ -0,0 +1,412 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "./spe10_coeff.cpp"
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_grad.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_grad.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+using namespace std;
+using namespace mfem;
+
+double kappa = 1.0;
+
+double u_exact(const Vector &x)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      return cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+   }
+   else { return 0.0; }
+}
+
+double f_exact(const Vector &x)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      return (kappa + 4.0 * M_PI*M_PI) * cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(
+                                                                              2))*cos(M_PI*x(3));
+   }
+   else { return 0.0; }
+}
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   bool set_bc = true;
+   bool standardCG = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   //   if(dim !=4 || sdim != 4)
+   //   {
+   //    MPI_Finalize();
+   //    return 0;
+   //   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      if (dim==4)
+      {
+         if (order==1) { fec = new LinearFECollection; }
+         else { fec = new QuadraticFECollection; }
+      }
+      else { fec = new H1_FECollection(order, dim); }
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = set_bc ? 1 : 0;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   FunctionCoefficient uExact(u_exact);
+   ParGridFunction x(fespace);
+
+   int NExpo =8;
+   for (int expo=-NExpo; expo<=NExpo; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      x.ProjectCoefficient(uExact);
+
+      ParLinearForm *b = new ParLinearForm(fespace);
+      FunctionCoefficient ffunc(f_exact);
+      b->AddDomainIntegrator(new DomainLFIntegrator(ffunc));
+      b->Assemble();
+
+      x = 0.0;
+
+      // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+      //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+      //     domain integrator.
+
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+      //    FunctionCoefficient *cspe10 = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      Coefficient *beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DiffusionIntegrator);
+      a->AddDomainIntegrator(new MassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreSolver *amg = new HypreBoomerAMG(A);
+
+      int iter = -1;
+      if (standardCG)
+      {
+         IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+         pcg->SetOperator(A);
+         pcg->SetRelTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetPrintLevel(1);
+         pcg->SetPreconditioner(*amg);
+         pcg->Mult(B, X);
+
+         iter = pcg->GetNumIterations();
+
+         delete pcg;
+      }
+      else
+      {
+         HyprePCG *pcg = new HyprePCG(A);
+         pcg->SetTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetResidualConvergenceOptions(1,tol);
+         pcg->SetPrintLevel(2);
+         pcg->SetPreconditioner(*amg);
+         pcg->Mult(B, X);
+
+         pcg->GetNumIterations(iter);
+
+         delete pcg;
+      }
+
+
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      {
+         double err = x.ComputeL2Error(uExact);
+         if (myid == 0)
+         {
+            cout << "\n|| u - u_h ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 14. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //   {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //   }
+
+      // 15. Send the solution by socket to a GLVis server.
+      //   if (visualization)
+      //   {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //   }
+
+      delete amg;
+      delete a;
+      delete beta;
+      delete b;
+   }
+
+   // 16. Free the used memory.
+
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
@@ -79,14 +79,14 @@ class GradT : public Operator
 {
 public:
   GradT() : Operator(1) {}
-   void Mult(const Vector &x, Vector &y) const override { y.Set(1.0/m_, x); }
+   void Mult(const Vector &x, Vector &y) const { y.Set(1.0/m_, x); }
 };

 class NegGradV : public TimeDependentOperator
 {
 public:
   NegGradV() : TimeDependentOperator(1) {}
-   void Mult(const Vector &x, Vector &y) const override;
+   void Mult(const Vector &x, Vector &y) const;
 };

 int main(int argc, char *argv[])
@@ -84,14 +84,14 @@ class GradT : public Operator
 {
 public:
   GradT() : Operator(1) {}
-   void Mult(const Vector &x, Vector &y) const override { y.Set(1.0/m_, x); }
+   void Mult(const Vector &x, Vector &y) const { y.Set(1.0/m_, x); }
 };

 class NegGradV : public TimeDependentOperator
 {
 public:
   NegGradV() : TimeDependentOperator(1) {}
-   void Mult(const Vector &x, Vector &y) const override;
+   void Mult(const Vector &x, Vector &y) const;
 };

 int main(int argc, char *argv[])
@@ -206,7 +206,7 @@ int main(int argc, char *argv[])
   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
   ComplexLinearForm b(fespace, conv);
-   b = 0.0;
+   b.Vector::operator=(0.0);

   // 8. Define the solution vector u as a complex finite element grid function
   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
@@ -235,7 +235,7 @@ int main(int argc, char *argv[])
   // 9. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
   ParComplexLinearForm b(fespace, conv);
-   b = 0.0;
+   b.Vector::operator=(0.0);

   // 10. Define the solution vector u as a parallel complex finite element grid
   //     function corresponding to fespace. Initialize u with initial guess of
@@ -44,7 +44,7 @@ protected:
   BilinearForm *M;
   BilinearForm *K;

-   SparseMatrix Mmat, Kmat;
+   SparseMatrix Mmat, Kmat, Kmat0;
   SparseMatrix *T; // T = M + dt K
   real_t current_dt;

@@ -61,20 +61,20 @@ public:
   WaveOperator(FiniteElementSpace &f, Array<int> &ess_bdr, real_t speed);

   using SecondOrderTimeDependentOperator::Mult;
-   void Mult(const Vector &u, const Vector &du_dt,
-             Vector &d2udt2) const override;
+   virtual void Mult(const Vector &u, const Vector &du_dt,
+                     Vector &d2udt2) const;

   /** Solve the Backward-Euler equation:
       d2udt2 = f(u + fac0*d2udt2,dudt + fac1*d2udt2, t),
       for the unknown d2udt2. */
   using SecondOrderTimeDependentOperator::ImplicitSolve;
-   void ImplicitSolve(const real_t fac0, const real_t fac1,
-                      const Vector &u, const Vector &dudt, Vector &d2udt2) override;
+   virtual void ImplicitSolve(const real_t fac0, const real_t fac1,
+                              const Vector &u, const Vector &dudt, Vector &d2udt2);

   ///
   void SetParameters(const Vector &u);

-   ~WaveOperator() override;
+   virtual ~WaveOperator();
 };


@@ -83,24 +83,25 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
   : SecondOrderTimeDependentOperator(f.GetTrueVSize(), (real_t) 0.0),
     fespace(f), M(NULL), K(NULL), T(NULL), current_dt(0.0), z(height)
 {
-   // Assemble Laplace matrix
+   const real_t rel_tol = 1e-8;
+
+   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
   c2 = new ConstantCoefficient(speed*speed);
+
   K = new BilinearForm(&fespace);
   K->AddDomainIntegrator(new DiffusionIntegrator(*c2));
   K->Assemble();

-   // Assemble Mass matrix
+   Array<int> dummy;
+   K->FormSystemMatrix(dummy, Kmat0);
+   K->FormSystemMatrix(ess_tdof_list, Kmat);
+
   M = new BilinearForm(&fespace);
   M->AddDomainIntegrator(new MassIntegrator());
   M->Assemble();
-
-   // Apply Bcs
-   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   K->FormSystemMatrix(ess_tdof_list, Kmat);
   M->FormSystemMatrix(ess_tdof_list, Mmat);

-   // Configure preconditioner
-   const real_t rel_tol = 1e-8;
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(rel_tol);
   M_solver.SetAbsTol(0.0);
@@ -109,13 +110,14 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
   M_solver.SetPreconditioner(M_prec);
   M_solver.SetOperator(Mmat);

-   // Configure solver
   T_solver.iterative_mode = false;
   T_solver.SetRelTol(rel_tol);
   T_solver.SetAbsTol(0.0);
   T_solver.SetMaxIter(100);
   T_solver.SetPrintLevel(0);
   T_solver.SetPreconditioner(T_prec);
+
+   T = NULL;
 }

 void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
@@ -124,11 +126,9 @@ void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
   // Compute:
   //    d2udt2 = M^{-1}*-K(u)
   // for d2udt2
-   K->FullMult(u, z);
+   Kmat.Mult(u, z);
   z.Neg(); // z = -z
-   z.SetSubVector(ess_tdof_list, 0.0);
   M_solver.Mult(z, d2udt2);
-   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }

 void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
@@ -142,11 +142,14 @@ void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
      T = Add(1.0, Mmat, fac0, Kmat);
      T_solver.SetOperator(*T);
   }
-   K->FullMult(u, z);
+   Kmat0.Mult(u, z);
   z.Neg();
-   z.SetSubVector(ess_tdof_list, 0.0);
+
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      z[ess_tdof_list[i]] = 0.0;
+   }
   T_solver.Mult(z, d2udt2);
-   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }

 void WaveOperator::SetParameters(const Vector &u)
@@ -201,7 +204,9 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  SecondOrderODESolver::Types.c_str());
+                  "ODE solver: [0--10] - GeneralizedAlpha(0.1 * s),\n\t"
+                  "\t   11 - Average Acceleration, 12 - Linear Acceleration\n"
+                  "\t   13 - CentralDifference, 14 - FoxGoodwin");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -236,7 +241,32 @@ int main(int argc, char *argv[])

   // 3. Define the ODE solver used for time integration. Several second order
   //    time integrators are available.
-   SecondOrderODESolver *ode_solver= SecondOrderODESolver::Select(ode_solver_type);
+   SecondOrderODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit methods
+      case 0: ode_solver = new GeneralizedAlpha2Solver(0.0); break;
+      case 1: ode_solver = new GeneralizedAlpha2Solver(0.1); break;
+      case 2: ode_solver = new GeneralizedAlpha2Solver(0.2); break;
+      case 3: ode_solver = new GeneralizedAlpha2Solver(0.3); break;
+      case 4: ode_solver = new GeneralizedAlpha2Solver(0.4); break;
+      case 5: ode_solver = new GeneralizedAlpha2Solver(0.5); break;
+      case 6: ode_solver = new GeneralizedAlpha2Solver(0.6); break;
+      case 7: ode_solver = new GeneralizedAlpha2Solver(0.7); break;
+      case 8: ode_solver = new GeneralizedAlpha2Solver(0.8); break;
+      case 9: ode_solver = new GeneralizedAlpha2Solver(0.9); break;
+      case 10: ode_solver = new GeneralizedAlpha2Solver(1.0); break;
+
+      case 11: ode_solver = new AverageAccelerationSolver(); break;
+      case 12: ode_solver = new LinearAccelerationSolver(); break;
+      case 13: ode_solver = new CentralDifferenceSolver(); break;
+      case 14: ode_solver = new FoxGoodwinSolver(); break;
+
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -284,6 +314,7 @@ int main(int argc, char *argv[])
         ess_bdr = 0;
      }
   }
+
   WaveOperator oper(fespace, ess_bdr, speed);

   u_gf.SetFromTrueDofs(u);
@@ -103,8 +103,8 @@ public:

   using VectorCoefficient::Eval;

-   void Eval(Vector &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
   {
      real_t x[3];
      Vector transip(x, 3);
@@ -347,7 +347,7 @@ int main(int argc, char *argv[])
   {
      b.AddDomainIntegrator(NULL, new VectorFEDomainLFIntegrator(f));
   }
-   b = 0.0;
+   b.Vector::operator=(0.0);
   b.Assemble();

   // 11. Define the solution vector x as a complex finite element grid function
@@ -102,8 +102,8 @@ public:

   using VectorCoefficient::Eval;

-   void Eval(Vector &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
   {
      real_t x[3];
      Vector transip(x, 3);
@@ -392,7 +392,7 @@ int main(int argc, char *argv[])
   {
      b.AddDomainIntegrator(NULL, new VectorFEDomainLFIntegrator(f));
   }
-   b = 0.0;
+   b.Vector::operator=(0.0);
   b.Assemble();

   // 13. Define the solution vector x as a parallel complex finite element grid
@@ -58,7 +58,7 @@ public:
      }
   }

-   ~DiffusionMultigrid() override
+   virtual ~DiffusionMultigrid()
   {
      delete amg;
   }
@@ -318,7 +318,7 @@ int main(int argc, char *argv[])
   // 10. Set up the parallel linear form b(.) which corresponds to the
   //     right-hand side of the FEM linear system.
   ParComplexLinearForm b(&fespace, conv);
-   b = 0.0;
+   b.Vector::operator=(0.0);

   // 11a. Define the solution vector u as a parallel complex finite element
   //      grid function corresponding to fespace. Initialize u to equal zero.
@@ -53,7 +53,7 @@ public:
                                    real_t min_val_=-36)
      : u(&u_), obstacle(&obst_), min_val(min_val_) { }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 class ExponentialGridFunctionCoefficient : public Coefficient
@@ -69,7 +69,7 @@ public:
                                      real_t min_val_=0.0, real_t max_val_=1e6)
      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 int main(int argc, char *argv[])
@@ -258,7 +258,7 @@ int main(int argc, char *argv[])
         MixedBilinearForm a10(&H1fes,&L2fes);
         a10.AddDomainIntegrator(new MixedScalarMassIntegrator());
         a10.Assemble();
-         a10.EliminateTrialEssentialBC(ess_bdr, x.GetBlock(0), rhs.GetBlock(1));
+         a10.EliminateTrialDofs(ess_bdr, x.GetBlock(0), rhs.GetBlock(1));
         a10.Finalize();
         SparseMatrix &A10 = a10.SpMat();

@@ -53,7 +53,7 @@ public:
                                    real_t min_val_=-36)
      : u(&u_), obstacle(&obst_), min_val(min_val_) { }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 class ExponentialGridFunctionCoefficient : public Coefficient
@@ -69,7 +69,7 @@ public:
                                      real_t min_val_=0.0, real_t max_val_=1e6)
      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
 };

 int main(int argc, char *argv[])
@@ -52,8 +52,8 @@ public:
       fun(fun_) {}


-   real_t Eval(ElementTransformation &T,
-               const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
   {
      return fun(GridFunctionCoefficient::Eval(T, ip));
   }
@@ -83,8 +83,8 @@ public:
       OtherGridF_cf(OtherGridF),
       fun(fun_) {}

-   real_t Eval(ElementTransformation &T,
-               const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
   {
      const real_t value1 = fun(GridFunctionCoefficient::Eval(T, ip));
      const real_t value2 = fun(OtherGridF_cf.Eval(T, ip));
@@ -108,7 +108,7 @@ public:
      : rho_filter(rho_filter_), min_val(min_val_), max_val(max_val_),
        exponent(exponent_) { }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip)
   {
      real_t val = rho_filter->GetValue(T, ip);
      real_t coeff = min_val + pow(val,exponent)*(max_val-min_val);
@@ -142,7 +142,7 @@ public:
      MFEM_ASSERT(rho_filter, "density field is not set");
   }

-   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override
+   virtual real_t Eval(ElementTransformation &T, const IntegrationPoint &ip)
   {
      real_t L = lambda->Eval(T, ip);
      real_t M = mu->Eval(T, ip);
@@ -176,8 +176,8 @@ public:

   using VectorCoefficient::Eval;

-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip)
   {
      Vector xx; xx.SetSize(T.GetDimension());
      T.Transform(ip,xx);
@@ -3,18 +3,18 @@
 // Compile with: make ex38
 //
 // Sample runs:
-// (since all sample runs require LAPACK or ALGOIM, the * symbol is used to
-//  exclude them from the automatically generated internal MFEM tests).
+// (since all sample runs require LAPACK, the * symbol is used to exclude them
+// from the automatically generated internal MFEM tests).
 //              * ex38
 //              * ex38 -i volumetric1d
 //              * ex38 -i surface2d
-//              * ex38 -i surface2d -o 4 -r 5 -m 1
+//              * ex38 -i surface2d -o 4 -r 5
 //              * ex38 -i volumetric2d
-//              * ex38 -i volumetric2d -o 4 -r 5 -m 1
+//              * ex38 -i volumetric2d -o 4 -r 5
 //              * ex38 -i surface3d
-//              * ex38 -i surface3d -o 3 -r 4 -m 1
+//              * ex38 -i surface3d -o 4 -r 5
 //              * ex38 -i volumetric3d
-//              * ex38 -i volumetric3d -o 3 -r 4 -m 1
+//              * ex38 -i volumetric3d -o 4 -r 5
 //
 // Description: This example code demonstrates the use of MFEM to integrate
 //              functions over implicit interfaces and subdomains bounded by
@@ -71,7 +71,7 @@ real_t integrand(const Vector& X)
   switch (itype)
   {
      case IntegrationType::Volumetric1D:
-         return pow(X(0), 2.);
+         return 1.;
      case IntegrationType::Surface2D:
         return 3. * pow(X(0), 2.) - pow(X(1), 2.);
      case IntegrationType::Volumetric2D:
@@ -91,7 +91,7 @@ real_t Surface()
   switch (itype)
   {
      case IntegrationType::Volumetric1D:
-         return .3025;
+         return 1.;
      case IntegrationType::Surface2D:
         return 2. * M_PI;
      case IntegrationType::Volumetric2D:
@@ -111,7 +111,7 @@ real_t Volume()
   switch (itype)
   {
      case IntegrationType::Volumetric1D:
-         return pow(.55, 3.) / 3.;
+         return .55;
      case IntegrationType::Surface2D:
         return NAN;
      case IntegrationType::Volumetric2D:
@@ -125,6 +125,7 @@ real_t Volume()
   }
 }

+#ifdef MFEM_USE_LAPACK
 /**
 @brief Class for surface IntegrationRule

@@ -134,14 +135,11 @@ real_t Volume()
 class SIntegrationRule : public IntegrationRule
 {
 protected:
-   /// method 0 is moments-based, 1 is Algoim.
-   int method, ir_order, ls_order;
-   Coefficient &level_set;
-   /// Space Dimension of the IntegrationRule
+   /// @brief Space Dimension of the IntegrationRule
   int dim;
-   /// Column-wise matrix of the quadtrature weights
+   /// @brief Column-wise matrix of the quadtrature weights
   DenseMatrix Weights;
-   /// Column-wise matrix of the transformation weights of the normal
+   /// @brief Column-wise matrix of the transformation weights of the normal
   DenseMatrix SurfaceWeights;

 public:
@@ -155,21 +153,15 @@ public:
    @param [in] lsOrder Polynomial degree for approx of level-set function
    @param [in] mesh Pointer to the mesh that is used
   */
-   SIntegrationRule(int method_, int Order,
-                    Coefficient& LvlSet, int lsOrder, Mesh* mesh)
-      : method(method_), ir_order(Order), ls_order(lsOrder),
-        level_set(LvlSet), dim(mesh->Dimension())
+   SIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
   {
-      // Nothing gets pre-computed for Algoim.
-      if (method == 1) { return; }
-
-#ifdef MFEM_USE_LAPACK
-      MomentFittingIntRules mf_ir(ir_order, level_set, ls_order);
+      dim = mesh->Dimension();

      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
      mesh->GetElementTransformation(0, &Tr);
      IntegrationRule ir;
-      mf_ir.GetSurfaceIntegrationRule(Tr, ir);
+      MFIRs.GetSurfaceIntegrationRule(Tr, ir);
      if (dim >1)
      {
         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
@@ -180,7 +172,7 @@ public:
      }
      SurfaceWeights.SetSize(ir.GetNPoints(), mesh->GetNE());
      Vector w;
-      mf_ir.GetSurfaceWeights(Tr, ir, w);
+      MFIRs.GetSurfaceWeights(Tr, ir, w);
      SurfaceWeights.SetCol(0, w);
      SetSize(ir.GetNPoints());

@@ -206,8 +198,8 @@ public:
      for (int elem = 1; elem < mesh->GetNE(); elem++)
      {
         mesh->GetElementTransformation(elem, &Tr);
-         mf_ir.GetSurfaceIntegrationRule(Tr, ir);
-         mf_ir.GetSurfaceWeights(Tr, ir, w);
+         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+         MFIRs.GetSurfaceWeights(Tr, ir, w);
         SurfaceWeights.SetCol(elem, w);

         for (int ip = 0; ip < GetNPoints(); ip++)
@@ -223,48 +215,48 @@ public:
            }
         }
      }
-#else
-      MFEM_ABORT("Moment-fitting requires MFEM to be built with LAPACK!");
-#endif
   }

   /**
    @brief Set the weights for the given element and multiply them with the
    transformation of the interface
    */
-   void SetElementAndSurfaceWeight(ElementTransformation &Tr)
+   void SetElementinclSurfaceWeight(int Element)
   {
-      if (method == 1)
-      {
-#ifdef MFEM_USE_ALGOIM
-         AlgoimIntegrationRules a_ir(ir_order, level_set, ls_order);
-         a_ir.GetSurfaceIntegrationRule(Tr, *this);
-         Vector w;
-         a_ir.GetSurfaceWeights(Tr, *this, w);
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntPoint(ip).weight *= w(ip);
-         }
-         return;
-#else
-         MFEM_ABORT("MFEM is not built with Algoim support!");
-#endif
-      }
-
      if (dim == 1)
      {
-         IntPoint(0).x = Weights(0, Tr.ElementNo);
-         IntPoint(0).weight = Weights(1, Tr.ElementNo);
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+         cout << intp.x << " " << Element << endl;
      }
      else
-      {
         for (int ip = 0; ip < GetNPoints(); ip++)
         {
-            IntPoint(ip).weight = Weights(ip, Tr.ElementNo) *
-                                  SurfaceWeights(ip, Tr.ElementNo);
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element) * SurfaceWeights(ip, Element);
         }
-      }
   }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of SIntegrationRule
+   ~SIntegrationRule() {}
 };

 /**
@@ -276,12 +268,9 @@ public:
 class CIntegrationRule : public IntegrationRule
 {
 protected:
-   /// method 0 is moments-based, 1 is Algoim.
-   int method, ir_order, ls_order;
-   Coefficient &level_set;
-   /// Space Dimension of the IntegrationRule
+   /// @brief Space Dimension of the IntegrationRule
   int dim;
-   /// Column-wise matrix of the quadtrature positions and weights.
+   /// @brief Column-wise matrix of the quadtrature weights
   DenseMatrix Weights;

 public:
@@ -295,21 +284,15 @@ public:
    @param [in] lsOrder Polynomial degree for approx of level-set function
    @param [in] mesh Pointer to the mesh that is used
   */
-   CIntegrationRule(int method_, int Order,
-                    Coefficient &LvlSet, int lsOrder, Mesh *mesh)
-      : method(method_), ir_order(Order), ls_order(lsOrder),
-        level_set(LvlSet), dim(mesh->Dimension())
+   CIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
   {
-      // Nothing gets pre-computed for Algoim.
-      if (method == 1) { return; }
-
-#ifdef MFEM_USE_LAPACK
-      MomentFittingIntRules mf_ir(ir_order, level_set, ls_order);
+      dim = mesh->Dimension();

      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
      mesh->GetElementTransformation(0, &Tr);
      IntegrationRule ir;
-      mf_ir.GetVolumeIntegrationRule(Tr, ir);
+      MFIRs.GetVolumeIntegrationRule(Tr, ir);
      if (dim > 1)
      {
         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
@@ -341,9 +324,9 @@ public:
      for (int elem = 1; elem < mesh->GetNE(); elem++)
      {
         mesh->GetElementTransformation(elem, &Tr);
-         mf_ir.GetVolumeIntegrationRule(Tr, ir);
+         MFIRs.GetVolumeIntegrationRule(Tr, ir);

-         for (int ip = 0; ip < ir.GetNPoints(); ip++)
+         for (int ip = 0; ip < GetNPoints(); ip++)
         {
            if (dim > 1)
            {
@@ -356,39 +339,29 @@ public:
            }
         }
      }
-#else
-      MFEM_ABORT("Moment-fitting requires MFEM to be built with LAPACK!");
-#endif
   }

   /// @brief Set the weights for the given element
-   void SetElement(ElementTransformation &Tr)
+   void SetElement(int Element)
   {
-      if (method == 1)
-      {
-#ifdef MFEM_USE_ALGOIM
-         AlgoimIntegrationRules a_ir(ir_order, level_set, ls_order);
-         a_ir.GetVolumeIntegrationRule(Tr, *this);
-         return;
-#else
-         MFEM_ABORT("MFEM is not built with Algoim support!");
-#endif
-      }
-
-      for (int ip = 0; ip < GetNPoints(); ip++)
-      {
-         IntegrationPoint &intp = IntPoint(ip);
-         if (dim == 1)
+      if (dim == 1)
+         for (int ip = 0; ip < GetNPoints(); ip++)
         {
-            intp.x = Weights(2 * ip, Tr.ElementNo);
-            intp.weight = Weights(2 * ip + 1, Tr.ElementNo);
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.x = Weights(2 * ip, Element);
+            intp.weight = Weights(2 * ip + 1, Element);
+         }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
         }
-         else { intp.weight = Weights(ip, Tr.ElementNo); }
-      }
   }
+
+   /// @brief Destructor of CIntegrationRule
+   ~CIntegrationRule() {}
 };
-
-
 /**
 @brief Class for surface linearform integrator

@@ -435,9 +408,9 @@ public:
    @param [in] Tr transformation of finite element
    @param [out] elvect vector containing the
   */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &Tr,
-                               Vector &elvect) override
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
   {
      int dof = el.GetDof();
      shape.SetSize(dof);
@@ -445,7 +418,7 @@ public:
      elvect = 0.;

      // Update the surface integration rule for the current element
-      SIntRule->SetElementAndSurfaceWeight(Tr);
+      SIntRule->SetElementinclSurfaceWeight(Tr.ElementNo);

      for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
      {
@@ -455,8 +428,6 @@ public:
         add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
      }
   }
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
 };

 /**
@@ -505,9 +476,9 @@ public:
    @param [in] Tr transformation of finite element
    @param [out] elvect vector containing the
   */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &Tr,
-                               Vector &elvect) override
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
   {
      int dof = el.GetDof();
      shape.SetSize(dof);
@@ -515,7 +486,7 @@ public:
      elvect = 0.;

      // Update the subdomain integration rule
-      CIntRule->SetElement(Tr);
+      CIntRule->SetElement(Tr.ElementNo);

      for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
      {
@@ -526,17 +497,18 @@ public:
         add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
      }
   }
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
 };
+#endif // MFEM_USE_LAPACK

 int main(int argc, char *argv[])
 {
-#if defined(MFEM_USE_LAPACK) || defined(MFEM_USE_ALGOIM)
+#ifndef MFEM_USE_LAPACK
+   cout << "MFEM must be built with LAPACK for this example." << endl;
+   return MFEM_SKIP_RETURN_VALUE;
+#else
   // 1. Parse he command-line options.
   int ref_levels = 3;
   int order = 2;
-   int method = 0;
   const char *inttype = "surface2d";
   bool visualization = true;
   itype = IntegrationType::Surface2D;
@@ -544,8 +516,6 @@ int main(int argc, char *argv[])
   OptionsParser args(argc, argv);
   args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
   args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
-   args.AddOption(&method, "-m", "--method",
-                  "Cut integration method: 0 for moments-based, 1 for Algoim.");
   args.AddOption(&inttype, "-i", "--integrationtype",
                  "IntegrationType to demonstrate");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -580,7 +550,7 @@ int main(int argc, char *argv[])
   }

   // 2. Construct and refine the mesh.
-   Mesh *mesh = nullptr;
+   Mesh *mesh;
   if (itype == IntegrationType::Volumetric1D)
   {
      mesh = new Mesh("../data/inline-segment.mesh");
@@ -628,14 +598,13 @@ int main(int argc, char *argv[])
   // 5. Define the necessary Integration rules on element 0.
   IsoparametricTransformation Tr;
   mesh->GetElementTransformation(0, &Tr);
-   SIntegrationRule* sir = new SIntegrationRule(method, order,
-                                                levelset, 2, mesh);
+   SIntegrationRule* sir = new SIntegrationRule(order, levelset, 2, mesh);
   CIntegrationRule* cir = NULL;
   if (itype == IntegrationType::Volumetric1D
       || itype == IntegrationType::Volumetric2D
       || itype == IntegrationType::Volumetric3D)
   {
-      cir = new CIntegrationRule(method, order, levelset, 2, mesh);
+      cir = new CIntegrationRule(order, levelset, 2, mesh);
   }

   // 6. Define and assemble the linear forms on the finite element space.
@@ -678,11 +647,11 @@ int main(int argc, char *argv[])
      cout << "Number of div free basis functions: " << nbasis << endl;
      cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
   }
-   cout << scientific << setprecision(10);
+   cout << scientific << setprecision(2);
   cout << "============================================" << endl;
   cout << "Computed value of surface integral: " << surface.Sum() << endl;
   cout << "True value of surface integral:     " << Surface() << endl;
-   cout << "Absolute Error (Surface):           ";
+   cout << "Absolute Error (Surface):            ";
   cout << abs(surface.Sum() - Surface()) << endl;
   cout << "Relative Error (Surface):           ";
   cout << abs(surface.Sum() - Surface()) / Surface() << endl;
@@ -693,7 +662,7 @@ int main(int argc, char *argv[])
      cout << "--------------------------------------------" << endl;
      cout << "Computed value of volume integral:  " << volume.Sum() << endl;
      cout << "True value of volume integral:      " << Volume() << endl;
-      cout << "Absolute Error (Volume):            ";
+      cout << "Absolute Error (Volume):             ";
      cout << abs(volume.Sum() - Volume()) << endl;
      cout << "Relative Error (Volume):            ";
      cout << abs(volume.Sum() - Volume()) / Volume() << endl;
@@ -722,8 +691,5 @@ int main(int argc, char *argv[])
   delete fespace;
   delete mesh;
   return EXIT_SUCCESS;
-#else
-   cout << "MFEM must be built with LAPACK or ALGOIM for this example." << endl;
-   return MFEM_SKIP_RETURN_VALUE;
-#endif // MFEM_USE_LAPACK
+#endif //MFEM_USE_LAPACK
 }
@@ -58,6 +58,180 @@ void f_exact(const Vector &, Vector &);
 real_t freq = 1.0, kappa;
 int dim;

+class Curl4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+
+   HypreParMatrix *idMat;
+   HypreParMatrix *H1VecLaplaceMat;
+   HypreBoomerAMG *amgVecH1;
+
+
+   HypreParMatrix *gradMat;
+   HypreParMatrix *H1LaplaceMat;
+   HypreBoomerAMG *amgH1;
+
+   HypreSmoother * smoother;
+   CGSolver *pcgGrad;
+   CGSolver *pcgH1Vec;
+
+   Vector *f;
+   Vector *fGrad, *uGrad;
+   Vector *fH1Vec, *uH1Vec;
+
+   bool exactSolves;
+
+public:
+   Curl4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+              const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //vecH1  --> H(curl)
+      int orderKer=orderKernel; //grad V --> H(curl)
+
+      smoother = new HypreSmoother(*A, 16, 3);
+
+      //    //for the pure dirichlet case
+      //    Array<int> essBnd(pmesh->bdr_attributes.Max()); essBnd = 1;
+
+      Array<int> HCurl_essDof(fespace->GetVSize()); HCurl_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HCurl_essDof);
+
+      //setup the H1 FESpace
+      FiniteElementCollection* fecH1;
+      if (orderKer==1) { fecH1 = new LinearFECollection; }
+      else { fecH1 = new QuadraticFECollection; }
+
+      ParFiniteElementSpace *H1FESpace = new ParFiniteElementSpace(pmesh, fecH1);
+      Array<int> H1_essDof(H1FESpace->GetVSize()); H1_essDof = 0;
+      H1FESpace->GetEssentialVDofs(essBnd, H1_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disGrad = new ParDiscreteLinearOperator(H1FESpace,
+                                                                         fespace);
+      disGrad->AddDomainInterpolator(new GradientInterpolator);
+      disGrad->Assemble();
+      disGrad->Finalize();
+      SparseMatrix* smat = &(disGrad->SpMat());
+      smat->EliminateCols(H1_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smat->EliminateRow(dof); }
+      gradMat = disGrad->ParallelAssemble();
+      delete disGrad;
+
+      //setup the H1 preconditioner
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1FESpace);
+      H1Varf->AddDomainIntegrator(new DiffusionIntegrator);
+      H1Varf->AddDomainIntegrator(new MassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1_essDof.Size(); dof++) if (H1_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1LaplaceMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1 = new HypreBoomerAMG(*H1LaplaceMat);
+
+
+      //setup the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1VecFESpace = new ParFiniteElementSpace(pmesh, fecH1Vec,
+                                                                      dim, Ordering::byVDIM);
+      Array<int> H1Vec_essDof(H1VecFESpace->GetVSize()); H1Vec_essDof = 0;
+      H1VecFESpace->GetEssentialVDofs(essBnd, H1Vec_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1VecFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Vec_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      idMat = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the H1-vec preconditioner
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1VecFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator);
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator);
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Vec_essDof.Size(); dof++) if (H1Vec_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1VecLaplaceMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgVecH1 = new HypreBoomerAMG(*H1VecLaplaceMat);
+      amgVecH1->SetSystemsOptions(dim);
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fGrad = new Vector(H1FESpace->GetTrueVSize());
+      uGrad = new Vector(H1FESpace->GetTrueVSize());
+
+      fH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+      uH1Vec = new Vector(H1VecFESpace->GetTrueVSize());;
+
+
+      amgH1->Mult(*fGrad, *uGrad);
+      amgVecH1->Mult(*fH1Vec, *uH1Vec);
+
+      pcgGrad = new CGSolver(MPI_COMM_WORLD);
+      pcgGrad->SetOperator(*H1LaplaceMat);
+      pcgGrad->SetPreconditioner(*amgH1);
+      pcgGrad->SetRelTol(1e-16);
+      pcgGrad->SetMaxIter(100000000);
+      pcgGrad->SetPrintLevel(-2);
+
+      pcgH1Vec = new CGSolver(MPI_COMM_WORLD);
+      pcgH1Vec->SetOperator(*H1VecLaplaceMat);
+      pcgH1Vec->SetPreconditioner(*amgVecH1);
+      pcgH1Vec->SetRelTol(1e-16);
+      pcgH1Vec->SetMaxIter(100000000);
+      pcgH1Vec->SetPrintLevel(-2);
+
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smoother->Mult(x,y);
+
+      idMat->MultTranspose(x,*fH1Vec);
+      *uH1Vec = 0.0;
+      if (exactSolves) { pcgH1Vec->Mult(*fH1Vec, *uH1Vec); }
+      else { amgVecH1->Mult(*fH1Vec, *uH1Vec); }
+      idMat->Mult(1.0, *uH1Vec, 1.0, y);
+
+      gradMat->MultTranspose(x,*fGrad);
+      *uGrad = 0.0;
+      if (exactSolves) { pcgGrad->Mult(*fGrad, *uGrad); }
+      else { amgH1->Mult(*fGrad, *uGrad); }
+      gradMat->Mult(1.0, *uGrad, 1.0, y);
+
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
@@ -162,7 +336,13 @@ int main(int argc, char *argv[])

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementCollection *fec;
+   if (dim==4)
+   {
+      if (order==1) { fec = new ND1_4DFECollection; }
+      else { fec = new ND2_4DFECollection; }
+   }
+   else { fec = new ND_FECollection(order, dim); }
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
   HYPRE_BigInt size = fespace->GlobalTrueVSize();
   if (myid == 0)
@@ -248,15 +428,29 @@ int main(int argc, char *argv[])

      ParFiniteElementSpace *prec_fespace =
         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      Solver *prec;
      HypreAMS ams(*A.As<HypreParMatrix>(), prec_fespace);
-      HyprePCG pcg(*A.As<HypreParMatrix>());
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(ams);
-      pcg.Mult(B, X);
+      if (dim <= 3)
+      {
+         prec = new HypreAMS(*A.As<HypreParMatrix>(), prec_fespace);
+      }
+      else if (dim == 4)
+      {
+         prec = new Curl4dPrec(A.As<HypreParMatrix>(), fespace, ess_bdr, order, false);
+      }
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(*A.As<HypreParMatrix>());
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete prec;
   }

+   // 13. Recover the parallel grid function corresponding to X. This is the
   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);
@@ -312,7 +506,14 @@ int main(int argc, char *argv[])

 void E_exact(const Vector &x, Vector &E)
 {
-   if (dim == 3)
+   if (dim==4)
+   {
+      E(0) =  sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(1) = -cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(2) =  cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(M_PI*x(3));
+      E(3) = -cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(M_PI*x(3));
+   }
+   else if (dim == 3)
   {
      E(0) = sin(kappa * x(1));
      E(1) = sin(kappa * x(2));
@@ -328,7 +529,19 @@ void E_exact(const Vector &x, Vector &E)

 void f_exact(const Vector &x, Vector &f)
 {
-   if (dim == 3)
+   //f_exact = E +  DivSkew P( curl E ), where P is the 4d permutation operator
+   if (dim==4)
+   {
+      f(0) =  (1.0+4.0*M_PI*M_PI)*sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(1) = -(1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                M_PI*x(3));
+      f(2) =  (1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(3) = -(1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(
+                M_PI*x(3));
+   }
+   else if (dim == 3)
   {
      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
@@ -0,0 +1,650 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_curl.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_curl.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa = 1.0;
+int dim;
+
+double osziCoeff(const Vector &x)
+{
+   return 1.0001 + sin(100*x(0))*sin(200*x(1))*sin(300*x(2))*sin(400*x(3));
+}
+
+class Curl4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+   Coefficient *alpha_, *beta_, *neg_beta_;
+
+   HypreParMatrix *idMat;
+   HypreParMatrix *H1VecLaplaceMat;
+   HypreBoomerAMG *amgVecH1;
+
+
+   HypreParMatrix *gradMat;
+   HypreParMatrix *H1LaplaceMat;
+   HypreBoomerAMG *amgH1;
+
+   HypreSmoother * smoother;
+   CGSolver *pcgGrad;
+   CGSolver *pcgH1Vec;
+
+   Vector *f;
+   Vector *fGrad, *uGrad;
+   Vector *fH1Vec, *uH1Vec;
+
+   bool exactSolves;
+
+public:
+   ~Curl4dPrec()
+   {
+      delete pcgH1Vec;
+      delete pcgGrad;
+
+      delete   f, fGrad, uGrad, fH1Vec, uH1Vec;
+
+      delete smoother;
+
+      delete amgVecH1, H1VecLaplaceMat;
+      delete idMat;
+      delete amgH1, H1LaplaceMat;
+      delete gradMat;
+   }
+
+   Curl4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+              Coefficient *alpha, Coefficient *beta, Coefficient *neg_beta,
+              const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+      neg_beta_=neg_beta;
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //vecH1  --> H(curl)
+      int orderKer=orderKernel; //grad V --> H(curl)
+
+      smoother = new HypreSmoother(*A, 16, 3);
+
+      //    //for the pure dirichlet case
+      //    Array<int> essBnd(pmesh->bdr_attributes.Max()); essBnd = 1;
+
+      Array<int> HCurl_essDof(fespace->GetVSize()); HCurl_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HCurl_essDof);
+
+      //setup the H1 FESpace
+      FiniteElementCollection* fecH1;
+      if (orderKer==1) { fecH1 = new LinearFECollection; }
+      else { fecH1 = new QuadraticFECollection; }
+
+      ParFiniteElementSpace *H1FESpace = new ParFiniteElementSpace(pmesh, fecH1);
+      Array<int> H1_essDof(H1FESpace->GetVSize()); H1_essDof = 0;
+      H1FESpace->GetEssentialVDofs(essBnd, H1_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disGrad = new ParDiscreteLinearOperator(H1FESpace,
+                                                                         fespace);
+      disGrad->AddDomainInterpolator(new GradientInterpolator);
+      disGrad->Assemble();
+      disGrad->Finalize();
+      SparseMatrix* smat = &(disGrad->SpMat());
+      smat->EliminateCols(H1_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smat->EliminateRow(dof); }
+      gradMat = disGrad->ParallelAssemble();
+      delete disGrad;
+
+      //setup the H1 preconditioner
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1FESpace);
+      H1Varf->AddDomainIntegrator(new DiffusionIntegrator(*beta_));
+      //    H1Varf->AddDomainIntegrator(new MassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1_essDof.Size(); dof++) if (H1_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1LaplaceMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1 = new HypreBoomerAMG(*H1LaplaceMat);
+
+
+      //setup the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1VecFESpace = new ParFiniteElementSpace(pmesh, fecH1Vec,
+                                                                      dim, Ordering::byVDIM);
+      Array<int> H1Vec_essDof(H1VecFESpace->GetVSize()); H1Vec_essDof = 0;
+      H1VecFESpace->GetEssentialVDofs(essBnd, H1Vec_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1VecFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Vec_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      idMat = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the H1-vec preconditioner
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1VecFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_));
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator(*neg_beta_));
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Vec_essDof.Size(); dof++) if (H1Vec_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1VecLaplaceMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgVecH1 = new HypreBoomerAMG(*H1VecLaplaceMat);
+      amgVecH1->SetSystemsOptions(dim);
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fGrad = new Vector(H1FESpace->GetTrueVSize());
+      uGrad = new Vector(H1FESpace->GetTrueVSize());
+
+      fH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+      uH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+
+
+      amgH1->Mult(*fGrad, *uGrad);
+      amgVecH1->Mult(*fH1Vec, *uH1Vec);
+
+      pcgGrad = new CGSolver(MPI_COMM_WORLD);
+      pcgGrad->SetOperator(*H1LaplaceMat);
+      pcgGrad->SetPreconditioner(*amgH1);
+      pcgGrad->SetRelTol(1e-16);
+      pcgGrad->SetMaxIter(100000000);
+      pcgGrad->SetPrintLevel(-2);
+
+      pcgH1Vec = new CGSolver(MPI_COMM_WORLD);
+      pcgH1Vec->SetOperator(*H1VecLaplaceMat);
+      pcgH1Vec->SetPreconditioner(*amgVecH1);
+      pcgH1Vec->SetRelTol(1e-16);
+      pcgH1Vec->SetMaxIter(100000000);
+      pcgH1Vec->SetPrintLevel(-2);
+
+      delete H1FESpace; delete fecH1;
+      delete H1VecFESpace; delete fecH1Vec;
+
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smoother->Mult(x,y);
+
+      idMat->MultTranspose(x,*fH1Vec);
+      *uH1Vec = 0.0;
+      if (exactSolves) { pcgH1Vec->Mult(*fH1Vec, *uH1Vec); }
+      else { amgVecH1->Mult(*fH1Vec, *uH1Vec); }
+      idMat->Mult(1.0, *uH1Vec, 1.0, y);
+
+      gradMat->MultTranspose(x,*fGrad);
+      *uGrad = 0.0;
+      if (exactSolves) { pcgGrad->Mult(*fGrad, *uGrad); }
+      else { amgH1->Mult(*fGrad, *uGrad); }
+      gradMat->Mult(1.0, *uGrad, 1.0, y);
+
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool exactH1Solver = false;
+   bool spe10Coeff = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   kappa = freq * M_PI;
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+
+   if (dim !=4 || sdim != 4)
+   {
+      MPI_Finalize();
+      return 0;
+   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->ReorientTetMesh();
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (dim==4)
+   {
+      if (order==1) { fec = new ND1_4DFECollection; }
+      else { fec = new ND2_4DFECollection; }
+   }
+   else { fec = new ND_FECollection(order, dim); }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      VectorFunctionCoefficient f(sdim, f_exact);
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      x.ProjectCoefficient(E);
+
+      // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+      //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+      //     mass domain integrators.
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+      Coefficient *neg_beta = new ConstantCoefficient(-weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new CurlCurlIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      Solver *prec;
+      if (dim<=3) { prec = new HypreAMS(A, prec_fespace); }
+      else if (dim==4) { prec = new Curl4dPrec(&A, fespace, alpha, beta, neg_beta, ess_bdr, order, false); }
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(A);
+      pcg->SetRelTol(tol);
+      pcg->SetMaxIter(5000);
+      pcg->SetPrintLevel(1);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      int iter = pcg->GetNumIterations();
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double err = x.ComputeL2Error(E);
+         if (myid == 0)
+         {
+            cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 15. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //   {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //   }
+
+      //   // 16. Send the solution by socket to a GLVis server.
+      //   if (visualization)
+      //   {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //   }
+
+      delete pcg;
+      delete prec;
+      delete a;
+      delete alpha;
+      delete beta;
+      delete b;
+
+   }
+
+   // 17. Free the used memory.
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim==4)
+   {
+      E(0) =  sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(1) = -cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(2) =  cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(M_PI*x(3));
+      E(3) = -cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(M_PI*x(3));
+   }
+   else if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   //f_exact = E +  DivSkew P( curl E ), where P is the 4d permutation operator
+   if (dim==4)
+   {
+      f(0) =  (kappa+4.0*M_PI*M_PI)*sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(1) = -(kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                M_PI*x(3));
+      f(2) =  (kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(3) = -(kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(
+                M_PI*x(3));
+   }
+   else if (dim == 3)
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}
@@ -67,10 +67,8 @@ public:
   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }

-   using VectorCoefficient::Eval;
-
-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
   void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };

@@ -84,8 +82,8 @@ public:
   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }

-   void Eval(DenseMatrix &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
   void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };

@@ -67,10 +67,8 @@ public:
   ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }

-   using VectorCoefficient::Eval;
-
-   void Eval(Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
   void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };

@@ -84,8 +82,8 @@ public:
   DZCoefficient(int height, ParGridFunction &psi_, real_t alpha_ = 1.0)
      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }

-   void Eval(DenseMatrix &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
   void SetAlpha(real_t alpha_) { alpha = alpha_; }
 };

@@ -0,0 +1,782 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_DivSkew.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_DivSkew.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact_vec(const Vector &x, Vector &E);
+void E_exact(const Vector &, DenseMatrix &);
+void f_exact(const Vector &, DenseMatrix &);
+
+
+class DivSkew4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+   Coefficient *alpha_, *beta_;
+
+   //kernel operators
+   HypreParMatrix *P_d_HCurl_HDivSkew;
+
+
+   HypreParMatrix *P_H1_HCurl;
+   HypreParMatrix *H1_KernelMat;
+   HypreBoomerAMG *amgH1_Kernel;
+
+   //"image" operators
+   HypreParMatrix *P_H1_HDivSkew;
+   HypreParMatrix *H1_ImageMat;
+   HypreBoomerAMG *amgH1_Image;
+
+
+   HypreParMatrix *HCurlMat;
+   HypreSmoother * smootherDivSkew;
+   HypreSmoother * smootherCurl;
+
+   CGSolver *pcgKernel;
+   CGSolver *pcgImage;
+
+   Vector *f;
+   Vector *fKernel, *uKernel;
+   Vector *fImage, *uImage;
+   Vector *fCurl, *uCurl;
+
+   bool exactSolves;
+
+   FiniteElementCollection* fecHCurlKernel;
+   ParFiniteElementSpace *HCurlKernelFESpace;
+
+
+public:
+   ~DivSkew4dPrec()
+   {
+      delete pcgImage, pcgKernel;
+
+      delete f, fKernel, uKernel, fImage, uImage, fCurl, uCurl;
+
+      delete smootherCurl, HCurlMat;
+
+      delete P_d_HCurl_HDivSkew, P_H1_HDivSkew, P_H1_HCurl;
+
+      delete amgH1_Image, H1_ImageMat;
+      delete amgH1_Kernel, H1_KernelMat;
+
+      delete smootherDivSkew;
+
+      delete HCurlKernelFESpace, fecHCurlKernel;
+   }
+   DivSkew4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+                 Coefficient *alpha, Coefficient *beta,
+                 const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //H1  --> H(divSkew)
+      int orderKer=orderKernel; //curl V --> H(divSkew)
+
+      smootherDivSkew = new HypreSmoother(*A, 16, 3);
+
+      Array<int> HDivSkew_essDof(fespace->GetVSize()); HDivSkew_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HDivSkew_essDof);
+
+      //setup the H1 FESpace for the kernel
+      FiniteElementCollection* fecH1Kernel = new H1_FECollection(orderKer, 4);
+
+      ParFiniteElementSpace *H1KernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Kernel, dim, Ordering::byVDIM);
+      Array<int> H1Kernel_essDof(H1KernelFESpace->GetVSize()); H1Kernel_essDof = 0;
+      H1KernelFESpace->GetEssentialVDofs(essBnd, H1Kernel_essDof);
+
+
+      //setup the H(curl) FESpace for the kernel
+      if (orderKer==1) { fecHCurlKernel = new ND1_4DFECollection; }
+      else { fecHCurlKernel = new ND2_4DFECollection; }
+
+      HCurlKernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                     fecHCurlKernel);
+      Array<int> HCurlKernel_essDof(HCurlKernelFESpace->GetVSize());
+      HCurlKernel_essDof = 0;
+      HCurlKernelFESpace->GetEssentialVDofs(essBnd, HCurlKernel_essDof);
+
+
+      //setup the FESpace for the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1_ImageFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Vec, 6, Ordering::byVDIM);
+      Array<int> H1Image_essDof(H1_ImageFESpace->GetVSize()); H1Image_essDof = 0;
+      H1_ImageFESpace->GetEssentialVDofs(essBnd, H1Image_essDof);
+
+
+
+      //setup the H1 preconditioner for the kernel
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1KernelFESpace);
+      H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*beta_));
+      //      H1Varf->AddDomainIntegrator(new VectorMassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof = 0; dof < H1Kernel_essDof.Size(); dof++)
+         if (H1Kernel_essDof[dof] < 0)
+         {
+            matH1.EliminateRowCol(dof);
+         }
+      H1_KernelMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1_Kernel = new HypreBoomerAMG(*H1_KernelMat);
+      amgH1_Kernel->SetSystemsOptions(dim);
+      amgH1_Kernel->SetPrintLevel(0);
+
+      //setup the H1 preconditioner for the image
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1_ImageFESpace);
+      VectorDiffusionIntegrator *alpha_integ = new VectorDiffusionIntegrator(*alpha_);
+      alpha_integ->SetVDim(6);
+      H1VecVarf->AddDomainIntegrator(alpha_integ);
+      VectorMassIntegrator *beta_integ = new VectorMassIntegrator(*beta);
+      beta_integ->SetVDim(6);
+      H1VecVarf->AddDomainIntegrator(beta_integ);
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Image_essDof.Size(); dof++) if (H1Image_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1_ImageMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgH1_Image = new HypreBoomerAMG(*H1_ImageMat);
+      amgH1_Image->SetSystemsOptions(6);
+      amgH1_Image->SetPrintLevel(0);
+
+
+      //setup the injection of H1 into H(curl)
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1KernelFESpace, HCurlKernelFESpace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Kernel_essDof);
+      for (int dof=0; dof<HCurlKernel_essDof.Size();
+           dof++) if (HCurlKernel_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      P_H1_HCurl = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the injection of H1 into H(DivSkew)
+      ParDiscreteLinearOperator *disInterpolIm = new ParDiscreteLinearOperator(
+         H1_ImageFESpace, fespace);
+      disInterpolIm->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpolIm->Assemble();
+      disInterpolIm->Finalize();
+      SparseMatrix* smatIDIm = &(disInterpolIm->SpMat());
+      smatIDIm->EliminateCols(H1Image_essDof);
+      for (int dof=0; dof<HDivSkew_essDof.Size(); dof++) if (HDivSkew_essDof[dof]<0) { smatIDIm->EliminateRow(dof); }
+      P_H1_HDivSkew = disInterpolIm->ParallelAssemble();
+      delete disInterpolIm;
+
+
+      //setup the injection of the curl(H(curl)) into H(DivSkew)
+      ParDiscreteLinearOperator *disCurl = new ParDiscreteLinearOperator(
+         HCurlKernelFESpace, fespace);
+      disCurl->AddDomainInterpolator(new CurlInterpolator);
+      disCurl->Assemble();
+      disCurl->Finalize();
+      SparseMatrix* smatCurl = &(disCurl->SpMat());
+      smatCurl->EliminateCols(HCurlKernel_essDof);
+      for (int dof=0; dof<HDivSkew_essDof.Size(); dof++) if (HDivSkew_essDof[dof]<0) { smatCurl->EliminateRow(dof); }
+      P_d_HCurl_HDivSkew = disCurl->ParallelAssemble();
+      delete disCurl;
+
+      //setup the smoother for H(curl)
+      //      Coefficient *massC = new ConstantCoefficient(1.0);
+      //      Coefficient *CurlCurlC = new ConstantCoefficient(1.0);
+      ParBilinearForm *a_HCurl = new ParBilinearForm(HCurlKernelFESpace);
+      a_HCurl->AddDomainIntegrator(new CurlCurlIntegrator(*beta_));
+      //      a_HCurl->AddDomainIntegrator(new CurlCurlIntegrator(*CurlCurlC));
+      //      a_HCurl->AddDomainIntegrator(new VectorFEMassIntegrator(*massC));
+      a_HCurl->Assemble();
+      a_HCurl->Finalize();
+      SparseMatrix &matHCurl(a_HCurl->SpMat());
+      for (int dof=0; dof<HCurlKernel_essDof.Size();
+           dof++) if (HCurlKernel_essDof[dof]<0) { matHCurl.EliminateRowCol(dof); }
+      HCurlMat = a_HCurl->ParallelAssemble();
+      delete a_HCurl;
+      smootherCurl = new HypreSmoother(*HCurlMat, 16, 3);
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+      uKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+
+      fImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+      uImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+
+      fCurl = new Vector(HCurlKernelFESpace->GetTrueVSize());
+      uCurl = new Vector(HCurlKernelFESpace->GetTrueVSize());
+
+
+      amgH1_Kernel->Mult(*fKernel, *uKernel);
+      amgH1_Image->Mult(*fImage, *uImage);
+
+      pcgKernel = new CGSolver(MPI_COMM_WORLD);
+      pcgKernel->SetOperator(*H1_KernelMat);
+      pcgKernel->SetPreconditioner(*amgH1_Kernel);
+      pcgKernel->SetRelTol(1e-16);
+      pcgKernel->SetMaxIter(100000000);
+      pcgKernel->SetPrintLevel(-2);
+
+      pcgImage = new CGSolver(MPI_COMM_WORLD);
+      pcgImage->SetOperator(*H1_ImageMat);
+      pcgImage->SetPreconditioner(*amgH1_Image);
+      pcgImage->SetRelTol(1e-16);
+      pcgImage->SetMaxIter(100000000);
+      pcgImage->SetPrintLevel(-2);
+
+      delete H1KernelFESpace, fecH1Kernel;
+      delete H1_ImageFESpace, fecH1Vec;
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smootherDivSkew->Mult(x,y);
+
+      P_H1_HDivSkew->MultTranspose(x,*fImage);
+      *uImage = 0.0;
+      if (exactSolves) { pcgImage->Mult(*fImage, *uImage); }
+      else { amgH1_Image->Mult(*fImage, *uImage); }
+      P_H1_HDivSkew->Mult(1.0, *uImage, 1.0, y);
+
+      *uCurl = 0.0;
+      P_d_HCurl_HDivSkew->MultTranspose(x,*fCurl);
+
+      smootherCurl->Mult(*fCurl, *uCurl);
+
+      P_H1_HCurl->MultTranspose(*fCurl,*fKernel);
+      *uKernel = 0.0;
+      if (exactSolves) { pcgKernel->Mult(*fKernel, *uKernel); }
+      else { amgH1_Kernel->Mult(*fKernel, *uKernel); }
+      P_H1_HCurl->Mult(1.0, *uKernel, 1.0, *uCurl);
+
+      P_d_HCurl_HDivSkew->Mult(1.0, *uCurl, 1.0, y);
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool exactH1Solver = false;
+   bool spe10Coeff = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   if (dim !=4 || sdim != 4)
+   {
+      MPI_Finalize();
+      return 0;
+   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (order==1) { fec = new DivSkew1_4DFECollection; }
+   //    else fec = new F2K1_4DFECollection;
+
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   fespace->SetUpdateOperatorType(Operator::Hypre_ParCSR);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   MatrixFunctionCoefficient f(sdim, f_exact);
+   MatrixFunctionCoefficient solMat(sdim, E_exact);
+   VectorFunctionCoefficient solVec(6, E_exact_vec);
+
+
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+
+      x.ProjectCoefficient(solVec);
+
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new MatFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      //   cout << x << endl;
+      //   x = 0.0;
+
+      // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+      //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+      //     mass domain integrators.
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFE_DivSkewMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      //Define the preconditioner
+
+      if (myid == 0) { cout << "Set up the preconditioner" << endl; }
+      Solver *prec;
+      if (dim==4) { prec = new DivSkew4dPrec(&A, fespace, alpha, beta, ess_bdr, order, exactH1Solver); }
+
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(A);
+      pcg->SetRelTol(tol);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(1);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      delete prec;
+
+      int iter = pcg->GetNumIterations();
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double error = 0.0;
+         for (int i = 0; i < fespace->GetNE(); i++)
+         {
+            const FiniteElement* fe = fespace->GetFE(i);
+            int fdof = fe->GetDof();
+            ElementTransformation* transf = fespace->GetElementTransformation(i);
+            DenseMatrix shape(fdof,dim*dim);
+
+            int intorder = 2*fe->GetOrder() + 1; // <----------
+            const IntegrationRule *ir;
+            ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+
+            Vector elSol(dim*dim);
+            DenseMatrix elSolMat(dim,dim);
+            DenseMatrix exactSol(dim,dim);
+            Vector exactSolVec(dim*dim);
+
+
+
+            Array<int> vdofs;
+            fespace->GetElementVDofs(i, vdofs);
+            for (int j = 0; j < ir->GetNPoints(); j++)
+            {
+               const IntegrationPoint &ip = ir->IntPoint(j);
+               transf->SetIntPoint(&ip);
+
+               fe->CalcVShape(*transf, shape);
+
+               elSol = 0.0;
+               for (int k = 0; k < fdof; k++)
+               {
+                  if (vdofs[k] >= 0)
+                  {
+                     for (int l=0; l<dim*dim; l++) { elSol(l) += shape(k,l)*x(vdofs[k]); }
+                  }
+                  else
+                  {
+                     for (int l=0; l<dim*dim; l++) { elSol(l) -= shape(k,l)*x(-1-vdofs[k]); }
+                  }
+               }
+               for (int k=0; k<dim; k++)
+                  for (int l=0; l<dim; l++)
+                  {
+                     elSolMat(k,l) = elSol(dim*k+l);
+                  }
+
+
+               solMat.Eval(exactSol,*transf, ip);
+               for (int k=0; k<dim; k++)
+                  for (int l=0; l<dim; l++)
+                  {
+                     exactSolVec(dim*k+l) = exactSol(k,l);
+                  }
+               elSol.Add(-1.0, exactSolVec);
+
+               error += ip.weight * fabs(transf->Weight()) * (elSol * elSol);
+            }
+         }
+         double globalError = 0.0;
+         MPI_Allreduce(&error, &globalError, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+         if (myid==0) { std::cout << "L2 error: " << sqrt(globalError) << std::endl; }
+
+
+      }
+
+      delete pcg;
+      delete a;
+      delete alpha;
+      delete beta;
+
+      delete b;
+   }
+
+   // 17. Free the used memory.
+
+
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+void E_exact_vec(const Vector &x, Vector &E)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      E.SetSize(6);
+
+      double s0 = sin(M_PI*x(0)), s1 = sin(M_PI*x(1)), s2 = sin(M_PI*x(2)),
+             s3 = sin(M_PI*x(3));
+      double c0 = cos(M_PI*x(0)), c1 = cos(M_PI*x(1)), c2 = cos(M_PI*x(2)),
+             c3 = cos(M_PI*x(3));
+
+      E(0) =  c0*c1*s2*s3;
+      E(1) = -c0*s1*c2*s3;
+      E(2) =  c0*s1*s2*c3;
+      E(3) =  s0*c1*c2*s3;
+      E(4) = -s0*c1*s2*c3;
+      E(5) =  s0*s1*c2*c3;
+   }
+}
+
+void E_exact(const Vector &x, DenseMatrix &E)
+{
+   int dim = x.Size();
+
+   E.SetSize(dim*dim);
+
+   if (dim==4)
+   {
+      Vector vecE; E_exact_vec(x, vecE);
+
+      E = 0.0;
+
+      E(0,1) = vecE(0);
+      E(0,2) = vecE(1);
+      E(0,3) = vecE(2);
+      E(1,2) = vecE(3);
+      E(1,3) = vecE(4);
+      E(2,3) = vecE(5);
+
+      E(1,0) =  -E(0,1);
+      E(2,0) =  -E(0,2);
+      E(3,0) =  -E(0,3);
+      E(2,1) =  -E(1,2);
+      E(3,1) =  -E(1,3);
+      E(3,2) =  -E(2,3);
+   }
+}
+
+
+
+//f_exact = E + 0.5 * P( curl DivSkew E ), where P is the 4d permutation operator
+void f_exact(const Vector &x, DenseMatrix &f)
+{
+   int dim = x.Size();
+
+   f.SetSize(dim,dim);
+
+   if (dim==4)
+   {
+      f = 0.0;
+
+      double s0 = sin(M_PI*x(0)), s1 = sin(M_PI*x(1)), s2 = sin(M_PI*x(2)),
+             s3 = sin(M_PI*x(3));
+      double c0 = cos(M_PI*x(0)), c1 = cos(M_PI*x(1)), c2 = cos(M_PI*x(2)),
+             c3 = cos(M_PI*x(3));
+
+      f(0,1) =  (1.0 + 1.0  * M_PI*M_PI)*c0*c1*s2*s3;
+      f(0,2) = -(1.0 + 0.0  * M_PI*M_PI)*c0*s1*c2*s3;
+      f(0,3) =  (1.0 + 1.0  * M_PI*M_PI)*c0*s1*s2*c3;
+      f(1,2) =  (1.0 - 1.0  * M_PI*M_PI)*s0*c1*c2*s3;
+      f(1,3) = -(1.0 + 0.0  * M_PI*M_PI)*s0*c1*s2*c3;
+      f(2,3) =  (1.0 + 1.0  * M_PI*M_PI)*s0*s1*c2*c3;
+
+      f(1,0) =  -f(0,1);
+      f(2,0) =  -f(0,2);
+      f(3,0) =  -f(0,3);
+      f(2,1) =  -f(1,2);
+      f(3,1) =  -f(1,3);
+      f(3,2) =  -f(2,3);
+   }
+}
@@ -0,0 +1,800 @@
+//                       MFEM Example 4 - Parallel Version
+//
+// Compile with: make ex4p
+//
+// Sample runs:  mpirun -np 4 ex4p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex4p -m ../data/star.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/beam-hex-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
+//
+// Description:  This example code solves a simple 2D/3D H(div) diffusion
+//               problem corresponding to the second order definite equation
+//               -grad(alpha div F) + beta F = f with boundary condition F dot n
+//               = <given normal field>. Here, we use a given exact solution F
+//               and compute the corresponding r.h.s. f.  We discretize with
+//               Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of H(div) finite element
+//               spaces with the grad-div and H(div) vector finite element mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Bilinear form
+//               hybridization and static condensation are also illustrated.
+//
+//               We recommend viewing examples 1-3 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_div.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_div.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+void F_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+
+
+
+class div4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+
+   Coefficient *alpha_, *beta_;
+
+   //kernel operators
+   HypreParMatrix *P_d_HSkewDiv_Hdiv;
+
+   HypreParMatrix *P_H1_HDivSkew;
+   HypreParMatrix *H1_KernelMat;
+   HypreBoomerAMG *amgH1_Kernel;
+
+   //"image" operators
+   HypreParMatrix *P_H1_Hdiv;
+   HypreParMatrix *H1_ImageMat;
+   HypreBoomerAMG *amgH1_Image;
+
+   HypreParMatrix *HDivSkewMat;
+   HypreSmoother * smootherdiv;
+   HypreSmoother * smootherDivSkew;
+
+   CGSolver *pcgKernel;
+   CGSolver *pcgImage;
+
+   Vector *f;
+   Vector *fKernel, *uKernel;
+   Vector *fImage, *uImage;
+   Vector *fDivSkew, *uDivSkew;
+
+   FiniteElementCollection* fecHDivSkewKernel;
+   ParFiniteElementSpace *HDivSkewKernelFESpace;
+
+   bool exactSolves;
+
+public:
+   ~div4dPrec()
+   {
+      delete pcgImage;
+      delete pcgKernel;
+
+      delete uDivSkew, fDivSkew, uImage, fImage, uKernel, fKernel, f;
+
+      delete smootherDivSkew;
+      delete HDivSkewMat;
+
+      delete P_d_HSkewDiv_Hdiv;
+      delete P_H1_Hdiv;
+      delete P_H1_HDivSkew;
+
+      delete amgH1_Image, H1_ImageMat;
+      delete amgH1_Kernel, H1_KernelMat;
+
+      delete smootherdiv;
+
+      delete HDivSkewKernelFESpace;
+      delete fecHDivSkewKernel;
+   }
+   div4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+             Coefficient *alpha, Coefficient *beta, const Array<int> &essBnd,
+             int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+
+
+
+      int orderIm=1;  //H1  --> H(div)
+      int orderKer=orderKernel; //DivSkew V --> H(div)
+
+
+
+      smootherdiv = new HypreSmoother(*A, 16, 3);
+
+      Array<int> Hdiv_essDof(fespace->GetVSize()); Hdiv_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, Hdiv_essDof);
+
+
+
+
+      //setup the H1 FESpace for the kernel
+      FiniteElementCollection* fecH1Kernel;
+      if (orderKer==1) { fecH1Kernel = new LinearFECollection; }
+      else { fecH1Kernel = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1KernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Kernel, 6, Ordering::byVDIM);
+      Array<int> H1Kernel_essDof(H1KernelFESpace->GetVSize()); H1Kernel_essDof = 0;
+      H1KernelFESpace->GetEssentialVDofs(essBnd, H1Kernel_essDof);
+
+
+      //setup the H(DivSkew) FESpace for the kernel
+      if (orderKer==1) { fecHDivSkewKernel = new DivSkew1_4DFECollection; }
+      //       else fecHDivSkewKernel = new DivSkewFull1_4DFECollection;
+      HDivSkewKernelFESpace = new ParFiniteElementSpace(pmesh, fecHDivSkewKernel);
+      Array<int> HDivSkewKernel_essDof(HDivSkewKernelFESpace->GetVSize());
+      HDivSkewKernel_essDof = 0;
+      HDivSkewKernelFESpace->GetEssentialVDofs(essBnd, HDivSkewKernel_essDof);
+
+
+      //setup the FESpace for the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1_ImageFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Vec, dim, Ordering::byVDIM);
+      Array<int> H1Image_essDof(H1_ImageFESpace->GetVSize()); H1Image_essDof = 0;
+      H1_ImageFESpace->GetEssentialVDofs(essBnd, H1Image_essDof);
+
+
+
+      //setup the H1 preconditioner for the kernel
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1KernelFESpace);
+      //    H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_, 6));
+      //    H1Varf->AddDomainIntegrator(new VectorMassIntegrator(6, beta_));
+
+      H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*beta_, 6));
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1Kernel_essDof.Size(); dof++) if (H1Kernel_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1_KernelMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1_Kernel = new HypreBoomerAMG(*H1_KernelMat);
+      amgH1_Kernel->SetSystemsOptions(6);
+
+      //setup the H1 preconditioner for the image
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1_ImageFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_));
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator(-1, beta_));
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Image_essDof.Size(); dof++) if (H1Image_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1_ImageMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgH1_Image = new HypreBoomerAMG(*H1_ImageMat);
+      amgH1_Image->SetSystemsOptions(dim);
+
+
+      //setup the injection of H1 into H(DivSkew)
+      ParDiscreteLinearOperator *disInterpolIm = new ParDiscreteLinearOperator(
+         H1KernelFESpace, HDivSkewKernelFESpace);
+      disInterpolIm->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpolIm->Assemble();
+      disInterpolIm->Finalize();
+      SparseMatrix* smatIDIm = &(disInterpolIm->SpMat());
+      smatIDIm->EliminateCols(H1Kernel_essDof);
+      for (int dof=0; dof<HDivSkewKernel_essDof.Size();
+           dof++) if (HDivSkewKernel_essDof[dof]<0) { smatIDIm->EliminateRow(dof); }
+      P_H1_HDivSkew = disInterpolIm->ParallelAssemble();
+      delete disInterpolIm;
+
+      //setup the injection of H1 into H(div)
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1_ImageFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Image_essDof);
+      for (int dof=0; dof<Hdiv_essDof.Size(); dof++) if (Hdiv_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      P_H1_Hdiv = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+
+
+
+      //setup the injection of the DivSkew(H(DivSkew)) into H(div)
+      ParDiscreteLinearOperator *disDivSkew = new ParDiscreteLinearOperator(
+         HDivSkewKernelFESpace, fespace);
+      disDivSkew->AddDomainInterpolator(new DivSkewInterpolator);
+      disDivSkew->Assemble();
+      disDivSkew->Finalize();
+      SparseMatrix* smatDivSkew= &(disDivSkew->SpMat());
+      smatDivSkew->EliminateCols(HDivSkewKernel_essDof);
+      for (int dof=0; dof<Hdiv_essDof.Size(); dof++) if (Hdiv_essDof[dof]<0) { smatDivSkew->EliminateRow(dof); }
+      P_d_HSkewDiv_Hdiv = disDivSkew->ParallelAssemble();
+      delete disDivSkew;
+
+
+      //setup the smoother for H(DivSkew)
+      ParBilinearForm *a_HDivSkew = new ParBilinearForm(HDivSkewKernelFESpace);
+      //    a_HDivSkew->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*alpha_));
+      //    a_HDivSkew->AddDomainIntegrator(new VectorFE_DivSkewMassIntegrator(*beta_));
+
+      a_HDivSkew->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*beta_));
+
+      a_HDivSkew->Assemble();
+      a_HDivSkew->Finalize();
+      SparseMatrix &matHDivSkew(a_HDivSkew->SpMat());
+      for (int dof=0; dof<HDivSkewKernel_essDof.Size();
+           dof++) if (HDivSkewKernel_essDof[dof]<0) { matHDivSkew.EliminateRowCol(dof); }
+      HDivSkewMat = a_HDivSkew->ParallelAssemble();
+      delete a_HDivSkew;
+      smootherDivSkew = new HypreSmoother(*HDivSkewMat, 16, 3);
+
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+      uKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+
+      fImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+      uImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+
+      fDivSkew = new Vector(HDivSkewKernelFESpace->GetTrueVSize());
+      uDivSkew = new Vector(HDivSkewKernelFESpace->GetTrueVSize());
+
+      amgH1_Kernel->Mult(*fKernel, *uKernel);
+      amgH1_Image->Mult(*fImage, *uImage);
+
+      pcgKernel = new CGSolver(MPI_COMM_WORLD);
+      pcgKernel->SetOperator(*H1_KernelMat);
+      pcgKernel->SetPreconditioner(*amgH1_Kernel);
+      pcgKernel->SetRelTol(1e-16);
+      pcgKernel->SetMaxIter(100000000);
+      pcgKernel->SetPrintLevel(-2);
+
+      pcgImage = new CGSolver(MPI_COMM_WORLD);
+      pcgImage->SetOperator(*H1_ImageMat);
+      pcgImage->SetPreconditioner(*amgH1_Image);
+      pcgImage->SetRelTol(1e-16);
+      pcgImage->SetMaxIter(100000000);
+      pcgImage->SetPrintLevel(-2);
+
+      delete H1_ImageFESpace;
+      delete H1KernelFESpace;
+      delete fecH1Kernel;
+      delete fecH1Vec;
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smootherdiv->Mult(x,y);
+
+      P_H1_Hdiv->MultTranspose(x,*fImage);
+      *uImage = 0.0;
+      if (exactSolves) { pcgImage->Mult(*fImage, *uImage); }
+      else { amgH1_Image->Mult(*fImage, *uImage); }
+      P_H1_Hdiv->Mult(1.0, *uImage, 1.0, y);
+
+
+      *uDivSkew = 0.0;
+      P_d_HSkewDiv_Hdiv->MultTranspose(x,*fDivSkew);
+
+      smootherDivSkew->Mult(*fDivSkew, *uDivSkew);
+
+      P_H1_HDivSkew->MultTranspose(*fDivSkew,*fKernel);
+      *uKernel = 0.0;
+      if (exactSolves) { pcgKernel->Mult(*fKernel, *uKernel); }
+      else { amgH1_Kernel->Mult(*fKernel, *uKernel); }
+      P_H1_HDivSkew->Mult(1.0, *uKernel, 1.0, *uDivSkew);
+
+      P_d_HSkewDiv_Hdiv->Mult(1.0, *uDivSkew, 1.0, y);
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool hybridization = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool spe10Coeff = false;
+   bool exactH1Solver = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
+                  "--no-hybridization", "Enable hybridization.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume, as well as periodic meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      for (int l = 0; l < sequ_ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Raviart-Thomas finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (dim==4) { fec = new RT0_4DFECollection; }
+   else { fec = new RT_FECollection(order-1, dim); }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary faces will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient F(sdim, F_exact);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      x.ProjectCoefficient(F);
+
+      VectorFunctionCoefficient f(sdim, f_exact);
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      // 10. Set up the parallel bilinear form corresponding to the H(div)
+      //     diffusion operator grad alpha div + beta I, by adding the div-div and
+      //     the mass domain integrators.
+
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation,
+      //     hybridization, etc.
+      FiniteElementCollection *hfec = NULL;
+      ParFiniteElementSpace *hfes = NULL;
+      if (static_cond)
+      {
+         a->EnableStaticCondensation();
+      }
+      else if (hybridization)
+      {
+         hfec = new DG_Interface_FECollection(order-1, dim);
+         hfes = new ParFiniteElementSpace(pmesh, hfec);
+         a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
+                                ess_tdof_list);
+      }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      HYPRE_Int glob_size = A.GetGlobalNumRows();
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << glob_size << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
+      //     the 3D ADS preconditioners from hypre. If using hybridization, the
+      //     system is preconditioned with hypre's BoomerAMG.
+      Solver *prec = NULL;
+      if (hybridization) { prec = new HypreBoomerAMG(A); }
+      else
+      {
+         ParFiniteElementSpace *prec_fespace =
+            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+         if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
+         else if (dim==3) { prec = new HypreADS(A, prec_fespace); }
+         else if (dim==4) { prec = new div4dPrec(&A, fespace, alpha, beta, ess_bdr, order, exactH1Solver); }
+         else { prec = NULL; }
+      }
+
+      int iter = -1;
+      if (standardCG)
+      {
+         IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+         pcg->SetOperator(A);
+         pcg->SetRelTol(tol);
+         pcg->SetMaxIter(500);
+         pcg->SetPrintLevel(1);
+         pcg->SetPreconditioner(*prec);
+         pcg->Mult(B, X);
+
+         iter = pcg->GetNumIterations();
+
+         delete pcg;
+      }
+      else
+      {
+         HyprePCG *pcg = new HyprePCG(A);
+         pcg->SetTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetResidualConvergenceOptions(1,tol);
+         pcg->SetPrintLevel(2);
+         //          pcg->SetPreconditioner(*prec);
+         pcg->Mult(B, X);
+
+         pcg->GetNumIterations(iter);
+
+         delete pcg;
+      }
+
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double err = x.ComputeL2Error(F);
+         if (myid == 0)
+         {
+            cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 15. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //    {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //    }
+
+      // 16. Send the solution by socket to a GLVis server.
+      //    if (visualization)
+      //    {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //    }
+
+      if (prec!=NULL) { delete prec; }
+      delete hfes;
+      delete hfec;
+      delete a;
+      delete alpha;
+      delete beta;
+
+      delete b;
+   }
+
+   // 17. Free the used memory.
+
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   if (dim==4)
+   {
+      double s0 = sin(M_PI*p(0)), s1 = sin(M_PI*p(1)), s2 = sin(M_PI*p(2)),
+             s3 = sin(M_PI*p(3));
+      double c0 = cos(M_PI*p(0)), c1 = cos(M_PI*p(1)), c2 = cos(M_PI*p(2)),
+             c3 = cos(M_PI*p(3));
+
+      F(0) = c0 * s1 * s2 * s3;
+      F(1) = s0 * c1 * s2 * s3;
+      F(2) = s0 * s1 * c2 * s3;
+      F(3) = s0 * s1 * s2 * c3;
+   }
+   else
+   {
+      double x = p(0);
+      double y = p(1);
+      // double z = (dim == 3) ? p(2) : 0.0;
+
+      F(0) = cos(kappa*x)*sin(kappa*y);
+      F(1) = cos(kappa*y)*sin(kappa*x);
+      if (dim == 3)
+      {
+         F(2) = 0.0;
+      }
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+   if (dim==4)
+   {
+      double s0 = sin(M_PI*p(0)), s1 = sin(M_PI*p(1)), s2 = sin(M_PI*p(2)),
+             s3 = sin(M_PI*p(3));
+      double c0 = cos(M_PI*p(0)), c1 = cos(M_PI*p(1)), c2 = cos(M_PI*p(2)),
+             c3 = cos(M_PI*p(3));
+
+      f(0) = c0 * s1 * s2 * s3;
+      f(1) = s0 * c1 * s2 * s3;
+      f(2) = s0 * s1 * c2 * s3;
+      f(3) = s0 * s1 * s2 * c3;
+
+      f *= (kappa + 4.0 * M_PI*M_PI);
+   }
+   else
+   {
+      double x = p(0);
+      double y = p(1);
+      // double z = (dim == 3) ? p(2) : 0.0;
+
+      double temp = 1 + 2*kappa*kappa;
+
+      f(0) = temp*cos(kappa*x)*sin(kappa*y);
+      f(1) = temp*cos(kappa*y)*sin(kappa*x);
+      if (dim == 3)
+      {
+         f(2) = 0;
+      }
+   }
+}
@@ -157,7 +157,7 @@ int main(int argc, char *argv[])
   MixedBilinearForm *B0 = new MixedBilinearForm(x0_space,test_space);
   B0->AddDomainIntegrator(new DiffusionIntegrator(one));
   B0->Assemble();
-   B0->EliminateTrialEssentialBC(ess_bdr, x.GetBlock(x0_var), F);
+   B0->EliminateTrialDofs(ess_bdr, x.GetBlock(x0_var), F);
   B0->Finalize();

   MixedBilinearForm *Bhat = new MixedBilinearForm(xhat_space,test_space);
@@ -9,7 +9,7 @@
 //    ex9 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 23 -tf 9
+//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 13 -tf 9
 //    ex9 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
@@ -77,7 +77,7 @@ public:
   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
      : M(M_),
        K(K_),
-        prec(fes.GetTypicalFE()->GetDof(),
+        prec(fes.GetFE(0)->GetDof(),
             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
        dt(-1.0)
   {
@@ -104,12 +104,12 @@ public:
      }
   }

-   void SetOperator(const Operator &op) override
+   void SetOperator(const Operator &op)
   {
      linear_solver.SetOperator(op);
   }

-   void Mult(const Vector &x, Vector &y) const override
+   virtual void Mult(const Vector &x, Vector &y) const
   {
      linear_solver.Mult(x, y);
   }
@@ -134,10 +134,10 @@ private:
 public:
   FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);

-   void Mult(const Vector &x, Vector &y) const override;
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);

-   ~FE_Evolution() override;
+   virtual ~FE_Evolution();
 };


@@ -182,7 +182,12 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - Backward Euler,\n\t"
+                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -219,7 +224,28 @@ int main(int argc, char *argv[])

   // 3. Define the ODE solver used for time integration. Several explicit
   //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // Explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      // Implicit (L-stable) methods
+      case 11: ode_solver = new BackwardEulerSolver; break;
+      case 12: ode_solver = new SDIRK23Solver(2); break;
+      case 13: ode_solver = new SDIRK33Solver; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         return 3;
+   }

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -414,6 +440,7 @@ int main(int argc, char *argv[])
   }

   // 10. Free the used memory.
+   delete ode_solver;
   delete pd;
   delete dc;

@@ -9,7 +9,7 @@
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-hexagon.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9
-//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 23 -tf 9
+//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 13 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-mixed.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
@@ -92,7 +92,7 @@ private:
 public:
   AIR_prec(int blocksize_) : AIR_solver(NULL), blocksize(blocksize_) { }

-   void SetOperator(const Operator &op) override
+   void SetOperator(const Operator &op)
   {
      width = op.Width();
      height = op.Height();
@@ -110,7 +110,7 @@ public:
      AIR_solver->SetMaxLevels(50);
   }

-   void Mult(const Vector &x, Vector &y) const override
+   virtual void Mult(const Vector &x, Vector &y) const
   {
      // Scale the rhs by block inverse and solve system
      HypreParVector z_s;
@@ -119,7 +119,7 @@ public:
      AIR_solver->Mult(z_s, y);
   }

-   ~AIR_prec() override
+   ~AIR_prec()
   {
      delete AIR_solver;
   }
@@ -145,7 +145,7 @@ public:
        linear_solver(M.GetComm()),
        dt(-1.0)
   {
-      int block_size = fes.GetTypicalFE()->GetDof();
+      int block_size = fes.GetFE(0)->GetDof();
      if (prec_type == PrecType::ILU)
      {
         prec = new BlockILU(block_size,
@@ -185,17 +185,17 @@ public:
      }
   }

-   void SetOperator(const Operator &op) override
+   void SetOperator(const Operator &op)
   {
      linear_solver.SetOperator(op);
   }

-   void Mult(const Vector &x, Vector &y) const override
+   virtual void Mult(const Vector &x, Vector &y) const
   {
      linear_solver.Mult(x, y);
   }

-   ~DG_Solver() override
+   ~DG_Solver()
   {
      delete prec;
      delete A;
@@ -223,10 +223,10 @@ public:
   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_,
                PrecType prec_type);

-   void Mult(const Vector &x, Vector &y) const override;
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override;
+   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);

-   ~FE_Evolution() override;
+   virtual ~FE_Evolution();
 };


@@ -285,7 +285,12 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::Types.c_str());
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - Backward Euler,\n\t"
+                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -333,7 +338,31 @@ int main(int argc, char *argv[])

   // 4. Define the ODE solver used for time integration. Several explicit
   //    Runge-Kutta methods are available.
-   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // Explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      // Implicit (L-stable) methods
+      case 11: ode_solver = new BackwardEulerSolver; break;
+      case 12: ode_solver = new SDIRK23Solver(2); break;
+      case 13: ode_solver = new SDIRK33Solver; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         if (Mpi::Root())
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         return 3;
+   }

   // 5. Refine the mesh in serial to increase the resolution. In this example
   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -613,6 +642,7 @@ int main(int argc, char *argv[])
   delete m;
   delete fes;
   delete pmesh;
+   delete ode_solver;
   delete pd;
 #ifdef MFEM_USE_ADIOS2
   if (adios2)
@@ -1,68 +0,0 @@
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI and HYPRE.
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   //Mesh mesh = Mesh::MakeCartesian3D(2, 2 ,2, Element::Type::HEXAHEDRON);
-   Mesh mesh = Mesh::MakeCartesian3D(2, 2, 2, Element::Type::TETRAHEDRON);
-
-   // Build faces and boundary
-   mesh.FinalizeTopology();
-   mesh.Finalize();
-
-   // Changing element and boundary attributes
-   for (int i=0; i<mesh.GetNE(); ++i)
-   {
-      mesh.SetAttribute(i, myid + 1);
-   }
-
-   for (int i=0; i<mesh.GetNBE(); ++i)
-   {
-      mesh.SetBdrAttribute(i, 100);
-   }
-
-   mesh.SetAttributes();
-
-   // Add internal boundary facets used for integrators
-   // TODO: what should be added here?
-
-   // Finalize connectivity and topology (is this even needed?)
-   mesh.FinalizeTopology();
-   mesh.Finalize(true);
-
-   // Make sure mesh is non-conforming
-   mesh.EnsureNCMesh(true);
-
-   // Make parallel mesh
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   pmesh.EnsureNCMesh(true);
-
-   // Refinement
-   Array<Refinement> refinements;
-   refinements.Append(Refinement(0));  // Local element 0 on this rank
-
-   pmesh.GeneralRefinement(refinements);
-   pmesh.SetAttributes();
-
-   {
-      ostringstream mesh_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-   }
-
-   return 0;
-}
@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ginkgo/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -96,7 +96,6 @@ public:
   {
      Vector w_glob(width);
      pfes.Dof_TrueDof_Matrix()->MultTranspose(w, w_glob);
-      w_glob.HostReadWrite(); // read+write -> can use w_glob(i) (non-const)
      for (int i = 0; i < width; i++) { grad(0, i) = w_glob(i); }
   }

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/hiop/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ..
 MFEM_BUILD_DIR ?= ..
-MFEM_INSTALL_DIR ?= ../mfem
 SRC = $(if $(MFEM_DIR:..=),$(MFEM_DIR)/examples/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -26,7 +27,8 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p ex40p
+   ex37p ex39p ex40p \
+   ex1p_4d ex3p_4d ex4D_DivSkew
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
   ex22p ex24p ex25p ex26p ex34p ex35p
@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/moonolith/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -206,7 +206,6 @@ int main(int argc, char *argv[])
   bool use_petsc = true;
   const char *petscrc_file = "";
   bool petsc_use_jfnk = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -244,8 +243,6 @@ int main(int argc, char *argv[])
   args.AddOption(&petsc_use_jfnk, "-jfnk", "--jfnk", "-no-jfnk",
                  "--no-jfnk",
                  "Use JFNK with user-defined preconditioner factory.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -260,12 +257,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
   if (use_petsc)
   {
      MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL);
@@ -67,7 +67,6 @@ int main(int argc, char *argv[])
   bool use_petsc = true;
   const char *petscrc_file = "";
   bool use_nonoverlapping = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -96,8 +95,6 @@ int main(int argc, char *argv[])
                  "-no-nonoverlapping", "--no-nonoverlapping",
                  "Use or not the block diagonal PETSc's matrix format "
                  "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -112,12 +109,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
   if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }

   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
@@ -61,7 +61,6 @@ int main(int argc, char *argv[])
   bool use_petsc = true;
   const char *petscrc_file = "";
   bool use_nonoverlapping = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -88,8 +87,6 @@ int main(int argc, char *argv[])
                  "-no-nonoverlapping", "--no-nonoverlapping",
                  "Use or not the block diagonal PETSc's matrix format "
                  "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -103,15 +100,9 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-   kappa = freq * M_PI;
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
   if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
+   kappa = freq * M_PI;

   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
@@ -58,7 +58,6 @@ int main(int argc, char *argv[])
   bool use_petsc = true;
   const char *petscrc_file = "";
   bool use_nonoverlapping = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -89,8 +88,6 @@ int main(int argc, char *argv[])
                  "-no-nonoverlapping", "--no-nonoverlapping",
                  "Use or not the block diagonal PETSc's matrix format "
                  "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -104,15 +101,9 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-   kappa = freq * M_PI;
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
   if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
+   kappa = freq * M_PI;

   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
@@ -59,8 +59,6 @@ int main(int argc, char *argv[])

   // 2. Parse command-line options.
   const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = -1;
-   int par_ref_levels = 2;
   int order = 1;
   bool par_format = false;
   bool visualization = 1;
@@ -68,22 +66,15 @@ int main(int argc, char *argv[])
   bool use_nonoverlapping = false;
   bool local_bdr_spec = false;
   const char *petscrc_file = "";
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&par_format, "-pf", "--parallel-format", "-sf",
                  "--serial-format",
                  "Format to use when saving the results for VisIt.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -112,13 +103,7 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
   if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }

   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
@@ -132,11 +117,9 @@ int main(int argc, char *argv[])
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 10,000 elements.
   {
-      if (ser_ref_levels < 0)
-      {
-         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      }
-      for (int l = 0; l < ser_ref_levels; l++)
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
      }
@@ -148,6 +131,7 @@ int main(int argc, char *argv[])
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
+      int par_ref_levels = 2;
      for (int l = 0; l < par_ref_levels; l++)
      {
         pmesh->UniformRefinement();
@@ -203,26 +187,21 @@ int main(int argc, char *argv[])

   // 9. Define the parallel grid function and parallel linear forms, solution
   //    vector and rhs.
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
-   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);

   ParLinearForm *fform(new ParLinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);
   fform->ParallelAssemble(trueRhs.GetBlock(0));
-   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);

   ParLinearForm *gform(new ParLinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);
   gform->ParallelAssemble(trueRhs.GetBlock(1));
-   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);

   // 10. Assemble the finite element matrices for the Darcy operator
   //
@@ -53,7 +53,6 @@ int main(int argc, char *argv[])
   bool use_petsc = true;
   const char *petscrc_file = "";
   bool use_nonoverlapping = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -74,8 +73,6 @@ int main(int argc, char *argv[])
                  "-no-nonoverlapping", "--no-nonoverlapping",
                  "Use or not the block diagonal PETSc's matrix format "
                  "for non-overlapping domain decomposition.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -89,13 +86,7 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-
-   // 2b. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 2c. We initialize PETSc
+   // 2b. We initialize PETSc
   if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }

   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/petsc/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/pumi/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -0,0 +1,352 @@
+/*
+ * spe10_coeff.cpp
+ *
+ *  Created on: Aug 23, 2017
+ *      Author: neumueller
+ */
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class InversePermeabilityFunction
+{
+public:
+
+   enum SliceOrientation {NONE, XY, XZ, YZ};
+
+   static void SetNumberCells(int Nx_, int Ny_, int Nz_);
+   static void SetMeshSizes(double hx, double hy, double hz);
+   static void Set2DSlice(SliceOrientation o, int npos );
+
+   static void ReadPermeabilityFile(const std::string fileName);
+#ifdef MFEM_USE_MPI
+   static void ReadPermeabilityFile(const std::string fileName, MPI_Comm comm);
+#endif
+   static void SetConstantInversePermeability(double ipx, double ipy, double ipz);
+
+   template<class F>
+   static void Transform(const F & f)
+   {
+      for (int i = 0; i < 3*Nx*Ny*Nz; ++i)
+      {
+         inversePermeability[i] = f(inversePermeability[i]);
+      }
+   }
+
+   static void InversePermeability(const Vector & x, Vector & val);
+   static double PermeabilityXY(Vector &x);
+   static void NegativeInversePermeability(const Vector & x, Vector & val);
+   static void Permeability(const Vector & x, Vector & val);
+
+   static double Norm2Permeability(const Vector & x);
+
+   static double Norm2InversePermeability(const Vector & x);
+   static double Norm1InversePermeability(const Vector & x);
+   static double NormInfInversePermeability(const Vector & x);
+
+   static double InvNorm2(const Vector & x);
+   static double InvNorm1(const Vector & x);
+   static double InvNormInf(const Vector & x);
+
+
+   static void ClearMemory();
+
+private:
+   static int Nx;
+   static int Ny;
+   static int Nz;
+   static double hx;
+   static double hy;
+   static double hz;
+   static double * inversePermeability;
+
+   static SliceOrientation orientation;
+   static int npos;
+};
+
+
+void InversePermeabilityFunction::SetNumberCells(int Nx_, int Ny_, int Nz_)
+{
+   Nx = Nx_;
+   Ny = Ny_;
+   Nz = Nz_;
+}
+
+void InversePermeabilityFunction::SetMeshSizes(double hx_, double hy_,
+                                               double hz_)
+{
+   hx = hx_;
+   hy = hy_;
+   hz = hz_;
+}
+
+void InversePermeabilityFunction::Set2DSlice(SliceOrientation o, int npos_ )
+{
+   orientation = o;
+   npos = npos_;
+}
+
+void InversePermeabilityFunction::SetConstantInversePermeability(double ipx,
+                                                                 double ipy, double ipz)
+{
+   int compSize = Nx*Ny*Nz;
+   int size = 3*compSize;
+   inversePermeability = new double [size];
+   double *ip = inversePermeability;
+
+   for (int i(0); i < compSize; ++i)
+   {
+      ip[i] = ipx;
+      ip[i+compSize] = ipy;
+      ip[i+2*compSize] = ipz;
+   }
+
+}
+
+#ifdef MFEM_USE_MPI
+void InversePermeabilityFunction::ReadPermeabilityFile(const std::string
+                                                       fileName, MPI_Comm comm)
+{
+   int num_procs, myid;
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   StopWatch chrono;
+
+   chrono.Start();
+   if (myid == 0)
+   {
+      ReadPermeabilityFile(fileName);
+   }
+   else
+   {
+      inversePermeability = new double [3*Nx*Ny*Nz];
+   }
+   chrono.Stop();
+
+   if (myid==0)
+   {
+      std::cout<<"Permeability file read in " << chrono.RealTime() << ".s \n";
+   }
+
+   chrono.Clear();
+
+   chrono.Start();
+   MPI_Bcast(inversePermeability, 3*Nx*Ny*Nz, MPI_DOUBLE, 0, comm);
+   chrono.Stop();
+
+   if (myid==0)
+   {
+      std::cout<<"Permeability field distributed in " << chrono.RealTime() <<
+               ".s \n";
+   }
+
+}
+#endif
+
+void InversePermeabilityFunction::ReadPermeabilityFile(const std::string
+                                                       fileName)
+{
+   std::ifstream permfile(fileName.c_str());
+
+   if (!permfile.is_open())
+   {
+      std::cout << "Error in opening file " << fileName << "\n";
+      mfem_error("File do not exists");
+   }
+
+   inversePermeability = new double [3*Nx*Ny*Nz];
+   double *ip = inversePermeability;
+   double tmp;
+   for (int l = 0; l < 3; l++)
+   {
+      for (int k = 0; k < Nz; k++)
+      {
+         for (int j = 0; j < Ny; j++)
+         {
+            for (int i = 0; i < Nx; i++)
+            {
+               permfile >> *ip;
+               *ip = 1./(*ip);
+               ip++;
+            }
+            for (int i = 0; i < 60-Nx; i++)
+            {
+               permfile >> tmp;   // skip unneeded part
+            }
+         }
+         for (int j = 0; j < 220-Ny; j++)
+            for (int i = 0; i < 60; i++)
+            {
+               permfile >> tmp;   // skip unneeded part
+            }
+      }
+
+      if (l < 2) // if not processing Kz, skip unneeded part
+         for (int k = 0; k < 85-Nz; k++)
+            for (int j = 0; j < 220; j++)
+               for (int i = 0; i < 60; i++)
+               {
+                  permfile >> tmp;
+               }
+   }
+
+}
+
+void InversePermeabilityFunction::InversePermeability(const Vector & x,
+                                                      Vector & val)
+{
+   val.SetSize(3);
+
+   unsigned int i,j,k;
+
+   switch (orientation)
+   {
+      case NONE:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      case XY:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = npos;
+         break;
+      case XZ:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = npos;
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      case YZ:
+         i = npos;
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      default:
+      {
+         mfem_error("InversePermeabilityFunction::InversePermeability");
+      }
+   }
+
+
+   int NMax = 3*Nx*Ny*Nz-1;
+   if (Ny*Nx*k + Nx*j + i>NMax || Ny*Nx*k + Nx*j + i + Nx*Ny*Nz>NMax ||
+       Ny*Nx*k + Nx*j + i + 2*Nx*Ny*Nz>NMax)
+   {
+      cout << " the indicies are wrong!" << endl;
+      cout << i << " " << j << " " << k << endl;
+   }
+
+   val[0] = inversePermeability[Ny*Nx*k + Nx*j + i];
+   val[1] = inversePermeability[Ny*Nx*k + Nx*j + i + Nx*Ny*Nz];
+
+   if (orientation == NONE)
+   {
+      val[2] = inversePermeability[Ny*Nx*k + Nx*j + i + 2*Nx*Ny*Nz];
+   }
+
+}
+
+double InversePermeabilityFunction::PermeabilityXY(Vector &x)
+{
+   unsigned int i,j,k;
+
+   i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+   j = (int)floor(x[1]/hy/(1.+3e-16));
+   k = npos;
+
+   return 1./inversePermeability[Ny*Nx*k + Nx*j + i];
+}
+
+void InversePermeabilityFunction::NegativeInversePermeability(const Vector & x,
+                                                              Vector & val)
+{
+   InversePermeability(x,val);
+   val *= -1.;
+}
+
+
+void InversePermeabilityFunction::Permeability(const Vector & x, Vector & val)
+{
+   InversePermeability(x,val);
+
+   for (double * it = val.GetData(), *end = val.GetData()+val.Size(); it != end;
+        ++it )
+   {
+      (*it) = 1./ (*it);
+   }
+}
+
+double InversePermeabilityFunction::Norm2Permeability(const Vector & x)
+{
+   Vector val(3);
+   Permeability(x,val);
+   return val.Norml2();
+}
+
+
+double InversePermeabilityFunction::Norm2InversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Norml2();
+}
+
+double InversePermeabilityFunction::Norm1InversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Norml1();
+}
+
+double InversePermeabilityFunction::NormInfInversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Normlinf();
+}
+
+double InversePermeabilityFunction::InvNorm2(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Norml2();
+}
+
+double InversePermeabilityFunction::InvNorm1(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Norml1();
+}
+
+double InversePermeabilityFunction::InvNormInf(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Normlinf();
+}
+
+
+void InversePermeabilityFunction::ClearMemory()
+{
+   delete[] inversePermeability;
+}
+
+int InversePermeabilityFunction::Nx(60);
+int InversePermeabilityFunction::Ny(220);
+int InversePermeabilityFunction::Nz(85);
+double InversePermeabilityFunction::hx(20);
+double InversePermeabilityFunction::hy(10);
+double InversePermeabilityFunction::hz(2);
+double * InversePermeabilityFunction::inversePermeability(NULL);
+InversePermeabilityFunction::SliceOrientation
+InversePermeabilityFunction::orientation( InversePermeabilityFunction::NONE );
+int InversePermeabilityFunction::npos(-1);
+
+
+
@@ -31,21 +31,11 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 add_custom_target(test_sundials
  ${CMAKE_CTEST_COMMAND} -R sundials USES_TERMINAL)

-# Add one executable per cpp file, adding "sundials_" as prefix so the CMake
-# target is unique from those in the non-SUNDIALS examples. Also sets
-# "test_sundials" as a target that depends on the given SUNDIALS examples.
+# Add one executable per cpp file, adding "sundials_" as prefix. Sets
+# "test_sundials" as a target that depends on the given examples.
 set(PFX sundials_)
 add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)

-# Remove "sundials_" prefix from exectuable name for consistency with GNU build
-# system.
-foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TARGET_NAME "${PFX}${SRC_FILENAME}")
-  string(REPLACE ${PFX} "" EXE_NAME ${TARGET_NAME})
-  set_target_properties(${TARGET_NAME} PROPERTIES OUTPUT_NAME ${EXE_NAME})
-endforeach()
-
 # Testing.
 # The SUNDIALS tests can be run separately using the target "test_sundials"
 # which builds the examples and runs:
@@ -61,10 +51,7 @@ if (MFEM_ENABLE_TESTING)
  set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
  set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
  set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
-  # Example 16: test ARKODE with implicit time stepping using mass form
-  set(EX16_COMMON_OPTS -s 15)
-  set(EX16_TEST_OPTS ${EX16_COMMON_OPTS})
-  set(EX16P_TEST_OPTS ${EX16_COMMON_OPTS})
+  # Example 16: use the default options

  # Add the tests: one test per source file.
  foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
@@ -1,17 +1,15 @@
 //                                MFEM Example 10
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex10             (GNU make)
-//    make sundials_ex10    (CMake)
+// Compile with: make ex10
 //
 // Sample runs:
 //    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 12 -dt 0.15 -vs 10
 //    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 16 -dt 0.3  -vs 5
 //    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 12 -dt 0.2  -vs 5
-//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  2 -dt 3 -nls 1
-//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  2 -dt 3 -nls 2
-//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  2 -dt 3 -nls 4
+//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  2 -dt 3 -nls kinsol
+//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  2 -dt 3 -nls kinsol
+//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  2 -dt 3 -nls kinsol
 //    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 14 -dt 0.15 -vs 10
 //    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 17 -dt 0.01 -vs 30
 //    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 14 -dt 0.15 -vs 10
@@ -99,11 +97,16 @@ protected:
   double saved_gamma; // saved gamma value from implicit setup

 public:
+   /// Solver type to use in the ImplicitSolve() method, used by SDIRK methods.
+   enum NonlinearSolverType
+   {
+      NEWTON = 0, ///< Use MFEM's plain NewtonSolver
+      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KINSolver)
+   };

   HyperelasticOperator(FiniteElementSpace &f, Array<int> &ess_bdr,
                        double visc, double mu, double K,
-                        int kinsol_nls_type = -1, double kinsol_damping = 0.0,
-                        int kinsol_aa_n = 0);
+                        NonlinearSolverType nls_type);

   /// Compute the right-hand side of the ODE system.
   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
@@ -221,10 +224,8 @@ int main(int argc, char *argv[])
   double mu = 0.25;
   double K = 5.0;
   bool visualization = true;
-   int nonlinear_solver_type = 0;
+   const char *nls = "newton";
   int vis_steps = 1;
-   double kinsol_damping = 0.0;
-   int kinsol_aa_n = -1;

   // Relative and absolute tolerances for CVODE and ARKODE.
   const double reltol = 1e-1, abstol = 1e-1;
@@ -261,18 +262,9 @@ int main(int argc, char *argv[])
                  "15 - ARKODE implicit, approximate Jacobian,\n\t"
                  "16 - ARKODE implicit, specified Jacobian,\n\t"
                  "17 - ARKODE explicit, 4th order.");
-   args.AddOption(&nonlinear_solver_type, "-nls", "--nonlinear-solver",
-                  "Nonlinear system solver:\n\t"
-                  "0  - MFEM Newton method,\n\t"
-                  "1  - KINSOL Newton method,\n\t"
-                  "2  - KINSOL Newton method with globalization,\n\t"
-                  "3  - KINSOL fixed-point method (with or without AA),\n\t"
-                  "4  - KINSOL Picard method (with or without AA).");
-   args.AddOption(&kinsol_damping, "-damp", "--kinsol-damping",
-                  "Picard or Fixed-Point damping parameter (only valid with KINSOL): "
-                  "0 < d <= 1.0");
-   args.AddOption(&kinsol_aa_n, "-aan", "--anderson-subspace",
-                  "Anderson Acceleration subspace size (only valid with KINSOL)");
+   args.AddOption(&nls, "-nls", "--nonlinear-solver",
+                  "Nonlinear systems solver: "
+                  "\"newton\" (plain Newton) or \"kinsol\" (KINSOL).");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -303,32 +295,22 @@ int main(int argc, char *argv[])
      return 1;
   }

-   // check for valid nonlinear solver options
-   if (nonlinear_solver_type < 0 || nonlinear_solver_type > 4)
-   {
-      cout << "Unknown nonlinear solver type: " << nonlinear_solver_type << "\n";
-      return 1;
-   }
-   if (kinsol_damping > 0.0 &&
-       !(nonlinear_solver_type == 3 || nonlinear_solver_type == 4))
-   {
-      cout << "Only KINSOL fixed-point and Picard methods can use damping\n";
-      return 1;
-   }
-   if (kinsol_aa_n > 0 &&
-       !(nonlinear_solver_type == 3 || nonlinear_solver_type == 4))
-   {
-      cout << "Only KINSOL fixed-point and Picard methods can use AA\n";
-      return 1;
-   }
-
-
   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 3. Setup the nonlinear solver
+   map<string,HyperelasticOperator::NonlinearSolverType> nls_map;
+   nls_map["newton"] = HyperelasticOperator::NEWTON;
+   nls_map["kinsol"] = HyperelasticOperator::KINSOL;
+   if (nls_map.find(nls) == nls_map.end())
+   {
+      cout << "Unknown type of nonlinear solver: " << nls << endl;
+      return 4;
+   }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
   //    command-line parameter.
   for (int lev = 0; lev < ref_levels; lev++)
@@ -336,7 +318,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 4. Define the vector finite element spaces representing the mesh
+   // 5. Define the vector finite element spaces representing the mesh
   //    deformation x, the velocity v, and the initial configuration, x_ref.
   //    Define also the elastic energy density, w, which is in a discontinuous
   //    higher-order space. Since x and v are integrated in time as a system,
@@ -364,7 +346,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace w_fespace(mesh, &w_fec);
   GridFunction w(&w_fespace);

-   // 5. Set the initial conditions for v and x, and the boundary conditions on
+   // 6. Set the initial conditions for v and x, and the boundary conditions on
   //    a beam-like mesh (see description above).
   VectorFunctionCoefficient velo(dim, InitialVelocity);
   v.ProjectCoefficient(velo);
@@ -377,34 +359,9 @@ int main(int argc, char *argv[])
   ess_bdr = 0;
   ess_bdr[0] = 1; // boundary attribute 1 (index 0) is fixed

-   // 6. Initialize the hyperelastic operator, the GLVis visualization and print
+   // 7. Initialize the hyperelastic operator, the GLVis visualization and print
   //    the initial energies.
-   std::unique_ptr<HyperelasticOperator> oper;
-   if (nonlinear_solver_type == 0)
-      oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr, visc, mu,
-                                                    K);
-   else
-   {
-      switch (nonlinear_solver_type)
-      {
-         case 1:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_NONE);
-            break;
-         case 2:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_LINESEARCH);
-            break;
-         case 3:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_FP, kinsol_damping, kinsol_aa_n);
-            break;
-         case 4:
-            oper = std::make_unique<HyperelasticOperator>(fespace, ess_bdr,
-                                                          visc, mu, K, KIN_PICARD, kinsol_damping, kinsol_aa_n);
-            break;
-      }
-   }
+   HyperelasticOperator oper(fespace, ess_bdr, visc, mu, K, nls_map[nls]);

   socketstream vis_v, vis_w;
   if (visualization)
@@ -418,23 +375,23 @@ int main(int argc, char *argv[])
      vis_w.open(vishost, visport);
      if (vis_w)
      {
-         oper->GetElasticEnergyDensity(x, w);
+         oper.GetElasticEnergyDensity(x, w);
         vis_w.precision(8);
         visualize(vis_w, mesh, &x, &w, "Elastic energy density", true);
      }
   }

-   double ee0 = oper->ElasticEnergy(x.GetTrueVector());
-   double ke0 = oper->KineticEnergy(v.GetTrueVector());
+   double ee0 = oper.ElasticEnergy(x.GetTrueVector());
+   double ke0 = oper.KineticEnergy(v.GetTrueVector());
   cout << "initial elastic energy (EE) = " << ee0 << endl;
   cout << "initial kinetic energy (KE) = " << ke0 << endl;
   cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;

-   // 7. Define the ODE solver used for time integration. Several implicit
+   // 8. Define the ODE solver used for time integration. Several implicit
   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
   //    explicit Runge-Kutta methods are available.
   double t = 0.0;
-   oper->SetTime(t);
+   oper.SetTime(t);

   ODESolver *ode_solver = NULL;
   CVODESolver *cvode = NULL;
@@ -458,7 +415,7 @@ int main(int argc, char *argv[])
      case 11:
      case 12:
         cvode = new CVODESolver(CV_BDF);
-         cvode->Init(*oper);
+         cvode->Init(oper);
         cvode->SetSStolerances(reltol, abstol);
         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
         cvode->SetMaxStep(dt);
@@ -471,7 +428,7 @@ int main(int argc, char *argv[])
      case 13:
      case 14:
         cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(*oper);
+         cvode->Init(oper);
         cvode->SetSStolerances(reltol, abstol);
         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
         cvode->SetMaxStep(dt);
@@ -484,13 +441,9 @@ int main(int argc, char *argv[])
      case 15:
      case 16:
         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
-         arkode->Init(*oper);
+         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
-#if MFEM_SUNDIALS_VERSION < 70100
         ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-#else
-         ARKodeSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-#endif
         arkode->SetMaxStep(dt);
         if (ode_solver_type == 15)
         {
@@ -500,16 +453,16 @@ int main(int argc, char *argv[])
      // ARKStep Explicit methods
      case 17:
         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(*oper);
+         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
         ode_solver = arkode; break;
   }

   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 11) { ode_solver->Init(*oper); }
+   if (ode_solver_type < 11) { ode_solver->Init(oper); }

-   // 8. Perform time-integration (looping over the time iterations, ti, with a
+   // 9. Perform time-integration (looping over the time iterations, ti, with a
   //    time-step dt).
   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -522,8 +475,8 @@ int main(int argc, char *argv[])

      if (last_step || (ti % vis_steps) == 0)
      {
-         double ee = oper->ElasticEnergy(x.GetTrueVector());
-         double ke = oper->KineticEnergy(v.GetTrueVector());
+         double ee = oper.ElasticEnergy(x.GetTrueVector());
+         double ke = oper.KineticEnergy(v.GetTrueVector());

         cout << "step " << ti << ", t = " << t << ", EE = " << ee << ", KE = "
              << ke << ", ΔTE = " << (ee+ke)-(ee0+ke0) << endl;
@@ -537,14 +490,14 @@ int main(int argc, char *argv[])
            visualize(vis_v, mesh, &x, &v);
            if (vis_w)
            {
-               oper->GetElasticEnergyDensity(x, w);
+               oper.GetElasticEnergyDensity(x, w);
               visualize(vis_w, mesh, &x, &w);
            }
         }
      }
   }

-   // 9. Save the displaced mesh, the velocity and elastic energy.
+   // 10. Save the displaced mesh, the velocity and elastic energy.
   {
      v.SetFromTrueVector(); x.SetFromTrueVector();
      GridFunction *nodes = &x;
@@ -559,11 +512,11 @@ int main(int argc, char *argv[])
      v.Save(velo_ofs);
      ofstream ee_ofs("elastic_energy.sol");
      ee_ofs.precision(8);
-      oper->GetElasticEnergyDensity(x, w);
+      oper.GetElasticEnergyDensity(x, w);
      w.Save(ee_ofs);
   }

-   // 10. Free the used memory.
+   // 11. Free the used memory.
   delete ode_solver;
   delete mesh;

@@ -647,9 +600,7 @@ ReducedSystemOperator::~ReducedSystemOperator()
 HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
                                           Array<int> &ess_bdr, double visc,
                                           double mu, double K,
-                                           int kinsol_nls_type,
-                                           double kinsol_damping,
-                                           int kinsol_aa_n)
+                                           NonlinearSolverType nls_type)
   : TimeDependentOperator(2*f.GetTrueVSize(), 0.0), fespace(f),
     M(&fespace), S(&fespace), H(&fespace),
     viscosity(visc), z(height/2),
@@ -700,28 +651,15 @@ HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
   J_prec = NULL;
 #endif

-   if (kinsol_nls_type > 0)
+   if (nls_type == KINSOL)
   {
-      KINSolver *kinsolver = new KINSolver(kinsol_nls_type, true);
-      if (kinsol_nls_type != KIN_PICARD)
-      {
-         kinsolver->SetJFNK(true);
-         kinsolver->SetLSMaxIter(100);
-      }
-      if (kinsol_aa_n > 0)
-      {
-         kinsolver->EnableAndersonAcc(kinsol_aa_n);
-      }
+      KINSolver *kinsolver = new KINSolver(KIN_NONE, true);
      newton_solver = kinsolver;
      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(200);
      newton_solver->SetRelTol(rel_tol);
      newton_solver->SetPrintLevel(0);
      kinsolver->SetMaxSetupCalls(4);
-      if (kinsol_damping > 0.0)
-      {
-         kinsolver->SetDamping(kinsol_damping);
-      }
   }
   else
   {
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Andreas Schafelner	bca9363172	Fixed some bugs introduced by the previous commit.	2026-08-10 11:36:14 +02:00
Tamas Horvath	85785a1d51	WIP: updating the branch	2025-11-28 20:16:14 -05:00
Andreas Schafelner	4f4c8bb000	Added more pentatope quadrature rules.	2024-09-17 18:32:36 +02:00
Andreas Schafelner	6245cb6df3	Added new pentatope quadratures and corresponding unit tests.	2024-05-29 21:16:47 +02:00
Andreas Schafelner	3839122306	Merge branch 'master' into 4d_dev	2021-04-12 09:33:58 +02:00
Andreas Schafelner	7be65f6362	Used the new constructor for 4D meshes in ex14 and ex14p.	2020-12-04 16:41:50 +01:00
Andreas Schafelner	7c580696ed	Added new constructor for 4D meshes which performs a tensor-product extension in time.	2020-12-04 13:37:28 +01:00
Andreas Schafelner	7efaaddb48	Rewrote the Make4D constructor for Mesh. Now you can create a 4-parallelotope in the same way as in 3D.	2020-12-03 13:41:39 +01:00
Andreas Schafelner	91ea8c521f	Fixes for 4D DG.	2020-12-02 13:04:01 +01:00
Andreas Schafelner	98f3cf0b7d	Fixes for 4D DG.	2020-12-02 12:30:47 +01:00
Andreas Schafelner	6fc5954c83	Fix for wrong DShape for L2_PentatopeElement	2020-12-02 12:00:19 +01:00
Andreas Schafelner	b27f142a8d	Fixed size of array for 4D.	2020-07-31 16:32:27 +02:00
aschaf	b7ac27d6a5	Small fix for hexahedra meshes.	2019-12-06 08:48:51 +01:00
aschaf	5cede87cc3	Added 4D case for three methods.	2019-11-18 13:23:05 +01:00
aschaf	b44ba89cc1	Fixed one example.	2019-10-21 14:24:32 +02:00
aschaf	20c7ddd40f	Fixed prolongation matrces in 4D.	2019-10-21 14:23:43 +02:00
aschaf	d2507c2d19	Small change for the CFOSLS fork of mfem.	2019-10-09 14:50:22 +02:00
Andreas Schafelner	daef4a707f	Changed output of FGMRES to resemble GMRES	2019-10-04 17:02:25 -07:00
Andreas Schafelner	3f4f2b5d7d	Bugfix for Mixedbilinearform::Update	2019-10-04 17:01:29 -07:00
Andreas Schafelner	b2e381995b	Small fixes. Changed the quad rule for pentatopes for some orders back to the Duffy transformed ones, to avoid negative weights.	2019-09-09 09:42:46 -07:00
Andreas Schafelner	ffac92f50f	Possible fix for initial ParMesh distribution in 4D	2019-08-22 14:29:49 -07:00
Andreas Schafelner	3ac0713e7d	Changed ordering of Pentatope faces.	2019-08-22 14:25:24 -07:00
Andreas Schafelner	91e2a491ac	Partial fix for ParMesh distribution in 4D.	2019-08-20 13:53:39 -07:00
Andreas Schafelner	e1dc14506a	Small test fix for load distribution when initializing the ParMesh.	2019-08-19 17:07:59 -07:00
Andreas Schafelner	b2a5a8818c	Fixed element transforms for pentatopes.	2019-08-16 10:17:18 -07:00
Andreas Schafelner	9a7d02214f	Added parallelisation for bisection in 4D > currently tested are first and second order H1 elements and first and second order RT elements > parallel tests so far up to 4 cores	2019-08-13 12:56:10 -07:00
Andreas Schafelner	e69123dbb2	Added parallel uniform bisection in 4D. Jacobi determinant can and will be still negative for some elements. So far only tested locally with up to 8 cores.	2019-08-02 15:16:06 -07:00
Andreas Schafelner	1b23c0b85f	Added a flag to indicate if mesh is reflected.	2019-07-30 14:06:50 -07:00
Andreas Schafelner	a80a46a45e	First working(?) version of Stevenson bisection. Why it works? I have no idea. :-)	2019-07-25 16:22:33 -07:00
Andreas Schafelner	58e416cd4c	Fixed the size of some arrays for 4d.	2019-07-18 13:05:40 -07:00
Andreas Schafelner	5431f28f2b	Added an additional option for HypreSmoother For the Chebyshev smoother, you can choose how many CG iterations to use in order to obtain an estimate for the eigenvalues.	2019-07-18 13:03:43 -07:00
aschaf	fc3775d549	Reworked the 4D conforming Mesh. Now the element gets reordered to keep the Jacobi determinant positive and it stays positive during subsequent uniform refinement. IMPORTANT: So far, the non-conforming 4D mesh only works when the fix_orientation flag in the constructor of Mesh is set to false. And the elements are thus not reordered and the Jacobi determinant can and will be negative.	2019-05-28 13:36:18 +02:00
aschaf	ee8f81e9b7	Added a Set4 method.	2019-05-28 13:33:20 +02:00
aschaf	ea5cbfa45d	Extended the Update method for MixedBilinearForm. You can now update the FESpaces from outside.	2019-05-28 11:31:23 +02:00
aschaf	0a178223f4	Removed the absolute value for the transformation weights.	2019-05-28 11:29:44 +02:00
aschaf	15b369d120	Added a new LinearFormIntegrator. DomainGradLFIntegrator represents right hand sides of the form f*a.grad(v), where a is some vector coeffcient.	2019-05-28 11:26:16 +02:00
aschaf	10d543419a	Added arbitrary order RT Elements for 4D.	2019-05-28 11:24:38 +02:00
Tzanio	2a2b5b3314	Default build for 4D AMR	2019-04-04 08:06:16 -07:00
Tzanio	0e2670e6ba	make style	2019-04-04 08:03:59 -07:00
aschaf	9cb6ac90f4	More bugfixes.	2019-04-01 19:18:33 +02:00
aschaf	dc92de7607	Bugfixes.	2019-04-01 18:08:59 +02:00
aschaf	838f32d052	Merge branch 'master' into 4d_dev	2019-03-29 23:50:55 +01:00
aschaf	6edbdb10b6	Added an example for 4D adaptivity. Its just ex6 plus the necessary options for 4D.	2019-03-18 16:53:43 +01:00
aschaf	051dd436a7	Added a simple example to solve the heat equation. Uses the HeatEquationIntegrator.	2019-03-15 11:09:00 +01:00
aschaf	ed0a53fc21	Corrected HeatEquationIntegrator	2019-03-15 11:08:19 +01:00
aschaf	bc3b20756a	Added three simple space-time domains. Unit interval, unit square and unit cube, with time interval [0,1]	2019-03-15 11:07:15 +01:00
aschaf	2ec070cfcd	Added entries for 4d examples.	2019-03-14 19:45:43 +01:00
aschaf	e247a48496	small fix.	2019-03-14 19:20:30 +01:00
aschaf	ce357b3d31	Added the absolute value to the Jacobi Determinants for some integrators.	2019-03-14 19:19:12 +01:00
aschaf	3b93842ed4	Modified to build.	2019-03-14 18:51:29 +01:00
aschaf	7247a056c0	Fixed enum in 4d geometries.	2019-03-14 17:35:34 +01:00
aschaf	1d5c78afaf	Added methods for interior face linearform integrators.	2019-03-08 15:11:32 +01:00
aschaf	51cabe4530	Fixed a bug for the point matrices of the slave planars.	2018-11-08 16:12:40 +01:00
aschaf	b5ad79948a	Degree 3 int. rule in 4D is not exact.	2018-10-25 16:51:16 +02:00
aschaf	13087639e4	Bugfix for parallel 4D. Also, the 4D NCMesh does currently not work with MFEM_USE_MEMALLOC, as i don't know how to transfer the MemAlloc object from one mesh to another.	2018-10-24 11:11:13 +02:00
aschaf	8bb3c1f88f	Small oversight	2018-10-19 09:58:18 +02:00
aschaf	1fb24abeb7	Bugfix for higher order elements	2018-10-18 11:23:24 +02:00
aschaf	3c84979ae1	Changed the 4D code to reflect the changes from prism-dev I added a new array & group for the shared faces in 4D (tetrahedra from simplical meshes). The 2 dimensional planars are now stored in the triangle array & group. Updated also RefineGroups4D and FinalizeParTopo.	2018-10-18 10:46:27 +02:00
aschaf	cc77cad832	Updated some functions to the 4D case	2018-10-18 10:37:00 +02:00
aschaf	39b4d088fc	changed return value of GetType to the new API	2018-10-18 10:05:50 +02:00
aschaf	994bf59afa	Some changes of int to Geometry::Type	2018-10-18 10:05:06 +02:00
aschaf	051642a9b8	Try to Merge branch 'master' into 4d_dev	2018-10-12 15:03:51 +02:00
aschaf	f27dbc30d5	Added a basic HeatEquationIntegrator Solves the heat equation (with scalar diffusion coeff) by a space-time method. I will add an example using this Integrator later.	2018-10-11 13:50:11 +02:00
aschaf	e945f6cf49	Added ParNCMesh for 4D (WIP!!) > so far, it just enables you to use ParNCMesh in 4D for 1 processor only (I wanted to wait for all the changes in the Prism PR) > added also the necessary basic functionalities to ParFESpace	2018-10-11 13:46:01 +02:00
aschaf	89f7f93f36	Fixed the missing planar cases for GetBoundrayClosure > now it should correctly find "hidden" master planars	2018-10-11 13:15:36 +02:00
aschaf	6da740baf0	Bugfix in GetFaceDofs for 4D In principle, non-conforming adaptive refinement for 4D meshes works now with an arbitrary order H1_FESpace. However, this needs to be tested more, as there are still some special cases which are not covered.	2018-10-11 09:48:29 +02:00
aschaf	61f084e34c	Some additions to NCMesh for 4D > Moved TraverseFace for 4D to its own implementation, with an additional argument to indicate its local face index (needed to generate the right path) > Added some first implementations of functions needed for ParNCMesh (WIP) > Removed a check in NCMesh::PrintElements to enable printing more than 8 children (16 in 4D) > Some minor fixes	2018-10-11 09:42:06 +02:00
aschaf	a426d939a3	Small fix and removal of some debug text	2018-10-10 09:04:56 +02:00
aschaf	f7e6beb99f	Some additions to 4d L2_FECollection	2018-10-01 09:35:24 +02:00
aschaf	d9824c7578	Some small fixes	2018-10-01 09:20:19 +02:00
kalchev1	23b3e7dcbc	Small additions by Chak and Kirill that allow the 4D ParElag to run. However, there seem to be a deeper issue currently, since the mass integrator in 4D returns some negative diagonal entries in the mass matrix.	2018-09-27 14:57:58 -07:00
aschaf	8f94150a3e	Fixed conforming 4D Mesh Refinement works now, but Weight of Jacobian can be negative Also added some additional tables for 4D planars	2018-09-24 14:46:12 +02:00
aschaf	0a19f04474	Nonconforming refinement in 4D Had to disable the swapping, so conforming 4D meshes do not work atm.	2018-09-17 13:41:29 +02:00
aschaf	aa1cb6f840	Changed MFEM_VERIFY to include 4D	2018-09-12 17:13:41 +02:00
aschaf	ac6016dd0c	Finished 4D NC refinement removed swapping of elements to get positive determinant	2018-09-12 17:09:10 +02:00
aschaf	ca4bd46f2f	Removed ex4D_DivSkew from makefile	2018-09-11 10:38:23 +02:00
aschaf	801430150b	Another bugfix	2018-09-11 09:53:21 +02:00
aschaf	9008f02d15	Minor bugfixes	2018-09-10 21:28:37 +02:00
aschaf	80f0b1f2d5	Merge branch 'master' into 4d_dev	2018-09-10 16:38:51 +02:00
aschaf	13ba92ccf1	Added GetFacePlanars and GetFacePlanarTable	2018-09-10 16:32:35 +02:00
aschaf	67480e6ca5	Finished BuildConformingInterpolation for 4D, still needs to be tested	2018-09-10 16:31:26 +02:00
aschaf	e4c43532b4	fixed constructor call	2018-09-07 17:03:05 +02:00
aschaf	3b2604e582	initialize pdofs and npdofs in default constructor	2018-09-07 15:30:24 +02:00
aschaf	42db0cd988	revert change from before	2018-09-07 14:57:11 +02:00
aschaf	0f7e0de3d5	Started implementing 4D NCMeshes. The handling of the tetrahedral faces is for now dirty, I will change this later. Nonconforming Refinement of 4D simplical meshes works for now only once.	2018-09-07 14:56:58 +02:00
aschaf	5c58758fda	New Hash object Hashed5 (works like Hashed4) For 4D NCMeshes, i needed to store all 4 vertices of a face, so Hashed5 is now 5 vertices where only 4 are hashed.	2018-09-07 14:54:21 +02:00
aschaf	de79f0243f	Implementation needed for NCMesh	2018-09-07 14:52:40 +02:00
aschaf	247b4d9fdf	Some preparations for 4D NCMeshes	2018-09-07 14:52:08 +02:00
aschaf	0eabd62aa9	Removed some not yet implemented call	2018-09-07 14:47:06 +02:00
aschaf	179e702e5b	Preparations for 4D NCMeshes Small bugfix in GetBdrElementDofs	2018-09-07 14:40:16 +02:00
aschaf	3dfaa64744	DeleteLazyTables also in 4D	2018-09-07 14:03:21 +02:00
aschaf	bf6262daf4	Added GeomToPerfGeomJac and PerfGeomToGeomJac for Pentatopes	2018-09-07 09:54:44 +02:00
aschaf	d09acbceff	Expanded CalcSingularValue(i) for higher dimensions	2018-09-07 09:53:05 +02:00
aschaf	e4f2b4fd0d	Restored ex4 for 2D/3D	2018-09-07 09:51:04 +02:00
aschaf	d60393edb0	Using now RefineGroups(HashTable<Hashed2>) for 4D refinement	2018-09-04 11:21:49 +02:00
aschaf	03bffe438b	Updated 4D refinement to use HashTable<Hashed2> Also restored original, working 4D refinement code	2018-09-04 09:45:01 +02:00
aschaf	e249c0889b	Changed asserts to include 4D	2018-09-04 09:44:26 +02:00
aschaf	be90d1179e	Removed some printf	2018-09-04 09:44:06 +02:00
aschaf	2da040c52a	Removed NCMesh for 4D, caused segfault before main Currently, the H1_FECollection is conforming in 4D, however uniform refinement in 4D does no longer work (WIP)	2018-08-31 16:19:05 +02:00
aschaf	9fdf71b028	Some bugfixes	2018-08-31 15:02:08 +02:00
aschaf	ecf5954f66	Merge branch 'master' into 4d_dev	2018-08-31 12:55:33 +02:00
aschaf	08cbd00509	Debug messages for 4D	2018-08-31 10:17:53 +02:00
aschaf	ceb7c8efbc	Changed order of Pentatope Vertices some DEBUG and WIP for 4D meshes	2018-08-31 10:16:56 +02:00
aschaf	b8aa9b705c	Started implementing 4D Ncmesh	2018-08-31 10:14:22 +02:00
aschaf	3ea14703e2	Started implementing NCMesh for pentatopes	2018-07-26 17:16:43 +02:00
aschaf	cc74c87a1a	Bugfix	2018-07-26 10:25:02 +02:00
aschaf	031722a50c	Some minor additions.	2018-07-24 11:23:57 +02:00
aschaf	4cac7186b3	Some additional functionalities Changed formula for 2nd deriv. of CalcChebyshev like in master Found bug in CalcHessian for H1_PentatopeElement Added tetrahedron orientations to Geometry::Constants Expanded GrundmannMollerSimplesRule for 4D Changed Pentatope integration rules to said GrundmannMollerSimplexRules Adapted some MFEM_ASSERTS in DenseMatrix for 4D Added Mesh::GetLocalTetToPentTransformation Extended Mesh::GetLocalFaceTransformatoin with the tetrahedron case Extended Mesh::GetRefinementTransforms for Pentatopes Added GetPointMatrix static method to Pentatope class FESpace::Update should now correctly create the Update Matrix for Gridfunctions	2018-07-24 11:04:32 +02:00
aschaf	92d12f61a2	Corrected the generation of TetDofOrd in H1_FECollection::H1_FECollection(...)	2018-06-25 11:04:32 +02:00
aschaf	da86aaa1fe	Added CalcHessian to H1_TriangleElement. Added CalcHessian to H1_PentatopeElement. Tried to extend the loops for the triangle dof orientations to the tetrahedron case, but I'm somewhat stuck. Currently H1_PentatopeElement produces correct results up to order 4.	2018-05-18 10:43:20 +02:00
aschaf	0f0d05726b	Added H1_PentatopeElement (still testing).	2018-05-14 17:29:26 +02:00
aschaf	90b7c2275b	Forgot parantheses.	2018-05-08 09:47:24 +02:00
Andreas Schafelner	fcf1e7ae09	Implemented CalcHessian for H1_Tetrahedron element. Also added a third Poly_1D::CalcChebyshev, which returns in addition the second derivative of the Chebyshev polynomials.	2018-05-08 09:23:11 +02:00
aschaf	5adf8ca172	Fixed a memory leak in Mesh::AddTetrahedralFaceElement.	2018-02-27 14:19:37 +01:00
aschaf	827eafe635	Added the case 4x3 for DenseMatrix::Weigth	2018-02-23 16:34:14 +01:00
aschaf	b2a3d478a7	Added the case Pentatope to Geometry::ProjectPoint	2018-02-23 16:33:28 +01:00
aschaf	ed02f2a25c	Implemented CalcHessian for Quadratic finite elements (3D,4D) and cubic finite elements (3D).	2018-02-23 16:31:35 +01:00
Freki	fa695a71c5	changed the allocated memory for TransformBack from the 3d case to the 4d case.	2018-02-23 16:19:31 +01:00
Tzanio	300bf6ebea	make style	2017-08-29 18:16:56 -07:00
Tzanio	35934ddd46	Merge branch 'master' into 4d_dev	2017-08-29 18:16:24 -07:00
neumueller	ab426339f1	changed the standard mesh-file to an existing one	2017-08-28 10:56:49 +02:00
neumueller	92b8ebbeaf	reordered the assembling of the rhs	2017-08-25 21:52:41 +02:00
neumueller	817fb5d549	removed memory leaks in the DivSkew example	2017-08-25 13:06:27 +02:00
neumueller	6c767f02f2	added the start and end weights for the H(curl) example	2017-08-25 11:12:14 +02:00
neumueller	88da01bd32	removed some memory leaks in the H(curl) example	2017-08-25 11:10:32 +02:00
neumueller	45dea7453f	modiefied the exact solution (such that it allows homogenuous neumann bc on a cube)	2017-08-25 10:25:35 +02:00
neumueller	779803d48f	removed some memory leaks	2017-08-25 10:00:33 +02:00
neumueller	6c728dfc40	reduced the number of max. pcg iterations	2017-08-24 16:51:58 +02:00
neumueller	47054c792f	minor modifications	2017-08-24 16:44:02 +02:00
neumueller	90d919c7a9	added an exact solution for the H1-problem	2017-08-24 14:36:38 +02:00
neumueller	06950f2249	some minor modifications	2017-08-24 11:26:00 +02:00
neumueller	1f92f750ed	added mass weights for the operators DivSkew and div	2017-08-24 11:16:52 +02:00
neumueller	620edaad46	modified the gradient example for weighted mass	2017-08-24 10:59:46 +02:00
neumueller	cb84230765	fixed a bug	2017-08-24 10:44:30 +02:00
neumueller	664959bc2a	modified the grad and curl example such that it runs several tests for different weights of the mass term	2017-08-24 10:38:01 +02:00
neumueller	e01483f8e4	modified the vector bilinear integrator to allow coefficients and vdims different from space-dim	2017-08-23 16:15:22 +02:00
neumueller	7a665f97ac	added some example files for 4d test problems with variable coefficients	2017-08-23 16:09:59 +02:00
neumueller	e8d880f923	added some files to the CMakeList.txt	2017-06-14 22:26:30 +02:00
neumueller	d8df32549e	added the header file pentatope.hpp in mesh.hpp	2017-06-14 22:11:34 +02:00
neumueller	3e34a1342c	removed a bug for the debug version	2017-06-14 22:04:25 +02:00
Tzanio	84e24629d8	make style	2017-06-13 10:30:08 -07:00
Martin Neumüller	30669fd6ee	Merge branch 'master' into 4d_dev	2017-06-13 18:18:53 +02:00
neumueller	cea505df7a	added the lowest order H(div) finite element space the included auxiliary space preconditioner has to be compared and checked with the theory	2017-04-21 01:04:44 +02:00
neumueller	498ef892ac	a first version of the lowest order H(DivSkew)-elements (2forms) with auxiliary space preconditioner	2017-04-20 21:07:01 +02:00
neumueller	8ad9e29dd6	added support for the lowest order 4d-H(curl) elements (1st and 2nd kind)	2017-04-20 00:14:19 +02:00
neumueller	3aa3ae43f3	a first simple version for handling 4d-simplex meshes (at least example ex1p is running)	2017-04-18 23:39:27 +02:00
neumueller	439436f44b	test for commiting the first simple change	2017-04-18 19:45:46 +02:00