Squashing commits on AIR_relax

.appveyor.yml

@@ -47,3 +47,4 @@ build_script:
 after_build:
 - cd build_serial
 - ctest -C Release --output-on-failure
+

.gitignore

@@ -72,10 +72,6 @@ examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
 examples/mode_*
-examples/ex5-p-*.bp
-examples/ex9-p-*.bp
-examples/ex12-p-*.bp
-examples/ex16-p-*.bp
 examples/ex16.mesh
 examples/ex16-mesh.*
 examples/ex16-init.*
@@ -96,9 +92,6 @@ examples/ex21p_*.*
 examples/ex23-*.gf
 examples/ex23*.mesh
 examples/Example23*
-examples/ex25.mesh
-examples/ex25-*.gf
-examples/ex25p-*.*
 
 examples/sundials/ex9
 examples/sundials/ex1[06]
@@ -163,19 +156,15 @@ miniapps/electromagnetics/Joule_*
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid
-miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
-miniapps/meshing/minimal-surface
-miniapps/meshing/pminimal-surface
 
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
-miniapps/meshing/twist-*.mesh
 miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
 miniapps/meshing/shaper.mesh
@@ -231,20 +220,10 @@ miniapps/gslib/field-diff
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts
 
-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
-tests/unit/punit_tests
 tests/unit/sedov_tests_*
-tests/unit/psedov_tests_*
 
 # Test script output
 tests/scripts/*.err

.gitlab-ci.yml

@@ -1,292 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# General GitLab pipelines configurations for supercomputers and Linux clusters
-# at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
-# LLNL-specific!
-
-# We define the following GitLab pipeline variables:
-#
-# BUILD_ROOT:
-# The path to the shared resources between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-#
-# BUILD_PATH:
-# In BUILD_ROOT, we want to separate builds depending on the machine used,
-# typically because we build on a set of dependencies per machine.
-#
-# REBASELINE:
-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-#
-# MFEM_ALLOC_NAME:
-# On LLNL's quartz, there is only one allocation shared among jobs in order to
-# save time and resources. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-#
-# TPLS_REPO & TESTS_REPO:
-# Git repositories used in the pipeline
-#
-# ARTIFACTS_DIR:
-# Directory used to place artifacts.
-
-variables:
-  BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
-  BUILD_PATH: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}/${PLAT}/${TOOLCHAIN}
-  REBASELINE: "NO"
-  MFEM_ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  ARTIFACTS_DIR: artifacts
-
-# The pipeline is divided into stages. Usually, these are also synchronization
-# points, however, we use "needs" keyword to express the DAG of jobs for more
-# efficiency.
-# - We use setup phase to download content outside of mfem directory.
-# - Allocate is where quartz resources are allocated once for all.
-# - Libs is where dependencies are built.
-# - Build is where we build MFEM for multiple toolchains.
-# - Test is where we perform a first set of tests on MFEM build.
-# - Baseline_checks gathers baseline-type test suites execution
-# - Baseline_publish, only available on master, allows to update baseline
-#   results
-# - Deallocate releases quartz resources
-stages:
-  - setup
-  - allocate
-  - libs
-  - build
-  - test
-  - deallocate
-  - baseline_check
-  - baseline_publish
-
-# The setup job in setup stage don’t rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO: updating tests and tpls is not necessary anymore since pipelines are
-# now using unique directories so repo are never shared with another pipeline.
-# This is not memory efficient (we keep a lot of data), hence this reminder.
-.setup:
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
-    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
-    - cd tpls && git pull && cd ..
-    - cd tests && git pull && cd ..
-
-# Share configurations (templates) for toolchain configuration.
-# Note: Any job using this template shouldn't redefine "before_script" key.
-.with_gcc_6_1_0:
-  variables:
-    TOOLCHAIN: gcc_6_1_0
-    CXX: g++
-    CC: gcc
-  before_script:
-    - module load gcc/6.1.0
-
-.with_gcc_4_9_3:
-  variables:
-    TOOLCHAIN: gcc_4_9_3
-    CXX: g++
-    CC: gcc
-  before_script:
-    - module load gcc/4.9.3
-
-.if_srun_alloc_for_make: &if_srun_alloc_for_make |
-  if [[ "${MPIEXEC}" == "srun" ]]
-  then
-    export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
-    [[ -n "JOB_ID" ]] && export JOB_ID="--jobid=${JOB_ID}"
-    RESOURCES="--extra-node-info=1:${MAKE_PAR}:1 --cpus-per-task=${MAKE_PAR} --ntasks=1"
-    # Allocation prefix:
-    export EXEC_PREFIX="srun ${JOB_ID} ${RESOURCES}"
-  fi
-
-# Build of dependencies and mfem:
-# if on quartz, uses a part of preallocated resources
-# Note: Those jobs don't rely on MFEM repository, hence the use of
-# "GIT_STRATEGY: none" preventing its clone.
-.build_hypre:
-  variables:
-    EXEC_PREFIX: ""
-    GIT_STRATEGY: none
-  stage: libs
-  script:
-    - cd ${BUILD_ROOT}/tpls
-    - mkdir -p ${BUILD_PATH}
-    - make clean
-    - *if_srun_alloc_for_make
-    - ${EXEC_PREFIX} make -j ${MAKE_PAR} hypre PREFIX=${BUILD_PATH}/hypre/src/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH}
-
-.build_metis:
-  variables:
-    EXEC_PREFIX: ""
-    GIT_STRATEGY: none
-  stage: libs
-  script:
-    - cd ${BUILD_ROOT}/tpls
-    - mkdir -p ${BUILD_PATH}
-    - *if_srun_alloc_for_make
-    - ${EXEC_PREFIX} make -j ${MAKE_PAR} ${METIS} PREFIX=${BUILD_PATH}/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH} CC=${CC} CXX=${CXX}
-
-# Building MFEM
-.build_mfem:
-  variables:
-    EXEC_PREFIX: ""
-  stage: build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
-    - *if_srun_alloc_for_make
-    - make config CXX=${CXX} MPICXX=mpicxx MFEM_USE_MPI=${MFEM_USE_MPI} MFEM_DEBUG=${MFEM_DEBUG} MFEM_MPIEXEC="${EXEC_PREFIX}" MFEM_MPIEXEC_NP=${MPIEXEC_NP}
-    - ${EXEC_PREFIX} make -j ${MAKE_PAR} all
-
-# TODO: Could this phase use a parallel run?
-.sanitycheck_mfem:
-  stage: test
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
-    - make test
-
-# Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
-# differentiates between the two tests.
-.baseline_script: &baseline_script |
-  # locals
-  _glob_out=${BASELINE_TEST}.out
-  _glob_err=${BASELINE_TEST}.err
-  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
-  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
-  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
-
-  _out=${BASELINE_TEST}-${SYS_TYPE}.out
-  _ref=../${BASELINE_TEST}-${SYS_TYPE}.saved
-  _out_txt=${BASELINE_TEST}.txt
-  _diff=${BASELINE_TEST}-diff.txt
-  # prepare
-  cd ${BUILD_ROOT}
-  ln -snf ${CI_PROJECT_DIR} mfem
-  cd tests
-  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
-  # run
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
-  # post
-  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
-  if [[ -s ${_glob_err} ]]
-  then
-    echo "ERROR during ${BASELINE_TEST} execution";
-    echo "Here is the ${_glob_err} file content";
-    cat ${_glob_err}
-    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}.txt
-    exit 1;
-  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
-  then
-    echo "Something went WRONG in ${BASELINE_TEST}:";
-    echo "Either ${_base_patch} or ${_base_out} should exists";
-    exit 1;
-  elif [[ -f ${_base_patch} ]]
-  then
-    echo "${BASELINE_TEST}: Differences found, patch generated"
-    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}.txt
-  elif [[ -f ${_base_out} ]]
-  then
-    echo "${BASELINE_TEST}: Differences found, replacement file generated"
-    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}.txt
-  fi
-  # _base_diff won’t even exist if there is no difference.
-  if [[ -f ${_base_diff} ]]
-  then
-    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
-    cat ${_base_diff}
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}.txt
-  fi
-  if [[ ! -s ${_base_diff} ]]
-  then
-    echo "${BASELINE_TEST}: PASSED"
-    true
-  else
-    echo "${BASELINE_TEST}: FAILED"
-    false
-  fi
-
-# Actual templates for baseline checks
-.baselinecheck_mfem:
-  stage: baseline_check
-  variables:
-    BASELINE_TEST: baseline
-    ADDITIONAL_DIR: ${BUILD_ROOT}/tpls
-  script:
-    - *baseline_script
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-.samplebaselinecheck_mfem:
-  stage: baseline_check
-  variables:
-    BASELINE_TEST: sample-runs-baseline
-    ADDITIONAL_DIR: ""
-  script:
-    - *baseline_script
-  timeout: 4h
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-# This job can only be manually triggers on a pipeline for master branch, or if
-# the pipeline was triggered with REBASELINE="YES"
-.rebaseline_mfem:
-  stage: baseline_publish
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-      when: manual
-  script:
-    - export PATCH_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.patch
-    - export FULL_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.out
-    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
-    - cd ${BUILD_ROOT}/tests
-    - |
-      if [[ ! -f "${DIFF_FILE}.txt" ]]
-      then
-          echo "Nothing to be done: no relevant change in baseline"
-          exit 0
-      elif [[ -f "${PATCH_FILE}.txt" ]]
-      then
-          mv ${PATCH_FILE}.txt ${PATCH_FILE}
-          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
-      elif [[ -f "${FULL_FILE}.txt" ]]
-      then
-          cp "${FULL_FILE}.txt" "./baseline-${SYS_TYPE}.saved"
-      else
-          echo "File missing: expected ${PATCH_FILE}.txt or ${FULL_FILE}.txt"
-          exit 1
-      fi
-    - git add baseline-${SYS_TYPE}.saved
-    - git commit -m "${SYS_TYPE} rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
-    - git push origin master
-
-# The list on jobs is defined in machine-specific files.
-include:
-  - local: .gitlab/quartz.yml

.gitlab/quartz.yml

@@ -1,149 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Quartz machine at LLNL
-
-.on_quartz:
-  tags:
-    - shell
-    - quartz
-  variables:
-    PLAT: quartz
-    MPIEXEC: srun
-    MPIEXEC_NP: "-n"
-    MAKE_PAR: 6
-    BASELINE_PAR: 18
-
-# Setup
-setup_quartz:
-  extends: [.setup, .on_quartz]
-
-# Allocate
-allocate_quartz:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: allocate
-  script:
-    - |
-      if [[ "${MPIEXEC}" == "srun" ]]
-      then
-      salloc --nodes=1 --extra-node-info=2:18:1 --partition=pdebug --time=30 --no-shell --job-name=${MFEM_ALLOC_NAME}
-      fi
-  timeout: 6h
-
-# Release
-deallocate_quartz:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: deallocate
-  script:
-    - |
-      if [[ "${MPIEXEC}" == "srun" ]]
-      then
-      export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
-      ([[ -n "${JOB_ID}" ]] && scancel ${JOB_ID})
-      fi
-  when: always
-
-# Build external libraries (tpls)
-build_hypre_gcc_6_1_0_quartz:
-  extends: [.build_hypre, .with_gcc_6_1_0, .on_quartz]
-
-build_parmetis_gcc_6_1_0_quartz:
-  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    METIS: parmetis
-
-build_metis4_gcc_6_1_0_quartz:
-  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    METIS: metis4
-
-build_metis5_gcc_6_1_0_quartz:
-  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    METIS: metis5
-  allow_failure: true
-
-# Build MFEM
-build_mfem_debug_ser_gcc_4_9_3_quartz:
-  extends: [.build_mfem, .with_gcc_4_9_3, .on_quartz]
-  variables:
-    MFEM_DEBUG: "YES"
-    MFEM_USE_MPI: "NO"
-
-build_mfem_debug_ser_gcc_6_1_0_quartz:
-  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    MFEM_DEBUG: "YES"
-    MFEM_USE_MPI: "NO"
-
-build_mfem_debug_par_gcc_6_1_0_quartz:
-  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    MFEM_DEBUG: "YES"
-    MFEM_USE_MPI: "YES"
-  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
-
-build_mfem_opt_ser_gcc_6_1_0_quartz:
-  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    MFEM_DEBUG: "NO"
-    MFEM_USE_MPI: "NO"
-
-build_mfem_opt_par_gcc_6_1_0_quartz:
-  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    MFEM_DEBUG: "NO"
-    MFEM_USE_MPI: "YES"
-  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
-
-### Tests
-
-# Sanity check
-sanitycheck_mfem_debug_ser_gcc_4_9_3_quartz:
-  extends: [.sanitycheck_mfem, .with_gcc_4_9_3, .on_quartz]
-  variables:
-    MFEM_DEBUG: "YES"
-    MFEM_USE_MPI: "NO"
-  needs: [build_mfem_debug_ser_gcc_4_9_3_quartz, allocate_quartz]
-
-sanitycheck_mfem_debug_ser_gcc_6_1_0_quartz:
-  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    MFEM_DEBUG: "YES"
-    MFEM_USE_MPI: "NO"
-  needs: [build_mfem_debug_ser_gcc_6_1_0_quartz, allocate_quartz]
-
-sanitycheck_opt_ser_mfem_gcc_6_1_0_quartz:
-  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    MFEM_DEBUG: "NO"
-    MFEM_USE_MPI: "NO"
-  needs: [build_mfem_opt_ser_gcc_6_1_0_quartz, allocate_quartz]
-
-sanitycheck_opt_par_mfem_gcc_6_1_0_quartz:
-  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
-  variables:
-    MFEM_DEBUG: "NO"
-    MFEM_USE_MPI: "YES"
-  needs: [build_mfem_opt_par_gcc_6_1_0_quartz, allocate_quartz]
-
-# Baseline
-baselinecheck_mfem_intel_quartz:
-  extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [allocate_quartz]
-
-baselinepublish_mfem_quartz:
-  extends: [.rebaseline_mfem, .on_quartz]
-  needs: [baselinecheck_mfem_intel_quartz]

.travis.yml

@@ -1,14 +1,3 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
 language: cpp
 
 sudo: false
@@ -145,8 +134,7 @@ jobs:
           - $TRAVIS_BUILD_DIR/../metis-4.0
       before_cache:
         - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
 
     - os: linux
       compiler: gcc
@@ -176,40 +164,7 @@ jobs:
           - $TRAVIS_BUILD_DIR/../metis-4.0
       before_cache:
         - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel (cmake)"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-           NPROCS=2
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - mkdir ${TRAVIS_BUILD_DIR}/build
-        - cd ${TRAVIS_BUILD_DIR}/build
-        - cmake ..
-             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../hypre-2.10.0b/src/hypre
-             -DMFEM_MPI_NP=$NPROCS
-        - make -j3 mfem examples
-        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
-        - make -j3
-        - ctest --output-on-failure
-      cache:
-        directories:
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
 
     # ========================
     #      Mac OS X tests
@@ -255,8 +210,7 @@ jobs:
           - $HOME/local-cached
       before_cache:
         - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
 
     - os: osx
       # osx_image: xcode7.3
@@ -276,8 +230,8 @@ jobs:
           - $HOME/local-cached
       before_cache:
         - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
+
 
 before_install:
    # No addon for brew yet, have to install OSX packages this way.

CHANGELOG

@@ -8,119 +8,43 @@
                                http://mfem.org
 
 
-Version 4.1.1 (development)
+Version 4.0.1 (development)
 ===========================
 
-Meshing improvements
---------------------
-- The graph linear ordering library Gecko, previously an external dependency, is
-  now included directly in MFEM. As a result, Mesh::GetGeckoElementOrdering is
-  always available. The interface has also been improved, see for example the
-  mesh-explorer miniapp.
-
-- Added support for finite difference-based gradient and Hessian approximation
-  in the TMOP mesh optimization algorithms. This improves the accuracy of the
-  Hessian for r-adaptivity using discrete fields, and allows use of skewness
-  and orientation based metrics.
-
-Improved GPU capabilities
--------------------------
-- Added support for Chebyshev accelerated polynomial smoother on GPU.
-
-Discretization improvements
----------------------------
-- Added support for matrix-free interpolation and restriction operators between
-  continuous H1 finite element spaces of different order on the same mesh or
-  with the same order on uniformly refined meshes.
-
-- Added support for simplices in GSLIB-FindPoints.
-
-Linear and nonlinear solvers
-----------------------------
-- Added power method to iteratively estimate the largest eigenvalue and the
-  corresponding eigenvector of an operator.
-
-- Added initial support for h- and p-multigrid solvers and preconditioners for
-  matrix-based and matrix-free discretizations with basic GPU capability.
-
-New and updated examples and miniapps
--------------------------------------
-- Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
-  Matched Layer (PML) for the simulation of electromagnetic wave propagation.
-  The example defines and solves several indefinite Maxwell problems.
-
-- Added a new Example 26/26p to demonstrate the construction of a matrix-free
-  geometric and p-multigrid preconditioner for the Laplace problem.
-
-- Added a new example, Example 27/27p, to demonstrate the enforcement of
-  various boundary conditions with the Laplace operator.  The example shows the
-  procedures for applying Dirichlet, Neumann (both homogeneous and
-  inhomogeneous), Robin, and periodic boundary conditions with either H1 or DG
-  discretizations.
-
-- Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
-  stitching together opposite surfaces of a mesh to create a topologically
-  periodic mesh.
-
-- Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
-  the Dirichlet problem for the minimal surface equation.
-
-Improved testing
-----------------
-- Added a GitLab pipeline that automates PR testing on supercomputing systems
-  and Linux clusters at Lawrence Livermore National Lab (LLNL). This can be
-  triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
-  and the updated CONTRIBUTING.md file.
-
-Miscellaneous
--------------
-- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
-  solver can work with non-SPD preconditioner B.
-
-- Added support for ADIOS2 for parallel I/O with ParaView visualization. The
-  classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
-  interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
-  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.
-
-
-Version 4.1, released on March 10, 2020
-=======================================
-
 Starting with this version, the MFEM open source license is changed to BSD-3.
 
-Improved GPU capabilities
--------------------------
+Improved GPU support
+--------------------
 - Added initial support for AMD GPUs based on HIP: a C++ runtime API and kernel
-  language that can run on both AMD and NVIDIA hardware.
+  language that can run on both AMD and NVIDIA hardware. With this change plus
+  the libCEED and debug devise additions below, the current list of available
+  backends is: "ceed-cuda", "occa-cuda", "raja-cuda", "cuda", "hip", "debug",
+  "occa-omp", "raja-omp", "omp", "ceed-cpu", "occa-cpu", "raja-cpu", and "cpu".
+
+- In addition to Examples 1 and 6, GPU/device acceleration is now available in
+  three additional example codes: 3/3p, 9/9p and 24/24p.
 
 - Added support for Umpire, a resource management library that allows the
   discovery, provision, and management of memory on machines with multiple
   memory devices like NUMA and GPUs, see https://github.com/LLNL/Umpire.
 
-- GPU acceleration is now available in 3 additional examples: 3, 9 and 24.
-
-- Improved RAJA backend and multi-GPU MPI communications.
-
 - Added a "debug" device designed specifically to aid in debugging GPU code by
   following the "device" code path (using separate host/device memory spaces and
   host <-> device transfers) without any GPU hardware.
 
-- Added support for matrix-free diagonal smoothers on GPUs.
-
-- The current list of available device backends is: "ceed-cuda", "occa-cuda",
-  "raja-cuda", "cuda", "hip", "debug", "occa-omp", "raja-omp", "omp",
-  "ceed-cpu", "occa-cpu", "raja-cpu", and "cpu".
-
 - The MFEM memory manager now supports different memory types, associated with
   the following memory backends:
   * Default host memory, using standard C++ new and delete,
   * CUDA pointers, using cudaMalloc and HIP pointers, using hipMalloc,
   * Managed CUDA/HIP memory (UVM), using cudaMallocManaged/hipMallocManaged,
   * Umpire-managed memory, including memory pools,
-  * 32- or 64-byte aligned memory, using posix_memalign (WIN32 also supported),
+  * 32- or 64-bit aligned memory, using posix_memalign (WIN32 also supported),
   * Debug memory with mmap/mprotect protection used by the new "debug" device.
 
+- Added support for matrix-free diagonal smoothers on GPUs.
+
+- Improved RAJA backend and multi-GPU MPI communications.
+
 libCEED support
 ---------------
 - Added support for libCEED, the portable library for high-order operator
@@ -135,35 +59,9 @@ libCEED support
   ceed-cuda), using some of the best performing CPU and GPU backends from
   libCEED, see the sample runs in examples 1 and 6.
 
-- NOTE: The current default libCEED GPU backend (ceed-cuda) uses atomics and
-  therefore is non-deterministic.
-
-Partial assembly and matrix-free discretizations
-------------------------------------------------
-- The support for matrix-free methods on both CPU and GPU devices based on a
-  partially assembled operator decomposition was extended to include:
-  * DG integrators, (for now only in the Gauss-Lobatto basis), see Example 9,
-  * H(curl) bilinear forms, see Example 3,
-  * vector mass and vector diffusion bilinear integrators,
-  * convection integrator with improved performance,
-  * gradient and vector divergence integrators for Stokes problems,
-  * initial partial assembly mode for NonlinearForms.
-
-- Diagonals of partially assembled operators can now be computed efficiently.
-  See the new methods AssembleDiagonal in BilinearForm, AssembleDiagonalPA in
-  BilinearFormIntegrator and the implementations in fem/bilininteg_*.cpp.
-
-- In many examples, the partial assembly algorithms provide significantly
-  improved performance, particularly in high-order 3D runs on GPUs.
-
 Meshing improvements
 --------------------
-- The algorithms for mesh element numbering were changed to have significantly
-  better caching and parallel partitioning properties. Both initial (see e.g.
-  Mesh::GetHilbertElementOrdering) and ordering after uniform refinement were
-  improved. NOTE: new ordering can have a round-off effect on solver results.
-
-- Added support for non-conforming AMR on both prisms and tetrahedra, including
+- Added support for non-conforming AMR on prisms and tetrahedra, including
   coarsening and parallel load balancing. Anisotropic prism refinement is only
   available in the serial version at the moment.
 
@@ -172,12 +70,18 @@ Meshing improvements
   (e.g. spatial dependence of size, aspect ratio, etc., for each element) or via
   a (Par)GridFunction specified on the original mesh.
 
-- The TMOP algorithms were also improved to support non-conforming AMR meshes.
+- New method Mesh::GetHilbertElementOrdering for sorting mesh elements along the
+  Hilbert curve. The ordering can be used to improve caching and parallel
+  partitioning in non-conforming AMR.
 
 - Added support for creating refined versions of periodic meshes, making use of
   the new L2ElementRestriction class. This class also allows for computing
   geometric factors on periodic meshes using partial assembly.
 
+- The TMOP mesh optimization algorithms were improved to support AMR meshes.
+
+- Improved element numbering after uniform mesh refinement.
+
 Discretization improvements
 ---------------------------
 - Added support for GSLIB-FindPoints, a general high-order interpolation utility
@@ -185,23 +89,32 @@ Discretization improvements
   in physical space. See INSTALL for details on building MFEM with GSLIB, and
   miniapps/gslib for examples of how to use this feature.
 
-- Added support for complex-valued finite element operators and fields using a
-  2x2 block structured linear system to mimic complex arithmetic. New classes
-  include: ComplexGridFunction, SesquilinearForm, ComplexLinearForm, and their
-  parallel counterparts.
-
-- Added second order derivatives of NURBS shape functions.
+- Extended the support for partial assembly to DG integrarors, H(curl) bilinear
+  forms, and the vector mass and vector diffusion bilinear integrators.
 
 - Added support for serendipity elements of arbitrary order on affinely-mapped
   square elements. Basis functions for these elements can be visualized using
   an option in the display-basis miniapp.
 
+- Added support for complex-valued finite element operators and fields using a
+  2x2 block structured linear system to mimic complex arithmetic. New classes
+  include: ComplexGridFunction, SesquilinearForm, ComplexLinearForm, and their
+  parallel counterparts.
+
 - Two integrators related to Stokes problems, (Q grad u, v) and (Q div v, u),
   where u and the components of v are in H1, were added/modified to support full
   and partial assembly modes. See the new GradientIntegrator and the updated
   VectorDivergenceIntegrator classes in fem/bilininteg.hpp, as well as the PA
   kernels in fem/bilininteg_gradient.cpp and fem/bilininteg_divergence.cpp.
 
+- Diagonals of partially assembled operators can now be computed efficiently.
+  See the new methods AssembleDiagonal in BilinearForm, AssembleDiagonalPA in
+  BilinearFormIntegrator and the implementations in fem/bilininteg_*.cpp.
+
+- Added second order derivatives of NURBS shape functions.
+
+- Added initial support for NonlinearForms to support the partial assembly mode.
+
 - Added a nonlinear vector valued convection integrator (Q u \cdot grad u, v)
   where u_i and v_i are in H1. This form occurs e.g. in the Navier-Stokes
   equations. The integrator supports the partial assembly mode for its
@@ -214,76 +127,76 @@ Discretization improvements
 
 Linear and nonlinear solvers
 ----------------------------
-- Added support for Ginkgo, a high-performance linear algebra library for GPU
-  and manycore nodes, with a focus on sparse solution of linear systems. For
-  more details see linalg/ginkgo.hpp and the example code in examples/gingko.
-
-- Added support for HiOp, a lightweight HPC solver for nonlinear optimization
-  problems, see class HiOpNLPOptimizer and the example codes in examples/hiop.
-
 - Added a general interface for specifying and solving nonlinear constrained
   optimization problems through the new classes OptimizationProblem and
   OptimizationSolver, see linalg/solver.hpp.
 
+- Added support for HiOp, a lightweight HPC solver for nonlinear optimization
+  problems see class HiOpNLPOptimizer and the example codes in examples/hiop.
+
+- Added support for Ginkgo, a high-performance linear algebra library for GPU
+  and manycore nodes, with a focus on sparse solution of linear systems. For
+  more details see linalg/ginkgo.hpp and the example code in examples/gingko.
+
+- Added Adams-Bashforth and Adams-Moulton time integrators.
+
+- Added several time integrators for 2nd order ODEs.
+
 - Added a block ILU(0) preconditioner for DG-type discretizations. Example 9
   (DG advection) now takes advantage of this for implicit time integration.
 
-- New time integrators: Adams-Bashforth, Adams-Moulton and several integrators
-  for 2nd order ODEs, see the new Example 23.
-
 - Added a LinearSolve(A,X) convenience method to solve dense linear systems. In
   the trivial cases, i.e., square matrices of size 1 or 2, the system is solved
   directly, otherwise, LU factorization is employed.
 
 New and updated examples and miniapps
 -------------------------------------
+- Added two new miniapps: Find Points (serial + parallel) and Field Diff in
+  miniapps/gslib that show how GSLIB-FindPoints can be used to interpolate a
+  (Par)GridFunction in an arbitrary number of physical space points in 2D and
+  3D. The GridFunction must be in H1 and in the same space as the mesh that is
+  used to find the points.
+
+- Added a new example, Example 22/22p, to demonstrate the use of the new
+  complex-valued finite element operators. The example defines and solves
+  a family of time-harmonic PDEs related to damped harmonic oscillators.
+
+- Added a new example, Example 24/24p, to demonstrate usage of mixed finite
+  element spaces in bilinear forms. Partial assembly is supported in this
+  example.
+
+- Added device support in Example 3/3p and Example 9/9p.
+
+- Updated Examples 1/1p and 3/3p to use diagonal preconditioning in partial
+  assembly mode.
+
+- The mesh-optimizer and pmesh-optimizer miniapps have been updated to
+  demonstrate the new r-adaptivity capabilities of TMOP.
+
+- New options to reorder and partition the mesh in the mesh-explorer miniapp.
+
+- The mesh-explorer miniapp now supports visualization of boundary attributes of
+  3D meshes (key 'b').
+
+- The (p)mesh-optimizer miniapp has been updated to demonstrate mesh
+  optimization for an AMR mesh.
+
+- Added a modification of Example 1 in examples/ginkgo that demonstrates the use
+  of the Gingko interface to solve a linear system.
+
+- Added a modification of ex9 in examples/hiop that demonstrates the nonlinear
+  constrained optimization interface and the use of the SLBQP and HiOp solvers.
+
 - Added a collection of 7 playful miniapps in miniapps/toys that illustrate the
   meshing and visualization features of the library in more relaxed settings.
   The toys include simulations of cellular automata, Rubik's cube, Mandelbrot
   set, a tool to convert any image to mfem mesh, and more.
 
-- Added 8 new example codes:
-  * Example 22/22p demonstrates the use of the new complex-valued finite element
-    operators by defining and solving a family of time-harmonic PDEs related to
-    damped harmonic oscillators.
-  * Example 23 solves a simple 2D/3D wave equation with the new second order
-    time integrators.
-  * Example 24/24p demonstrates usage of mixed finite element spaces in bilinear
-    forms. Partial assembly is supported in this example.
-  * A version of Example 1 in examples/ginkgo demonstrating the use of the
-    Gingko interface to solve a linear system.
-  * A version of Example 9/9p in examples/hiop demonstrating the nonlinear
-    constrained optimization interface and use of the SLBQP and HiOp solvers.
-
-- Added two new miniapps: Find Points and Field Diff in miniapps/gslib that show
-  how GSLIB-FindPoints can be used to interpolate a (Par) GridFunction in an
-  arbitrary number of physical space points in 2D and 3D. The GridFunction must
-  be in H1 and in the same space as the mesh that is used to find the points.
-
-- Added a simple miniapp, Get Values, that extracts field values at a set of
-  points, from previously saved data via DataCollection classes.
-
-- Several examples and miniapps were updated:
-  * Added device support in Example 3/3p and Example 9/9p.
-  * Example 1/1p and Example 3/3p now use diagonal preconditioning in partial
-    assembly mode.
-  * Example 9/9p now supports implicit time integration, using the new block
-    ILU(0) solvers as preconditioners for the linear system.
-  * The mesh-optimizer and pmesh-optimizer miniapps now include the new
-    r-adaptivity capabilities of TMOP. They were also updated to support mesh
-    optimization on non-conforming AMR meshes.
-  * New options to reorder and partition the mesh and boundary attribute
-    visualization (key 'b') are now available in the mesh-explorer miniapp.
-
-- Collected object files from the miniapps/common directory into a new library,
-  libmfem-common for the convenience of application developers. The new library
-  is now used in several miniapps in the electromagnetic and tools directories.
+- Example 9 and 9p now support implicit time integration, using the new block
+  ILU(0) solvers as preconditioners for the linear system.
 
 Improved testing
 ----------------
-- Added a large number of unit tests in the tests/unit directory, including
-  several parallel unit tests.
-
 - Added a new directory, tests/scripts, with several shell scripts that perform
   simple checks on the code including: code styling, documentation formatting,
   proper use of .gitignore, and preventing the accidental commit of large files.
@@ -298,13 +211,7 @@ Improved testing
 
 Miscellaneous
 -------------
-- Added support for output in the ParaView XML format. Both low-order and
-  high-order Lagrange elements are supported. Output can be in ASCII or binary
-  format. The binary output can be compressed if MFEM is compiled with zlib
-  support (MFEM_USE_ZLIB). See the new ParaViewDataCollection class and the
-  updated Examples 5/5p and 9/9p.
-
-- Upgraded the SUNDIALS interface to utilize SUNDIALS 5.0. This necessitated a
+- Upgraded the SUNDIALS interface to utilize version 5.0. This necessitated a
   complete rework of the interface and requires changes at the application
   level. Example usage of the new interface can be found in examples/sundials.
 
@@ -312,13 +219,23 @@ Miscellaneous
   output stream. The build system definition now uses MFEM_USE_ZLIB instead of
   MFEM_USE_GZSTREAM, but the code interface (e.g. ofgzstream) remains the same.
 
-- Various other simplifications, extensions, and bugfixes in the code.
+- Added support for output in the ParaView XML format. Both low-order and
+  high-order Lagrange elements are supported. Output can be in ASCII or binary
+  format. The binary output can be compressed if MFEM is compiled with zlib
+  support (MFEM_USE_ZLIB). See the new ParaViewDataCollection class and the
+  updated Examples 5/5p and 9/9p.
+
+- Collected object files from the miniapps/common directory into a new library,
+  libmfem-common for the convenience of application developers. The new library
+  is now used in several miniapps in the electromagnetic and tools directories.
+
+- Added unit tests for time integrators.
 
 API changes
 -----------
-- In the enum classes MemoryType and MemoryClass, "CUDA" was renamed to "DEVICE"
-  which now denotes either "CUDA" or "HIP" depending on the build configuration.
-  In the same enum classes, "CUDA_UVM" was renamed to "MANAGED".
+- In the enum classes MemoryType and MemoryClass, 'CUDA' was renamed to 'DEVICE'
+  which now denotes either 'CUDA' or 'HIP' depending on the build configuration.
+  In the same enum classes, 'CUDA_UVM' was renamed to 'MANAGED'.
 
 
 Version 4.0, released on May 24, 2019

CMakeLists.txt

@@ -50,7 +50,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.1.1)
+set(${PROJECT_NAME}_VERSION 4.0.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -237,16 +237,16 @@ if (MFEM_USE_STRUMPACK)
   endif()
 endif()
 
+# Gecko
+if (MFEM_USE_GECKO)
+  find_package(Gecko REQUIRED)
+endif()
+
 # GnuTLS
 if (MFEM_USE_GNUTLS)
   find_package(_GnuTLS REQUIRED)
 endif()
 
-# GSLIB
-if (MFEM_USE_GSLIB)
-  find_package(GSLIB REQUIRED)
-endif()
-
 # NetCDF
 if (MFEM_USE_NETCDF)
   find_package(NetCDF REQUIRED)
@@ -317,17 +317,12 @@ endif()
 if (MFEM_USE_RAJA)
    find_package(RAJA REQUIRED)
 endif()
-
+   
 # UMPIRE
 if (MFEM_USE_UMPIRE)
    find_package(UMPIRE REQUIRED)
 endif()
 
-# ADIOS2 for parallel I/O
-if (MFEM_USE_ADIOS2)
-   find_package(ADIOS2 REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -352,8 +347,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO Ginkgo GNUTLS NETCDF MPFR
+    PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

CONTRIBUTING.md

@@ -370,9 +370,6 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   checks in Travis enforce MFEM-specific rules which are explained in the
   error messages and the `tests/scripts` directory.
 
-- If triggered, track the status of the LLNL GitLab tests. If failing, ask
-  one of the _LLNL developers_ for details.
-
 ### Pull Request Checklist
 
 Before a PR can be merged, it should satisfy the following:
@@ -383,13 +380,9 @@ Before a PR can be merged, it should satisfy the following:
     - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
     - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
 - [ ] Update `INSTALL`:
-    - [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
-    - [ ] Have the version ranges for any required or optional libraries changed?
+    - [ ] Had a new optional library been added? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
     - [ ] Does `make` or `cmake` have a new target?
     - [ ] Did the requirements or the installation process change? *(rare)*
-- [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.travis.yml`
-    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
     - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
     - [ ] Add new patterns (just for the new files above) and re-run the above test.
@@ -503,10 +496,6 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - [ ] `makefile`
     - [ ] `CMakeLists.txt`
     - [ ] `doc/CodeDocumentation.conf.in`
-- [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
-- [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.travis.yml`
-    - [ ] `.appveyor.yml`
 - [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:
 
@@ -547,17 +536,6 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   - `mfem:gh-next` -- Bleeding-edge development version, may be broken, use at
      your own risk.
 
-### Mirroring on GitLab
-
-- MFEM repository is also mirrored on the LLNL GitLab instance, in a
-  semi-automated manner.
-
-- This instance is meant to complete CI testing with tests on Livermore
-  Computing systems. Gitlab pipeline status is reported in the corresponding
-  GitHub pull request.
-
-- No change can be made on this instance.
-
 ## Automated Testing
 
 MFEM has several levels of automated testing running on GitHub, as well as on
@@ -585,14 +563,10 @@ and debug build is performed with a simple run of `ex1` to verify the executable
 
 ### Tests at LLNL
 
-- We mirror the `master` and `next` branches internally (to `gh-master` and
-  `gh-next`) and run longer nightly tests via cron. On the weekends, a more
-  extensive test is run which extracts and executes all the different sample
-  runs from each example.
-
-- We also mirror PRs on the LLNL GitLab instance. PR mirroring can only be
-  triggered by _LLNL developers_, but test status is publicly available. Only
-  _LLNL developers_ can access the detailed test report.
+- We mirror the `master` and `next` branches internally (to `gh-master`
+and `gh-next`) and run longer nightly tests via cron. On the weekends, a more
+extensive test is run which extracts and executes all the different sample runs
+from each example.
 
 ## Contact Information
 

INSTALL

@@ -30,8 +30,7 @@ the option MFEM_USE_METIS.
 MFEM also includes support for devices such as GPUs, and programming models such
 as CUDA, HIP, OCCA, OpenMP and RAJA.
 
-- Starting with version 4.0, MFEM requires a C++11 compiler. We recommend using
-  a newer compiler, e.g. GCC version 4.9 or higher.
+- Starting with version 4.0, MFEM requires a C++11 compiler
 
 - CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
   https://developer.nvidia.com/cuda-toolkit
@@ -403,11 +402,6 @@ MFEM_USE_CONDUIT = YES/NO
    an installation of Conduit. If Conduit was built with HDF5 support, it also
    requires an installation of HDF5 (see also MFEM_USE_NETCDF).
 
-MFEM_USE_ADIOS2 = YES/NO
-   Enables support for ADIOS2, version 2 of the adaptable input output system
-   for scientific data management. In MFEM, ADIOS2 provides parallel I/O with
-   ParaView visualization.
-
 MFEM_USE_ZLIB = YES/NO
    Enables use of on-the-fly gzip compressed streams. With this feature enabled
    (YES), MFEM can compress its output files on-the-fly. In addition, it can
@@ -497,13 +491,11 @@ The specific libraries and their options are:
 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
   URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b.
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
   URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
   Options: METIS_OPT, METIS_LIB.
-  Versions: METIS 4.0.3 or 5.1.0.
 
 - LAPACK (optional), used when MFEM_USE_LAPACK = YES. Alternative, optimized
   implementations can also be used, e.g. the ATLAS project.
@@ -527,7 +519,6 @@ The specific libraries and their options are:
   both MPI and hypre.
   URL: http://computation.llnl.gov/projects/sundials/sundials-software
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0.
 
 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
   URL: http://trilinos.org/oldsite/packages/mesquite
@@ -536,7 +527,6 @@ The specific libraries and their options are:
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
   Options: SUITESPARSE_OPT, SUITESPARSE_LIB.
-  Versions: SuiteSparse >= 4.5.4, older versions may work too.
 
 - SuperLU_DIST (optional), used when MFEM_USE_SUPERLU = YES. Note that
   SuperLU_DIST requires ParMETIS, which includes METIS 5 in its distribution.
@@ -544,7 +534,6 @@ The specific libraries and their options are:
   same location.
   URL: http://crd-legacy.lbl.gov/~xiaoye/SuperLU
   Options: SUPERLU_OPT, SUPERLU_LIB.
-  Versions: SuperLU_DIST >= 5.1.0.
 
 - STRUMPACK (optional), used when MFEM_USE_STRUMPACK = YES. Note that STRUMPACK
   requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
@@ -554,7 +543,6 @@ The specific libraries and their options are:
   2.0.0 or later.
   URL: http://portal.nersc.gov/project/sparse/strumpack
   Options: STRUMPACK_OPT, STRUMPACK_LIB.
-  Versions: STRUMPACK >= 3.0.0, requires HYPRE < 2.16.0.
 
 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
   C++ compiler that supports the C++-11 standard. For additional requirements
@@ -567,7 +555,6 @@ The specific libraries and their options are:
   one can get the library through the Homebrew package manager (http://brew.sh).
   URL: http://gnutls.org
   Options: GNUTLS_OPT, GNUTLS_LIB.
-  Versions: GnuTLS >= 2.12.0, older versions may work too.
 
 - NetCDF (optional), used when MFEM_USE_NETCDF = YES, required for reading Cubit
   mesh files. Also requires installation of HDF5 and ZLIB, as explained at the
@@ -575,7 +562,6 @@ The specific libraries and their options are:
   don't need the C++ or parallel versions.
   URL: www.unidata.ucar.edu/software/netcdf
   Options: NETCDF_OPT, NETCDF_LIB.
-  Versions: NetCDF >= 4.4.0.
 
 - PETSc (optional), used when MFEM_USE_PETSC = YES. Version 3.8 or higher of
   the PETSC dev branch is required. The MFEM and PETSc builds can share common
@@ -587,7 +573,6 @@ The specific libraries and their options are:
                 --with-shared-libraries=0
   URL: https://www.mcs.anl.gov/petsc
   Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0.
 
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
   Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
@@ -595,22 +580,16 @@ The specific libraries and their options are:
        https://github.com/LLNL/conduit (Conduit)
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: SIDRE_OPT, SIDRE_LIB.
-  Versions: Axom >= 0.3.1.
 
 - Conduit (optional), used when MFEM_USE_CONDUIT = YES. Conduit Mesh Blueprint
   support requires Conduit >= v0.3.1 and VisIt >= v2.13.1 to read the output.
   URL: https://github.com/LLNL/conduit (Conduit)
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: CONDUIT_OPT, CONDUIT_LIB.
-  Versions: Conduit >= 0.3.1.
-
-- ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
-  URL: https://adios2.readthedocs.io/
 
 - PUMI (optional), used when MFEM_USE_PUMI = YES.
   URL: https://scorec.rpi.edu/pumi
   Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI >= 2.2.0.
 
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
@@ -624,12 +603,10 @@ The specific libraries and their options are:
   MFEM_USE_GSLIB=YES.
   URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
   Options: GSLIB_OPT, GSLIB_LIB.
-  Versions: GSLIB >= 1.0.5.
 
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
   URL: https://developer.nvidia.com/cuda-toolkit
   Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
-  Versions: CUDA >= 9.1, older versions may work too.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
   URL: https://rocm.github.io/ROCmInstall.html
@@ -638,25 +615,21 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
   URL: https://libocca.org
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.0.9.
 
-- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
-  or later version, specifically, git-hash 3d05795 or later.
+- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED's master
+  branch, specifically, git-hash c00ee0d or later.
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.6.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
   URL: https://github.com/LLNL/RAJA
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.10.0.
 
 - Umpire, used when MFEM_USE_UMPIRE = YES.
   URL: https://github.com/LLNL/Umpire
   Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-  Versions: Umpire >= 2.0.0.
 
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org

LICENSE

@@ -27,3 +27,12 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Additionally this project distributes 3 libraries with their own respective
+licenses which can be found in their code and attached license files. These
+libraries and their licenses are as follows:
+
+* Picojson (fem/picojson.h) -- Custom 2-clause license
+* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
+* Zstr (general/zstr.hpp) -- MIT license
+

NOTICE

@@ -18,14 +18,3 @@ The views and opinions of authors expressed herein do not necessarily
 state or reflect those of the United States Government or Lawrence
 Livermore National Security, LLC, and shall not be used for advertising
 or product endorsement purposes.
-
-Inclusion of external software:
-
-This project distributes the sources of several external software products with
-their own respective licenses which can be found in their code and attached
-license files. These software products and their licenses are as follows:
-
-* Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
-* Picojson (fem/picojson.h) -- Custom 2-clause license
-* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
-* Zstr (general/zstr.hpp) -- MIT license

config/XSDKDefaults.cmake

@@ -228,6 +228,10 @@ IF (DEFINED TPL_ENABLE_STRUMPACK)
   SET(MFEM_USE_STRUMPACK ${TPL_ENABLE_STRUMPACK} CACHE BOOL "Enable STRUMPACK usage" FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_GECKO)
+  SET(MFEM_USE_GECKO ${TPL_ENABLE_GECKO} CACHE BOOL "Enable GECKO usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_GINKGO)
   SET(MFEM_USE_GINKGO ${TPL_ENABLE_GINKGO} CACHE BOOL "Enable GINKGO usage" FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -33,9 +33,9 @@ set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
+set(MFEM_USE_GECKO @MFEM_USE_GECKO@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
-set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
@@ -47,7 +47,6 @@ set(MFEM_USE_OCCA @MFEM_USE_OCCA@)
 set(MFEM_USE_RAJA @MFEM_USE_RAJA@)
 set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
-set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_CONFIG_HEADER
@@ -89,15 +89,15 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC
 
+// Enable functionality based on the Gecko library
+#cmakedefine MFEM_USE_GECKO
+
 // Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO
 
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS
 
-// Enable MFEM functionality based on the GSLIB library
-#cmakedefine MFEM_USE_GSLIB
-
 // Enable MFEM functionality based on the NetCDF library
 #cmakedefine MFEM_USE_NETCDF
 
@@ -132,9 +132,6 @@
 // Enable MFEM functionality based on the Umpire library
 #cmakedefine MFEM_USE_UMPIRE
 
-// Enable MFEM functionality based on the ADIOS2 library
-#cmakedefine MFEM_USE_ADIOS2
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.

config/cmake/modules/FindADIOS2.cmake

@@ -1,52 +0,0 @@
-#------------------------------------------------------------------------------#
-# Distributed under the OSI-approved Apache License, Version 2.0.  See
-# accompanying file Copyright.txt for details.
-#------------------------------------------------------------------------------#
-#
-# FindADIOS2
-# -----------
-#
-# Try to find the ADIOS2 library
-#
-# This module defines the following variables:
-#
-#   ADIOS2_FOUND        - System has ADIOS2
-#   ADIOS2_INCLUDE_DIRS - The ADIOS2 include directory
-#   ADIOS2_LIBRARIES    - Link these to use ADIOS2
-#
-# and the following imported targets:
-#   ADIOS2::ADIOS2 - The ADIOS2 compression library target
-#
-# You can also set the following variable to help guide the search:
-#   ADIOS2_DIR - The install prefix for ADIOS2 containing the
-#              include and lib folders
-#              Note: this can be set as a CMake variable or an
-#                    environment variable.  If specified as a CMake
-#                    variable, it will override any setting specified
-#                    as an environment variable.
-
-if(NOT ADIOS2_FOUND)
-  if((NOT ADIOS2_DIR) AND (NOT (ENV{ADIOS2_DIR} STREQUAL "")))
-    set(ADIOS2_DIR "$ENV{ADIOS2_DIR}")
-  endif()
-  if(ADIOS2_DIR)
-    set(ADIOS2_INCLUDE_OPTS HINTS ${ADIOS2_DIR}/include NO_DEFAULT_PATHS)
-    set(ADIOS2_LIBRARY_OPTS
-      HINTS ${ADIOS2_DIR}/lib ${ADIOS2_DIR}/lib64
-      NO_DEFAULT_PATHS
-    )
-  endif()
-
-  find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 
-
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(ADIOS2
-    FOUND_VAR ADIOS2_FOUND
-    REQUIRED_VARS ADIOS2_LIBRARY ADIOS2_INCLUDE_DIR
-  )
-  if(ADIOS2_FOUND)
-    set(ADIOS2_INCLUDE_DIRS ${ADIOS2_INCLUDE_DIR})
-    set(ADIOS2_LIBRARIES ${ADIOS2_LIBRARY})
-  endif()
-endif()

config/cmake/modules/FindGecko.cmake

@@ -10,10 +10,10 @@
 # CONTRIBUTING.md for details.
 
 # Defines the following variables:
-#   - GSLIB_FOUND
-#   - GSLIB_LIBRARIES
-#   - GSLIB_INCLUDE_DIRS
+#   - GECKO_FOUND
+#   - GECKO_LIBRARIES
+#   - GECKO_INCLUDE_DIRS
 
 include(MfemCmakeUtilities)
-mfem_find_package(gslib GSLIB GSLIB_DIR "include" gslib.h "lib" gs
-  "Paths to headers required by GSLIB." "Libraries required by GSLIB.")
+mfem_find_package(Gecko GECKO GECKO_DIR "include;inc" graph.h "lib" gecko
+  "Paths to headers required by Gecko." "Libraries required by Gecko.")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -730,8 +730,8 @@ function(mfem_export_mk_files)
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GECKO MFEM_USE_GNUTLS
+      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
       MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
       MFEM_USE_UMPIRE)
   foreach(var ${CONFIG_MK_BOOL_VARS})

config/config.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 

config/config.hpp.in

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_CONFIG_HEADER
@@ -97,6 +97,9 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
 
+// Enable functionality based on the Gecko library
+// #define MFEM_USE_GECKO
+
 // Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO
 
@@ -147,9 +150,6 @@
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE
 
-// Enable IO functionality based on the ADIOS2 library.
-// #define MFEM_USE_ADIOS2
-
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 

config/config.mk.in

@@ -33,6 +33,7 @@ MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
+MFEM_USE_GECKO         = @MFEM_USE_GECKO@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
@@ -49,7 +50,6 @@ MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
 MFEM_USE_CEED          = @MFEM_USE_CEED@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
-MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -34,9 +34,9 @@ option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
+option(MFEM_USE_GECKO "Enable GECKO usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
-option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
@@ -49,7 +49,6 @@ option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
-option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 
@@ -139,12 +138,12 @@ set(ScaLAPACK_TARGET_NAMES scalapack)
 # set(ScaLAPACK_TARGET_FORCE)
 # set(ScaLAPACK_IMPORT_CONFIG DEBUG)
 
+set(GECKO_DIR "${MFEM_DIR}/../gecko" CACHE PATH "Path to the Gecko library.")
+
 set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")
 
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
-set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
-
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
 # May need to add "HDF5" as requirement.
 set(NetCDF_REQUIRED_PACKAGES "" CACHE STRING

config/defaults.mk

@@ -121,6 +121,7 @@ MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_STRUMPACK     = NO
+MFEM_USE_GECKO         = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
@@ -137,7 +138,6 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
-MFEM_USE_ADIOS2        = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -244,6 +244,11 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
  $(SCOTCH_LIB) $(SCALAPACK_LIB)
 
+# Gecko library configuration
+GECKO_DIR = @MFEM_DIR@/../gecko
+GECKO_OPT = -I$(GECKO_DIR)/inc
+GECKO_LIB = -L$(GECKO_DIR)/lib -lgecko
+
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include

config/get_hypre_version.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "HYPRE_config.h"

config/sample-runs.sh

@@ -47,7 +47,7 @@ groups_serial=(
    "Meshing miniapps:"
    "miniapps/meshing"
    "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    mesh-optimizer.cpp minimal-surface.cpp"'
+    mesh-optimizer.cpp"'
 )
 # Parallel groups
 groups_parallel=(
@@ -72,7 +72,7 @@ groups_parallel=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "pmesh-optimizer.cpp pminimal-surface.cpp"'
+   "pmesh-optimizer.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
@@ -101,7 +101,7 @@ groups_all=(
    "Meshing miniapps:"
    "miniapps/meshing"
    "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    {,p}mesh-optimizer.cpp {,p}minimal-surface.cpp"'
+    {,p}mesh-optimizer.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
@@ -201,11 +201,11 @@ function help_message()
       -s|-show    Show all configured sample runs and exit
       -n          Dry run: replace "\$sample_run" with "echo \$sample_run"
       <var>=<value>
-                  Set a shell script variable; see below for valid variables
+                  Set a shell script varible; see below for valid variables
        *          Any other parameter is treated as <mfem_dir>
       <mfem_dir>  [${mfem_dir}] is the MFEM source directory
 
-   This script tests all the sample runs listed in the beginning comments of
+   This script tests all the sample runs listed in the begining comments of
    MFEM's serial or parallel example and miniapp codes. The list of sample runs
    is auto-generated and can be viewed with the -s|-show option.
 

config/tconfig.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_TEMPLATE_CONFIG

config/test.mk

@@ -70,10 +70,9 @@ mfem-test = \
 mfem-test-file = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
-   err="$$3"; \
    if [ "$$3" = 0 ] && [ -e $(4) ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
-   rm -f $(1).stderr; exit $$err
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
+   rm -f $(1).stderr; exit $$3
 
 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
 

data/rt-2d-p4-tri.mesh

@@ -1,658 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-32
-1 2 0 6 5
-1 2 6 0 1
-1 2 1 7 6
-1 2 7 1 2
-1 2 2 8 7
-1 2 8 2 3
-1 2 3 9 8
-1 2 9 3 4
-1 2 5 11 10
-1 2 11 5 6
-1 2 6 12 11
-1 2 12 6 7
-1 2 7 13 12
-1 2 13 7 8
-1 2 8 14 13
-1 2 14 8 9
-1 2 10 16 15
-1 2 16 10 11
-1 2 11 17 16
-1 2 17 11 12
-1 2 12 18 17
-1 2 18 12 13
-1 2 13 19 18
-1 2 19 13 14
-1 2 15 21 20
-1 2 21 15 16
-1 2 16 22 21
-1 2 22 16 17
-1 2 17 23 22
-1 2 23 17 18
-1 2 18 24 23
-1 2 24 18 19
-
-boundary
-16
-1 1 0 1
-1 1 1 2
-1 1 2 3
-1 1 3 4
-1 1 21 20
-1 1 22 21
-1 1 23 22
-1 1 24 23
-1 1 5 0
-1 1 10 5
-1 1 15 10
-1 1 20 15
-1 1 4 9
-1 1 9 14
-1 1 14 19
-1 1 19 24
-
-vertices
-25
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P4
-VDim: 2
-Ordering: 0
-
-0
-0.080037555
-0.18891599
-0.33468826
-0.5
-0
-0.068735129
-0.15432123
-0.26160657
-0.5
-0
-0.29654082
-0.30546623
-0.3058964
-0.5
-0
-0.20767624
-0.36127438
-0.44146415
-0.5
-0
-0.13483947
-0.26369712
-0.38683816
-0.5
-0.014952234
-0.041153003
-0.059682322
-0.0094260723
-0.029495442
-0.05352801
-0
-0
-0
-0.012522467
-0.037217335
-0.064269681
-0.087609647
-0.08723084
-0.076130492
-0.10350129
-0.13573231
-0.15245099
-0.08322337
-0.11165341
-0.14025324
-0.098506321
-0.13189839
-0.16793065
-0.19870689
-0.19690918
-0.17000006
-0.21946868
-0.25930152
-0.26834534
-0.16770468
-0.19757636
-0.23626491
-0.21173907
-0.25453506
-0.30293787
-0.33377239
-0.3257268
-0.29597076
-0.36376506
-0.39473641
-0.43647098
-0.28072394
-0.33792261
-0.42659702
-0.36947063
-0.42570548
-0.47596105
-0.5
-0.5
-0.5
-0.0099864538
-0.063795839
-0.20414266
-0.08893824
-0.21002116
-0.2762338
-0
-0
-0
-0.054898508
-0.10157741
-0.2284324
-0.064584618
-0.10597019
-0.23249836
-0.29981305
-0.30245895
-0.3041069
-0.11707613
-0.13803749
-0.2415601
-0.12965761
-0.15738912
-0.2354581
-0.30218595
-0.29671399
-0.29863017
-0.20412323
-0.20453954
-0.26863508
-0.22539134
-0.24787467
-0.34860698
-0.33837263
-0.4043122
-0.469932
-0.5
-0.5
-0.5
-0.11844871
-0.24101622
-0.25197733
-0.031500653
-0.10358535
-0.17066571
-0
-0
-0
-0.31794342
-0.32743739
-0.27710165
-0.33607374
-0.37471009
-0.37354724
-0.23992073
-0.29585331
-0.34187884
-0.36550349
-0.4073581
-0.387609
-0.37180134
-0.43316502
-0.44343776
-0.37758077
-0.40585611
-0.43000882
-0.3933336
-0.45407946
-0.45249351
-0.40898455
-0.47651657
-0.49206447
-0.45256956
-0.47260482
-0.49092561
-0.5
-0.5
-0.5
-0.026362903
-0.073878701
-0.11613778
-0.023122535
-0.066695566
-0.1108041
-0
-0
-0
-0.16793289
-0.14434036
-0.13987182
-0.19656602
-0.21275812
-0.24994776
-0.1578417
-0.19934979
-0.24083284
-0.32320543
-0.27914423
-0.27084466
-0.3439122
-0.33953819
-0.36877629
-0.28631832
-0.3269526
-0.36633079
-0.42319581
-0.39501579
-0.38817981
-0.43758677
-0.44675508
-0.48109078
-0.406478
-0.44380678
-0.48087362
-0.5
-0.5
-0.5
-0.016969126
-0.041983544
-0.014198448
-0.065312723
-0.035548117
-0.069355718
-0.10857895
-0.13201179
-0.095829809
-0.16607293
-0.13147499
-0.1755789
-0.22583324
-0.24098225
-0.19443999
-0.29139212
-0.24983404
-0.30111347
-0.35852094
-0.38184769
-0.3233027
-0.44295857
-0.40231794
-0.46329465
-0.022440932
-0.15935137
-0.069456975
-0.11343049
-0.023880252
-0.044943579
-0.08204175
-0.21920653
-0.21861503
-0.14933649
-0.077488899
-0.10124637
-0.13138589
-0.22210914
-0.22639497
-0.19848199
-0.14524467
-0.17849541
-0.21264375
-0.31502785
-0.2802125
-0.34654237
-0.2609936
-0.38139942
-0.11500094
-0.20970793
-0.068148195
-0.2948225
-0.21148716
-0.29602829
-0.34953362
-0.35609711
-0.31302869
-0.39332167
-0.3512777
-0.36560449
-0.41299688
-0.43208187
-0.40693217
-0.44872186
-0.38511057
-0.39496487
-0.45690758
-0.48133813
-0.46346986
-0.490474
-0.44166593
-0.48147819
-0.030282336
-0.083792186
-0.02910331
-0.11410518
-0.061710433
-0.12834537
-0.17607001
-0.21998313
-0.168306
-0.24854215
-0.22965745
-0.28950386
-0.31259845
-0.34167622
-0.29599772
-0.36840013
-0.36351431
-0.40168148
-0.42128125
-0.45466143
-0.4113499
-0.47687155
-0.45469921
-0.48567374
--1
--1
--1
--1
--1
--0.15965432
--0.32197482
--0.41918151
--0.52678756
--0.77133994
-0.40860878
-0.072728863
--0.14848828
--0.26931819
--0.7021443
-0.57766762
-0.57081481
-0.44393876
-0.36669313
-0.29774286
-1
-1
-1
-1
-1
--0.86569445
--0.63683784
--0.41390431
--0.1637733
--0.1944228
--0.25939651
--0.86922175
--0.6201834
--0.3431334
--1
--1
--1
--0.88868738
--0.67663975
--0.44834472
--0.90397029
--0.72975264
--0.54422009
--0.32327853
--0.35864361
--0.40291224
--1
--1
--1
--0.91491529
--0.74947265
--0.55773662
--0.91593956
--0.7515123
--0.60003971
--0.43409189
--0.46919283
--0.50807492
--1
--1
--1
--0.94329396
--0.81124577
--0.63976678
--0.94400926
--0.80597931
--0.72924317
--0.53646582
--0.59031025
--0.68652559
--1
--1
--1
--0.93104934
--0.84569627
--0.79296547
-0.041999904
-0.29850158
-0.30729195
-0.45728782
-0.35764094
-0.16913525
-0.061137722
-0.29127294
-0.38099644
--0.088384262
-0.25650801
-0.24561704
--0.16435602
-0.15009489
-0.11237637
-0.025288955
--0.048972923
--0.11406787
--0.21720563
-0.08488915
-0.066506492
--0.24209903
--0.043575157
--0.086589095
--0.16153884
--0.18523687
--0.22932127
--0.38122961
--0.16461293
--0.19243227
--0.41865735
--0.34156248
--0.48356303
--0.37201441
--0.53268808
--0.65308607
--0.75430416
--0.73023922
--0.71120622
-0.47884311
-0.4178142
-0.49072846
-0.58725278
-0.59584798
-0.58239064
-0.4234016
-0.46095327
-0.52171434
-0.070761641
-0.22139345
-0.4728164
--0.0061031166
-0.08139472
-0.31389427
-0.56360444
-0.52568701
-0.47004678
--0.24367032
--0.046748649
-0.29680271
--0.28444339
--0.1549505
-0.19113109
-0.43088152
-0.39665287
-0.37100684
--0.44177181
--0.25461873
-0.18475263
--0.48136759
--0.29140221
-0.1123432
-0.37161554
-0.35259093
-0.32059835
--0.61384042
--0.31147032
-0.090567115
-0.65220034
-0.78355045
-0.92418305
-1
-1
-1
-0.64832621
-0.78550418
-0.92400425
-0.63797415
-0.77596776
-0.92314707
-0.63104533
-0.7660646
-0.91965726
-1
-1
-1
-0.57545921
-0.75383088
-0.9173847
-0.55774932
-0.7380311
-0.90968212
-1
-1
-1
-0.51253666
-0.7278085
-0.90770935
-0.50727947
-0.72218234
-0.90535401
-1
-1
-1
-0.48486496
-0.73073829
-0.90287375
--0.6962467
--0.40228898
--0.37574172
--0.58649911
--0.84441703
--0.85738994
--0.75308434
--0.5319245
--0.48294944
--0.68520148
--0.88206789
--0.89093514
--0.79488362
--0.60754966
--0.58805034
--0.73294422
--0.90141216
--0.91752348
--0.85154118
--0.69385203
--0.65973283
--0.78922213
--0.92138639
--0.91934097
-0.24934006
-0.37486984
-0.4198655
-0.31832202
-0.045830659
--0.0077671847
-0.13803158
-0.16930028
-0.22884866
-0.15849229
--0.12626684
--0.15760342
--0.027255655
--0.016529794
-0.056743578
--0.083681581
--0.24140643
--0.31096294
--0.26404238
--0.40050205
--0.2660714
--0.50456441
--0.44620407
--0.60109934
-0.50320805
-0.51265344
-0.55244235
-0.34530401
-0.40091936
-0.18310352
-0.093141548
-0.32926871
-0.41233045
-0.09370592
--0.080921737
--0.19241964
--0.18262835
-0.17261584
-0.23021507
--0.10375706
--0.33420072
--0.40930232
--0.37227789
-0.095851556
-0.13123474
--0.20809881
--0.52360734
--0.57587426
-0.74756029
-0.90938574
-0.90980172
-0.81942673
-0.65843419
-0.662674
-0.7329696
-0.9039993
-0.90589299
-0.80179047
-0.64060982
-0.61482888
-0.69893809
-0.89438167
-0.89767922
-0.78163371
-0.56765352
-0.5444136
-0.66910116
-0.88940521
-0.89031668
-0.77444781
-0.538213
-0.52498697

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.1.1
+PROJECT_NUMBER         = v4.0.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -774,7 +774,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
                          @MFEM_SOURCE_DIR@/miniapps/performance \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
@@ -842,7 +841,7 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        = mfem::internal mfem::kernels::internal
+EXCLUDE_SYMBOLS        = mfem::internal
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

doc/CodeDocumentation.dox

@@ -83,13 +83,7 @@ namespace mfem {
  * - <a class="el" href="ex21p_8cpp_source.html">Example 21p</a>: parallel adaptive mesh refinement for linear elasticity
  * - <a class="el" href="ex22_8cpp_source.html">Example 22</a>: complex-valued linear systems for damped harmonic oscillators
  * - <a class="el" href="ex22p_8cpp_source.html">Example 22p</a>: parallel complex-valued linear systems for damped harmonic oscillators
- * - <a class="el" href="ex23_8cpp_source.html">Example 23</a>: second order in time wave equation
- * - <a class="el" href="ex24_8cpp_source.html">Example 24</a>: mixed finite element spaces and interpolators
- * - <a class="el" href="ex24p_8cpp_source.html">Example 24p</a>: parallel mixed finite element spaces and interpolators
- * - <a class="el" href="ex25_8cpp_source.html">Example 25</a>: simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
- * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
- * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM 
- * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM 
+ * - <a class="el" href="ex23_8cpp_source.html">Example 23</a>: 2nd order in time the wave equation
  *
  * <H4>SUNDIALS Examples</H4>
  * - Variants of Examples
@@ -130,11 +124,6 @@ namespace mfem {
  *   <a class="el" href="hiop_2ex9p_8cpp_source.html">9p</a>,
  *   demonstrating the use of MFEM's \link hiop.hpp HiOp classes\endlink
  *
- * <H4>Ginkgo Examples</H4>
- * - Variants of Example
- *   <a class="el" href="ginkgo_2ex1_8cpp_source.html">1</a>
- *   demonstrating the use of MFEM's \link ginkgo.hpp Ginkgo classes\endlink
- *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
@@ -143,8 +132,6 @@ namespace mfem {
  * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
  * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
- * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
- * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
  * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
  * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
  * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
@@ -156,7 +143,6 @@ namespace mfem {
  * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
  * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
  * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
- * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *

examples/CMakeLists.txt

@@ -31,9 +31,6 @@ list(APPEND ALL_EXE_SRCS
   ex22.cpp
   ex23.cpp
   ex24.cpp
-  ex25.cpp
-  ex26.cpp
-  ex27.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -61,11 +58,6 @@ if (MFEM_USE_MPI)
     ex21p.cpp
     ex22p.cpp
     ex24p.cpp
-    ex25p.cpp
-    ex26p.cpp
-    ex27p.cpp
-    pa_oper.cpp
-    io_benchmark.cpp
     )
 endif()
 
@@ -85,8 +77,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
   elseif(${TEST_NAME} MATCHES "ex15p*")
     list(APPEND THIS_TEST_OPTIONS "-e" "1")
-  elseif(${TEST_NAME} MATCHES "ex27p*")
-    list(APPEND THIS_TEST_OPTIONS "-dg")
   endif()
 
   if (NOT (${TEST_NAME} MATCHES ".*p$"))

examples/ex1.cpp

@@ -35,7 +35,6 @@
 //               ex1 -pa -d ceed-cuda
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem

examples/ex12p.cpp

@@ -33,8 +33,7 @@
 //               The example highlights the use of the LOBPCG eigenvalue solver
 //               together with the BoomerAMG preconditioner in HYPRE. Reusing a
 //               single GLVis visualization window for multiple eigenfunctions
-//               and optional saving with ADIOS2 (adios2.readthedocs.io) streams
-//               are also illustrated.
+//               is also illustrated.
 //
 //               We recommend viewing examples 2 and 11 before viewing this
 //               example.
@@ -61,7 +60,6 @@ int main(int argc, char *argv[])
    int seed = 66;
    bool visualization = 1;
    bool amg_elast = 0;
-   bool adios2 = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,9 +77,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
    args.Parse();
    if (!args.Good())
    {
@@ -291,28 +286,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 13. Optionally output a BP (binary pack) file using ADIOS2. This can be
-   //     visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-
-      adios2stream adios2output("ex12-p-" + postfix + ".bp",
-                                adios2stream::openmode::out, MPI_COMM_WORLD);
-      pmesh->Print(adios2output);
-      for (int i=0; i<nev; i++)
-      {
-         x = lobpcg->GetEigenvector(i);
-         // x is a temporary that must be saved immediately
-         x.Save(adios2output, "mode_" + std::to_string(i));
-      }
-   }
-#endif
-
-   // 14. Send the above data by socket to a GLVis server. Use the "n" and "b"
+   // 13. Send the above data by socket to a GLVis server. Use the "n" and "b"
    //     keys in GLVis to visualize the displacements.
    if (visualization)
    {
@@ -352,7 +326,7 @@ int main(int argc, char *argv[])
       mode_sock.close();
    }
 
-   // 15. Free the used memory.
+   // 14. Free the used memory.
    delete lobpcg;
    delete amg;
    delete M;

examples/ex16p.cpp

@@ -24,8 +24,7 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. Optional saving
-//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -109,7 +108,6 @@ int main(int argc, char *argv[])
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
-   bool adios2 = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -142,9 +140,6 @@ int main(int argc, char *argv[])
                   "Save data files for VisIt (visit.llnl.gov) visualization.");
    args.AddOption(&vis_steps, "-vs", "--visualization-steps",
                   "Visualize every n-th timestep.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
    args.Parse();
    if (!args.Good())
    {
@@ -253,27 +248,6 @@ int main(int argc, char *argv[])
       visit_dc.Save();
    }
 
-   // Optionally output a BP (binary pack) file using ADIOS2. This can be
-   // visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection* adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      postfix += "_solver" + std::to_string(ode_solver_type);
-      const std::string collection_name = "ex16-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      adios2_dc->RegisterField("temperature", &u_gf);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
    socketstream sout;
    if (visualization)
    {
@@ -343,26 +317,10 @@ int main(int argc, char *argv[])
             visit_dc.SetTime(t);
             visit_dc.Save();
          }
-
-#ifdef MFEM_USE_ADIOS2
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
       }
       oper.SetParameters(u);
    }
 
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      delete adios2_dc;
-   }
-#endif
-
    // 11. Save the final solution in parallel. This output can be viewed later
    //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
    {

examples/ex1p.cpp

@@ -70,7 +70,6 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   int nfiles = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -87,7 +86,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&nfiles, "-nf", "--num-files", "Number of files to write.");
    args.Parse();
    if (!args.Good())
    {
@@ -160,7 +158,7 @@ int main(int argc, char *argv[])
    {
       fec = new H1_FECollection(order = 1, dim);
    }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec, 1, 0);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
    HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
@@ -239,52 +237,20 @@ int main(int argc, char *argv[])
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
 
-   std::string filename("nranks_");
-   filename += to_string(num_procs);
-   filename += ".gf";
-   {
-      double t1;
-      t1 = MPI_Wtime();
-      x.Save(filename.c_str(), nfiles);
-      double t2 = MPI_Wtime();
-
-      if (myid == 0)
-      {
-         err << "elapsed write time: " << t2 - t1 << endl;
-      }
-   }
-   {
-      double t1;
-      t1 = MPI_Wtime();
-      ParGridFunction new_x(fespace, filename.c_str());
-      double t2 = MPI_Wtime();
-      if (myid == 0)
-      {
-         err << "elapsed read time: " << t2 - t1 << endl;
-      }
-      // new_x -= x;
-      // out << "GF difference: " << new_x.Norml1() << endl;
-   }
-
    // 15. Save the refined mesh and the solution in parallel. This output can
    //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_name;
-      //mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << num_procs << setfill('0') << setw(6) << myid;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
 
-      //ofstream mesh_ofs(mesh_name.str().c_str());
-      //mesh_ofs.precision(8);
-      //pmesh->Print(mesh_ofs);
-      double t1 = MPI_Wtime();
       ofstream sol_ofs(sol_name.str().c_str());
       sol_ofs.precision(8);
       x.Save(sol_ofs);
-      double t2 = MPI_Wtime();
-      if (myid == 0)
-      {
-         err << t2 - t1 << endl;
-      }
    }
 
    // 16. Send the solution by socket to a GLVis server.

examples/ex22p.cpp

@@ -401,7 +401,7 @@ int main(int argc, char *argv[])
    //     diagonal preconditioner based on the appropriate multigrid
    //     preconditioner from hypre.
    {
-      Array<int> blockTrueOffsets;
+      Array<HYPRE_Int> blockTrueOffsets;
       blockTrueOffsets.SetSize(3);
       blockTrueOffsets[0] = 0;
       blockTrueOffsets[1] = PCOp.Ptr()->Height();

examples/ex23.cpp

@@ -3,21 +3,14 @@
 // Compile with: make ex23
 //
 // Sample runs:  ex23
-//               ex23 -o 4 -tf 5
-//               ex23 -m ../data/square-disc.mesh -o 2 -tf 2 --neumann
-//               ex23 -m ../data/disc-nurbs.mesh -r 3 -o 4 -tf 2
-//               ex23 -m ../data/inline-hex.mesh -o 1 -tf 2 --neumann
-//               ex23 -m ../data/inline-tet.mesh -o 1 -tf 2 --neumann
+//               ex23 -m ../data/inline-tri.mesh
+//               ex23 -m ../data/disc-nurbs.mesh -r 3 -o 2 -tf 2
 //
 // Description:  This example solves the wave equation problem of the form:
 //
 //                               d^2u/dt^2 = c^2 \Delta u.
 //
-//               The example demonstrates the use of time dependent operators,
-//               implicit solvers and second order time integration.
-//
-//               We recommend viewing examples 9 and 10 before viewing this
-//               example.
+//               The example demonstrates the use of 2nd order time integration.
 
 #include "mfem.hpp"
 #include <fstream>
@@ -61,14 +54,10 @@ protected:
 public:
    WaveOperator(FiniteElementSpace &f, Array<int> &ess_bdr,double speed);
 
-   using SecondOrderTimeDependentOperator::Mult;
    virtual void Mult(const Vector &u, const Vector &du_dt,
                      Vector &d2udt2) const;
-
    /** Solve the Backward-Euler equation:
-       d2udt2 = f(u + fac0*d2udt2,dudt + fac1*d2udt2, t),
-       for the unknown d2udt2. */
-   using SecondOrderTimeDependentOperator::ImplicitSolve;
+       d2udt2 = f(u + fac0*d2udt2,dudt + fac1*d2udt2, t), for the unknown d2udt2.*/
    virtual void ImplicitSolve(const double fac0, const double fac1,
                               const Vector &u, const Vector &dudt, Vector &d2udt2);
 
@@ -170,7 +159,14 @@ WaveOperator::~WaveOperator()
 
 double InitialSolution(const Vector &x)
 {
-   return exp(-x.Norml2()*x.Norml2()*30);
+   if (x.Norml2() < 0.5)
+   {
+      return 1.0;
+   }
+   else
+   {
+      return 0.0;
+   }
 }
 
 double InitialRate(const Vector &x)
@@ -185,7 +181,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    const char *ref_dir  = "";
    int ref_levels = 2;
-   int order = 2;
+   int order = 1;
    int ode_solver_type = 10;
    double t_final = 0.5;
    double dt = 1.0e-2;
@@ -218,8 +214,10 @@ int main(int argc, char *argv[])
    args.AddOption(&dirichlet, "-dir", "--dirichlet", "-neu",
                   "--neumann",
                   "BC switch.");
+
    args.AddOption(&ref_dir, "-r", "--ref",
                   "Reference directory for checking final solution.");
+
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -228,6 +226,7 @@ int main(int argc, char *argv[])
                   "Save data files for VisIt (visit.llnl.gov) visualization.");
    args.AddOption(&vis_steps, "-vs", "--visualization-steps",
                   "Visualize every n-th timestep.");
+
    args.Parse();
    if (!args.Good())
    {
@@ -241,8 +240,9 @@ int main(int argc, char *argv[])
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Define the ODE solver used for time integration. Several second order
-   //    time integrators are available.
+   // 3. Define the ODE solver used for time integration. Several implicit
+   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //    explicit Runge-Kutta methods are available.
    SecondOrderODESolver *ode_solver;
    switch (ode_solver_type)
    {
@@ -415,3 +415,5 @@ int main(int argc, char *argv[])
 
    return 0;
 }
+
+

examples/ex24.cpp

@@ -193,8 +193,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 10. Second approach: compute the same solution by applying
-   //     GradientInterpolator in H(curl).
+   // 10. Compute the same solution by applying GradientInterpolator in H(curl).
    DiscreteLinearOperator grad(H1fespace, fespace);
    grad.AddDomainInterpolator(new GradientInterpolator());
    grad.Assemble();

examples/ex24p.cpp

@@ -242,8 +242,7 @@ int main(int argc, char *argv[])
 
    x.SetFromTrueDofs(X);
 
-   // 12. Second approach: compute the same solution by applying
-   //     GradientInterpolator in H(curl).
+   // 12. Compute the same solution by applying GradientInterpolator in H(curl).
    ParDiscreteLinearOperator grad(H1fespace, fespace);
    grad.AddDomainInterpolator(new GradientInterpolator());
    grad.Assemble();

examples/ex25.cpp

@@ -1,995 +0,0 @@
-//                                MFEM Example 25
-//
-// Compile with: make ex25
-//
-// Sample runs:  ex25 -o 2 -f 1.0 -ref 2 -prob 0
-//               ex25 -o 3 -f 10.0 -ref 2 -prob 1
-//               ex25 -o 2 -f 5.0 -ref 4 -prob 2
-//               ex25 -o 2 -f 1.0 -ref 2 -prob 3
-//               ex25 -o 2 -f 1.0 -ref 2 -prob 0 -m ../data/beam-quad.mesh
-//               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
-//               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh
-//
-// Description:  This example code solves a simple electromagnetic wave
-//               propagation problem corresponding to the second order
-//               indefinite Maxwell equation
-//                  (1/mu) * curl curl E - \omega^2 * epsilon E = f
-//               with a Perfectly Matched Layer (PML).
-//
-//               The example demonstrates discretization with Nedelec finite
-//               elements in 2D or 3D, as well as the use of complex-valued
-//               bilinear and linear forms. Several test problems are included,
-//               with prob = 0-3 having known exact solutions, see "On perfectly
-//               matched layers for discontinuous Petrov–Galerkin methods" by
-//               Vaziri Astaneh, Keith, Demkowicz, Comput Mech 63, 2019.
-//
-//               We recommend viewing Example 22 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifdef _WIN32
-#define jn(n, x) _jn(n, x)
-#define yn(n, x) _yn(n, x)
-#endif
-
-using namespace std;
-using namespace mfem;
-
-// Class for setting up a simple Cartesian PML region
-class CartesianPML
-{
-private:
-   Mesh *mesh;
-
-   int dim;
-
-   // Length of the PML Region in each direction
-   Array2D<double> length;
-
-   // Computational Domain Boundary
-   Array2D<double> comp_dom_bdr;
-
-   // Domain Boundary
-   Array2D<double> dom_bdr;
-
-   // Integer Array identifying elements in the PML
-   // 0: in the PML, 1: not in the PML
-   Array<int> elems;
-
-   // Compute Domain and Computational Domain Boundaries
-   void SetBoundaries();
-
-public:
-   // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
-
-   // Return Computational Domain Boundary
-   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
-
-   // Return Domain Boundary
-   Array2D<double> GetDomainBdr() {return dom_bdr;}
-
-   // Return Markers list for elements
-   Array<int> * GetMarkedPMLElements() {return &elems;}
-
-   // Mark elements in the PML region
-   void SetAttributes(Mesh *mesh_);
-
-   // PML complex stretching function
-   void StretchFunction(const Vector &x, vector<complex<double>> &dxs);
-};
-
-// Class for returning the PML coefficients of the bilinear form
-class PMLMatrixCoefficient : public MatrixCoefficient
-{
-private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
-public:
-   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
-   {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
-                     const IntegrationPoint &ip)
-   {
-      double x[3];
-      Vector transip(x, 3);
-      T.Transform(ip, transip);
-      K.SetSize(height, width);
-      (*Function)(transip, pml, K);
-   }
-};
-
-void maxwell_solution(const Vector &x, vector<complex<double>> &Eval);
-
-void E_bdr_data_Re(const Vector &x, Vector &E);
-void E_bdr_data_Im(const Vector &x, Vector &E);
-
-void E_exact_Re(const Vector &x, Vector &E);
-void E_exact_Im(const Vector &x, Vector &E);
-
-void source(const Vector &x, Vector & f);
-
-// Functions for computing the necessary coefficients after PML stretching.
-// J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-
-Array2D<double> comp_domain_bdr;
-Array2D<double> domain_bdr;
-
-double mu = 1.0;
-double epsilon = 1.0;
-double omega;
-int dim;
-bool exact_known = false;
-
-enum prob_type
-{
-   beam,     // Wave propagating in a beam-like domain
-   disc,     // Point source propagating in the square-disc domain
-   lshape,   // Point source propagating in the L-shape domain
-   fichera,  // Point source propagating in the fichera domain
-   load_src  // Approximated point source with PML all around
-};
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = nullptr;
-   int order = 1;
-   int ref_levels = 3;
-   int iprob = 4;
-   double freq = 5.0;
-   bool herm_conv = true;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: beam, 1: disc, 2: lshape, 3: fichera, 4: General");
-   args.AddOption(&ref_levels, "-ref", "--refinements",
-                  "Number of refinements");
-   args.AddOption(&mu, "-mu", "--permeability",
-                  "Permeability of free space (or 1/(spring constant)).");
-   args.AddOption(&epsilon, "-eps", "--permittivity",
-                  "Permittivity of free space (or mass constant).");
-   args.AddOption(&freq, "-f", "--frequency",
-                  "Frequency (in Hz).");
-   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
-                  "--no-hermitian", "Use convention for Hermitian operators.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-
-   if (iprob > 4) { iprob = 4; }
-   prob = (prob_type)iprob;
-
-   // 2. Setup the mesh
-   if (!mesh_file)
-   {
-      exact_known = true;
-      switch (prob)
-      {
-         case beam:
-            mesh_file = "../data/beam-hex.mesh";
-            break;
-         case disc:
-            mesh_file = "../data/square-disc.mesh";
-            break;
-         case lshape:
-            mesh_file = "../data/l-shape.mesh";
-            break;
-         case fichera:
-            mesh_file = "../data/fichera.mesh";
-            break;
-         default:
-            exact_known = false;
-            mesh_file = "../data/inline-quad.mesh";
-            break;
-      }
-   }
-
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Mesh * mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-
-   // Angular frequency
-   omega = 2.0 * M_PI * freq;
-
-   // Setup PML length
-   Array2D<double> length(dim, 2); length = 0.0;
-
-   // 3. Setup the Cartesian PML region.
-   switch (prob)
-   {
-      case disc:
-         length = 0.2;
-         break;
-      case lshape:
-         length(0, 0) = 0.1;
-         length(1, 0) = 0.1;
-         break;
-      case fichera:
-         length(0, 1) = 0.5;
-         length(1, 1) = 0.5;
-         length(2, 1) = 0.5;
-         break;
-      case beam:
-         length(0, 1) = 2.0;
-         break;
-      default:
-         length = 0.25;
-         break;
-   }
-   CartesianPML * pml = new CartesianPML(mesh,length);
-   comp_domain_bdr = pml->GetCompDomainBdr();
-   domain_bdr = pml->GetDomainBdr();
-
-   // 4. Refine the mesh to increase the resolution.
-   for (int l = 0; l < ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Reorient mesh in case of a tet mesh
-   mesh->ReorientTetMesh();
-
-   // Set element attributes in order to distinguish elements in the PML region
-   pml->SetAttributes(mesh);
-
-   // 6. Define a finite element space on the mesh. Here we use the Nedelec
-   //    finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   int size = fespace->GetTrueVSize();
-
-   cout << "Number of finite element unknowns: " << size << endl;
-
-   // 7. Determine the list of true essential boundary dofs. In this example,
-   //    the boundary conditions are defined based on the specific mesh and the
-   //    problem type.
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      if (prob == lshape || prob == fichera)
-      {
-         ess_bdr = 0;
-         for (int j = 0; j < mesh->GetNBE(); j++)
-         {
-            Vector center(dim);
-            int bdrgeom = mesh->GetBdrElementBaseGeometry(j);
-            ElementTransformation * tr = mesh->GetBdrElementTransformation(j);
-            tr->Transform(Geometries.GetCenter(bdrgeom),center);
-            int k = mesh->GetBdrAttribute(j);
-            switch (prob)
-            {
-               case lshape:
-                  if (center[0] == 1.0 || center[0] == 0.5 || center[1] == 0.5)
-                  {
-                     ess_bdr[k - 1] = 1;
-                  }
-                  break;
-               case fichera:
-                  if (center[0] == -1.0 || center[0] == 0.0 ||
-                      center[1] ==  0.0 || center[2] == 0.0)
-                  {
-                     ess_bdr[k - 1] = 1;
-                  }
-                  break;
-               default:
-                  break;
-            }
-         }
-      }
-   }
-   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   // 8. Setup Complex Operator convention
-   ComplexOperator::Convention conv =
-      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
-
-   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
-   VectorFunctionCoefficient f(dim, source);
-   ComplexLinearForm b(fespace, conv);
-   if (prob == load_src)
-   {
-      b.AddDomainIntegrator(NULL, new VectorFEDomainLFIntegrator(f));
-   }
-   b.Vector::operator=(0.0);
-   b.Assemble();
-
-   // 10. Define the solution vector x as a complex finite element grid function
-   //     corresponding to fespace.
-   ComplexGridFunction x(fespace);
-   x = 0.0;
-   VectorFunctionCoefficient E_Re(dim, E_bdr_data_Re);
-   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
-   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
-
-   // 11. Set up the sesquilinear form a(.,.)
-   //
-   //     In Comp
-   //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
-   //
-   //     In PML:   1/mu (1/det(J) J^T J Curl E, Curl F)
-   //               - omega^2 * epsilon (det(J) * (J^T J)^-1 * E, F)
-   //
-   //     where J denotes the Jacobian Matrix of the PML Stretching function
-   Array<int> attr;
-   Array<int> attrPML;
-   if (mesh->attributes.Size())
-   {
-      attr.SetSize(mesh->attributes.Max());
-      attrPML.SetSize(mesh->attributes.Max());
-      attr = 0;   attr[0] = 1;
-      attrPML = 0;
-      if (mesh->attributes.Max() > 1)
-      {
-         attrPML[1] = 1;
-      }
-   }
-
-   ConstantCoefficient muinv(1.0/mu);
-   ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
-   RestrictedCoefficient restr_muinv(muinv,attr);
-   RestrictedCoefficient restr_omeg(omeg,attr);
-
-   // Integrators inside the computational domain (excluding the PML region)
-   SesquilinearForm a(fespace, conv);
-   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv),NULL);
-   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
-
-   int cdim = (dim == 2) ? 1 : dim;
-   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
-   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
-
-   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
-   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
-
-   // Integrators inside the PML region
-   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
-                         new CurlCurlIntegrator(restr_c1_Im));
-   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
-                         new VectorFEMassIntegrator(restr_c2_Im));
-
-   // 12. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: assembly, eliminating
-   //     boundary conditions, applying conforming constraints for
-   //     non-conforming AMR, etc.
-   a.Assemble();
-
-   OperatorHandle Ah;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
-
-   // 13. Transform to monolithic SparseMatrix
-   SparseMatrix *A = Ah.As<ComplexSparseMatrix>()->GetSystemMatrix();
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 14. Solve using a direct or an iterative solver
-#ifdef MFEM_USE_SUITESPARSE
-   {
-      UMFPackSolver  solver(*A);
-      solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      solver.Mult(B, X);
-   }
-#else
-
-   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
-   //
-   //    In Comp
-   //    Domain:   1/mu (Curl E, Curl F) + omega^2 * epsilon (E,F)
-   //
-   //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
-   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   {
-      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
-      RestrictedCoefficient restr_absomeg(absomeg,attr);
-
-      BilinearForm prec(fespace);
-      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
-      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
-
-      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
-      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
-
-      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
-
-      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
-      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
-
-      prec.Assemble();
-
-      OperatorHandle PCOpAh;
-      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
-      //      preconditioner based on the Gauss-Seidel sparse smoother.
-      Array<int> offsets(3);
-      offsets[0] = 0;
-      offsets[1] = fespace->GetTrueVSize();
-      offsets[2] = fespace->GetTrueVSize();
-      offsets.PartialSum();
-
-      GSSmoother gs00(*PCOpAh.As<SparseMatrix>());
-      BlockDiagonalPreconditioner BlockGS(offsets);
-      ScaledOperator gs11(&gs00,
-                          (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
-      BlockGS.SetDiagonalBlock(0,&gs00);
-      BlockGS.SetDiagonalBlock(1,&gs11);
-
-      GMRESSolver gmres;
-      gmres.SetPrintLevel(1);
-      gmres.SetKDim(200);
-      gmres.SetMaxIter(2000);
-      gmres.SetRelTol(1e-5);
-      gmres.SetAbsTol(0.0);
-      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockGS);
-      gmres.Mult(B, X);
-   }
-#endif
-
-   // 15. Recover the solution as a finite element grid function and compute the
-   //     errors if the exact solution is known.
-   a.RecoverFEMSolution(X, b, x);
-
-   // If exact is known compute the error
-   if (exact_known)
-   {
-      ComplexGridFunction x_gf(fespace);
-      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
-      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
-      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
-      int order_quad = max(2, 2 * order + 1);
-      const IntegrationRule *irs[Geometry::NumGeom];
-      for (int i = 0; i < Geometry::NumGeom; ++i)
-      {
-         irs[i] = &(IntRules.Get(i, order_quad));
-      }
-
-      double L2Error_Re = x.real().ComputeL2Error(E_ex_Re, irs,
-                                                  pml->GetMarkedPMLElements());
-      double L2Error_Im = x.imag().ComputeL2Error(E_ex_Im, irs,
-                                                  pml->GetMarkedPMLElements());
-
-      ComplexGridFunction x_gf0(fespace);
-      x_gf0 = 0.0;
-      double norm_E_Re, norm_E_Im;
-      norm_E_Re = x_gf0.real().ComputeL2Error(E_ex_Re, irs,
-                                              pml->GetMarkedPMLElements());
-      norm_E_Im = x_gf0.imag().ComputeL2Error(E_ex_Im, irs,
-                                              pml->GetMarkedPMLElements());
-
-      cout << "\n Relative Error (Re part): || E_h - E || / ||E|| = "
-           << L2Error_Re / norm_E_Re
-           << "\n Relative Error (Im part): || E_h - E || / ||E|| = "
-           << L2Error_Im / norm_E_Im
-           << "\n Total Error: "
-           << sqrt(L2Error_Re*L2Error_Re + L2Error_Im*L2Error_Im) << "\n\n";
-   }
-
-   // 16. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m mesh -g sol".
-   {
-      ofstream mesh_ofs("ex25.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-
-      ofstream sol_r_ofs("ex25-sol_r.gf");
-      ofstream sol_i_ofs("ex25-sol_i.gf");
-      sol_r_ofs.precision(8);
-      sol_i_ofs.precision(8);
-      x.real().Save(sol_r_ofs);
-      x.imag().Save(sol_i_ofs);
-   }
-
-   // 17. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      // Define visualization keys for GLVis (see GLVis documentation)
-      string keys;
-      keys = (dim == 3) ? "keys macF\n" : keys = "keys amrRljcUUuu\n";
-      if (prob == beam && dim == 3) {keys = "keys macFFiYYYYYYYYYYYYYYYYYY\n";}
-      if (prob == beam && dim == 2) {keys = "keys amrRljcUUuuu\n"; }
-
-      char vishost[] = "localhost";
-      int visport = 19916;
-
-      socketstream sol_sock_re(vishost, visport);
-      sol_sock_re.precision(8);
-      sol_sock_re << "solution\n"
-                  << *mesh << x.real() << keys
-                  << "window_title 'Solution real part'" << flush;
-
-      socketstream sol_sock_im(vishost, visport);
-      sol_sock_im.precision(8);
-      sol_sock_im << "solution\n"
-                  << *mesh << x.imag() << keys
-                  << "window_title 'Solution imag part'" << flush;
-
-      GridFunction x_t(fespace);
-      x_t = x.real();
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n"
-               << *mesh << x_t << keys << "autoscale off\n"
-               << "window_title 'Harmonic Solution (t = 0.0 T)'"
-               << "pause\n" << flush;
-      cout << "GLVis visualization paused."
-           << " Press space (in the GLVis window) to resume it.\n";
-      int num_frames = 32;
-      int i = 0;
-      while (sol_sock)
-      {
-         double t = (double)(i % num_frames) / num_frames;
-         ostringstream oss;
-         oss << "Harmonic Solution (t = " << t << " T)";
-
-         add(cos(2.0 * M_PI * t), x.real(),
-             sin(2.0 * M_PI * t), x.imag(), x_t);
-         sol_sock << "solution\n"
-                  << *mesh << x_t
-                  << "window_title '" << oss.str() << "'" << flush;
-         i++;
-      }
-   }
-
-   // 18. Free the used memory.
-   delete A;
-   delete pml;
-   delete fespace;
-   delete fec;
-   delete mesh;
-   return 0;
-}
-
-void source(const Vector &x, Vector &f)
-{
-   Vector center(dim);
-   double r = 0.0;
-   for (int i = 0; i < dim; ++i)
-   {
-      center(i) = 0.5 * (comp_domain_bdr(i, 0) + comp_domain_bdr(i, 1));
-      r += pow(x[i] - center[i], 2.);
-   }
-   double n = 5.0 * omega * sqrt(epsilon * mu) / M_PI;
-   double coeff = pow(n, 2) / M_PI;
-   double alpha = -pow(n, 2) * r;
-   f = 0.0;
-   f[0] = coeff * exp(alpha);
-}
-
-void maxwell_solution(const Vector &x, vector<complex<double>> &E)
-{
-   // Initialize
-   for (int i = 0; i < dim; ++i)
-   {
-      E[i] = 0.0;
-   }
-
-   complex<double> zi = complex<double>(0., 1.);
-   double k = omega * sqrt(epsilon * mu);
-   switch (prob)
-   {
-      case disc:
-      case lshape:
-      case fichera:
-      {
-         Vector shift(dim);
-         shift = 0.0;
-         if (prob == fichera) { shift =  1.0; }
-         if (prob == disc)    { shift = -0.5; }
-         if (prob == lshape)  { shift = -1.0; }
-
-         if (dim == 2)
-         {
-            double x0 = x(0) + shift(0);
-            double x1 = x(1) + shift(1);
-            double r = sqrt(x0 * x0 + x1 * x1);
-            double beta = k * r;
-
-            // Bessel functions
-            complex<double> Ho, Ho_r, Ho_rr;
-            Ho = jn(0, beta) + zi * yn(0, beta);
-            Ho_r = -k * (jn(1, beta) + zi * yn(1, beta));
-            Ho_rr = -k * k * (1.0 / beta *
-                              (jn(1, beta) + zi * yn(1, beta)) -
-                              (jn(2, beta) + zi * yn(2, beta)));
-
-            // First derivatives
-            double r_x = x0 / r;
-            double r_y = x1 / r;
-            double r_xy = -(r_x / r) * r_y;
-            double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
-
-            complex<double> val, val_xx, val_xy;
-            val = 0.25 * zi * Ho;
-            val_xx = 0.25 * zi * (r_xx * Ho_r + r_x * r_x * Ho_rr);
-            val_xy = 0.25 * zi * (r_xy * Ho_r + r_x * r_y * Ho_rr);
-            E[0] = zi / k * (k * k * val + val_xx);
-            E[1] = zi / k * val_xy;
-         }
-         else if (dim == 3)
-         {
-            double x0 = x(0) + shift(0);
-            double x1 = x(1) + shift(1);
-            double x2 = x(2) + shift(2);
-            double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
-
-            double r_x = x0 / r;
-            double r_y = x1 / r;
-            double r_z = x2 / r;
-            double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
-            double r_yx = -(r_y / r) * r_x;
-            double r_zx = -(r_z / r) * r_x;
-
-            complex<double> val, val_r, val_rr;
-            val = exp(zi * k * r) / r;
-            val_r = val / r * (zi * k * r - 1.0);
-            val_rr = val / (r * r) * (-k * k * r * r
-                                      - 2.0 * zi * k * r + 2.0);
-
-            complex<double> val_xx, val_yx, val_zx;
-            val_xx = val_rr * r_x * r_x + val_r * r_xx;
-            val_yx = val_rr * r_x * r_y + val_r * r_yx;
-            val_zx = val_rr * r_x * r_z + val_r * r_zx;
-
-            complex<double> alpha = zi * k / 4.0 / M_PI / k / k;
-            E[0] = alpha * (k * k * val + val_xx);
-            E[1] = alpha * val_yx;
-            E[2] = alpha * val_zx;
-         }
-         break;
-      }
-      case beam:
-      {
-         // T_10 mode
-         if (dim == 3)
-         {
-            double k10 = sqrt(k * k - M_PI * M_PI);
-            E[1] = -zi * k / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
-         }
-         else if (dim == 2)
-         {
-            E[1] = -zi * k / M_PI * exp(zi * k * x(0));
-         }
-         break;
-      }
-      default:
-         break;
-   }
-}
-
-void E_exact_Re(const Vector &x, Vector &E)
-{
-   vector<complex<double>> Eval(E.Size());
-   maxwell_solution(x, Eval);
-   for (int i = 0; i < dim; ++i)
-   {
-      E[i] = Eval[i].real();
-   }
-}
-
-void E_exact_Im(const Vector &x, Vector &E)
-{
-   vector<complex<double>> Eval(E.Size());
-   maxwell_solution(x, Eval);
-   for (int i = 0; i < dim; ++i)
-   {
-      E[i] = Eval[i].imag();
-   }
-}
-
-void E_bdr_data_Re(const Vector &x, Vector &E)
-{
-   E = 0.0;
-   bool in_pml = false;
-
-   for (int i = 0; i < dim; ++i)
-   {
-      // check if in PML
-      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
-          x(i) - comp_domain_bdr(i, 1) > 0.0)
-      {
-         in_pml = true;
-         break;
-      }
-   }
-   if (!in_pml)
-   {
-      vector<complex<double>> Eval(E.Size());
-      maxwell_solution(x, Eval);
-      for (int i = 0; i < dim; ++i)
-      {
-         E[i] = Eval[i].real();
-      }
-   }
-}
-
-// Define bdr_data solution
-void E_bdr_data_Im(const Vector &x, Vector &E)
-{
-   E = 0.0;
-   bool in_pml = false;
-
-   for (int i = 0; i < dim; ++i)
-   {
-      // check if in PML
-      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
-          x(i) - comp_domain_bdr(i, 1) > 0.0)
-      {
-         in_pml = true;
-         break;
-      }
-   }
-   if (!in_pml)
-   {
-      vector<complex<double>> Eval(E.Size());
-      maxwell_solution(x, Eval);
-      for (int i = 0; i < dim; ++i)
-      {
-         E[i] = Eval[i].imag();
-      }
-   }
-}
-
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
-   pml->StretchFunction(x, dxs);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M = 0.0;
-   for (int i = 0; i < dim; ++i)
-   {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
-   }
-}
-
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M = 0.0;
-   for (int i = 0; i < dim; ++i)
-   {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
-   }
-}
-
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M = 0.0;
-   for (int i = 0; i < dim; ++i)
-   {
-      M(i, i) = abs(det / pow(dxs[i], 2));
-   }
-}
-
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
-   pml->StretchFunction(x, dxs);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   // in the 2D case the coefficient is scalar 1/det(J)
-   if (dim == 2)
-   {
-      M = (1.0 / det).real();
-   }
-   else
-   {
-      M = 0.0;
-      for (int i = 0; i < dim; ++i)
-      {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
-      }
-   }
-}
-
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   if (dim == 2)
-   {
-      M = (1.0 / det).imag();
-   }
-   else
-   {
-      M = 0.0;
-      for (int i = 0; i < dim; ++i)
-      {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
-      }
-   }
-}
-
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   if (dim == 2)
-   {
-      M = abs(1.0 / det);
-   }
-   else
-   {
-      M = 0.0;
-      for (int i = 0; i < dim; ++i)
-      {
-         M(i, i) = abs(pow(dxs[i], 2) / det);
-      }
-   }
-}
-
-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
-   : mesh(mesh_), length(length_)
-{
-   dim = mesh->Dimension();
-   SetBoundaries();
-}
-
-void CartesianPML::SetBoundaries()
-{
-   comp_dom_bdr.SetSize(dim, 2);
-   dom_bdr.SetSize(dim, 2);
-   Vector pmin, pmax;
-   mesh->GetBoundingBox(pmin, pmax);
-   for (int i = 0; i < dim; i++)
-   {
-      dom_bdr(i, 0) = pmin(i);
-      dom_bdr(i, 1) = pmax(i);
-      comp_dom_bdr(i, 0) = dom_bdr(i, 0) + length(i, 0);
-      comp_dom_bdr(i, 1) = dom_bdr(i, 1) - length(i, 1);
-   }
-}
-
-void CartesianPML::SetAttributes(Mesh *mesh_)
-{
-   int nrelem = mesh_->GetNE();
-   elems.SetSize(nrelem);
-
-   // Loop through the elements and identify which of them are in the PML
-   for (int i = 0; i < nrelem; ++i)
-   {
-      elems[i] = 1;
-      bool in_pml = false;
-      Element *el = mesh_->GetElement(i);
-      Array<int> vertices;
-
-      // Initialize attribute
-      el->SetAttribute(1);
-      el->GetVertices(vertices);
-      int nrvert = vertices.Size();
-
-      // Check if any vertex is in the PML
-      for (int iv = 0; iv < nrvert; ++iv)
-      {
-         int vert_idx = vertices[iv];
-         double *coords = mesh_->GetVertex(vert_idx);
-         for (int comp = 0; comp < dim; ++comp)
-         {
-            if (coords[comp] > comp_dom_bdr(comp, 1) ||
-                coords[comp] < comp_dom_bdr(comp, 0))
-            {
-               in_pml = true;
-               break;
-            }
-         }
-      }
-      if (in_pml)
-      {
-         elems[i] = 0;
-         el->SetAttribute(2);
-      }
-   }
-   mesh_->SetAttributes();
-}
-
-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs)
-{
-   complex<double> zi = complex<double>(0., 1.);
-
-   double n = 2.0;
-   double c = 5.0;
-   double coeff;
-   double k = omega * sqrt(epsilon * mu);
-
-   // Stretch in each direction independently
-   for (int i = 0; i < dim; ++i)
-   {
-      dxs[i] = 1.0;
-      if (x(i) >= comp_domain_bdr(i, 1))
-      {
-         coeff = n * c / k / pow(length(i, 1), n);
-         dxs[i] = 1.0 + zi * coeff *
-                  abs(pow(x(i) - comp_domain_bdr(i, 1), n - 1.0));
-      }
-      if (x(i) <= comp_domain_bdr(i, 0))
-      {
-         coeff = n * c / k / pow(length(i, 0), n);
-         dxs[i] = 1.0 + zi * coeff *
-                  abs(pow(x(i) - comp_domain_bdr(i, 0), n - 1.0));
-      }
-   }
-}

examples/ex26.cpp

@@ -1,255 +0,0 @@
-//                                MFEM Example 26
-//
-// Compile with: make ex26
-//
-// Sample runs:  ex26 -m ../data/star.mesh
-//               ex26 -m ../data/fichera.mesh
-//               ex26 -m ../data/beam-hex.mesh
-//
-// Device sample runs:
-//               ex26 -d cuda
-//               ex26 -d raja-cuda
-//               ex26 -d occa-cuda
-//               ex26 -d raja-omp
-//               ex26 -d occa-omp
-//               ex26 -d ceed-cpu
-//               ex26 -d ceed-cuda
-//               ex26 -m ../data/beam-hex.mesh -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions
-//               as in Example 1.
-//
-//               It highlights on the creation of a hierarchy of discretization
-//               spaces with partial assembly and the construction of an
-//               efficient multigrid preconditioner for the iterative solver.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Class for constructing a multigrid preconditioner for the diffusion operator.
-// This example multigrid preconditioner class demonstrates the creation of the
-// diffusion bilinear forms and operators using partial assembly for all spaces
-// in the FiniteElementSpaceHierarchy. The preconditioner uses a CG solver on
-// the coarsest level and second order Chebyshev accelerated smoothers on the
-// other levels.
-class DiffusionMultigrid : public Multigrid
-{
-private:
-   ConstantCoefficient one;
-
-public:
-   // Constructs a diffusion multigrid for the given FiniteElementSpaceHierarchy
-   // and the array of essential boundaries
-   DiffusionMultigrid(FiniteElementSpaceHierarchy& fespaces, Array<int>& ess_bdr)
-      : Multigrid(fespaces), one(1.0)
-   {
-      ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);
-
-      for (int level = 1; level < fespaces.GetNumLevels(); ++level)
-      {
-         ConstructOperatorAndSmoother(fespaces.GetFESpaceAtLevel(level), ess_bdr);
-      }
-   }
-
-private:
-   void ConstructBilinearForm(FiniteElementSpace& fespace, Array<int>& ess_bdr)
-   {
-      BilinearForm* form = new BilinearForm(&fespace);
-      form->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      form->AddDomainIntegrator(new DiffusionIntegrator(one));
-      form->Assemble();
-      bfs.Append(form);
-
-      essentialTrueDofs.Append(new Array<int>());
-      fespace.GetEssentialTrueDofs(ess_bdr, *essentialTrueDofs.Last());
-   }
-
-   void ConstructCoarseOperatorAndSolver(FiniteElementSpace& coarse_fespace,
-                                         Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(coarse_fespace, ess_bdr);
-
-      OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
-      opr.SetOperatorOwner(false);
-
-      CGSolver* pcg = new CGSolver();
-      pcg->SetPrintLevel(-1);
-      pcg->SetMaxIter(200);
-      pcg->SetRelTol(sqrt(1e-4));
-      pcg->SetAbsTol(0.0);
-      pcg->SetOperator(*opr.Ptr());
-
-      AddLevel(opr.Ptr(), pcg, true, true);
-   }
-
-   void ConstructOperatorAndSmoother(FiniteElementSpace& fespace,
-                                     Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(fespace, ess_bdr);
-
-      OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
-      opr.SetOperatorOwner(false);
-
-      Vector diag(fespace.GetTrueVSize());
-      bfs.Last()->AssembleDiagonal(diag);
-
-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
-                                                       *essentialTrueDofs.Last(), 2);
-      AddLevel(opr.Ptr(), smoother, true, true);
-   }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int geometric_refinements = 0;
-   int order_refinements = 2;
-   const char *device_config = "cpu";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&geometric_refinements, "-gr", "--geometric-refinements",
-                  "Number of geometric refinements done prior to order refinements.");
-   args.AddOption(&order_refinements, "-or", "--order-refinements",
-                  "Number of order refinements. Finest level in the hierarchy has order 2^{or}.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(5000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space hierarchy on the mesh. Here we use
-   //    continuous Lagrange finite elements. We start with order 1 on the
-   //    coarse level and geometrically refine the spaces by the specified
-   //    amount. Afterwards, we increase the order of the finite elements
-   //    by a factor of 2 for each additional level.
-   FiniteElementCollection *fec = new H1_FECollection(1, dim);
-   FiniteElementSpace *coarse_fespace = new FiniteElementSpace(mesh, fec);
-   FiniteElementSpaceHierarchy fespaces(mesh, coarse_fespace, true, true);
-
-   Array<FiniteElementCollection*> collections;
-   collections.Append(fec);
-   for (int level = 0; level < geometric_refinements; ++level)
-   {
-      fespaces.AddUniformlyRefinedLevel();
-   }
-   for (int level = 0; level < order_refinements; ++level)
-   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
-      fespaces.AddOrderRefinedLevel(collections.Last());
-   }
-
-   cout << "Number of finite element unknowns: "
-        << fespaces.GetFinestFESpace().GetTrueVSize() << endl;
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(&fespaces.GetFinestFESpace());
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespaces.GetFinestFESpace());
-   x = 0.0;
-
-   // 8. Create the multigrid operator using the previously created
-   //    FiniteElementSpaceHierarchy and additional boundary information. This operator
-   //    is then used to create the MultigridSolver as a preconditioner in the
-   //    iterative solver.
-   Array<int> ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 1;
-
-   DiffusionMultigrid M(fespaces, ess_bdr);
-   M.SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1);
-
-   OperatorPtr A;
-   Vector B, X;
-   M.FormFineLinearSystem(x, *b, A, X, B);
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 9. Solve the linear system A X = B.
-   PCG(*A, M, B, X, 1, 2000, 1e-12, 0.0);
-
-   // 10. Recover the solution as a finite element grid function.
-   M.RecoverFineFEMSolution(X, *b, x);
-
-   // 11. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   fespaces.GetFinestFESpace().GetMesh()->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 12. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *fespaces.GetFinestFESpace().GetMesh() << x <<
-               flush;
-   }
-
-   // 13. Free the used memory.
-   delete b;
-   for (int level = 0; level < collections.Size(); ++level)
-   {
-      delete collections[level];
-   }
-
-   return 0;
-}

examples/ex26p.cpp

@@ -1,317 +0,0 @@
-//                       MFEM Example 26 - Parallel Version
-//
-// Compile with: make ex26p
-//
-// Sample runs:  mpirun -np 4 ex26p -m ../data/star.mesh
-//               mpirun -np 4 ex26p -m ../data/fichera.mesh
-//               mpirun -np 4 ex26p -m ../data/beam-hex.mesh
-//
-// Device sample runs:
-//               mpirun -np 4 ex26p -d cuda
-//               mpirun -np 4 ex26p -d occa-cuda
-//               mpirun -np 4 ex26p -d raja-omp
-//               mpirun -np 4 ex26p -d ceed-cpu
-//               mpirun -np 4 ex26p -d ceed-cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions
-//               as in Example 1.
-//
-//               It highlights on the creation of a hierarchy of discretization
-//               spaces with partial assembly and the construction of an
-//               efficient multigrid preconditioner for the iterative solver.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Class for constructing a multigrid preconditioner for the diffusion operator.
-// This example multigrid preconditioner class demonstrates the creation of the
-// parallel diffusion bilinear forms and operators using partial assembly for
-// all spaces except the coarsest one in the ParFiniteElementSpaceHierarchy.
-// The multigrid uses a PCG solver preconditioned with AMG on the coarsest level
-// and second order Chebyshev accelerated smoothers on the other levels.
-class DiffusionMultigrid : public Multigrid
-{
-private:
-   ConstantCoefficient one;
-   HypreBoomerAMG* amg;
-
-public:
-   // Constructs a diffusion multigrid for the ParFiniteElementSpaceHierarchy
-   // and the array of essential boundaries
-   DiffusionMultigrid(ParFiniteElementSpaceHierarchy& fespaces,
-                      Array<int>& ess_bdr)
-      : Multigrid(fespaces), one(1.0)
-   {
-      ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);
-
-      for (int level = 1; level < fespaces.GetNumLevels(); ++level)
-      {
-         ConstructOperatorAndSmoother(fespaces.GetFESpaceAtLevel(level), ess_bdr);
-      }
-   }
-
-   virtual ~DiffusionMultigrid()
-   {
-      delete amg;
-   }
-
-private:
-   void ConstructBilinearForm(ParFiniteElementSpace& fespace, Array<int>& ess_bdr,
-                              bool partial_assembly)
-   {
-      ParBilinearForm* form = new ParBilinearForm(&fespace);
-      if (partial_assembly)
-      {
-         form->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      }
-      form->AddDomainIntegrator(new DiffusionIntegrator(one));
-      form->Assemble();
-      bfs.Append(form);
-
-      essentialTrueDofs.Append(new Array<int>());
-      fespace.GetEssentialTrueDofs(ess_bdr, *essentialTrueDofs.Last());
-   }
-
-   void ConstructCoarseOperatorAndSolver(ParFiniteElementSpace& coarse_fespace,
-                                         Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(coarse_fespace, ess_bdr, false);
-
-      HypreParMatrix* hypreCoarseMat = new HypreParMatrix();
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), *hypreCoarseMat);
-
-      amg = new HypreBoomerAMG(*hypreCoarseMat);
-      amg->SetPrintLevel(-1);
-
-      CGSolver* pcg = new CGSolver(MPI_COMM_WORLD);
-      pcg->SetPrintLevel(-1);
-      pcg->SetMaxIter(10);
-      pcg->SetRelTol(sqrt(1e-4));
-      pcg->SetAbsTol(0.0);
-      pcg->SetOperator(*hypreCoarseMat);
-      pcg->SetPreconditioner(*amg);
-
-      AddLevel(hypreCoarseMat, pcg, true, true);
-   }
-
-   void ConstructOperatorAndSmoother(ParFiniteElementSpace& fespace,
-                                     Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(fespace, ess_bdr, true);
-
-      OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
-      opr.SetOperatorOwner(false);
-
-      Vector diag(fespace.GetTrueVSize());
-      bfs.Last()->AssembleDiagonal(diag);
-
-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
-                                                       *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());
-
-      AddLevel(opr.Ptr(), smoother, true, true);
-   }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int geometric_refinements = 0;
-   int order_refinements = 2;
-   const char *device_config = "cpu";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&geometric_refinements, "-gr", "--geometric-refinements",
-                  "Number of geometric refinements done prior to order refinements.");
-   args.AddOption(&order_refinements, "-or", "--order-refinements",
-                  "Number of order refinements. Finest level in the hierarchy has order 2^{or}.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space hierarchy on the parallel mesh.
-   //    Here we use continuous Lagrange finite elements. We start with order 1
-   //    on the coarse level and geometrically refine the spaces by the specified
-   //    amount. Afterwards, we increase the order of the finite elements by a
-   //    factor of 2 for each additional level.
-   FiniteElementCollection *fec = new H1_FECollection(1, dim);
-   ParFiniteElementSpace *coarse_fespace = new ParFiniteElementSpace(pmesh, fec);
-
-   Array<FiniteElementCollection*> collections;
-   collections.Append(fec);
-   ParFiniteElementSpaceHierarchy* fespaces = new ParFiniteElementSpaceHierarchy(
-      pmesh, coarse_fespace, true, true);
-   for (int level = 0; level < geometric_refinements; ++level)
-   {
-      fespaces->AddUniformlyRefinedLevel();
-   }
-   for (int level = 0; level < order_refinements; ++level)
-   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
-      fespaces->AddOrderRefinedLevel(collections.Last());
-   }
-
-   HYPRE_Int size = fespaces->GetFinestFESpace().GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(&fespaces->GetFinestFESpace());
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(&fespaces->GetFinestFESpace());
-   x = 0.0;
-
-   // 10. Create the multigrid operator using the previously created parallel
-   //     FiniteElementSpaceHierarchy and additional boundary information. This operator
-   //     is then used to create the MultigridSolver as a preconditioner in the
-   //     iterative solver.
-   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr = 1;
-   }
-
-   DiffusionMultigrid* M = new DiffusionMultigrid(*fespaces, ess_bdr);
-   M->SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1);
-
-   OperatorPtr A;
-   Vector X, B;
-   M->FormFineLinearSystem(x, *b, A, X, B);
-
-   // 11. Solve the linear system A X = B.
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetOperator(*A);
-   cg.SetPreconditioner(*M);
-   cg.Mult(B, X);
-
-   // 12. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   M->RecoverFineFEMSolution(X, *b, x);
-
-   // 13. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      fespaces->GetFinestFESpace().GetParMesh()->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *fespaces->GetFinestFESpace().GetParMesh()
-               << x << flush;
-   }
-
-   // 15. Free the used memory.
-   delete M;
-   delete b;
-   delete fespaces;
-   for (int level = 0; level < collections.Size(); ++level)
-   {
-      delete collections[level];
-   }
-
-   MPI_Finalize();
-
-   return 0;
-}

examples/ex27.cpp

@@ -1,736 +0,0 @@
-//                       MFEM Example 27 - Serial Version
-//
-// Compile with: make ex27
-//
-// Sample runs:  ex27
-//               ex27 -dg
-//               ex27 -dg -dbc 8 -nbc -2
-//               ex27 -rbc-a 1 -rbc-b 8
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 0 with a variety of boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order using a continuous or discontinuous space.  We then
-//               apply Dirichlet, Neumann (both homogeneous and inhomogeneous),
-//               Robin, and Periodic boundary conditions on different portions
-//               of a predefined mesh.
-//
-//               The predefined mesh consists of a rectangle with two
-//               holes removed (see below).  The narrow ends of the
-//               mesh are connected to form a Periodic boundary
-//               condition.  The lower edge (tagged with attribute 1)
-//               receives an inhomogeneous Neumann boundary condition.
-//               A Robin boundary condition is applied to upper edge
-//               (attribute 2).  The circular hole on the left
-//               (attribute 3) enforces a Dirichlet boundary
-//               condition.  Finally, a natural boundary condition, or
-//               homogeneous Neumann BC, is applied to the circular
-//               hole on the right (attribute 4).
-//
-//                  Attribute 3    ^ y  Attribute 2
-//                        \        |      /
-//                     +-----------+-----------+
-//                     |    \_     |     _     |
-//                     |    / \    |    / \    |
-//                  <--+---+---+---+---+---+---+--> x
-//                     |    \_/    |    \_/    |
-//                     |           |      \    |
-//                     +-----------+-----------+       (hole radii are
-//                          /      |        \            adjustable)
-//                  Attribute 1    v    Attribute 4
-//
-//               The boundary conditions are defined as (where u is
-//               the solution field):
-//                  Dirichlet: u = d
-//                  Neumann:   n.Grad(u) = g
-//                  Robin:     n.Grad(u) + a u = b
-//
-//               The user can adjust the values of 'd', 'g', 'a', and
-//               'b' with command line options.
-//
-//               This example highlights the differing implementations of
-//               boundary conditions with continuous and discontinuous Galerkin
-//               formulations of the Laplace problem.
-//
-//               We recommend viewing examples 1 and 14 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static double a_ = 0.2;
-
-// Normal to hole with boundary attribute 4
-void n4Vec(const Vector &x, Vector &n) { n = x; n[0] -= 0.5; n /= -n.Norml2(); }
-
-Mesh * GenerateSerialMesh(int ref);
-
-// Compute the average value of alpha*n.Grad(sol) + beta*sol over the boundary
-// attributes marked in bdr_marker.  Also computes the L2 norm of
-// alpha*n.Grad(sol) + beta*sol - gamma  over the same boundary.
-double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
-                   double alpha, double beta, double gamma,
-                   double &err);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int ser_ref_levels = 2;
-   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   bool h1 = true;
-   bool visualization = true;
-
-   double mat_val = 1.0;
-   double dbc_val = 0.0;
-   double nbc_val = 1.0;
-   double rbc_a_val = 1.0; // du/dn + a * u = b
-   double rbc_b_val = 1.0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&h1, "-h1", "--continuous", "-dg", "--discontinuous",
-                  "Select continuous \"H1\" or discontinuous \"DG\" basis.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&mat_val, "-mat", "--material-value",
-                  "Constant value for material coefficient "
-                  "in the Laplace operator.");
-   args.AddOption(&dbc_val, "-dbc", "--dirichlet-value",
-                  "Constant value for Dirichlet Boundary Condition.");
-   args.AddOption(&nbc_val, "-nbc", "--neumann-value",
-                  "Constant value for Neumann Boundary Condition.");
-   args.AddOption(&rbc_a_val, "-rbc-a", "--robin-a-value",
-                  "Constant 'a' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&rbc_b_val, "-rbc-b", "--robin-b-value",
-                  "Constant 'b' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&a_, "-a", "--radius",
-                  "Radius of holes in the mesh.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(mfem::out);
-      return 1;
-   }
-   if (kappa < 0 && !h1)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   args.PrintOptions(mfem::out);
-
-   if (a_ < 0.01)
-   {
-      mfem::out << "Hole radius too small, resetting to 0.01.\n";
-      a_ = 0.01;
-   }
-   if (a_ > 0.49)
-   {
-      mfem::out << "Hole radius too large, resetting to 0.49.\n";
-      a_ = 0.49;
-   }
-
-   // 2. Construct the (serial) mesh and refine it if requested.
-   Mesh *mesh = GenerateSerialMesh(ser_ref_levels);
-   int dim = mesh->Dimension();
-
-   // 3. Define a finite element space on the serial mesh.  Here we
-   //    use either continuous Lagrange finite elements or discontinuous
-   //    Galerkin finite elements of the specified order.
-   FiniteElementCollection *fec =
-      h1 ? (FiniteElementCollection*)new H1_FECollection(order, dim) :
-      (FiniteElementCollection*)new DG_FECollection(order, dim);
-   FiniteElementSpace fespace(mesh, fec);
-   int size = fespace.GetTrueVSize();
-   mfem::out << "Number of finite element unknowns: " << size << endl;
-
-   // 4. Create "marker arrays" to define the portions of the boundary
-   //    associated with each type of boundary condition.  These arrays
-   //    have an entry corresponding to each boundary attribute.
-   //    Placing a '1' in entry i marks attribute i+1 as being
-   //    active, '0' is inactive.
-   Array<int> nbc_bdr(mesh->bdr_attributes.Max());
-   Array<int> rbc_bdr(mesh->bdr_attributes.Max());
-   Array<int> dbc_bdr(mesh->bdr_attributes.Max());
-
-   nbc_bdr = 0; nbc_bdr[0] = 1;
-   rbc_bdr = 0; rbc_bdr[1] = 1;
-   dbc_bdr = 0; dbc_bdr[2] = 1;
-
-   Array<int> ess_tdof_list(0);
-   if (h1 && mesh->bdr_attributes.Size())
-   {
-      // For a continuous basis the linear system must be modifed to enforce
-      // an essential (Dirichlet) boundary condition.  In the DG case this is
-      // not necessary as the boundary condition will only be enforced weakly.
-      fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);
-   }
-
-   // 5. Setup the various coefficients needed for the Laplace operator and
-   //    the various boundary conditions.  In general these coefficients could
-   //    be functions of position but here we use only constants.
-   ConstantCoefficient matCoef(mat_val);
-   ConstantCoefficient dbcCoef(dbc_val);
-   ConstantCoefficient nbcCoef(nbc_val);
-   ConstantCoefficient rbcACoef(rbc_a_val);
-   ConstantCoefficient rbcBCoef(rbc_b_val);
-
-   // Since the n.Grad(u) terms arise by integrating -Div(m Grad(u)) by parts
-   // we must introduce the coefficient 'm' into the boundary conditions.
-   // Therefore, in the case of the Neumann BC, we actually enforce
-   // m n.Grad(u) = m g rather than simply n.Grad(u) = g.
-   ProductCoefficient m_nbcCoef(matCoef, nbcCoef);
-   ProductCoefficient m_rbcACoef(matCoef, rbcACoef);
-   ProductCoefficient m_rbcBCoef(matCoef, rbcBCoef);
-
-   // 6. Define the solution vector u as a finite element grid function
-   //    corresponding to fespace. Initialize u with initial guess of zero.
-   GridFunction u(&fespace);
-   u = 0.0;
-
-   // 7. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(matCoef));
-   if (h1)
-   {
-      // Add a Mass integrator on the Robin boundary
-      a.AddBoundaryIntegrator(new MassIntegrator(m_rbcACoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the interfacial portion of the Lapalce operator
-      a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(matCoef,
-                                                            sigma, kappa));
-
-      // Counteract the n.Grad(u) term on the Dirichlet portion of the boundary
-      a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(matCoef, sigma, kappa),
-                             dbc_bdr);
-
-      // Augment the n.Grad(u) term with a*u on the Robin portion of boundary
-      a.AddBdrFaceIntegrator(new BoundaryMassIntegrator(m_rbcACoef),
-                             rbc_bdr);
-   }
-   a.Assemble();
-
-   // 8. Assemble the linear form for the right hand side vector.
-   LinearForm b(&fespace);
-
-   if (h1)
-   {
-      // Set the Dirchlet values in the solution vector
-      u.ProjectBdrCoefficient(dbcCoef, dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_nbcCoef), nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_rbcBCoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the desired value for the Dirchlet boundary
-      b.AddBdrFaceIntegrator(new DGDirichletLFIntegrator(dbcCoef, matCoef,
-                                                         sigma, kappa),
-                             dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_nbcCoef),
-                             nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_rbcBCoef),
-                             rbc_bdr);
-   }
-   b.Assemble();
-
-   // 9. Construct the linear system.
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, u, b, A, X, B);
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //     solve the system AX=B with PCG in the symmetric case, and GMRES in the
-   //     non-symmetric one.
-   {
-      GSSmoother M((SparseMatrix&)(*A));
-      if (sigma == -1.0)
-      {
-         PCG(*A, M, B, X, 1, 500, 1e-12, 0.0);
-      }
-      else
-      {
-         GMRES(*A, M, B, X, 1, 500, 10, 1e-12, 0.0);
-      }
-   }
-#else
-   // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
-   //     system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(*A);
-   umf_solver.Mult(B, X);
-#endif
-
-   // 12. Recover the grid function corresponding to U. This is the
-   //     local finite element solution.
-   a.RecoverFEMSolution(X, b, u);
-
-   // 13. Build a mass matrix to help solve for n.Grad(u) where 'n' is
-   //     a surface normal.
-   BilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 14. Compute the various boundary integrals.
-   mfem::out << endl
-             << "Verifying boundary conditions" << endl
-             << "=============================" << endl;
-   {
-      // Integrate the solution on the Dirichlet boundary and compare
-      // to the expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
-
-      bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
-      mfem::out << "Average of solution on Gamma_dbc:\t"
-                << avg << ", \t"
-                << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and
-      // compare to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
-
-      bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare
-      // to the expected value of zero.
-      Array<int> nbc0_bdr(mesh->bdr_attributes.Max());
-      nbc0_bdr = 0;
-      nbc0_bdr[3] = 1;
-
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
-
-      bool hom_nbc = true;
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to
-      // the expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
-
-      bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
-      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
-                << avg << ", \t"
-                << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-
-   // 15. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
-   {
-      ofstream mesh_ofs("refined.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-      ofstream sol_ofs("sol.gf");
-      sol_ofs.precision(8);
-      u.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      string title_str = h1 ? "H1" : "DG";
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << u
-               << "window_title '" << title_str << " Solution'"
-               << " keys 'mmc'" << flush;
-   }
-
-   // 17. Free the used memory.
-   delete fec;
-   delete mesh;
-
-   return 0;
-}
-
-void quad_trans(double u, double v, double &x, double &y, bool log = false)
-{
-   double a = a_; // Radius of disc
-
-   double d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);
-
-   double v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
-               ((4.0 - 3 * M_SQRT2) * a +
-                (8.0 * (M_SQRT2 - 1.0) * a - 2.0) * v) / d;
-
-   double r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
-                     2.0 * (1.0 + M_SQRT2 *
-                            (1.0 + 2.0 * (2.0 * a - M_SQRT2 - 1.0) * a)) * v * v
-                    ) / d;
-
-   double t = asin(v / r) * u / v;
-   if (log)
-   {
-      mfem::out << "u, v, r, v0, t "
-                << u << " " << v << " " << r << " " << v0 << " " << t
-                << endl;
-   }
-   x = r * sin(t);
-   y = r * cos(t) - v0;
-}
-
-void trans(const Vector &u, Vector &x)
-{
-   double tol = 1e-4;
-
-   if (u[1] > 0.5 - tol || u[1] < -0.5 + tol)
-   {
-      x = u;
-      return;
-   }
-   if (u[0] > 1.0 - tol || u[0] < -1.0 + tol || fabs(u[0]) < tol)
-   {
-      x = u;
-      return;
-   }
-
-   if (u[0] > 0.0)
-   {
-      if (u[1] > fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, u[1], x[0], x[1]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, -u[1], x[0], x[1]);
-         x[0] += 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] - 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] - 0.5, x[1], x[0]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[0] - 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], 0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] += 0.5;
-         return;
-      }
-   }
-   else
-   {
-      if (u[1] > fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, -u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] + 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] + 0.5, x[1], x[0]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[0] + 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], -0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] -= 0.5;
-         return;
-      }
-   }
-   x = u;
-}
-
-Mesh * GenerateSerialMesh(int ref)
-{
-   Mesh * mesh = new Mesh(2, 29, 16, 24, 2);
-
-   int vi[4];
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o + 0; vi[1] = o + 3; vi[2] = o + 4; vi[3] = o + 1;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 1; vi[1] = o + 4; vi[2] = o + 5; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 5; vi[1] = o + 8; vi[2] = o + 9; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 8; vi[1] = o + 12; vi[2] = o + 15; vi[3] = o + 9;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 11; vi[1] = o + 14; vi[2] = o + 15; vi[3] = o + 12;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 10; vi[1] = o + 13; vi[2] = o + 14; vi[3] = o + 11;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 6; vi[1] = o + 13; vi[2] = o + 10; vi[3] = o + 7;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 0; vi[1] = o + 6; vi[2] = o + 7; vi[3] = o + 3;
-      mesh->AddQuad(vi);
-   }
-
-   vi[0] =  0; vi[1] =  6; mesh->AddBdrSegment(vi, 1);
-   vi[0] =  6; vi[1] = 13; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 13; vi[1] = 19; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 19; vi[1] = 26; mesh->AddBdrSegment(vi, 1);
-
-   vi[0] = 28; vi[1] = 22; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 22; vi[1] = 15; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 15; vi[1] =  9; mesh->AddBdrSegment(vi, 2);
-   vi[0] =  9; vi[1] =  2; mesh->AddBdrSegment(vi, 2);
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o +  7; vi[1] = o +  3; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 10; vi[1] = o +  7; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 11; vi[1] = o + 10; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 12; vi[1] = o + 11; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  8; vi[1] = o + 12; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  5; vi[1] = o +  8; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  4; vi[1] = o +  5; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  3; vi[1] = o +  4; mesh->AddBdrSegment(vi, 3 + i);
-   }
-
-   double d[2];
-   double a = a_ / M_SQRT2;
-
-   d[0] = -1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] = -0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   mesh->FinalizeTopology();
-
-   mesh->SetCurvature(1, true);
-
-   // Stitch the ends of the stack together
-   {
-      Array<int> v2v(mesh->GetNV());
-      for (int i = 0; i < v2v.Size() - 3; i++)
-      {
-         v2v[i] = i;
-      }
-      // identify vertices on the narrow ends of the rectangle
-      v2v[v2v.Size() - 3] = 0;
-      v2v[v2v.Size() - 2] = 1;
-      v2v[v2v.Size() - 1] = 2;
-
-      // renumber elements
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         Element *el = mesh->GetElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      // renumber boundary elements
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         Element *el = mesh->GetBdrElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      mesh->RemoveUnusedVertices();
-      mesh->RemoveInternalBoundaries();
-   }
-   mesh->SetCurvature(3, true);
-
-   for (int l = 0; l < ref; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   mesh->Transform(trans);
-
-   return mesh;
-}
-
-double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
-                   double alpha, double beta, double gamma,
-                   double &err)
-{
-   double nrm = 0.0;
-   double avg = 0.0;
-   err = 0.0;
-
-   const bool a_is_zero = alpha == 0.0;
-   const bool b_is_zero = beta == 0.0;
-
-   const FiniteElementSpace &fes = *x.FESpace();
-   MFEM_ASSERT(fes.GetVDim() == 1, "");
-   Mesh &mesh = *fes.GetMesh();
-   Vector shape, loc_dofs, w_nor;
-   DenseMatrix dshape;
-   Array<int> dof_ids;
-   for (int i = 0; i < mesh.GetNBE(); i++)
-   {
-      if (bdr[mesh.GetBdrAttribute(i)-1] == 0) { continue; }
-
-      FaceElementTransformations *FTr = mesh.GetBdrFaceTransformations(i);
-      if (FTr == nullptr) { continue; }
-
-      const FiniteElement &fe = *fes.GetFE(FTr->Elem1No);
-      MFEM_ASSERT(fe.GetMapType() == FiniteElement::VALUE, "");
-      const int int_order = 2*fe.GetOrder() + 3;
-      const IntegrationRule &ir = IntRules.Get(FTr->FaceGeom, int_order);
-
-      fes.GetElementDofs(FTr->Elem1No, dof_ids);
-      x.GetSubVector(dof_ids, loc_dofs);
-      if (!a_is_zero)
-      {
-         const int sdim = FTr->Face->GetSpaceDim();
-         w_nor.SetSize(sdim);
-         dshape.SetSize(fe.GetDof(), sdim);
-      }
-      if (!b_is_zero)
-      {
-         shape.SetSize(fe.GetDof());
-      }
-      for (int j = 0; j < ir.GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(j);
-         IntegrationPoint eip;
-         FTr->Loc1.Transform(ip, eip);
-         FTr->Face->SetIntPoint(&ip);
-         double face_weight = FTr->Face->Weight();
-         double val = 0.0;
-         if (!a_is_zero)
-         {
-            FTr->Elem1->SetIntPoint(&eip);
-            fe.CalcPhysDShape(*FTr->Elem1, dshape);
-            CalcOrtho(FTr->Face->Jacobian(), w_nor);
-            val += alpha * dshape.InnerProduct(w_nor, loc_dofs) / face_weight;
-         }
-         if (!b_is_zero)
-         {
-            fe.CalcShape(eip, shape);
-            val += beta * (shape * loc_dofs);
-         }
-
-         // Measure the length of the boundary
-         nrm += ip.weight * face_weight;
-
-         // Integrate alpha * n.Grad(x) + beta * x
-         avg += val * ip.weight * face_weight;
-
-         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
-         val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
-      }
-   }
-
-   // Normalize by the length of the boundary
-   if (std::abs(nrm) > 0.0)
-   {
-      err /= nrm;
-      avg /= nrm;
-   }
-
-   // Compute l2 norm of the error in the boundary condition
-   // (negative quadrature weights may produce negative 'err')
-   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);
-
-   // Return the average value of alpha * n.Grad(x) + beta * x
-   return avg;
-}

examples/ex27p.cpp

@@ -1,773 +0,0 @@
-//                       MFEM Example 27 - Parallel Version
-//
-// Compile with: make ex27p
-//
-// Sample runs:  mpirun -np 4 ex27p
-//               mpirun -np 4 ex27p -dg
-//               mpirun -np 4 ex27p -dg -dbc 8 -nbc -2
-//               mpirun -np 4 ex27p -rbc-a 1 -rbc-b 8
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 0 with a variety of boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order using a continuous or discontinuous space.  We then
-//               apply Dirichlet, Neumann (both homogeneous and inhomogeneous),
-//               Robin, and Periodic boundary conditions on different portions
-//               of a predefined mesh.
-//
-//               The predefined mesh consists of a rectangle with two
-//               holes removed (see below).  The narrow ends of the
-//               mesh are connected to form a Periodic boundary
-//               condition.  The lower edge (tagged with attribute 1)
-//               receives an inhomogeneous Neumann boundary condition.
-//               A Robin boundary condition is applied to upper edge
-//               (attribute 2).  The circular hole on the left
-//               (attribute 3) enforces a Dirichlet boundary
-//               condition.  Finally, a natural boundary condition, or
-//               homogeneous Neumann BC, is applied to the circular
-//               hole on the right (attribute 4).
-//
-//                  Attribute 3    ^ y  Attribute 2
-//                        \        |      /
-//                     +-----------+-----------+
-//                     |    \_     |     _     |
-//                     |    / \    |    / \    |
-//                  <--+---+---+---+---+---+---+--> x
-//                     |    \_/    |    \_/    |
-//                     |           |      \    |
-//                     +-----------+-----------+       (hole radii are
-//                          /      |        \            adjustable)
-//                  Attribute 1    v    Attribute 4
-//
-//               The boundary conditions are defined as (where u is
-//               the solution field):
-//                  Dirichlet: u = d
-//                  Neumann:   n.Grad(u) = g
-//                  Robin:     n.Grad(u) + a u = b
-//
-//               The user can adjust the values of 'd', 'g', 'a', and
-//               'b' with command line options.
-//
-//               This example highlights the differing implementations of
-//               boundary conditions with continuous and discontinuous Galerkin
-//               formulations of the Laplace problem.
-//
-//               We recommend viewing examples 1 and 14 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static double a_ = 0.2;
-
-// Normal to hole with boundary attribute 4
-void n4Vec(const Vector &x, Vector &n) { n = x; n[0] -= 0.5; n /= -n.Norml2(); }
-
-Mesh * GenerateSerialMesh(int ref);
-
-// Compute the average value of alpha*n.Grad(sol) + beta*sol over the boundary
-// attributes marked in bdr_marker.  Also computes the L2 norm of
-// alpha*n.Grad(sol) + beta*sol - gamma  over the same boundary.
-double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
-                   double alpha, double beta, double gamma,
-                   double &err);
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   MPI_Session mpi;
-   if (!mpi.Root()) { mfem::out.Disable(); mfem::err.Disable(); }
-
-   // 2. Parse command-line options.
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   bool h1 = true;
-   bool visualization = true;
-
-   double mat_val = 1.0;
-   double dbc_val = 0.0;
-   double nbc_val = 1.0;
-   double rbc_a_val = 1.0; // du/dn + a * u = b
-   double rbc_b_val = 1.0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&h1, "-h1", "--continuous", "-dg", "--discontinuous",
-                  "Select continuous \"H1\" or discontinuous \"DG\" basis.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&mat_val, "-mat", "--material-value",
-                  "Constant value for material coefficient "
-                  "in the Laplace operator.");
-   args.AddOption(&dbc_val, "-dbc", "--dirichlet-value",
-                  "Constant value for Dirichlet Boundary Condition.");
-   args.AddOption(&nbc_val, "-nbc", "--neumann-value",
-                  "Constant value for Neumann Boundary Condition.");
-   args.AddOption(&rbc_a_val, "-rbc-a", "--robin-a-value",
-                  "Constant 'a' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&rbc_b_val, "-rbc-b", "--robin-b-value",
-                  "Constant 'b' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&a_, "-a", "--radius",
-                  "Radius of holes in the mesh.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(mfem::out);
-      return 1;
-   }
-   if (kappa < 0 && !h1)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   args.PrintOptions(mfem::out);
-
-   if (a_ < 0.01)
-   {
-      mfem::out << "Hole radius too small, resetting to 0.01.\n";
-      a_ = 0.01;
-   }
-   if (a_ > 0.49)
-   {
-      mfem::out << "Hole radius too large, resetting to 0.49.\n";
-      a_ = 0.49;
-   }
-
-   // 3. Construct the (serial) mesh and refine it if requested.
-   Mesh *mesh = GenerateSerialMesh(ser_ref_levels);
-   int dim = mesh->Dimension();
-
-   // 4. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int l = 0; l < par_ref_levels; l++)
-   {
-      pmesh.UniformRefinement();
-   }
-
-   // 5. Define a parallel finite element space on the parallel mesh. Here we
-   //    use either continuous Lagrange finite elements or discontinuous
-   //    Galerkin finite elements of the specified order.
-   FiniteElementCollection *fec =
-      h1 ? (FiniteElementCollection*)new H1_FECollection(order, dim) :
-      (FiniteElementCollection*)new DG_FECollection(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   mfem::out << "Number of finite element unknowns: " << size << endl;
-
-   // 6. Create "marker arrays" to define the portions of the boundary
-   //    associated with each type of boundary condition.  These arrays
-   //    have an entry corresponding to each boundary attribute.
-   //    Placing a '1' in entry i marks attribute i+1 as being
-   //    active, '0' is inactive.
-   Array<int> nbc_bdr(pmesh.bdr_attributes.Max());
-   Array<int> rbc_bdr(pmesh.bdr_attributes.Max());
-   Array<int> dbc_bdr(pmesh.bdr_attributes.Max());
-
-   nbc_bdr = 0; nbc_bdr[0] = 1;
-   rbc_bdr = 0; rbc_bdr[1] = 1;
-   dbc_bdr = 0; dbc_bdr[2] = 1;
-
-   Array<int> ess_tdof_list(0);
-   if (h1 && pmesh.bdr_attributes.Size())
-   {
-      // For a continuous basis the linear system must be modifed to enforce
-      // an essential (Dirichlet) boundary condition.  In the DG case this is
-      // not necessary as the boundary condition will only be enforced weakly.
-      fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);
-   }
-
-   // 7. Setup the various coefficients needed for the Laplace operator and
-   //    the various boundary conditions.  In general these coefficients could
-   //    be functions of position but here we use only constants.
-   ConstantCoefficient matCoef(mat_val);
-   ConstantCoefficient dbcCoef(dbc_val);
-   ConstantCoefficient nbcCoef(nbc_val);
-   ConstantCoefficient rbcACoef(rbc_a_val);
-   ConstantCoefficient rbcBCoef(rbc_b_val);
-
-   // Since the n.Grad(u) terms arise by integrating -Div(m Grad(u)) by parts
-   // we must introduce the coefficient 'm' into the boundary conditions.
-   // Therefore, in the case of the Neumann BC, we actually enforce
-   // m n.Grad(u) = m g rather than simply n.Grad(u) = g.
-   ProductCoefficient m_nbcCoef(matCoef, nbcCoef);
-   ProductCoefficient m_rbcACoef(matCoef, rbcACoef);
-   ProductCoefficient m_rbcBCoef(matCoef, rbcBCoef);
-
-   // 8. Define the solution vector u as a parallel finite element grid function
-   //    corresponding to fespace. Initialize u with initial guess of zero.
-   ParGridFunction u(&fespace);
-   u = 0.0;
-
-   // 9. Set up the parallel bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(matCoef));
-   if (h1)
-   {
-      // Add a Mass integrator on the Robin boundary
-      a.AddBoundaryIntegrator(new MassIntegrator(m_rbcACoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the interfacial portion of the Lapalce operator
-      a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(matCoef,
-                                                            sigma, kappa));
-
-      // Counteract the n.Grad(u) term on the Dirichlet portion of the boundary
-      a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(matCoef, sigma, kappa),
-                             dbc_bdr);
-
-      // Augment the n.Grad(u) term with a*u on the Robin portion of boundary
-      a.AddBdrFaceIntegrator(new BoundaryMassIntegrator(m_rbcACoef),
-                             rbc_bdr);
-   }
-   a.Assemble();
-
-   // 10. Assemble the parallel linear form for the right hand side vector.
-   ParLinearForm b(&fespace);
-
-   if (h1)
-   {
-      // Set the Dirchlet values in the solution vector
-      u.ProjectBdrCoefficient(dbcCoef, dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_nbcCoef), nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_rbcBCoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the desired value for the Dirchlet boundary
-      b.AddBdrFaceIntegrator(new DGDirichletLFIntegrator(dbcCoef, matCoef,
-                                                         sigma, kappa),
-                             dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_nbcCoef),
-                             nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_rbcBCoef),
-                             rbc_bdr);
-   }
-   b.Assemble();
-
-   // 11. Construct the linear system.
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, u, b, A, X, B);
-
-   // 12. Solve the linear system A X = B.
-   HypreSolver *amg = new HypreBoomerAMG;
-   if (h1 || sigma == -1.0)
-   {
-      HyprePCG pcg(MPI_COMM_WORLD);
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(200);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(*amg);
-      pcg.SetOperator(*A);
-      pcg.Mult(B, X);
-   }
-   else
-   {
-      GMRESSolver gmres(MPI_COMM_WORLD);
-      gmres.SetAbsTol(0.0);
-      gmres.SetRelTol(1e-12);
-      gmres.SetMaxIter(200);
-      gmres.SetKDim(10);
-      gmres.SetPrintLevel(1);
-      gmres.SetPreconditioner(*amg);
-      gmres.SetOperator(*A);
-      gmres.Mult(B, X);
-   }
-   delete amg;
-
-   // 13. Recover the parallel grid function corresponding to U. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, u);
-
-   // 14. Build a mass matrix to help solve for n.Grad(u) where 'n' is
-   //     a surface normal.
-   ParBilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 15. Compute the various boundary integrals.
-   mfem::out << endl
-             << "Verifying boundary conditions" << endl
-             << "=============================" << endl;
-   {
-      // Integrate the solution on the Dirichlet boundary and compare
-      // to the expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
-
-      bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
-      mfem::out << "Average of solution on Gamma_dbc:\t"
-                << avg << ", \t"
-                << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and
-      // compare to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
-
-      bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare
-      // to the expected value of zero.
-      Array<int> nbc0_bdr(pmesh.bdr_attributes.Max());
-      nbc0_bdr = 0;
-      nbc0_bdr[3] = 1;
-
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
-
-      bool hom_nbc = true;
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to
-      // the expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
-
-      bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
-      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
-                << avg << ", \t"
-                << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-
-   // 16. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << mpi.WorldRank();
-      sol_name << "sol." << setfill('0') << setw(6) << mpi.WorldRank();
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      u.Save(sol_ofs);
-   }
-
-   // 17. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      string title_str = h1 ? "H1" : "DG";
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << mpi.WorldSize()
-               << " " << mpi.WorldRank() << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << u
-               << "window_title '" << title_str << " Solution'"
-               << " keys 'mmc'" << flush;
-   }
-
-   // 18. Free the used memory.
-   delete fec;
-
-   return 0;
-}
-
-void quad_trans(double u, double v, double &x, double &y, bool log = false)
-{
-   double a = a_; // Radius of disc
-
-   double d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);
-
-   double v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
-               ((4.0 - 3 * M_SQRT2) * a +
-                (8.0 * (M_SQRT2 - 1.0) * a - 2.0) * v) / d;
-
-   double r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
-                     2.0 * (1.0 + M_SQRT2 *
-                            (1.0 + 2.0 * (2.0 * a - M_SQRT2 - 1.0) * a)) * v * v
-                    ) / d;
-
-   double t = asin(v / r) * u / v;
-   if (log)
-   {
-      mfem::out << "u, v, r, v0, t "
-                << u << " " << v << " " << r << " " << v0 << " " << t
-                << endl;
-   }
-   x = r * sin(t);
-   y = r * cos(t) - v0;
-}
-
-void trans(const Vector &u, Vector &x)
-{
-   double tol = 1e-4;
-
-   if (u[1] > 0.5 - tol || u[1] < -0.5 + tol)
-   {
-      x = u;
-      return;
-   }
-   if (u[0] > 1.0 - tol || u[0] < -1.0 + tol || fabs(u[0]) < tol)
-   {
-      x = u;
-      return;
-   }
-
-   if (u[0] > 0.0)
-   {
-      if (u[1] > fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, u[1], x[0], x[1]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, -u[1], x[0], x[1]);
-         x[0] += 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] - 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] - 0.5, x[1], x[0]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[0] - 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], 0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] += 0.5;
-         return;
-      }
-   }
-   else
-   {
-      if (u[1] > fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, -u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] + 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] + 0.5, x[1], x[0]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[0] + 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], -0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] -= 0.5;
-         return;
-      }
-   }
-   x = u;
-}
-
-Mesh * GenerateSerialMesh(int ref)
-{
-   Mesh * mesh = new Mesh(2, 29, 16, 24, 2);
-
-   int vi[4];
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o + 0; vi[1] = o + 3; vi[2] = o + 4; vi[3] = o + 1;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 1; vi[1] = o + 4; vi[2] = o + 5; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 5; vi[1] = o + 8; vi[2] = o + 9; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 8; vi[1] = o + 12; vi[2] = o + 15; vi[3] = o + 9;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 11; vi[1] = o + 14; vi[2] = o + 15; vi[3] = o + 12;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 10; vi[1] = o + 13; vi[2] = o + 14; vi[3] = o + 11;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 6; vi[1] = o + 13; vi[2] = o + 10; vi[3] = o + 7;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 0; vi[1] = o + 6; vi[2] = o + 7; vi[3] = o + 3;
-      mesh->AddQuad(vi);
-   }
-
-   vi[0] =  0; vi[1] =  6; mesh->AddBdrSegment(vi, 1);
-   vi[0] =  6; vi[1] = 13; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 13; vi[1] = 19; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 19; vi[1] = 26; mesh->AddBdrSegment(vi, 1);
-
-   vi[0] = 28; vi[1] = 22; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 22; vi[1] = 15; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 15; vi[1] =  9; mesh->AddBdrSegment(vi, 2);
-   vi[0] =  9; vi[1] =  2; mesh->AddBdrSegment(vi, 2);
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o +  7; vi[1] = o +  3; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 10; vi[1] = o +  7; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 11; vi[1] = o + 10; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 12; vi[1] = o + 11; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  8; vi[1] = o + 12; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  5; vi[1] = o +  8; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  4; vi[1] = o +  5; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  3; vi[1] = o +  4; mesh->AddBdrSegment(vi, 3 + i);
-   }
-
-   double d[2];
-   double a = a_ / M_SQRT2;
-
-   d[0] = -1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] = -0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   mesh->FinalizeTopology();
-
-   mesh->SetCurvature(1, true);
-
-   // Stitch the ends of the stack together
-   {
-      Array<int> v2v(mesh->GetNV());
-      for (int i = 0; i < v2v.Size() - 3; i++)
-      {
-         v2v[i] = i;
-      }
-      // identify vertices on the narrow ends of the rectangle
-      v2v[v2v.Size() - 3] = 0;
-      v2v[v2v.Size() - 2] = 1;
-      v2v[v2v.Size() - 1] = 2;
-
-      // renumber elements
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         Element *el = mesh->GetElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      // renumber boundary elements
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         Element *el = mesh->GetBdrElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      mesh->RemoveUnusedVertices();
-      mesh->RemoveInternalBoundaries();
-   }
-   mesh->SetCurvature(3, true);
-
-   for (int l = 0; l < ref; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   mesh->Transform(trans);
-
-   return mesh;
-}
-
-double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
-                   double alpha, double beta, double gamma,
-                   double &glb_err)
-{
-   double loc_vals[3];
-   double &nrm = loc_vals[0];
-   double &avg = loc_vals[1];
-   double &err = loc_vals[2];
-
-   nrm = 0.0;
-   avg = 0.0;
-   err = 0.0;
-
-   const bool a_is_zero = alpha == 0.0;
-   const bool b_is_zero = beta == 0.0;
-
-   const ParFiniteElementSpace &fes = *x.ParFESpace();
-   MFEM_ASSERT(fes.GetVDim() == 1, "");
-   ParMesh &mesh = *fes.GetParMesh();
-   Vector shape, loc_dofs, w_nor;
-   DenseMatrix dshape;
-   Array<int> dof_ids;
-   for (int i = 0; i < mesh.GetNBE(); i++)
-   {
-      if (bdr[mesh.GetBdrAttribute(i)-1] == 0) { continue; }
-
-      FaceElementTransformations *FTr = mesh.GetBdrFaceTransformations(i);
-      if (FTr == nullptr) { continue; }
-
-      const FiniteElement &fe = *fes.GetFE(FTr->Elem1No);
-      MFEM_ASSERT(fe.GetMapType() == FiniteElement::VALUE, "");
-      const int int_order = 2*fe.GetOrder() + 3;
-      const IntegrationRule &ir = IntRules.Get(FTr->FaceGeom, int_order);
-
-      fes.GetElementDofs(FTr->Elem1No, dof_ids);
-      x.GetSubVector(dof_ids, loc_dofs);
-      if (!a_is_zero)
-      {
-         const int sdim = FTr->Face->GetSpaceDim();
-         w_nor.SetSize(sdim);
-         dshape.SetSize(fe.GetDof(), sdim);
-      }
-      if (!b_is_zero)
-      {
-         shape.SetSize(fe.GetDof());
-      }
-      for (int j = 0; j < ir.GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(j);
-         IntegrationPoint eip;
-         FTr->Loc1.Transform(ip, eip);
-         FTr->Face->SetIntPoint(&ip);
-         double face_weight = FTr->Face->Weight();
-         double val = 0.0;
-         if (!a_is_zero)
-         {
-            FTr->Elem1->SetIntPoint(&eip);
-            fe.CalcPhysDShape(*FTr->Elem1, dshape);
-            CalcOrtho(FTr->Face->Jacobian(), w_nor);
-            val += alpha * dshape.InnerProduct(w_nor, loc_dofs) / face_weight;
-         }
-         if (!b_is_zero)
-         {
-            fe.CalcShape(eip, shape);
-            val += beta * (shape * loc_dofs);
-         }
-
-         // Measure the length of the boundary
-         nrm += ip.weight * face_weight;
-
-         // Integrate alpha * n.Grad(x) + beta * x
-         avg += val * ip.weight * face_weight;
-
-         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
-         val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
-      }
-   }
-
-   double glb_vals[3];
-   MPI_Allreduce(loc_vals, glb_vals, 3, MPI_DOUBLE, MPI_SUM, fes.GetComm());
-
-   double glb_nrm = glb_vals[0];
-   double glb_avg = glb_vals[1];
-   glb_err = glb_vals[2];
-
-   // Normalize by the length of the boundary
-   if (std::abs(glb_nrm) > 0.0)
-   {
-      glb_err /= glb_nrm;
-      glb_avg /= glb_nrm;
-   }
-
-   // Compute l2 norm of the error in the boundary condition
-   // (negative quadrature weights may produce negative 'err')
-   glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);
-
-   // Return the average value of alpha * n.Grad(x) + beta * x
-   return glb_avg;
-}

examples/ex2p.cpp

@@ -274,13 +274,6 @@ int main(int argc, char *argv[])
       pmesh->SetNodalFESpace(fespace);
    }
 
-   {
-      x.Save("ex2p.gf", 1);
-      ParGridFunction new_x(fespace, "ex2p.gf");
-      new_x -= x;
-      out << "GF difference: " << new_x.Norml1() << endl;
-   }
-
    // 16. Save in parallel the displaced mesh and the inverted solution (which
    //     gives the backward displacements to the original grid). This output
    //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".

examples/ex5.cpp

@@ -20,7 +20,7 @@
 //               polynomials (pressure p).
 //
 //               The example demonstrates the use of the BlockMatrix class, as
-//               well as the collective saving of several grid functions in
+//               well as the collective saving of several grid functions in a
 //               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

examples/ex5p.cpp

@@ -20,10 +20,8 @@
 //               polynomials (pressure p).
 //
 //               The example demonstrates the use of the BlockMatrix class, as
-//               well as the collective saving of several grid functions in
+//               well as the collective saving of several grid functions in a
 //               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
-//               Optional saving with ADIOS2 (adios2.readthedocs.io) streams is
-//               also illustrated.
 //
 //               We recommend viewing examples 1-4 before viewing this example.
 
@@ -57,7 +55,6 @@ int main(int argc, char *argv[])
    int order = 1;
    bool par_format = false;
    bool visualization = 1;
-   bool adios2 = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -70,9 +67,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
    args.Parse();
    if (!args.Good())
    {
@@ -343,27 +337,7 @@ int main(int argc, char *argv[])
    paraview_dc.RegisterField("pressure",p);
    paraview_dc.Save();
 
-   // 17. Optionally output a BP (binary pack) file using ADIOS2. This can be
-   //     visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex5-p_" + postfix + ".bp";
-
-      ADIOS2DataCollection adios2_dc(MPI_COMM_WORLD, collection_name, pmesh);
-      adios2_dc.SetLevelsOfDetail(1);
-      adios2_dc.SetCycle(1);
-      adios2_dc.SetTime(0.0);
-      adios2_dc.RegisterField("velocity",u);
-      adios2_dc.RegisterField("pressure",p);
-      adios2_dc.Save();
-   }
-#endif
-
-   // 18. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -383,7 +357,7 @@ int main(int argc, char *argv[])
              << endl;
    }
 
-   // 19. Free the used memory.
+   // 18. Free the used memory.
    delete fform;
    delete gform;
    delete u;

examples/ex6.cpp

@@ -20,8 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//               * ex6 -pa -d ceed-cuda
-//               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex6 -pa -d ceed-cuda
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace

examples/ex6p.cpp

@@ -20,8 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//               * mpirun -np 4 ex6p -pa -d ceed-cuda
-//               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex6p -pa -d ceed-cuda
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace

examples/ex7.cpp

@@ -279,11 +279,4 @@ void SnapNodes(Mesh &mesh)
          nodes(nodes.FESpace()->DofToVDof(i, d)) = node(d);
       }
    }
-   if (mesh.Nonconforming())
-   {
-      // Snap hanging nodes to the master side.
-      Vector tnodes;
-      nodes.GetTrueDofs(tnodes);
-      nodes.SetFromTrueDofs(tnodes);
-   }
 }

examples/ex7p.cpp

@@ -348,11 +348,4 @@ void SnapNodes(Mesh &mesh)
          nodes(nodes.FESpace()->DofToVDof(i, d)) = node(d);
       }
    }
-   if (mesh.Nonconforming())
-   {
-      // Snap hanging nodes to the master side.
-      Vector tnodes;
-      nodes.GetTrueDofs(tnodes);
-      nodes.SetFromTrueDofs(tnodes);
-   }
 }

examples/ex9p.cpp

@@ -31,10 +31,9 @@
 //               and explicit ODE time integrators, the definition of periodic
 //               boundary conditions through periodic meshes, as well as the use
 //               of GLVis for persistent visualization of a time-evolving
-//               solution. Saving of time-dependent data files for visualization
-//               with VisIt (visit.llnl.gov) and ParaView (paraview.org), as
-//               well as the optional saving with ADIOS2 (adios2.readthedocs.io)
-//               are also illustrated.
+//               solution. The saving of time-dependent data files for external
+//               visualization with VisIt (visit.llnl.gov) and ParaView
+//               (paraview.org) is also illustrated.
 
 #include "mfem.hpp"
 #include <fstream>
@@ -59,6 +58,81 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
+struct AIR_parameters
+{
+   int blocksize;
+   int distanceR;
+   std::string prerelax;
+   std::string postrelax;
+   int interp_type;
+   int relax_type;
+   int coarsen_type;
+   double strength_tolC;
+   double strength_tolR;
+   double filter_tolR;
+   double filterA_tol;
+};
+
+class AIR_prec : public Solver
+{
+private:
+   const HypreParMatrix *A;
+   HypreParMatrix A_s;
+
+   // Preconditioner/solvers for A
+   HypreBoomerAMG *AIR_solver;
+   const AIR_parameters &AIR;
+   int blocksize;
+
+public:
+
+   AIR_prec(const AIR_parameters &_AIR) :
+      AIR_solver(NULL), AIR(_AIR)
+   {
+      blocksize = AIR.blocksize;
+   }
+
+   void SetOperator(const Operator &op)
+   {
+      A = dynamic_cast<const HypreParMatrix *>(&op);
+      delete AIR_solver;
+
+      // Scale A by block-diagonal inverse
+#if MFEM_HYPRE_VERSION >= 21800
+      BlockInverseScale(A, &A_s, NULL, NULL, blocksize, 0);
+      AIR_solver = new HypreBoomerAMG(A_s);
+      AIR_solver->SetLAIROptions(AIR.distanceR, AIR.prerelax,
+                                 AIR.postrelax, AIR.strength_tolC,
+                                 AIR.strength_tolR, AIR.filter_tolR,
+                                 AIR.interp_type, AIR.relax_type,
+                                 AIR.filterA_tol, AIR.coarsen_type,
+                                 -1, 1);
+#else
+      MFEM_ABORT("Must have MFEM_HYPRE_VERSION >= 21800 to use AIR.\n");
+#endif
+      AIR_solver->SetPrintLevel(0);
+      AIR_solver->SetMaxLevels(50);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      // scale the rhs by block inverse and solve system
+      HypreParVector z_s;
+#if MFEM_HYPRE_VERSION >= 21800
+      BlockInverseScale(A, NULL, &x, &z_s, blocksize, 2);
+#endif
+      AIR_solver->Mult(z_s, y);
+   }
+
+   ~AIR_prec()
+   {
+#if MFEM_HYPRE_VERSION >= 21800
+      BlockInverseScale(NULL, NULL, NULL, NULL, 0, -1);
+#endif
+      delete AIR_solver;
+   }
+};
+
 class DG_Solver : public Solver
 {
 private:
@@ -66,7 +140,7 @@ private:
    SparseMatrix M_diag;
    HypreParMatrix *A;
    GMRESSolver linear_solver;
-   BlockILU prec;
+   Solver *prec;
    double dt;
 public:
    DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
@@ -74,20 +148,40 @@ public:
         K(K_),
         A(NULL),
         linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
         dt(-1.0)
    {
+      prec = new BlockILU(fes.GetFE(0)->GetDof(),
+                          BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
       linear_solver.iterative_mode = false;
       linear_solver.SetRelTol(1e-9);
       linear_solver.SetAbsTol(0.0);
       linear_solver.SetMaxIter(100);
       linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
+      linear_solver.SetPreconditioner(*prec);
 
       M.GetDiag(M_diag);
    }
 
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes,
+             const AIR_parameters &_AIR)
+      : M(M_),
+        K(K_),
+        A(NULL),
+        linear_solver(M.GetComm()),
+        dt(-1.0)
+   {
+      prec = new AIR_prec(_AIR);
+      linear_solver.iterative_mode = false;
+      linear_solver.SetRelTol(1e-9);
+      linear_solver.SetAbsTol(0.0);
+      linear_solver.SetMaxIter(100);
+      linear_solver.SetPrintLevel(0);
+      linear_solver.SetPreconditioner(*prec);
+
+      M.GetDiag(M_diag);
+   }
+
+
    void SetTimeStep(double dt_)
    {
       if (dt_ != dt)
@@ -116,10 +210,12 @@ public:
 
    ~DG_Solver()
    {
+      delete prec;
       delete A;
    }
 };
 
+
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
     and advection matrices, and b describes the flow on the boundary. This can
@@ -137,6 +233,8 @@ private:
    mutable Vector z;
 
 public:
+   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b,
+                const AIR_parameters &_AIR);
    FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);
 
    virtual void Mult(const Vector &x, Vector &y) const;
@@ -168,10 +266,12 @@ int main(int argc, char *argv[])
    bool visualization = true;
    bool visit = false;
    bool paraview = false;
-   bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
-
+   int solver_type = 1;
+   AIR_parameters AIR0 = {-1, 1, "", "FA", 100, 10, 10,
+                          0.1, 0.01, 0.0, 1e-4
+                         };
    int precision = 8;
    cout.precision(precision);
 
@@ -201,6 +301,8 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption(&solver_type, "-st", "--solver-type",
+                  "Solver for implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -210,9 +312,6 @@ int main(int argc, char *argv[])
    args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
                   "--no-paraview-datafiles",
                   "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
    args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                   "--ascii-datafiles",
                   "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -305,6 +404,9 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << global_vSize << endl;
    }
 
+   // Get DG blocksize for AIR
+   AIR0.blocksize = fes->GetFE(0)->GetDof();
+
    // 8. Set up and assemble the parallel bilinear and linear forms (and the
    //    parallel hypre matrices) corresponding to the DG discretization. The
    //    DGTraceIntegrator involves integrals over mesh interior faces.
@@ -399,28 +501,6 @@ int main(int argc, char *argv[])
       pd->Save();
    }
 
-   // Optionally output a BP (binary pack) file using ADIOS2. This can be
-   // visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection *adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex9-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      // output data substreams are half the number of mpi processes
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      // adios2_dc->SetLevelsOfDetail(2);
-      adios2_dc->RegisterField("solution", u);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
    socketstream sout;
    if (visualization)
    {
@@ -454,11 +534,19 @@ int main(int argc, char *argv[])
    // 10. Define the time-dependent evolution operator describing the ODE
    //     right-hand side, and perform time-integration (looping over the time
    //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution *adv;
+   if (solver_type == 1)
+   {
+      adv = new FE_Evolution(*m, *k, *B, AIR0);
+   }
+   else
+   {
+      adv = new FE_Evolution(*m, *k, *B);
+   }
 
    double t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
+   adv->SetTime(t);
+   ode_solver->Init(*adv);
 
    bool done = false;
    for (int ti = 0; !done; )
@@ -499,16 +587,6 @@ int main(int argc, char *argv[])
             pd->SetTime(t);
             pd->Save();
          }
-
-#ifdef MFEM_USE_ADIOS2
-         // transient solutions can be visualized with ParaView
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
       }
    }
 
@@ -534,13 +612,8 @@ int main(int argc, char *argv[])
    delete pmesh;
    delete ode_solver;
    delete pd;
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      delete adios2_dc;
-   }
-#endif
    delete dc;
+   delete adv;
 
    MPI_Finalize();
    return 0;
@@ -550,8 +623,7 @@ int main(int argc, char *argv[])
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
                            const Vector &_b)
-   : TimeDependentOperator(_M.Height()),
-     b(_b),
+   : TimeDependentOperator(_M.Height()), b(_b),
      M_solver(_M.ParFESpace()->GetComm()),
      z(_M.Height())
 {
@@ -594,6 +666,45 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
    M_solver.SetPrintLevel(0);
 }
 
+// Implementation of class FE_Evolution
+FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
+                           const Vector &_b, const AIR_parameters &_AIR)
+   : TimeDependentOperator(_M.Height()), b(_b),
+     M_solver(_M.ParFESpace()->GetComm()),
+     z(_M.Height())
+{
+   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
+
+   if (pa)
+   {
+      MFEM_ABORT("AIR solver not available for partial assembly.\n");
+   }
+   else
+   {
+      M.Reset(_M.ParallelAssemble(), true);
+      K.Reset(_K.ParallelAssemble(), true);
+   }
+
+   HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
+   HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
+   HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::GS);
+   M_prec = hypre_prec;
+
+   dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace(), _AIR);
+
+   M_solver.SetPreconditioner(*M_prec);
+   M_solver.SetOperator(*M);
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(1e-9);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(100);
+   M_solver.SetPrintLevel(0);
+}
+
+// Solve the equation:
+//    u_t = M^{-1}(Ku + b),
+// by solving associated linear system
+//    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
 {
    K->Mult(x, z);

examples/ginkgo/README

@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               http://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on the Ginkgo high-performance linear algebra library
-for GPU and manycore nodes.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_GINKGO = YES", see the top-level INSTALL file for details.
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.

examples/hiop/README

@@ -1,19 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               http://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM for solving nonlinear constrained optimization problems, including
-features based on the HiOp, a lightweight HPC solver for nonlinear optimization
-problems.
-
-To use the HiOp featuress, make sure that MFEM is configured with the option
-"MFEM_USE_HIOP = YES", see the top-level INSTALL file for details.
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.

examples/hiop/ex9.cpp

@@ -1,9 +1,9 @@
-//                                MFEM Example 9
-//               Nonlinear Constrained Optimization Modification
+//            MFEM Example 9 with Nonlinear Constrained Optimization
 //
 // Compile with: make ex9
 //
 // Sample runs:
+//
 //    ex9 -m ../../data/periodic-segment.mesh -r 3 -p 0 -o 2 -dt 0.002 -opt 1
 //    ex9 -m ../../data/periodic-segment.mesh -r 3 -p 0 -o 2 -dt 0.002 -opt 2
 //

examples/hiop/ex9p.cpp

@@ -1,9 +1,9 @@
-//                       MFEM Example 9 - Parallel Version
-//               Nonlinear Constrained Optimization Modification
+//  MFEM Example 9 with Nonlinear Constrained Optimization - Parallel Version
 //
 // Compile with: make ex9p
 //
 // Sample runs:
+//
 //    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -rs 3 -p 0 -o 2 -dt 0.002 -opt 1
 //    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -rs 3 -p 0 -o 2 -dt 0.002 -opt 2
 //

examples/io_benchmark.cpp

@@ -1,292 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include "mpi.h"
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   // const char *mesh_file = "../data/star.mesh";
-   const char *mesh_file = "../data/square-disc.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = false;
-   int nfiles = 1;
-   // const char *out_file = "0_0.gf";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&nfiles, "-nf", "--num-files", "Number of files to write.");
-   // args.AddOption(&out_file, "-o", "--outfile",
-   //                "Name of file to write.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec, 1, 0);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   // 13. Solve the linear system A X = B.
-   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use Jacobi smoothing, for now.
-   Solver *prec = NULL;
-   if (pa)
-   {
-      if (UsesTensorBasis(*fespace))
-      {
-         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
-      }
-   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   std::string filename = to_string(num_procs) + "_" + to_string(nfiles) + "_";
-   {
-      double t1;
-      t1 = MPI_Wtime();
-      x.Save(filename.c_str(), nfiles);
-      double t2 = MPI_Wtime();
-
-      double write_time = t2 - t1;
-      double average_write_time;
-      MPI_Reduce(&write_time, &average_write_time, 1,
-               MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-      if (myid == 0)
-      {
-         std::cout << "Average write time: " << average_write_time / num_procs << " for "
-                   << nfiles << " files and "  << num_procs << " ranks\n";
-      }
-   }
-   {
-      double t1;
-      t1 = MPI_Wtime();
-      ParGridFunction temp_gf(fespace, filename.c_str());
-      double t2 = MPI_Wtime();
-
-      double read_time = t2 - t1;
-      double average_read_time;
-      MPI_Reduce(&read_time, &average_read_time, 1,
-                 MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-      if (myid == 0)
-      {
-         std::cout << "Average read time: " << average_read_time / num_procs << " for "
-                   << nfiles << " files and "  << num_procs << " ranks\n";
-      }
-   }
-
-   // 17. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}

examples/makefile

@@ -22,10 +22,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
-  ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27
+  ex18 ex19 ex20 ex21 ex22 ex23 ex24
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
-  ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p ex25p\
-  ex26p ex27p
+ ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -104,10 +103,6 @@ ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-e 1)
-ex27-test-seq: ex27
-	@$(call mfem-test,$<,, Serial example,-dg)
-ex27p-test-par: ex27p
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-dg)
 # Testing: optional tests
 ifeq ($(MFEM_USE_STRUMPACK),YES)
 ex11p-test-strumpack: ex11p
@@ -133,11 +128,9 @@ clean-exec:
 	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
 	@rm -f deformed.* velocity.* elastic_energy.* mode_*
-	@rm -f ex5-p-*.bp ex9-p-*.bp ex12-p-*.bp ex16-p-*.bp
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*
 	@rm -f vortex-mesh.* vortex.mesh vortex-?-init.* vortex-?-final.*
 	@rm -f deformation.* pressure.*
 	@rm -f ex20.dat ex20p_?????.dat gnuplot_ex20.inp gnuplot_ex20p.inp
 	@rm -f ex21*.mesh ex21*.sol ex21p_*.*
 	@rm -f ex23.mesh ex23-*.gf
-	@rm -f ex25.mesh ex25-*.gf ex25p-*.*

examples/pa_oper.cpp

@@ -1,906 +0,0 @@
-//                                MFEM Example 9
-//
-// Compile with: make serial_nogpu
-//
-// Description:  This code solves the time-dependent advection-diffusion
-//               equation:
-//               \frac(\partial u}{\partial t}
-//                                 = \mathbf{a} \cdot \Nabla u - \nu \Nabla^2 u
-//               where a is a given advection velocity, \nu is the diffusion
-//               parameter, and u0(x) = u(0,x) is a given initial condition.
-//
-//               The demonstrates explicit time marching with H1 elements of
-//               arbitrary order. Periodic boundary conditions are used through
-//               periodic meshes. GLVis can be used for visualization of a
-//               time-evolving solution.
-
-#include <fstream>
-#include <iostream>
-#include <algorithm>
-
-#include "mfem.hpp"
-#include "mpi.h"
-
-using namespace std;
-using namespace mfem;
-
-/** A time-dependent operator for the right-hand side of the ODE. The weak
-    form of du/dt = -a.grad(u) + nu Delta(u) is M du/dt = K u + b, where M and
-    K are the mass and advection-diffusion matrices, and b describes the flow
-    on the boundary. This can be written as a general ODE,
-    du/dt = M^{-1} (K u + b), and this class is used to evaluate the right-hand
-    side. */
-class AdvectionDiffusionEvolution : public mfem::TimeDependentOperator
-{
-public:
-   /// \param[in] M - bilinear form for mass matrix
-   /// \param[in] K - bilinear form for stiffness matrix
-   /// \param[in] b - load vector
-   AdvectionDiffusionEvolution(mfem::BilinearForm &M, mfem::BilinearForm &K,
-                               const mfem::Vector &b);
-
-   /// Perform the action of the operator: y = k = f(x, t), where k solves
-   /// Compute k = M^-1(Kx + l)
-   void Mult(const mfem::Vector &x, mfem::Vector &y) const override;
-
-   /// Solve the implicit equation: k = f(x + dt k, t), for the unknown k at
-   /// the current time t. 
-   void ImplicitSolve(const double dt, const mfem::Vector &x,
-                      mfem::Vector &k) override;
-
-   virtual ~AdvectionDiffusionEvolution();
-
-private:
-   mfem::BilinearForm &M, &K;
-   const mfem::Vector &b;
-   /// solver for inverting mass matrix for explicit time-marching
-   std::unique_ptr<mfem::Solver> M_prec;
-   mfem::CGSolver M_solver;
-   /// solver for implicit time-marching
-   mfem::GSSmoother prec;
-   mfem::GMRESSolver linear_solver;
-   mfem::NewtonSolver newton;
-
-   mutable mfem::Vector z;
-
-   /// pointer-to-implementation idiom
-   /// Hides implementation details of this operator
-   class SystemOperator;
-   /// Operator that combines the linear spatial discretization with
-   /// the load vector into one operator used for implicit solves
-   std::unique_ptr<SystemOperator> combined_oper;
-
-   /// sets the state and dt for the combined operator
-   /// \param[in] dt - time increment
-   /// \param[in] x - the current state
-   void setOperParameters(double dt, const mfem::Vector *x);
-
-};
-
-class PAJacobianOperator : public mfem::Operator
-{
-public:
-   PAJacobianOperator(mfem::ParBilinearForm &_mass,
-                      mfem::ParBilinearForm &_stiff);
-
-   /// Compute r = J@k = M@k + dt*K@k
-   /// \param[in] k - dx/dt
-   /// \param[out] r - J@k = M@k + dt*K@k
-   void Mult(const mfem::Vector &k, mfem::Vector &r) const override;
-      
-   /// Set current dt values - needed to compute action of Jacobian.
-   void setParameters(double dt);
-
-private:
-   mfem::ParBilinearForm &mass;
-   mfem::ParBilinearForm &stiff;
-
-   double dt;
-};
-
-class ParSystemOperator : public mfem::Operator
-{
-public:
-   /// Nonlinear operator of the form that combines the mass, res, stiff,
-   /// and load elements for implicit/explicit ODE integration
-   /// \param[in] ess_bdr - array of boundaries attributes marked essential
-   /// \param[in] mass - bilinear form for mass matrix (not owned)
-   /// \param[in] res - nonlinear residual operator (not owned)
-   /// \param[in] stiff - bilinear form for stiffness matrix (not owned)
-   /// \param[in] load - load vector (not owned)
-   /// \param[in] a - used to move the spatial residual to the rhs
-   ParSystemOperator(mfem::ParBilinearForm &_mass,
-                     mfem::ParBilinearForm &_stiff);
-
-   /// Compute r = M@k + K@(x+dt*k)
-   /// (with `@` denoting matrix-vector multiplication)
-   /// \param[in] k - dx/dt 
-   /// \param[out] r - the residual
-   /// \note the signs on each operator must be accounted for elsewhere
-   void Mult(const mfem::Vector &k, mfem::Vector &r) const override;
-
-   /// Compute J = M + dt * K
-   /// \param[in] k - dx/dt 
-   mfem::Operator &GetGradient(const mfem::Vector &k) const override;
-
-   /// Set current dt and x values - needed to compute action and Jacobian.
-   void setParameters(double _dt, const mfem::Vector *_x);
-
-   ~ParSystemOperator();
-
-private:
-   mfem::ParBilinearForm &mass;
-   mfem::ParBilinearForm &stiff;
-   mutable mfem::HypreParMatrix *jacobian, *stiff_jacobian;
-
-   double dt;
-   const mfem::Vector *x;
-
-   mutable mfem::Vector work, work2;
-
-   std::unique_ptr<PAJacobianOperator> pa_jac;
-
-};
-
-/** A time-dependent operator for the right-hand side of the ODE. The weak
-    form of du/dt = -a.grad(u) + nu Delta(u) is M du/dt = K u + b, where M and
-    K are the mass and advection-diffusion matrices, and b describes the flow
-    on the boundary. This can be written as a general ODE,
-    du/dt = M^{-1} (K u + b), and this class is used to evaluate the right-hand
-    side. */
-class ParAdvectionDiffusionEvolution : public mfem::TimeDependentOperator
-{
-public:
-   /// \param[in] M - parallel bilinear form for mass matrix
-   /// \param[in] K - parallel bilinear form for stiffness matrix
-   ParAdvectionDiffusionEvolution(mfem::ParBilinearForm &M,
-                                  mfem::ParBilinearForm &K);
-
-   /// Perform the action of the operator: y = k = f(x, t), where k solves
-   /// Compute k = M^-1(Kx + l)
-   void Mult(const mfem::Vector &x, mfem::Vector &y) const override;
-
-   /// Solve the implicit equation: k = f(x + dt k, t), for the unknown k at
-   /// the current time t. 
-   void ImplicitSolve(const double dt, const mfem::Vector &x,
-                      mfem::Vector &k) override;
-
-   virtual ~ParAdvectionDiffusionEvolution();
-
-private:
-   mfem::OperatorHandle M_;
-   mfem::ParBilinearForm &M, &K;
-   /// solver for inverting mass matrix for explicit time-marching
-   std::unique_ptr<mfem::Solver> M_prec;
-   mfem::CGSolver M_solver;
-   /// solver for implicit time-marching
-   mfem::Solver *prec;
-   mfem::GMRESSolver linear_solver;
-   mfem::NewtonSolver newton;
-
-   mfem::Vector diag;
-   mutable mfem::Vector z, work, work2;
-
-   /// pointer-to-implementation idiom
-   /// Hides implementation details of this operator
-   /// Operator that combines the linear spatial discretization with
-   /// the load vector into one operator used for implicit solves
-   std::unique_ptr<ParSystemOperator> combined_oper;
-
-   /// sets the state and dt for the combined operator
-   /// \param[in] dt - time increment
-   /// \param[in] x - the current state
-   void setOperParameters(double dt, const mfem::Vector *x);
-
-};
-
-// Choice for the problem setup. The fluid velocity, initial condition and
-// inflow boundary condition are chosen based on this parameter.
-int problem;
-
-// Velocity coefficient
-void velocity_function(const Vector &X, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &X);
-
-// Inflow boundary condition
-double inflow_function(const Vector &X, const double t);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   problem = 3;
-   const char *mesh_file = "../data/periodic-square.mesh";
-   int ser_ref_levels = 0;
-   int par_ref_levels = 0;
-   int order = 3;
-   const char *device_config = "cpu";
-   int ode_solver_type = 22;
-   double t_final = 3 * 2*M_PI;
-   double dt = 0.01;
-   bool glvis = false;
-   bool paraview = false;
-   int vis_steps = 5;
-
-   double nu_val = 0.001;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&glvis, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&nu_val, "-nu", "--nu-value",
-                  "Value for \nu, the parameter that controls diffusion.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      std::cout << "Num ranks: " << num_procs << "\n";
-      args.PrintOptions(cout);
-   }
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 3. Read the serial mesh from the given mesh file on all processors. We can
-   //    handle geometrically periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 5. Refine the mesh in serial to increase the resolution. In this example
-   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-   //    a command-line parameter. If the mesh is of NURBS type, we convert it
-   //    to a (piecewise-polynomial) high-order mesh.
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 7. Define the finite element space of the given
-   //    polynomial order on the refined mesh.
-   H1_FECollection fec(order, dim, BasisType::GaussLobatto);
-   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
-
-   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << global_vSize << endl;
-   }
-
-   // 8. Set up and assemble the bilinear and linear forms corresponding to the
-   //    CG discretization.
-   /// negative to move the diffusion terms to the right side
-   ConstantCoefficient nu(-nu_val);
-   ConstantCoefficient one(1.0);
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient u0(u0_function);
-
-   ParBilinearForm *m_pa = new ParBilinearForm(fes);
-   ParBilinearForm *k_pa = new ParBilinearForm(fes);
-   m_pa->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   k_pa->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-
-   /// create mass matrix
-   m_pa->AddDomainIntegrator(new MassIntegrator(one));
-   /// add advection terms to stiffness matrix
-   k_pa->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   /// add diffusion terms to stiffness matrix
-   k_pa->AddDomainIntegrator(new DiffusionIntegrator(nu));
-
-   m_pa->Assemble();
-   int skip_zeros = 0;
-   k_pa->Assemble(skip_zeros);
-   m_pa->Finalize();
-   k_pa->Finalize(skip_zeros);
-
-   ParBilinearForm *m = new ParBilinearForm(fes);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   /// create mass matrix
-   m->AddDomainIntegrator(new MassIntegrator);
-   /// add advection terms to stiffness matrix
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   /// add diffusion terms to stiffness matrix
-   k->AddDomainIntegrator(new DiffusionIntegrator(nu));
-
-   m->Assemble();
-   k->Assemble(skip_zeros);
-   m->Finalize();
-   k->Finalize(skip_zeros);
-
-
-   ParGridFunction *u = new ParGridFunction(fes);
-   u->UseDevice(true);
-   u->ProjectCoefficient(u0);
-
-
-   HypreParVector *U = u->GetTrueDofs();
-
-   ParSystemOperator pso(*m, *k);
-   ParSystemOperator pso_pa(*m_pa, *k_pa);
-
-   pso.setParameters(dt, U);
-   pso_pa.setParameters(dt, U);
-
-   MPI_Barrier(MPI_COMM_WORLD);
-   mfem::Vector pso_r(U->Size());
-   double t1 = MPI_Wtime();
-   pso.Mult(*U, pso_r);
-   double t2 = MPI_Wtime();
-   double fa_mult_time = t2 - t1;
-   double average_fa_mult_time;
-   MPI_Reduce(&fa_mult_time, &average_fa_mult_time, 1,
-              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   if (myid == 0)
-      std::cout << "FA Mult time: " << average_fa_mult_time / num_procs << endl;
-
-   MPI_Barrier(MPI_COMM_WORLD);
-   mfem::Vector pso_pa_r(U->Size());
-   double t3 = MPI_Wtime();
-   pso_pa.Mult(*U, pso_pa_r);
-   double t4 = MPI_Wtime();
-   double pa_mult_time = t4 - t3;
-   double average_pa_mult_time;
-   MPI_Reduce(&pa_mult_time, &average_pa_mult_time, 1,
-              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   if (myid == 0)
-      std::cout << "FA Mult time: " << average_pa_mult_time / num_procs << endl;
-   
-   double local_mult_speedup = (t2-t1) / (t4-t3);
-   double global_mult_speedup;
-   MPI_Reduce(&local_mult_speedup, &global_mult_speedup, 1,
-              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-
-   if (myid == 0)
-      std::cout << "PA mult speedup: " << global_mult_speedup / num_procs << endl;
-
-   mfem::Vector diff_r(pso_pa_r);
-   diff_r -= pso_r;
-   // std::cout << "r diff: " << diff_r.Norml2() << std::endl;
-
-   mfem::Operator &pso_jac = pso.GetGradient(*U);
-   mfem::Operator &pso_pa_jac = pso_pa.GetGradient(*U);
-
-   MPI_Barrier(MPI_COMM_WORLD);
-   mfem::Vector pso_jac_r(U->Size());
-   double t5 = MPI_Wtime();
-   pso_jac.Mult(*U, pso_jac_r);
-   double t6 = MPI_Wtime();
-   double fa_jac_mult_time = t6-t5;
-   double average_fa_jac_time;
-   MPI_Reduce(&fa_jac_mult_time, &average_fa_jac_time, 1,
-              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   if (myid == 0)
-      std::cout << "FA Jac Mult time: " << average_fa_jac_time / num_procs << endl;
-
-   MPI_Barrier(MPI_COMM_WORLD);
-   mfem::Vector pso_pa_jac_r(U->Size());
-   double t7 = MPI_Wtime();
-   pso_pa_jac.Mult(*U, pso_pa_jac_r);
-   double t8 = MPI_Wtime();
-   double pa_jac_mult_time = t8-t7;
-   double average_pa_jac_time;
-   MPI_Reduce(&pa_jac_mult_time, &average_pa_jac_time, 1,
-              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   if (myid == 0)
-      std::cout << "PA Jac Mult time: " << average_pa_jac_time / num_procs << endl;
-
-   double local_jac_speedup = (t6-t5) / (t8-t7);
-   double global_jac_speedup;
-   MPI_Reduce(&local_jac_speedup, &global_jac_speedup, 1,
-              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   if (myid == 0)
-      std::cout << "PA Jac mult speedup: " << global_jac_speedup / num_procs << endl;
-
-   // 13. Free the used memory.
-   delete U;
-   delete u;
-   delete k;
-   delete m;
-   delete fes;
-   delete pmesh;
-
-   MPI_Finalize();
-   return 0;
-}
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 3:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 0:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-double u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return exp(-40.*pow(X(0)-0.5,2));
-            case 2:
-            case 3:
-            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const double f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x, const double t)
-{
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      case 2:
-      case 3: return 0.0;
-   }
-   return 0.0;
-}
-
-class AdvectionDiffusionEvolution::SystemOperator : public mfem::Operator
-{
-public:
-   /// Nonlinear operator of the form that combines the mass, res, stiff,
-   /// and load elements for implicit/explicit ODE integration
-   /// \param[in] mass - bilinear form for mass matrix (not owned)
-   /// \param[in] res - nonlinear residual operator (not owned)
-   /// \param[in] stiff - bilinear form for stiffness matrix (not owned)
-   /// \param[in] load - load vector (not owned)
-   /// \param[in] a - used to move the spatial residual to the rhs
-   SystemOperator(BilinearForm &_mass, BilinearForm &_stiff,
-                  const mfem::Vector &b)
-      : Operator(_mass.Height()), mass(_mass), stiff(_stiff),
-        load(b), Jacobian(NULL), dt(0.0), x(NULL), work(height)
-      { }
-
-   /// Compute r = M@k + K@(x+dt*k) + l
-   /// (with `@` denoting matrix-vector multiplication)
-   /// \param[in] k - dx/dt 
-   /// \param[out] r - the residual
-   /// \note the signs on each operator must be accounted for elsewhere
-   void Mult(const mfem::Vector &k, mfem::Vector &r) const override
-   {
-      /// work = x+dt*k = x+dt*dx/dt = x+dx
-      add(1.0, *x, dt, k, work);
-      r = 0.0;
-      stiff.AddMult(work, r);
-      r += load;
-      mass.AddMult(k, r, -1.0);
-   }
-
-   /// Compute J = M + dt * K
-   /// \param[in] k - dx/dt 
-   mfem::Operator &GetGradient(const mfem::Vector &k) const override
-   {
-      delete Jacobian;
-      Jacobian = Add(-1.0, mass.SpMat(), dt, stiff.SpMat());
-      return *Jacobian;
-   }
-
-   /// Set current dt and x values - needed to compute action and Jacobian.
-   void setParameters(double _dt, const mfem::Vector *_x)
-   {
-      dt = _dt;
-      x = _x;
-   };
-
-   ~SystemOperator() {delete Jacobian;};
-
-private:
-   BilinearForm &mass;
-   BilinearForm &stiff;
-   const mfem::Vector &load;
-   mutable mfem::SparseMatrix *Jacobian;
-
-   double dt;
-   const mfem::Vector *x;
-
-   mutable mfem::Vector work, work2;
-
-};
-
-AdvectionDiffusionEvolution::AdvectionDiffusionEvolution(
-      BilinearForm &_M, BilinearForm &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b),
-     z(_M.Height())
-{
-   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
-   Array<int> ess_tdof_list;
-   if (pa)
-   {
-      M_prec.reset(new OperatorJacobiSmoother(M, ess_tdof_list));
-      M_solver.SetOperator(M);
-   }
-   else
-   {
-      M_prec.reset(new DSmoother(M.SpMat()));
-      M_solver.SetOperator(M.SpMat());
-   }
-
-   combined_oper.reset(new SystemOperator(_M, _K, _b));
-
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-
-   linear_solver.iterative_mode = true;
-   linear_solver.SetRelTol(1e-12);
-   linear_solver.SetAbsTol(0.0);
-   linear_solver.SetMaxIter(100);
-   linear_solver.SetPrintLevel(0);
-   linear_solver.SetPreconditioner(prec);
-
-   newton.iterative_mode = false;
-   newton.SetRelTol(1e-9);
-   newton.SetAbsTol(0.0);
-   newton.SetMaxIter(100);
-   newton.SetPrintLevel(-1);
-   newton.SetSolver(linear_solver);
-   newton.SetOperator(*combined_oper);
-}
-
-void AdvectionDiffusionEvolution::Mult(const Vector &x, Vector &y) const
-{
-   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-void AdvectionDiffusionEvolution::ImplicitSolve(const double dt,
-                                                const Vector &x,
-                                                Vector &k)
-{
-   setOperParameters(dt, &x);
-   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
-   newton.Mult(zero, k);
-   MFEM_VERIFY(newton.GetConverged(), "Newton solver did not converge!");
-}
-
-void AdvectionDiffusionEvolution::setOperParameters(double dt,
-                                                    const mfem::Vector *x)
-{
-   combined_oper->setParameters(dt, x);
-}
-
-AdvectionDiffusionEvolution::~AdvectionDiffusionEvolution() {}
-
-
-PAJacobianOperator::PAJacobianOperator(ParBilinearForm &_mass, ParBilinearForm &_stiff)
- : Operator(_mass.ParFESpace()->GetTrueVSize()), mass(_mass), stiff(_stiff),
-   dt(0.0) { }
-
-
-void PAJacobianOperator::Mult(const mfem::Vector &k, mfem::Vector &r) const
-{
-   r.UseDevice(true);
-   r = 0.0;
-   stiff.TrueAddMult(k, r, dt);
-   mass.TrueAddMult(k, r, -1.0);
-}
-   
-void PAJacobianOperator::setParameters(const double _dt)
-{
-   dt = _dt;
-};
-
-ParSystemOperator::ParSystemOperator(ParBilinearForm &_mass, ParBilinearForm &_stiff)
-   : Operator(_mass.ParFESpace()->GetTrueVSize()), mass(_mass), stiff(_stiff),
-   jacobian(NULL), stiff_jacobian(NULL), dt(0.0), x(NULL),
-   work(height)
-{
-   pa_jac.reset(new PAJacobianOperator(mass, stiff));
-}
-
-/// Compute r = M@k + K@(x+dt*k)
-/// (with `@` denoting matrix-vector multiplication)
-/// \param[in] k - dx/dt 
-/// \param[out] r - the residual
-/// \note the signs on each operator must be accounted for elsewhere
-void ParSystemOperator::Mult(const mfem::Vector &k, mfem::Vector &r) const
-{
-   r = 0.0;
-   work.UseDevice(true);
-   work = 0.0;
-   /// work = x+dt*k = x+dt*dx/dt = x+dx
-   if (x)
-   {
-      add(1.0, *x, dt, k, work);
-   }
-
-   stiff.TrueAddMult(work, r);
-   mass.TrueAddMult(k, r, -1.0);
-}
-
-/// Compute J = M + dt * K
-/// \param[in] k - dx/dt 
-mfem::Operator &ParSystemOperator::GetGradient(const mfem::Vector &k) const
-{
-   bool mass_pa = mass.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
-   bool stiff_pa = stiff.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
-   
-   if (mass_pa && stiff_pa)
-   {
-      return *pa_jac.get();
-   }
-   else
-   {
-      delete stiff_jacobian;
-      delete jacobian;
-      jacobian = mass.ParallelAssemble();
-      *jacobian *= -1.0; //alpha;
-      stiff_jacobian = stiff.ParallelAssemble();
-      jacobian->Add(dt, *stiff_jacobian);
-      return *jacobian;
-   }
-}
-
-/// Set current dt and x values - needed to compute action and Jacobian.
-void ParSystemOperator::setParameters(const double _dt, const mfem::Vector *_x)
-{
-   dt = _dt;
-   x = _x;
-   pa_jac->setParameters(_dt);
-};
-
-ParSystemOperator::~ParSystemOperator()
-{
-   delete jacobian;
-   delete stiff_jacobian;
-};
-
-ParAdvectionDiffusionEvolution::ParAdvectionDiffusionEvolution(
-      ParBilinearForm &_M, ParBilinearForm &_K)
-   : TimeDependentOperator(_M.ParFESpace()->GetTrueVSize()), M(_M), K(_K), z(_M.Height())
-{
-   bool mass_pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
-   bool stiff_pa = K.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
-
-   Array<int> ess_tdof_list;
-   M_solver = CGSolver(MPI_COMM_WORLD);
-   if (mass_pa)
-   {
-      M_prec.reset(new OperatorJacobiSmoother(M, ess_tdof_list));
-      M_solver.SetOperator(M);
-   }
-   else
-   {
-      M_.Reset(_M.ParallelAssemble(), true);
-
-      // M_prec.reset(new HypreSmoother());
-      // M_solver.SetOperator(M.As<HypreParMatrix>());
-      HypreParMatrix &M_mat = *M_.As<HypreParMatrix>();
-      // HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
-      M_prec.reset(new HypreSmoother(M_mat, HypreSmoother::Jacobi));
-   }
-
-   combined_oper.reset(new ParSystemOperator(_M, _K));
-
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-
-   if (mass_pa && stiff_pa)
-   {
-      diag.UseDevice(true);
-      diag.SetSize(M.ParFESpace()->GetTrueVSize());
-      diag = 0.0;
-      work.UseDevice(true);
-      work2.UseDevice(true);
-      work.SetSize(M.ParFESpace()->GetTrueVSize());
-      work2.SetSize(M.ParFESpace()->GetTrueVSize());
-      work = 0.0;
-      work2 = 0.0;
-      M.AssembleDiagonal(work);
-
-      ParBilinearForm k(M.ParFESpace());
-      ConstantCoefficient nu(-0.01);
-      k.AddDomainIntegrator(new mfem::DiffusionIntegrator(nu));
-      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k.Assemble(0);
-      k.Finalize(0);
-      k.AssembleDiagonal(work2);
-
-      double dt = 0.1;
-      add(-1.0, work, dt, work2, diag);
-
-      prec = new OperatorChebyshevSmoother(combined_oper.get(), diag,
-                                           ess_tdof_list, 5,
-                                           M.ParFESpace()->GetComm());
-   }
-   else
-   {
-      prec = new HypreSmoother();
-   }
-   
-   linear_solver = GMRESSolver(MPI_COMM_WORLD);
-   linear_solver.iterative_mode = true;
-   linear_solver.SetRelTol(1e-12);
-   linear_solver.SetAbsTol(0.0);
-   linear_solver.SetMaxIter(2000);
-   linear_solver.SetPrintLevel(0);
-   linear_solver.SetPreconditioner(*prec);
-   linear_solver.SetKDim(2000);
-
-   newton.iterative_mode = true;
-   newton.SetRelTol(1e-9);
-   newton.SetAbsTol(0.0);
-   newton.SetMaxIter(10);
-   newton.SetPrintLevel(-1);
-   newton.SetSolver(linear_solver);
-   newton.SetOperator(*combined_oper);
-}
-
-void ParAdvectionDiffusionEvolution::Mult(const Vector &x, Vector &y) const
-{
-   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
-   M_solver.Mult(z, y);
-}
-
-void ParAdvectionDiffusionEvolution::ImplicitSolve(const double dt,
-                                                   const Vector &x,
-                                                   Vector &k)
-{
-   setOperParameters(dt, &x);
-   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
-   newton.Mult(zero, k);
-   MFEM_VERIFY(newton.GetConverged(), "Newton solver did not converge!");
-}
-
-void ParAdvectionDiffusionEvolution::setOperParameters(const double dt,
-                                                       const mfem::Vector *x)
-{
-   combined_oper->setParameters(dt, x);
-}
-
-ParAdvectionDiffusionEvolution::~ParAdvectionDiffusionEvolution() {delete prec;}

fem/CMakeLists.txt

@@ -17,11 +17,11 @@ set(SRCS
   bilininteg_dgtrace.cpp
   bilininteg_diffusion.cpp
   bilininteg_divergence.cpp
-  bilininteg_hcurl.cpp
   bilininteg_gradient.cpp
   bilininteg_mass.cpp
   bilininteg_vecdiffusion.cpp
   bilininteg_vecmass.cpp
+  bilininteg_hcurl.cpp
   coefficient.cpp
   complex_fem.cpp
   datacollection.cpp
@@ -36,20 +36,16 @@ set(SRCS
   intrules.cpp
   linearform.cpp
   lininteg.cpp
-  multigrid.cpp
   nonlinearform.cpp
   nonlinearform_ext.cpp
   nonlininteg.cpp
-  fespacehierarchy.cpp
   nonlininteg_vectorconvection.cpp
   quadinterpolator.cpp
-  quadinterpolator_face.cpp
   restriction.cpp
   staticcond.cpp
   tmop.cpp
   tmop_tools.cpp
   gslib.cpp
-  transfer.cpp
   )
 
 set(HDRS
@@ -71,14 +67,11 @@ set(HDRS
   intrules.hpp
   linearform.hpp
   lininteg.hpp
-  multigrid.hpp
   nonlinearform.hpp
   nonlinearform_ext.hpp
   nonlininteg.hpp
   quadinterpolator.hpp
-  quadinterpolator_face.hpp
   restriction.hpp
-  fespacehierarchy.hpp
   staticcond.hpp
   tbilinearform.hpp
   tbilininteg.hpp
@@ -91,7 +84,6 @@ set(HDRS
   tmop.hpp
   tmop_tools.hpp
   gslib.hpp
-  transfer.hpp
   )
 
 if (MFEM_USE_SIDRE)
@@ -104,19 +96,13 @@ if (MFEM_USE_CONDUIT)
   list(APPEND HDRS conduitdatacollection.hpp)
 endif()
 
-if (MFEM_USE_ADIOS2)
-  list(APPEND SRCS adios2datacollection.cpp)
-  list(APPEND HDRS adios2datacollection.hpp)
-endif()
-
 if (MFEM_USE_MPI)
   list(APPEND SRCS
     pbilinearform.cpp
     pfespace.cpp
     pgridfunc.cpp
     plinearform.cpp
-    pnonlinearform.cpp
-    prestriction.cpp)
+    pnonlinearform.cpp)
   # If this list (HDRS -> HEADERS) is used for install, we probably want the
   # headers added all the time.
   list(APPEND HDRS
@@ -124,8 +110,7 @@ if (MFEM_USE_MPI)
     pfespace.hpp
     pgridfunc.hpp
     plinearform.hpp
-    pnonlinearform.hpp
-    prestriction.hpp)
+    pnonlinearform.hpp)
 endif()
 
 if (MFEM_USE_CEED)

fem/adios2datacollection.cpp

@@ -1,90 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-//
-// Created on: Jan 7, 2020
-// Author: William F Godoy godoywf@ornl.gov
-// adios2: Adaptable Input/Output System https://github.com/ornladios/ADIOS2
-
-#include "adios2datacollection.hpp"
-
-namespace mfem
-{
-
-#ifdef MFEM_USE_MPI
-ADIOS2DataCollection::ADIOS2DataCollection(MPI_Comm comm,
-                                           const std::string& collection_name, Mesh* mesh,
-                                           const std::string engine_type) : DataCollection(collection_name, mesh),
-   stream( new adios2stream(name, adios2stream::openmode::out, comm, engine_type) )
-{
-   SetMesh(mesh);
-}
-#else
-ADIOS2DataCollection::ADIOS2DataCollection(
-   const std::string& collection_name, Mesh* mesh,
-   const std::string engine_type): DataCollection(collection_name, mesh),
-   stream( new adios2stream(name, adios2stream::openmode::out, engine_type) )
-{
-   SetMesh(mesh);
-}
-#endif
-
-ADIOS2DataCollection::~ADIOS2DataCollection()
-{
-   stream->Close();
-}
-
-void ADIOS2DataCollection::Save()
-{
-   stream->BeginStep();
-
-   // only save mesh once (moving mesh, not yet supported)
-   if (stream->CurrentStep() == 0)
-   {
-      if (mesh == nullptr)
-      {
-         const std::string error_message =
-            "MFEM ADIOS2DataCollection Save error: Mesh is null. Please call SetMesh before Save\n";
-         mfem_error(error_message.c_str());
-      }
-      stream->Print(*mesh);
-   }
-
-   // reduce footprint
-   if (myid == 0)
-   {
-      stream->SetTime(time);
-      stream->SetCycle(cycle);
-   }
-
-   for (const auto& field : field_map)
-   {
-      const std::string& variable_name = field.first;
-      field.second->Save(*stream.get(), variable_name);
-   }
-
-   stream->EndStep();
-}
-
-void ADIOS2DataCollection::SetParameter(const std::string key,
-                                        const std::string value) noexcept
-{
-   stream->SetParameter(key, value);
-}
-
-void ADIOS2DataCollection::SetLevelsOfDetail(const int levels_of_detail)
-noexcept
-{
-   stream->SetRefinementLevel(levels_of_detail);
-}
-
-} //end namespace mfem
-
-

fem/adios2datacollection.hpp

@@ -1,88 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-//
-// Created on: Jan 7, 2020
-// Author: William F Godoy godoywf@ornl.gov
-// adios2: Adaptable Input/Output System https://github.com/ornladios/ADIOS2
-
-#ifndef MFEM_ADIOS2DATACOLLECTION
-#define MFEM_ADIOS2DATACOLLECTION
-
-#include "../config/config.hpp"
-#include "../general/adios2stream.hpp"
-#include "datacollection.hpp"
-
-#include <memory>  // std::unique_ptr
-#include <string>
-
-namespace mfem
-{
-
-class ADIOS2DataCollection : public DataCollection
-{
-public:
-#ifdef MFEM_USE_MPI
-   /**
-    * Parallel constructor. Important: scope of this object must be within
-    * MPI_Init and MPI_Finalize otherwise. The destructor will call the Close
-    * function. Either object must live in a try/catch block (inside try) or use
-    * raw pointers calling delete before MPI_Finalize.
-    * @param comm MPI communicator setting the datacollection domain
-    * @param collection_name unique name for saving data
-    * @param mesh can be set at the constructor level or later by calling
-    * SetMesh()
-    * @param engine_type adios2 engine type
-    */
-   ADIOS2DataCollection(MPI_Comm comm, const std::string& collection_name,
-                        Mesh* mesh = nullptr,
-                        const std::string engine_type = "BPFile");
-#else
-   /**
-    * Serial constructor
-    * @param collection_name unique name for saving data
-    * @param mesh can be set at the constructor level or later by calling
-    * SetMesh()
-    * @param engine_type adios2 engine type
-    * @throws std::invalid_argument (user input error) or std::runtime_error
-    *         (system error)
-    */
-   ADIOS2DataCollection(const std::string& collection_name, Mesh* mesh = nullptr,
-                        const std::string engine_type = "BPFile");
-#endif
-
-   virtual ~ADIOS2DataCollection();
-
-   /** Save the collection */
-   virtual void Save();
-
-   /**
-    * Pass a parameter unique to adios2datacollection
-    * For available parameters:
-    * See https://adios2.readthedocs.io/en/latest/engines/engines.html
-    * The most common is: key=SubStreams value=1 to nprocs (MPI processes)
-    * @param key parameter key
-    * @param value parameter value
-    */
-   void SetParameter(const std::string key, const std::string value) noexcept;
-
-   /**
-    * Sets the levels of detail for the global grid refinement
-    * @param levels_of_detail (default = 1)
-    */
-   void SetLevelsOfDetail(const int levels_of_detail) noexcept;
-
-private:
-   std::unique_ptr<adios2stream> stream;
-};
-
-}  // namespace mfem
-
-#endif /* MFEM_ADIOS2DATACOLLECTION */

fem/bilinearform.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 // Implementation of class BilinearForm
@@ -467,17 +467,8 @@ void BilinearForm::Assemble(int skip_zeros)
          const FiniteElement &be = *fes->GetBE(i);
          fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
-         int k = 0;
-         for (; k < bbfi.Size(); k++)
-         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elmat);
-            k++;
-            break;
-         }
-         for (; k < bbfi.Size(); k++)
+         bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
+         for (int k = 1; k < bbfi.Size(); k++)
          {
             if (bbfi_marker[k] &&
                 (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }

fem/bilinearform.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_BILINEARFORM
@@ -96,9 +96,11 @@ protected:
    StaticCondensation *static_cond; ///< Owned.
    Hybridization *hybridization; ///< Owned.
 
-   /** This data member allows one to specify what should be done to the
-       diagonal matrix entries and corresponding RHS values upon elimination of
-       the constrained DoFs. */
+   /**
+    * This member allows one to specify what should be done
+    * to the diagonal matrix entries and corresponding RHS
+    * values upon elimination of the constrained DoFs.
+    */
    DiagonalPolicy diag_policy;
 
    int precompute_sparsity;
@@ -151,8 +153,8 @@ public:
    /** This method must be called before assembly. */
    void SetAssemblyLevel(AssemblyLevel assembly_level);
 
-   /// Returns the assembly level
-   AssemblyLevel GetAssemblyLevel() const { return assembly; }
+   /// Get the assembly level
+   AssemblyLevel GetAssemblyLevel() {return assembly;}
 
    /** Enable the use of static condensation. For details see the description
        for class StaticCondensation in fem/staticcond.hpp This method should be
@@ -530,7 +532,7 @@ public:
 
    /// (DEPRECATED) Return the FE space associated with the BilinearForm.
    /** @deprecated Use FESpace() instead. */
-   MFEM_DEPRECATED FiniteElementSpace *GetFES() { return fes; }
+   FiniteElementSpace *GetFES() { return fes; }
 
    /// Return the FE space associated with the BilinearForm.
    FiniteElementSpace *FESpace() { return fes; }

fem/bilinearform_ext.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 // Implementations of classes FABilinearFormExtension, EABilinearFormExtension,

fem/bilinearform_ext.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_BILINEARFORM_EXT

fem/bilininteg.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 // Implementation of Bilinear Form Integrators
@@ -2135,17 +2135,17 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
    ElementTransformation &Trans,
    DenseMatrix &elmat)
 {
-   const int dim = el.GetDim();
-   const int dof = el.GetDof();
-   const int sdim = Trans.GetSpaceDim();
-   const bool square = (dim == sdim);
-   double w;
+   int dim = el.GetDim();
+   int dof = el.GetDof();
 
-   elmat.SetSize(sdim * dof);
+   double norm;
 
-   dshape.SetSize(dof, dim);
-   dshapedxt.SetSize(dof, sdim);
-   pelmat.SetSize(dof);
+   elmat.SetSize (dim * dof);
+
+   Jinv.  SetSize (dim);
+   dshape.SetSize (dof, dim);
+   gshape.SetSize (dof, dim);
+   pelmat.SetSize (dof);
 
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
@@ -2163,29 +2163,35 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
    }
 
    elmat = 0.0;
-   pelmat = 0.0;
 
    for (int i = 0; i < ir -> GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
+
       el.CalcDShape (ip, dshape);
+
       Trans.SetIntPoint (&ip);
-      w = Trans.Weight();
-      w = ip.weight / (square ? w : w*w*w);
-      // AdjugateJacobian = / adj(J),         if J is square
-      //                    \ adj(J^t.J).J^t, otherwise
-      Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
-      if (Q) {  w *= Q -> Eval (Trans, ip); }
-      AddMult_a_AAt(w, dshapedxt, pelmat);
-   }
-   for (int d = 0; d < sdim; d++)
-   {
-      for (int k = 0; k < dof; k++)
+      norm = ip.weight * Trans.Weight();
+      CalcInverse (Trans.Jacobian(), Jinv);
+
+      Mult (dshape, Jinv, gshape);
+
+      MultAAt (gshape, pelmat);
+
+      if (Q)
       {
-         for (int l = 0; l < dof; l++)
-         {
-            elmat(dof*d+k, dof*d+l) = pelmat(k, l);
-         }
+         norm *= Q -> Eval (Trans, ip);
+      }
+
+      pelmat *= norm;
+
+      for (int d = 0; d < dim; d++)
+      {
+         for (int k = 0; k < dof; k++)
+            for (int l = 0; l < dof; l++)
+            {
+               elmat (dof*d+k, dof*d+l) += pelmat (k, l);
+            }
       }
    }
 }
@@ -2210,7 +2216,7 @@ void VectorDiffusionIntegrator::AssembleElementVector(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      // integrand is rational function if det(J) is not constant
+      // integrant is rational function if det(J) is not constant
       int order = 2 * Tr.OrderGrad(&el); // order of the numerator
       ir = (el.Space() == FunctionSpace::rQk) ?
            &RefinedIntRules.Get(el.GetGeomType(), order) :

fem/bilininteg.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_BILININTEG
@@ -453,16 +453,6 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 
-   /// Support for use in BilinearForm. Can be used only when appropriate.
-   /** Appropriate use cases are classes derived from
-       MixedScalarVectorIntegrator where the trial and test spaces can be the
-       same. Examples of such classes are: MixedVectorDivergenceIntegrator,
-       MixedScalarWeakDivergenceIntegrator, etc. */
-   virtual void AssembleElementMatrix(const FiniteElement &fe,
-                                      ElementTransformation &Trans,
-                                      DenseMatrix &elmat)
-   { AssembleElementMatrix2(fe, fe, Trans, elmat); }
-
 protected:
 
    MixedScalarVectorIntegrator(VectorCoefficient &vq, bool _transpose = false,
@@ -684,12 +674,10 @@ protected:
              "vector field";
    }
 
-   // Subtract one due to the divergence and add one for the coefficient
-   // which is assumed to be at least linear.
    inline virtual int GetIntegrationOrder(const FiniteElement & trial_fe,
                                           const FiniteElement & test_fe,
                                           ElementTransformation &Trans)
-   { return trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW() - 1 + 1; }
+   { return trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW() - 1; }
 
    inline virtual void CalcShape(const FiniteElement & scalar_fe,
                                  ElementTransformation &Trans,
@@ -723,11 +711,6 @@ protected:
              "and the test space must be H(Div)";
    }
 
-   inline virtual int GetIntegrationOrder(const FiniteElement & trial_fe,
-                                          const FiniteElement & test_fe,
-                                          ElementTransformation &Trans)
-   { return trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW() - 1; }
-
    virtual void CalcTestShape(const FiniteElement & test_fe,
                               ElementTransformation &Trans,
                               Vector & shape)
@@ -888,13 +871,6 @@ public:
              "and the test space must be a vector field with a divergence";
    }
 
-   // Subtract one due to the gradient and add one for the coefficient
-   // which is assumed to be at least linear.
-   inline virtual int GetIntegrationOrder(const FiniteElement & trial_fe,
-                                          const FiniteElement & test_fe,
-                                          ElementTransformation &Trans)
-   { return trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW() - 1 + 1; }
-
    inline virtual void CalcShape(const FiniteElement & scalar_fe,
                                  ElementTransformation &Trans,
                                  Vector & shape)
@@ -1855,8 +1831,8 @@ public:
 #endif
    }
 
-   /** Given a particular Finite Element computes the element stiffness matrix
-       elmat. */
+   /** Given a particular Finite Element
+       computes the element stiffness matrix elmat. */
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
@@ -1944,8 +1920,8 @@ public:
       delete ceedDataPtr;
 #endif
    }
-   /** Given a particular Finite Element computes the element mass matrix
-       elmat. */
+   /** Given a particular Finite Element
+       computes the element mass matrix elmat. */
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
@@ -2066,9 +2042,10 @@ public:
    /// Construct an integrator with coefficient 1.0
    VectorMassIntegrator()
       : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
-   /** Construct an integrator with scalar coefficient q.  If possible, save
-       memory by using a scalar integrator since the resulting matrix is block
-       diagonal with the same diagonal block repeated. */
+   /** Construct an integrator with scalar coefficient q.
+       If possible, save memory by using a scalar integrator since
+       the resulting matrix is block diagonal with the same diagonal
+       block repeated. */
    VectorMassIntegrator(Coefficient &q, int qo = 0)
       : vdim(-1), Q(&q) { VQ = NULL; MQ = NULL; Q_order = qo; }
    VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir)
@@ -2098,13 +2075,13 @@ public:
 };
 
 
-/** Class for integrating (div u, p) where u is a vector field given by
-    VectorFiniteElement through Piola transformation (for RT elements); p is
-    scalar function given by FiniteElement through standard transformation.
-    Here, u is the trial function and p is the test function.
-
-    Note: the element matrix returned by AssembleElementMatrix2 does NOT depend
-    on the ElementTransformation Trans. */
+/** Class for integrating (div u, p) where u is a vector field given
+    by VectorFiniteElement through Piola transformation (for RT
+    elements); p is scalar function given by FiniteElement through
+    standard transformation. Here, u is the trial function and p is
+    the test function.
+    Note: the element matrix returned by AssembleElementMatrix2
+    does NOT depend on the ElementTransformation Trans. */
 class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2415,12 +2392,14 @@ protected:
    // PA extension
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
-   int dim, sdim, ne, dofs1D, quad1D;
+   int dim, ne, dofs1D, quad1D;
    Vector pa_data;
 
 private:
-   DenseMatrix dshape, dshapedxt, pelmat;
-   DenseMatrix Jinv, gshape;
+   DenseMatrix Jinv;
+   DenseMatrix dshape;
+   DenseMatrix gshape;
+   DenseMatrix pelmat;
 
 public:
    VectorDiffusionIntegrator() { Q = NULL; }

fem/bilininteg_convection.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"

fem/bilininteg_dgtrace.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"
@@ -743,7 +743,7 @@ void PADGTraceApplyTranspose2D(const int NF,
    });
 }
 
-// PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
+// PA DGTrace Apply Transpoe 3D kernel for Gauss-Lobatto/Bernstein
 template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApplyTranspose3D(const int NF,
                                const Array<double> &b,

fem/bilininteg_diffusion.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"
@@ -81,20 +81,12 @@ static void OccaPADiffusionSetup3D(const int D1D,
 #endif // MFEM_USE_OCCA
 
 // PA Diffusion Assemble 2D kernel
-template<const int T_SDIM>
 static void PADiffusionSetup2D(const int Q1D,
                                const int NE,
                                const Array<double> &w,
                                const Vector &j,
                                const Vector &c,
-                               Vector &d);
-template<>
-void PADiffusionSetup2D<2>(const int Q1D,
-                           const int NE,
-                           const Array<double> &w,
-                           const Vector &j,
-                           const Vector &c,
-                           Vector &d)
+                               Vector &d)
 {
    const int NQ = Q1D*Q1D;
    const bool const_c = c.Size() == 1;
@@ -120,48 +112,6 @@ void PADiffusionSetup2D<2>(const int Q1D,
    });
 }
 
-// PA Diffusion Assemble 2D kernel with 3D node coords
-template<>
-void PADiffusionSetup2D<3>(const int Q1D,
-                           const int NE,
-                           const Array<double> &w,
-                           const Vector &j,
-                           const Vector &c,
-                           Vector &d)
-{
-   constexpr int DIM = 2;
-   constexpr int SDIM = 3;
-   const int NQ = Q1D*Q1D;
-   const bool const_c = c.Size() == 1;
-
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, SDIM, DIM, NE);
-   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
-   auto D = Reshape(d.Write(), NQ, 3, NE);
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double wq = W[q];
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double E = J11*J11 + J21*J21 + J31*J31;
-         const double G = J12*J12 + J22*J22 + J32*J32;
-         const double F = J11*J12 + J21*J22 + J31*J32;
-         const double iw = 1.0 / sqrt(E*G - F*F);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double alpha = wq * coeff * iw;
-         D(q,0,e) =  alpha * G; // 1,1
-         D(q,1,e) = -alpha * F; // 1,2
-         D(q,2,e) =  alpha * E; // 2,2
-      }
-   });
-}
-
 // PA Diffusion Assemble 3D kernel
 static void PADiffusionSetup3D(const int Q1D,
                                const int NE,
@@ -216,7 +166,6 @@ static void PADiffusionSetup3D(const int Q1D,
 }
 
 static void PADiffusionSetup(const int dim,
-                             const int sdim,
                              const int D1D,
                              const int Q1D,
                              const int NE,
@@ -234,11 +183,8 @@ static void PADiffusionSetup(const int dim,
          OccaPADiffusionSetup2D(D1D, Q1D, NE, W, J, C, D);
          return;
       }
-#else
-      MFEM_CONTRACT_VAR(D1D);
 #endif // MFEM_USE_OCCA
-      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, NE, W, J, C, D); }
-      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, NE, W, J, C, D); }
+      PADiffusionSetup2D(Q1D, NE, W, J, C, D);
    }
    if (dim == 3)
    {
@@ -271,8 +217,6 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
       InitCeedCoeff(Q, ptr);
       return CeedPADiffusionAssemble(fes, *ir, *ptr);
    }
-#else
-   MFEM_CONTRACT_VAR(force);
 #endif
    const int dims = el.GetDim();
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
@@ -280,7 +224,6 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    dim = mesh->Dimension();
    ne = fes.GetNE();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   const int sdim = mesh->SpaceDimension();
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
@@ -309,8 +252,8 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
          }
       }
    }
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
-                    coeff, pa_data);
+   PADiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J, coeff,
+                    pa_data);
 }
 
 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
@@ -967,6 +910,8 @@ template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionApply2D(const int NE,
                                    const Array<double> &b_,
                                    const Array<double> &g_,
+                                   const Array<double> &bt_,
+                                   const Array<double> &gt_,
                                    const Vector &d_,
                                    const Vector &x_,
                                    Vector &y_,
@@ -1312,6 +1257,8 @@ template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
                                    const Array<double> &b_,
                                    const Array<double> &g_,
+                                   const Array<double> &bt_,
+                                   const Array<double> &gt_,
                                    const Vector &d_,
                                    const Vector &x_,
                                    Vector &y_,
@@ -1578,14 +1525,14 @@ static void PADiffusionApply(const int dim,
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
-         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
-         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,B,G,D,X,Y);
-         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,B,G,D,X,Y);
-         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,B,G,D,X,Y);
-         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,B,G,D,X,Y);
-         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,B,G,D,X,Y);
-         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,B,G,D,X,Y);
+         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,B,G,Bt,Gt,D,X,Y);
          default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
       }
    }
@@ -1593,15 +1540,15 @@ static void PADiffusionApply(const int dim,
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,D,X,Y);
-         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,D,X,Y);
-         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,B,G,D,X,Y);
-         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,B,G,D,X,Y);
-         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,B,G,D,X,Y);
-         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,B,G,D,X,Y);
-         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,B,G,D,X,Y);
+         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,B,G,Bt,Gt,D,X,Y);
          default:   return PADiffusionApply3D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
       }
    }

fem/bilininteg_divergence.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"

fem/bilininteg_gradient.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"

fem/bilininteg_hcurl.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"
@@ -747,7 +747,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
    {
-      // pa_data_2.SetSize(6 * nq * ne, Device::GetMemoryType());
+      //pa_data_2.SetSize(6 * nq * ne, Device::GetMemoryType());
 
       PACurlCurlSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                         coeff, pa_data);
@@ -997,7 +997,7 @@ static void PACurlCurlApply3D(const int D1D,
                   for (int qx = 0; qx < Q1D; ++qx)
                   {
                      // \hat{\nabla}\times\hat{u} is [0, (u_0)_{x_2}, -(u_0)_{x_1}]
-                     curl[qz][qy][qx][1] += gradXY[qy][qx][1] * wDz; // (u_0)_{x_2}
+                     curl[qz][qy][qx][1] += gradXY[qy][qx][1] * wDz;  // (u_0)_{x_2}
                      curl[qz][qy][qx][2] -= gradXY[qy][qx][0] * wz;  // -(u_0)_{x_1}
                   }
                }
@@ -1066,7 +1066,7 @@ static void PACurlCurlApply3D(const int D1D,
                   for (int qx = 0; qx < Q1D; ++qx)
                   {
                      // \hat{\nabla}\times\hat{u} is [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
-                     curl[qz][qy][qx][0] -= gradXY[qy][qx][1] * wDz; // -(u_1)_{x_2}
+                     curl[qz][qy][qx][0] -= gradXY[qy][qx][1] * wDz;  // -(u_1)_{x_2}
                      curl[qz][qy][qx][2] += gradXY[qy][qx][0] * wz;  // (u_1)_{x_0}
                   }
                }
@@ -1137,7 +1137,7 @@ static void PACurlCurlApply3D(const int D1D,
                   {
                      // \hat{\nabla}\times\hat{u} is [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
                      curl[qz][qy][qx][0] += gradYZ[qz][qy][1] * wx;  // (u_2)_{x_1}
-                     curl[qz][qy][qx][1] -= gradYZ[qz][qy][0] * wDx; // -(u_2)_{x_0}
+                     curl[qz][qy][qx][1] -= gradYZ[qz][qy][0] * wDx;  // -(u_2)_{x_0}
                   }
                }
             }

fem/bilininteg_mass.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"
@@ -912,8 +912,9 @@ static void SmemPAMassApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
+      const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -930,188 +931,130 @@ static void SmemPAMassApply3D(const int NE,
       double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
       double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
       double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
          }
-         MFEM_FOREACH_THREAD(dx,x,Q1D)
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            B[dx][dy] = b(dx,dy);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               u[dz] = 0;
+               double u = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  u += X[dz][dy][dx] * B[qx][dx];
+               }
+               DDQ[dz][dy][qx] = u;
             }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               MFEM_UNROLL(MD1)
+               double u = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  u += DDQ[dz][dy][qx] * B[qy][dy];
+               }
+               DQQ[dz][qy][qx] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  u[dz] += X[dz][dy][dx] * B[qx][dx];
+                  u += DQQ[dz][qy][qx] * B[qz][dz];
                }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
+               QQQ[qz][qy][qx] = u * d(qx,qy,qz,e);
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
+            MFEM_FOREACH_THREAD(q,x,Q1D)
             {
-               u[dz] = 0;
+               Bt[d][q] = b(q,d);
             }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
+               double u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
                {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+                  u += QQQ[qz][qy][qx] * Bt[dx][qx];
                }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
+               QQD[qz][qy][dx] = u;
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
+               double u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
                {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+                  u += QQD[qz][qy][dx] * Bt[dy][qy];
                }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+               QDD[qz][dy][dx] = u;
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(MQ1)
+               double u = 0.0;
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+                  u += QDD[qz][dy][dx] * Bt[dz][qz];
                }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += u[dz];
+               y(dx,dy,dz,e) += u;
             }
          }
       }
@@ -1142,20 +1085,14 @@ static void PAMassApply(const int dim,
       MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
    }
 #endif // MFEM_USE_OCCA
-   const int id = (D1D << 4) | Q1D;
    if (dim == 2)
    {
-      switch (id)
+      switch ((D1D << 4) | Q1D)
       {
          case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
          case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
          case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
          case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
          case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
          case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
          case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
@@ -1165,25 +1102,19 @@ static void PAMassApply(const int dim,
    }
    else if (dim == 3)
    {
-      switch (id)
+      switch ((D1D << 4) | Q1D)
       {
          case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
          case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
          case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
          case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
          case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
          case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
          case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
    }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
    MFEM_ABORT("Unknown kernel.");
 }
 

fem/bilininteg_vecdiffusion.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"
@@ -101,6 +101,7 @@ static void PAVectorDiffusionSetup3D(const int Q1D,
 }
 
 static void PAVectorDiffusionSetup(const int dim,
+                                   const int D1D,
                                    const int Q1D,
                                    const int NE,
                                    const Array<double> &W,
@@ -133,7 +134,6 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
    const int nq = ir->GetNPoints();
    dim = mesh->Dimension();
-   sdim = mesh->SpaceDimension();
    ne = fes.GetNE();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -147,83 +147,46 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
       MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
       coeff = cQ->constant;
    }
-   const Array<double> &w = ir->GetWeights();
-   const Vector &j = geom->J;
-   Vector &d = pa_data;
-   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PAVectorDiffusionSetup"); }
-   if (dim == 2 && sdim == 3)
-   {
-      constexpr int DIM = 2;
-      constexpr int SDIM = 3;
-      const int NQ = quad1D*quad1D;
-      auto W = w.Read();
-      auto J = Reshape(j.Read(), NQ, SDIM, DIM, ne);
-      auto D = Reshape(d.Write(), NQ, SDIM, ne);
-      MFEM_FORALL(e, ne,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double wq = W[q];
-            const double J11 = J(q,0,0,e);
-            const double J21 = J(q,1,0,e);
-            const double J31 = J(q,2,0,e);
-            const double J12 = J(q,0,1,e);
-            const double J22 = J(q,1,1,e);
-            const double J32 = J(q,2,1,e);
-            const double E = J11*J11 + J21*J21 + J31*J31;
-            const double G = J12*J12 + J22*J22 + J32*J32;
-            const double F = J11*J12 + J21*J22 + J31*J32;
-            const double iw = 1.0 / sqrt(E*G - F*F);
-            const double alpha = wq * coeff * iw;
-            D(q,0,e) =  alpha * G; // 1,1
-            D(q,1,e) = -alpha * F; // 1,2
-            D(q,2,e) =  alpha * E; // 2,2
-         }
-      });
-   }
-   else
-   {
-      PAVectorDiffusionSetup(dim, quad1D, ne, w, j, coeff, d);
-   }
+   PAVectorDiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
+                          coeff, pa_data);
 }
 
 // PA Diffusion Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0> static
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PAVectorDiffusionApply2D(const int NE,
                               const Array<double> &b,
                               const Array<double> &g,
                               const Array<double> &bt,
                               const Array<double> &gt,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
+                              const Vector &_op,
+                              const Vector &_x,
+                              Vector &_y,
                               const int d1d = 0,
-                              const int q1d = 0,
-                              const int vdim = 0)
+                              const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   constexpr int VDIM = 2;
    MFEM_VERIFY(D1D <= MAX_D1D, "");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE);
+   auto op = Reshape(_op.Read(), Q1D*Q1D, 3, NE);
+   auto x = Reshape(_x.Read(), D1D, D1D, VDIM, NE);
+   auto y = Reshape(_y.ReadWrite(), D1D, D1D, VDIM, NE);
    MFEM_FORALL(e, NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      // the following variables are evaluated at compile time
       constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
       constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
-      double grad[max_Q1D][max_Q1D][2];
-      for (int c = 0; c < VDIM; c++)
+      for (int c = 0; c < VDIM; ++ c)
       {
+         double grad[max_Q1D][max_Q1D][2];
          for (int qy = 0; qy < Q1D; ++qy)
          {
             for (int qx = 0; qx < Q1D; ++qx)
@@ -266,11 +229,14 @@ void PAVectorDiffusionApply2D(const int NE,
             for (int qx = 0; qx < Q1D; ++qx)
             {
                const int q = qx + qy * Q1D;
-               const double O11 = D(q,0,e);
-               const double O12 = D(q,1,e);
-               const double O22 = D(q,2,e);
+
+               const double O11 = op(q,0,e);
+               const double O12 = op(q,1,e);
+               const double O22 = op(q,2,e);
+
                const double gradX = grad[qy][qx][0];
                const double gradY = grad[qy][qx][1];
+
                grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
                grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
             }
@@ -280,8 +246,8 @@ void PAVectorDiffusionApply2D(const int NE,
             double gradX[max_D1D][2];
             for (int dx = 0; dx < D1D; ++dx)
             {
-               gradX[dx][0] = 0.0;
-               gradX[dx][1] = 0.0;
+               gradX[dx][0] = 0;
+               gradX[dx][1] = 0;
             }
             for (int qx = 0; qx < Q1D; ++qx)
             {
@@ -503,36 +469,35 @@ void PAVectorDiffusionApply3D(const int NE,
    });
 }
 
+static void PAVectorDiffusionApply(const int dim,
+                                   const int D1D,
+                                   const int Q1D,
+                                   const int NE,
+                                   const Array<double> &B,
+                                   const Array<double> &G,
+                                   const Array<double> &Bt,
+                                   const Array<double> &Gt,
+                                   const Vector &op,
+                                   const Vector &x,
+                                   Vector &y)
+{
+   if (dim == 2)
+   {
+      return PAVectorDiffusionApply2D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
+   }
+   if (dim == 3)
+   {
+      return PAVectorDiffusionApply3D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
 // PA Diffusion Apply kernel
 void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const int D1D = dofs1D;
-   const int Q1D = quad1D;
-   const Array<double> &B = maps->B;
-   const Array<double> &G = maps->G;
-   const Array<double> &Bt = maps->Bt;
-   const Array<double> &Gt = maps->Gt;
-   const Vector &D = pa_data;
-
-   if (dim == 2 && sdim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
-         default:
-            return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
-      }
-   }
-   if (dim == 2 && sdim == 2)
-   { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
-
-   if (dim == 3 && sdim == 3)
-   { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
-
-   MFEM_ABORT("Unknown kernel.");
+   PAVectorDiffusionApply(dim, dofs1D, quad1D, ne,
+                          maps->B, maps->G, maps->Bt, maps->Gt,
+                          pa_data, x, y);
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>

fem/bilininteg_vecmass.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"

fem/coefficient.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 // Implementation of Coefficient class
@@ -174,24 +174,7 @@ void VectorGridFunctionCoefficient::SetGridFunction(GridFunction *gf)
 void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
-   Mesh *mesh = GridFunc->FESpace()->GetMesh();
-   if (mesh->Dimension() == T.GetDimension())
-   {
-      GridFunc->GetVectorValue(T.ElementNo, ip, V);
-   }
-   else // Assuming T is a boundary element transformation
-   {
-      int el_id, el_info;
-      mesh->GetBdrElementAdjacentElement(T.ElementNo, el_id, el_info);
-      IntegrationPointTransformation loc_T;
-      mesh->GetLocalFaceTransformation(mesh->GetBdrElementType(T.ElementNo),
-                                       mesh->GetElementType(el_id),
-                                       loc_T.Transf,
-                                       el_info);
-      IntegrationPoint eip;
-      loc_T.Transform(ip, eip);
-      GridFunc->GetVectorValue(el_id, eip, V);
-   }
+   GridFunc->GetVectorValue(T.ElementNo, ip, V);
 }
 
 void VectorGridFunctionCoefficient::Eval(

fem/coefficient.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_COEFFICIENT
@@ -91,7 +91,7 @@ public:
    { constants = 0.0; }
 
    /** c should be a vector defined by attributes, so for region with
-       attribute i  c[i-1] is the coefficient in that region */
+       attribute i  c[i] is the coefficient in that region */
    PWConstCoefficient(Vector &c)
    { constants.SetSize(c.Size()); constants=c; }
 
@@ -138,7 +138,7 @@ public:
    /// (DEPRECATED) Define a time-independent coefficient from a C-function
    /** @deprecated Use the method where the C-function, @a f, uses a const
        Vector argument instead of Vector. */
-   MFEM_DEPRECATED FunctionCoefficient(double (*f)(Vector &))
+   FunctionCoefficient(double (*f)(Vector &))
    {
       Function = reinterpret_cast<double(*)(const Vector&)>(f);
       TDFunction = NULL;
@@ -147,7 +147,7 @@ public:
    /// (DEPRECATED) Define a time-dependent coefficient from a C-function
    /** @deprecated Use the method where the C-function, @a tdf, uses a const
        Vector argument instead of Vector. */
-   MFEM_DEPRECATED FunctionCoefficient(double (*tdf)(Vector &, double))
+   FunctionCoefficient(double (*tdf)(Vector &, double))
    {
       Function = NULL;
       TDFunction = reinterpret_cast<double(*)(const Vector&,double)>(tdf);

fem/complex_fem.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "complex_fem.hpp"
@@ -65,8 +65,9 @@ ComplexGridFunction::Update()
       gfr->NewDataAndSize(data, vsize);
       gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
 
-      // These updates will only set the proper 'sequence' value within the
-      // individual GridFunction objects because their sizes are already correct
+      // These updates will only set the proper 'sequence' value within
+      // the individual GridFunction objects because their sizes are
+      // already correct
       gfr->Update();
       gfi->Update();
    }
@@ -522,8 +523,8 @@ ParComplexGridFunction::Update()
       pgfr->Update();
       pgfi->Update();
 
-      // Our data array now contains old data as well as being the wrong size so
-      // reallocate it.
+      // Our data array now contains old data as well as being the wrong size
+      // so reallocate it.
       this->SetSize(2 * vsize);
 
       // Create temporary vectors which point to the new data array
@@ -1043,9 +1044,9 @@ ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
       MFEM_ABORT("Both Real and Imaginary part of the Sesquilinear form are empty");
    }
 
-   // Modify offdiagonal blocks (Imaginary parts of the matrix) to conform with
-   // standard essential BC treatment i.e. zero out rows and columns and place
-   // ones on the diagonal.
+   // Modify offdiagonal blocks (Imaginary parts of the matrix) to
+   // conform with standard essential BC treatment i.e. zero out rows and
+   // columns and place ones on the diagonal.
    if (RealInteg() && ImagInteg())
    {
       if ( A_i.Type() == Operator::Hypre_ParCSR )

fem/complex_fem.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_COMPLEX_FEM

fem/conduitdatacollection.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "../config/config.hpp"

fem/conduitdatacollection.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_CONDUITDATACOLLECTION

fem/datacollection.cpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #include "fem.hpp"

fem/datacollection.hpp

@@ -6,7 +6,7 @@
 // availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// terms of the BSD-3 license.  We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
 #ifndef MFEM_DATACOLLECTION