renaming and alternative interface

example test
support coordinates in qfunc kernel
2020-12-03 08:49:51 -08:00 · 2020-11-30 13:43:45 -08:00 · 2020-11-25 09:59:55 -08:00 · 2020-11-24 14:29:27 -08:00 · 2020-11-24 12:05:45 -08:00 · 2020-11-23 18:10:52 -08:00
382 changed files with 36937 additions and 18055 deletions
@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 30
+
+# Number of days of inactivity before an Issue or Pull Request with the stale
+# label is closed.  Set to false to disable. If disabled, issues still need to
+# be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale.
+# Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set
+# to `[]` to disable
+exemptLabels:
+  - bug
+  - WIP
+  - ready-for-review
+  - in-review
+  - in-next
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking an issue as stale
+staleLabel: stale
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  :warning: This issue or PR has been automatically marked as stale because it has not
+  had any activity in the last month. *If no activity occurs in the next week, it will
+  be automatically closed.* Thank you for your contributions.
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed
@@ -102,28 +102,29 @@ examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*

-examples/sundials/ex9
-examples/sundials/ex1[06]
-examples/sundials/ex9p
-examples/sundials/ex1[06]p
+examples/amgx/ex1
+examples/amgx/ex1p
+examples/amgx/.logamgx
+examples/amgx/refined.mesh
+examples/amgx/sol.gf
+examples/amgx/mesh.*
+examples/amgx/sol.*

-examples/sundials/ex9.mesh
-examples/sundials/ex9-mesh.*
-examples/sundials/ex9-init.*
-examples/sundials/ex9-final.*
-examples/sundials/Example9*
-examples/sundials/deformed.*
-examples/sundials/velocity.*
-examples/sundials/elastic_energy.*
-examples/sundials/ex16.mesh
-examples/sundials/ex16-mesh.*
-examples/sundials/ex16-init.*
-examples/sundials/ex16-final.*
-examples/sundials/Example16*
+examples/gingko/ex1
+examples/gingko/refined.mesh
+examples/gingko/sol.gf
+examples/gingko/mesh.*
+examples/gingko/sol.*
+
+examples/hiop/ex9
+examples/hiop/ex9p
+examples/hiop/ex9.mesh
+examples/hiop/ex9-mesh.*
+examples/hiop/ex9-init.*
+examples/hiop/ex9-final.*

 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
-
 examples/petsc/mesh.*
 examples/petsc/sol.*
 examples/petsc/sol_p.*
@@ -141,28 +142,51 @@ examples/petsc/mode_*

 examples/pumi/ex1
 examples/pumi/ex[126]p
-
-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
-
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
 examples/pumi/sol.*
 examples/pumi/displaced.mesh

+examples/sundials/ex9
+examples/sundials/ex1[06]
+examples/sundials/ex9p
+examples/sundials/ex1[06]p
+examples/sundials/ex9.mesh
+examples/sundials/ex9-mesh.*
+examples/sundials/ex9-init.*
+examples/sundials/ex9-final.*
+examples/sundials/Example9*
+examples/sundials/deformed.*
+examples/sundials/velocity.*
+examples/sundials/elastic_energy.*
+examples/sundials/ex16.mesh
+examples/sundials/ex16-mesh.*
+examples/sundials/ex16-init.*
+examples/sundials/ex16-final.*
+examples/sundials/Example16*
+
+examples/superlu/ex1p
+examples/superlu/mesh.*
+examples/superlu/sol.*
+
+miniapps/adjoint/cvsRoberts_ASAi_dns
+miniapps/adjoint/adjoint_advection_diffusion
+
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
-
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*

+miniapps/gslib/field-diff
+miniapps/gslib/field-interp
+miniapps/gslib/findpts
+miniapps/gslib/pfindpts
+
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid
@@ -175,7 +199,7 @@ miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
-
+miniapps/meshing/polar-nc
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -187,10 +211,27 @@ miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
+miniapps/meshing/polar-nc.mesh
+
+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output
+
+miniapps/nurbs/nurbs_ex1
+miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/refined.mesh
+miniapps/nurbs/mesh.*
+miniapps/nurbs/sol.*
+miniapps/nurbs/mode_*
+miniapps/nurbs/Example1*

 miniapps/performance/ex1
 miniapps/performance/ex1p
-
 miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
@@ -208,7 +249,6 @@ miniapps/toys/rubik
 miniapps/toys/snake
 miniapps/toys/lissajous
 miniapps/toys/mondrian
-
 miniapps/toys/snake-init.mesh
 miniapps/toys/snake-user.mesh
 miniapps/toys/snake-joined.mesh
@@ -223,41 +263,23 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh

-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/refined.mesh
-miniapps/nurbs/mesh.*
-miniapps/nurbs/sol.*
-miniapps/nurbs/mode_*
-miniapps/nurbs/Example1*
-
-miniapps/gslib/field-diff
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
+tests/unit/ceed_tests

 # Test script output
 tests/scripts/*.err
 tests/scripts/*.out
 tests/scripts/*.msg

+# Other tests
+tests/convergence/rates
+tests/convergence/prates
+tests/par-mesh-format/ex1p
+
 # VPATH builds
 build-*/*
@@ -71,15 +71,21 @@ stages:
  - build
  - test
  - deallocate
+  - lassen_build
+  - lassen_test
  - baseline_check
  - baseline_publish

-# The setup job in setup stage don’t rely on MFEM git repo. It prepares a
+# The setup job in setup stage don't rely on MFEM git repo. It prepares a
 # pipeline-wide working directory downloading/updating external repos.
 # TODO: updating tests and tpls is not necessary anymore since pipelines are
 # now using unique directories so repo are never shared with another pipeline.
 # This is not memory efficient (we keep a lot of data), hence this reminder.
-.setup:
+# Setup
+setup:
+  tags:
+    - shell
+    - quartz
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -100,6 +106,15 @@ stages:
  before_script:
    - module load gcc/6.1.0

+# On lassen
+.with_gcc_8_3_1:
+  variables:
+    TOOLCHAIN: gcc_8_3_1
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/8.3.1
+
 .with_gcc_4_9_3:
  variables:
    TOOLCHAIN: gcc_4_9_3
@@ -211,7 +226,7 @@ stages:
    echo "${BASELINE_TEST}: Differences found, replacement file generated"
    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}.txt
  fi
-  # _base_diff won’t even exist if there is no difference.
+  # _base_diff won't even exist if there is no difference.
  if [[ -f ${_base_diff} ]]
  then
    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
@@ -290,3 +305,4 @@ stages:
 # The list on jobs is defined in machine-specific files.
 include:
  - local: .gitlab/quartz.yml
+  - local: .gitlab/lassen.yml
@@ -0,0 +1,57 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Lassen machine at LLNL
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  variables:
+    PLAT: lassen
+
+# Build MFEM
+build_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
+
+build_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70
+
+# Sanity check
+sanitycheck_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 15 -q pdebug make -j test
+
+sanitycheck_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_debug_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 30 -q pdebug make -j test
@@ -22,10 +22,6 @@
    MAKE_PAR: 6
    BASELINE_PAR: 18

-# Setup
-setup_quartz:
-  extends: [.setup, .on_quartz]
-
 # Allocate
 allocate_quartz:
  variables:
@@ -5,12 +5,142 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org


-Version 4.1.1 (development)
+Version 4.2.1 (development)
 ===========================

+- Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
+  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
+
+- Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
+  ESDIRK-32, and A-stable ESDIRK-33.
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+
+Version 4.2, released on October 30, 2020
+=========================================
+
+High-Performance Computing
+--------------------------
+- Added support for explicit vectorization in the high-performance templated
+  code, which can now take advantage of specific classes on the following
+  architectures:
+  * x86 (SSE/AVX/AVX2/AVX512),
+  * Power8 & Power9 (VSX),
+  * BG/Q (QPX).
+  These are disabled by default, but can be enabled with MFEM_USE_SIMD=YES.
+  See the new file linalg/simd.hpp and the new directory linalg/simd.
+
+- Added an Element Assembly mode compatible with GPU device execution for H1 and
+  L2 spaces in the mass, convection, diffusion, transpose, and the face DG trace
+  integrators. See option '-ea' in Example 9. When enabled, this assembly level
+  stores independent dense matrices for the elements, and independent dense
+  matrices for the faces in the DG case.
+
+- Added a Full Assembly mode compatible with GPU device execution. This assembly
+  level builds on top of the Element Assembly kernels to compute a global sparse
+  matrix. All integrators supported by element assembly are also supported by
+  full assembly. See the '-fa' option in Example 9.
+
+- Optimized the AMD/HIP kernel support and enabled HIP support in the libCEED
+  integration. This is now available via the "ceed-hip" device backend.
+
+- Improved the libCEED integration to support:
+  * AssemblyLevel::NONE for Mass, Diffusion, VectorMass, and VectorDiffusion
+    Integrators. This level computes the full operator evaluation "on the fly".
+  * VectorMassIntegrator and VectorDiffusionIntegrator.
+  * All types of (scalar) Coefficients.
+
+- Added partial assembly / device support for:
+  * H(div) bilinear forms and VectorFEDivergenceIntegrator.
+  * BlockOperator, see the updated Example 5.
+  * Complex operators, see the updated Example 22.
+  * Chebyshev accelerated polynomial smoother.
+  * Convergent diagonal preconditioning on non-conforming adaptively refined
+    meshes, see Example 6/6p.
+
+- Added CUDA support for:
+  * Sparse matrix-vector multiplication with cuSPARSE,
+  * SUNDIALS ODE integrators, see updated SUNDIALS modification of Example 9/9p.
+
+Linear and nonlinear solvers
+----------------------------
+- Added a new solver class for simple integration with NVIDIA's multigrid
+  library, AmgX. The AmgX class is designed to work as a standalone solver or
+  preconditioner for existing MFEM solvers. It uses MFEM's sparse matrix format
+  for serial runs and the HypreParMatrix format for parallel runs. The new
+  solver may be configured to run with one GPU per MPI rank or with more MPI
+  ranks than GPUs. In the latter case, matrices and vectors are consolidated to
+  ranks communicating with the GPUs and the solution is then broadcasted.
+
+  Although CUDA is required to build, the AmgX support is compatible with the
+  MFEM CPU device configuration. The examples/amgx folder illustrates how to
+  integrate AmgX in existing MFEM applications. The AmgX solver class is
+  partially based on: "AmgXWrapper: An interface between PETSc and the NVIDIA
+  AmgX library", by Pi-Yueh Chuang and Lorena A. Barba, doi:10.21105/joss.00280.
+
+- Added initial support for geometric h- and p-multigrid preconditioners for
+  matrix-based and matrix-free discretizations with basic GPU capability, see
+  Example 26/26p.
+
+- Added support for the CVODES package in SUNDIALS which provides ODE solvers
+  with sensitivity analysis capabilities. See the CVODESSolver class and the new
+  adjoint miniapps in the miniapps/adjoint directory.
+
+- Added an interface to the MKL CPardiso solver, an MPI-parallel sparse direct
+  solver developed by Intel. See Example 11p for an illustration of its usage.
+
+- Added support for the SLEPc eigensolver package, https://slepc.upv.es.
+
+- Upgraded SuperLU interface to use SuperLU_DIST 6.3.1. Added a simple SuperLU
+  example in the new directory examples/superlu.
+
+- Extended the KINSOL (SUNDIALS) nonlinear solver interface to support the
+  Jacobian-free Newton-Krylov method. A usage example is shown in Example 10p.
+
+- Block arrays of parallel matrices can now be merged into a single parallel
+  matrix with the function HypreParMatrixFromBlocks. This could be useful for
+  solving block systems with parallel direct solvers such as STRUMPACK.
+
+- Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
+  preconditioner. The latter requires hypre version 2.19.0 or later.
+
+Discretization improvements
+---------------------------
+- Extended GSLIB-FindPoints integration to support simplices and interpolation
+  of functions from L2, H(div) and H(curl) spaces.
+
+- Added support for computing asymptotic error estimates and convergence rates
+  for the whole de Rham sequence based on the new class ConvergenceStudy and new
+  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
+  the tests/convergence directory for sample usage.
+
+- Extended the GetValue and GetVectorValue methods of GridFunction to support
+  evaluation on boundary elements and, in the continuous field case, arbitrary
+  mesh edges and faces. This requires passing an ElementTransformation argument.
+
+- Added support for matrix-free interpolation and restriction operators between
+  continuous H1 finite element spaces of different order on the same mesh or
+  with the same order on uniformly refined meshes.
+
+- The Coefficient classes based on a C-function pointer (FunctionCoefficient,
+  VectorFunctionCoefficient and MatrixFunctionCoefficient) now use the more
+  general std::function class template. This allows the classes to be backward
+  compatible (i.e. they can still work with C-functions) and, in addition,
+  support any "callable", e.g. lambda functions.
+
+- Non-conforming meshes are now supported with block nonlinear forms. See the
+  updated Example 19/19p.
+
 Meshing improvements
 --------------------
 - The graph linear ordering library Gecko, previously an external dependency, is
@@ -18,124 +148,55 @@ Meshing improvements
  always available. The interface has also been improved, see for example the
  Mesh Explorer miniapp.

- Improved Gmsh reader (version 2.2), which now supports both high-order and
-  periodic meshes. Segments, triangles, quadrilaterals, and tetrahedra are
-  supported up to order 10. Wedges and hexahedra are supported up to order 9.
-  For sample periodic meshes, see the periodic*.msh files in the data directory.
-
 - Added support for finite difference-based gradient and Hessian approximation
  in the TMOP mesh optimization algorithms. This improves the accuracy of the
  Hessian for r-adaptivity using discrete fields, and allows use of skewness
  and orientation based metrics.

+- Improved Gmsh reader (version 2.2), which now supports both high-order and
+  periodic meshes. Segments, triangles, quadrilaterals, and tetrahedra are
+  supported up to order 10. Wedges and hexahedra are supported up to order 9.
+  For sample periodic meshes, see the periodic*.msh files in the data directory.
+
+- Added support for construction of (serial) non-conforming meshes. Hanging
+  nodes can be marked with Mesh::AddVertexParents when building the mesh with
+  the "init" constructor. The usage is demonstrated in a new meshing miniapp,
+  Polar NC, which generates meshes that are non-conforming from the start.
+
 - Added support for r-adaptivity with more than one discrete field. This allows
-  the user to specify different discrete functions for controlling the
-  size, aspect-ratio, orientation, and skew of elements in the mesh.
+  the user to specify different discrete functions for controlling the size,
+  aspect-ratio, orientation, and skew of elements in the mesh.

- Added TMOP capability for approximate tangential mesh relaxation. Added
-  support and examples for using TMOP on mixed meshes.
-
- Added complete action of the TMOP Integrator to account for the spatial
-  derivatives of discrete and analytic targets.
-
-Performance improvements
------------------------
- Added support for explicit vectorization in the high-performance templated
-  code, which can now take advantage of specific intrinsics classes on the
-  following architectures:
-    - x86 (SSE/AVX/AVX2/AVX512),
-    - Power8 & Power9 (VSX),
-    - BG/Q (QPX).
-  These are disabled by default, and can be enabled with MFEM_USE_SIMD=YES.
-  See the new file linalg/simd.hpp and the new directory linalg/simd.
-
-Improved GPU capabilities
-------------------------
- Added support for Chebyshev accelerated polynomial smoother on GPU.
-
- Optimized AMD/HIP kernel support.
-
- Added a Full Assembly mode compatible with Device kernel execution. This
-  assembly level builds on top of the current Element Assembly kernels to
-  compute a global sparse matrix. All integrators supported by element assembly
-  are also supported by full assembly. See the '-fa' option in Example 9.
-
- Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.
-
- Added support for BlockOperator on GPU. See the updated Example 5.
-
-Discretization improvements
---------------------------
- Added support for matrix-free interpolation and restriction operators between
-  continuous H1 finite element spaces of different order on the same mesh or
-  with the same order on uniformly refined meshes.
-
- Added support for simplices in GSLIB-FindPoints.
-
- Added support for H1 and L2 element matrix assembly in the mass, convection,
-  diffusion, transpose, and the face DG trace integrators. This is compatible
-  with GPU device execution and is illustrated in Example 9/9p, see the option
-  '-ea'. When enabled, this level of assembly stores independent dense matrices
-  for the elements, and independent dense matrices for the faces in the DG case.
-
- Added new partial assembly kernels for H(div) bilinear forms, as well as
-  VectorFEDivergenceIntegrator.
-
- Improved the documentation of the GridFunction GetValue and GetVectorValue
-  methods. Expanded the GetValue and GetVectorValue methods which accept an
-  ElementTransformation argument to support evaluation on boundary elements
-  and, in the continuous field case, arbitrary mesh edges and faces.
-
- Added new coefficient and vector coefficient classes for QuadratureFunctions.
-  Additionally, new LinearForm integrators were also added which make use of
-  these new QuadratureFunction coefficient classes.
-
- Added support face integrals on the boundaries of NURBS meshes.
-
-Linear and nonlinear solvers
----------------------------
- Added power method to iteratively estimate the largest eigenvalue and the
-  corresponding eigenvector of an operator.
-
- Added initial support for h- and p-multigrid solvers and preconditioners for
-  matrix-based and matrix-free discretizations with basic GPU capability.
-
- Added a new IterativeSolverMonitor class that allows to monitor the residual
-  and solution during the solving process of an IterativeSolver after every
-  iteration.
-
- Added support for the CVODES package in SUNDIALS which provides ODE
-  solvers with sensitivity analysis capabilities. See the CVODESSolver
-  class and the new adjoint miniapps below.
-
- Block arrays of parallel matrices can now be merged into a single parallel
-  matrix with the function HypreParMatrixFromBlocks. This could be useful for
-  solving block systems with parallel direct solvers such as STRUMPACK.
-
- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
-  solver can work with non-SPD preconditioner B.
-
- Added support for the SLEPc eigensolver package.
+- Additional TMOP improvements:
+  * Capability for approximate tangential mesh relaxation.
+  * Support and examples for using TMOP on mixed meshes.
+  * Complete integrator action accounting for spatial derivatives of discrete
+    and analytic targets.

 New and updated examples and miniapps
 -------------------------------------
- Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
-  Matched Layer (PML) for the simulation of electromagnetic wave propagation.
-  The example defines and solves several indefinite Maxwell problems.
-
- Added a new Example 26/26p to demonstrate the construction of a matrix-free
-  geometric and p-multigrid preconditioner for the Laplace problem.
-
- Added a new example, Example 27/27p, to demonstrate the enforcement of various
-  boundary conditions with the Laplace operator. The example shows the procedure
-  for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
-  and periodic boundary conditions with either H1 or DG discretizations.
-
 - Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
  equations of incompressible fluid dynamics. See the miniapps/navier directory
  for more details.

- Added a new miniapps/adjoint directory with two miniapps demonstrating how to
+- Added 10 new example codes:
+  * Example 25/25p demonstrates the use of a Perfectly Matched Layer (PML) for
+    electromagnetic wave propagation (indefinite Maxwell).
+  * Example 26/26p shows how to construct matrix-free geometric and p-multigrid
+    preconditioner for the Laplace problem.
+  * Example 27/27p demonstrates the enforcement of Dirichlet, Neumann, Robin,
+    and periodic boundary conditions with either H1 or DG Laplace problems.
+  * Versions of Example 1/1p in examples/amgx demonstrating the use of AmgX,
+    to solve the Laplace problem with AMG preconditioning on GPUs.
+  * A version of Example 11p in examples/petsc demonstrating the use of SLEPc,
+    to solve the Laplace eigenproblem with shift-and-invert transformation.
+  * A version of Example 1 in examples/superlu demonstrating the use of SuperLU
+    to solve the Laplace problem.
+
+- Added a new Field Interpolation miniapp in miniapps/gslib that demonstrates
+  transfer of grid functions between different meshes using GSLIB-FindPoints.
+
+- Added 2 miniapps in the new miniapps/adjoint directory demonstrating how to
  solve adjoint problems in MFEM using the CVODES package in SUNDIALS. Both of
  these miniapps require the MFEM_USE_SUNDIALS configuration option.
  * The cvsRoberts_ASAi_dns miniapp solves a backward adjoint problem for a
@@ -143,33 +204,31 @@ New and updated examples and miniapps
  * The adjoint_advection_diffusion miniapp solves a backward adjoint problem
    for an advection diffusion PDE, evaluating adjoint quadratures in parallel.

- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
-  equation with the shift-and-invert spectral transformation method.
+- Added 4 additional meshing miniapps:
+  * The Minimal Surface miniapp solves Plateau's problem: the Dirichlet problem
+    for the minimal surface equation.
+  * The Twist miniapp demonstrates how to stitch together opposite surfaces of a
+    mesh to create a topologically periodic mesh.
+  * The Trimmer miniapp trims away parts of a mesh based on element attributes.
+  * Polar NC shows the construction of polar non-conforming meshes.

- Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
-  stitching together opposite surfaces of a mesh to create a topologically
-  periodic mesh.
+- Several examples and miniapps were updated to include:
+  * Full and element assembly support in Example 9/9p.
+  * Partial assembly with diagonal preconditioning in Examples 4/4p/5/5p/22/22p.
+  * Diagonal preconditioner in Example 6/6p for partial assembly with AMR.
+  * The option to plot a function in Mesh Explorer.
+  * A new test problem showing a mixed bilinear form for H1, H(curl), H(div) and
+    L2, with partial assembly support in Example 24/24p.
+  * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

- Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
-  the Dirichlet problem for the minimal surface equation.
+Data management and Visualization
+---------------------------------
+- Added support for ADIOS2 for parallel I/O with ParaView visualization. See
+  Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection
+  provide the interface to generate ADIOS2 Binary Pack (BP4) directory datasets.

- Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
-  preconditioning.
-
- Added full assembly support in Example 9/9p.
-
- Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
-  form for H1, H(curl), H(div) and L_2, with partial assembly support.
-
- Added weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
-
- Added a simple mesh editing miniapp, Trimmer, which trims away portions of a
-  mesh based on element attributes. Any newly exposed boundary elements are
-  assigned attribute numbers related to the trimmed element attributes.
-
- Added device support in Example 5/5p.
-
- Added the option to plot a function in Mesh Explorer.
+- Added VTU output of boundary elements and attributes and parallel VTU (PVTU)
+  output of parallel meshes for visualization using ParaView.

 Improved testing
 ----------------
@@ -178,17 +237,39 @@ Improved testing
  triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
  and the updated CONTRIBUTING.md file.

+- Added additional testing for convergence, the parallel mesh I/O, and for the
+  libCEED integration in MFEM in the tests/ directory.
+
+- Upgraded the Catch unit test framework from version 1.6.1 to version 2.13.0.
+
 Miscellaneous
 -------------
- Added support for ADIOS2 for parallel I/O with ParaView visualization. The
-  classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
-  interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
-  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.
+- Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
+  as DEBUG is sometimes used as a macro.
+
+- Added a new IterativeSolverMonitor class that allows to monitor the residual
+  and solution with an IterativeSolver after every iteration.
+
+- Added power method to iteratively estimate the largest eigenvalue and the
+  corresponding eigenvector of an operator.
+
+- Added support for face integrals on the boundaries of NURBS meshes.
+
+- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
+  solver can work with non-SPD preconditioner B.
+
+- Added new coefficient and vector coefficient classes for QuadratureFunctions,
+  with new LinearForm integrators which use them.

 - The integration order used in the ComputeLpError and ComputeElementLpError
  methods of class GridFunction has been increased.

+- Change the IntegrationRule inside VectorDiffusionIntegrator to use the same
+  quadrature as DiffusionIntegrator.
+
+- The README.html files previously included in several source directories have
+  been removed. Use the corresponding pages at mfem.org instead.
+
 - Various other simplifications, extensions, and bugfixes in the code.


@@ -521,7 +602,7 @@ Other meshing improvements
  follows precisely the paper:

     D. Arnold, A. Mukherjee, and L. Pouly, "Locally Adapted Tetrahedral Meshes
-     Using Bisection", SIAM J. Sci. Comput. 22 (2000), 431–448.
+     Using Bisection", SIAM J. Sci. Comput. 22 (2000), 431-448.

  This guarantees that the shape regularity of the elements will be preserved
  under refinement.
@@ -666,7 +747,7 @@ Discretization improvements

 - New specialized time integrators: symplectic integrators of orders 1-4 for
  systems of first order ODEs derived from a Hamiltonian and generalized-alpha
-  ODE solver for the filtered Navier–Stokes equations with stabilization. See
+  ODE solver for the filtered Navier-Stokes equations with stabilization. See
  classes SIASolver and GeneralizedAlphaSolver in linalg/ode.hpp.

 - Inherit finite element classes from the new base class TensorBasisElement,
@@ -1156,7 +1237,7 @@ New and improved linear solvers
  it can only be enabled along with MFEM_USE_MPI. When MFEM is configured with
  MFEM_USE_SUPERLU, one also needs to alter the version of METIS, since SuperLU
  requires ParMETIS (which comes packaged with a serial version of METIS).  See
-  http://http://crd-legacy.lbl.gov/~xiaoye/SuperLU for SuperLU_DIST details.
+  http://crd-legacy.lbl.gov/~xiaoye/SuperLU for SuperLU_DIST details.

 - Added a wrapper for the KLU solver in SuiteSparse see
  http://faculty.cse.tamu.edu/davis/suitesparse.html for details of KLU.
@@ -14,7 +14,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
  "Path to optional user configuration file.")

 # Require C++11 and disable compiler-specific extensions
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)

@@ -50,7 +50,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.1.1)
+set(${PROJECT_NAME}_VERSION 4.2.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -185,7 +185,7 @@ if (MFEM_USE_MPI)
    endif()
  endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU MUMPS PETSC SLEPC STRUMPACK PUMI)
  foreach(PKG IN LISTS PKGS_NEED_MPI)
    if (MFEM_USE_${PKG})
      message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -240,12 +240,14 @@ endif()

 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
-  if (NOT MFEM_USE_MPI)
-    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
-  else()
-    find_package(SUNDIALS REQUIRED
-      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
+  set(SUNDIALS_COMPONENTS CVODES ARKODE KINSOL NVector_Serial)
+  if (MFEM_USE_MPI)
+    list(APPEND SUNDIALS_COMPONENTS NVector_Parallel NVector_MPIPlusX)
  endif()
+  if (MFEM_USE_CUDA)
+    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
+  endif()
+  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

 # Mesquite
@@ -262,6 +264,15 @@ if (MFEM_USE_SUPERLU)
  endif()
 endif()

+# MUMPS can only be enabled in parallel
+if (MFEM_USE_MUMPS)
+  if (MFEM_USE_MPI)
+    find_package(MUMPS REQUIRED mumps_common pord)
+  else()
+    message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
+  endif()
+endif()
+
 # STRUMPACK can only be enabled in parallel
 if (MFEM_USE_STRUMPACK)
  if (MFEM_USE_MPI)
@@ -295,6 +306,10 @@ if (MFEM_USE_CEED)
  find_package(libCEED REQUIRED)
 endif()

+if (MFEM_USE_AMGX)
+  find_package(AMGX REQUIRED)
+endif()
+
 if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -346,6 +361,12 @@ if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
 endif()

+if (MFEM_USE_MKL_CPARDISO)
+  if (MFEM_USE_MPI)
+   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
+  endif()
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -370,9 +391,9 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE SuperLUDist MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE)
+    CUSPARSE MKL_CPARDISO AMGX)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -1,12 +1,12 @@
 <p align="center">
-<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
+<a href="https://mfem.org/"><img alt="mfem" src="https://mfem.org/img/logo-300.png"></a>
 </p>

 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
 <a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>


@@ -26,7 +26,7 @@ request (PR) toward the `mfem:master` branch to propose your contribution. If
 you are planning significant code changes or have questions, you may want to
 open an [issue](https://github.com/mfem/mfem/issues) before issuing a PR. In
 addition to technical contributions, we are also interested in your results and
-[simulation images](http://mfem.org/gallery/), which you can share via a pull
+[simulation images](https://mfem.org/gallery/), which you can share via a pull
 request in the [mfem/web](https://github.com/mfem/web) repo.

 See the [Quick Summary](#quick-summary) section for the main highlights of our
@@ -51,7 +51,7 @@ back to them before issuing pull requests:
 Contributing to MFEM requires knowledge of Git and, likely, finite elements. If
 you are new to Git, see the [GitHub learning
 resources](https://help.github.com/articles/git-and-github-learning-resources/).
-To learn more about the finite element method, see our [FEM page](http://mfem.org/fem).
+To learn more about the finite element method, see our [FEM page](https://mfem.org/fem).

 *By submitting a pull request, you are affirming the [Developer's Certificate of
 Origin](#developers-certificate-of-origin-11) at the end of this file.*
@@ -92,21 +92,22 @@ The MFEM source code has the following structure:
  .
  ├── config
  │   └── cmake
-  │       └── modules
+  │       └── ...
  ├── data
  ├── doc
-  │   └── web
-  │       └── examples
  ├── examples
+  │   ├── amgx
  │   ├── ginkgo
  │   ├── hiop
  │   ├── petsc
  │   ├── pumi
-  │   └── sundials
+  │   ├── sundials
+  |   └── superlu
  ├── fem
  │   └── libceed
  ├── general
  ├── linalg
+  │   └── simd
  ├── mesh
  ├── miniapps
  │   ├── adjoint
@@ -114,15 +115,17 @@ The MFEM source code has the following structure:
  │   ├── electromagnetics
  │   ├── gslib
  │   ├── meshing
+  │   ├── navier
  │   ├── nurbs
  │   ├── performance
  │   ├── tools
  │   └── toys
  └── tests
+      ├── convergence
+      ├── par-mesh-format
      ├── scripts
-      ├── unit
-      │   ├── ...
-      └── ...
+      └── unit
+          └── ...
 ```

 #### Main directories and classes
@@ -132,24 +135,24 @@ classes implementing the finite element, mesh and linear algebra concepts
 respectively.

 - The main mesh classes are:
-  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
-  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
-  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
-  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
+  + [`Mesh`](https://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](https://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](https://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)

 - The main finite element classes are:
-  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
-  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
-  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
+  + [`FiniteElement`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)

 - The main linear algebra classes and sources are
-  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
-  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
-  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`Operator`](https://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](https://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](https://mfem.github.io/doxygen/html/solvers_8hpp.html)

 #### Parallel implementation

@@ -159,13 +162,13 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.

 - The main parallel classes are
-  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
-  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
-  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
-  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
-  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
-  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
-  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
+  + [`ParMesh`](https://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://mfem.github.io/doxygen/html/hypre_8hpp.html)

 #### GPU and general device support

@@ -174,10 +177,10 @@ backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.

 - The main device-relevant classes and sources are:
-  + [`Device`](http://mfem.github.io/doxygen/html/device_8hpp.html)
-  + [`MemoryManager`](http://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](http://mfem.github.io/doxygen/html/forall_8hpp.html) macro
-  + the [`cuda.hpp`](http://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](http://mfem.github.io/doxygen/html/occa_8hpp.html) files
+  + [`Device`](https://mfem.github.io/doxygen/html/device_8hpp.html)
+  + [`MemoryManager`](https://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
+  + the [`MFEM_FORALL`](https://mfem.github.io/doxygen/html/forall_8hpp.html) macro
+  + the [`cuda.hpp`](https://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](https://mfem.github.io/doxygen/html/occa_8hpp.html) files

 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -186,7 +189,7 @@ device/host memory manager.
  Makefile and the CMake build options.
 - The `doc/` directory contains configuration for the Doxygen code documentation
  that can either be built locally or browsed online at
-  http://mfem.github.io/doxygen/html/index.html.
+  https://mfem.github.io/doxygen/html/index.html.

 #### Examples and tests
 - `examples` and `miniapps` respectively gather simple and more fully-featured
@@ -195,7 +198,7 @@ device/host memory manager.
 - The `tests/` directory contains a unit test suite and will later contain more
  tests that run example codes.

-See also the [code overview](http://mfem.org/code-overview/) section on the MFEM
+See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
 website.

 ## GitHub Workflow
@@ -435,7 +438,6 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] Run `make unittest` to make sure all unit tests pass.
 - [ ] Run the tests in `tests/scripts`.
 - [ ] (LLNL only) After merging:
-   - [ ] Regenerate `README.html` files from companion documentation pull requests.
   - [ ] Update internal tests to include the new features.

 ### Master/Next Workflow
@@ -508,7 +510,10 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
- [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
+- [ ] Update the `CHANGELOG` to organize all release contributions
+- [ ] Review the whole source code once over
+- [ ] Ask MFEM-based applications to test the pre-release branch
+- [ ] Test on additional platforms and compilers
 - [ ] Tag the repository:

  ```
@@ -519,14 +524,13 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Recreate the `next` branch as described in previous section.
 - [ ] Update and push documentation  to `mfem/doxygen`.
 - [ ] Update URL shortlinks:
-    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. http://mfem.github.io/releases/mfem-3.1.tgz.
+    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
    - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
    - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
 - [ ] Update website in `mfem/web` repo:
    - Update version and shortlinks in `src/index.md` and `src/download.md`.
    -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.

-
 ## LLNL Workflow

 ### Mirroring on Bitbucket
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
@@ -19,10 +19,10 @@ requires an MPI C++ compiler, as well as the following external libraries:
  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview

 The hypre dependency can be downloaded as a tarball from GitHub or from the
-project webpage https://www.llnl.gov/casc/hypre. For example, the 2.16.0 release
+project webpage https://www.llnl.gov/casc/hypre. For example, the 2.20.0 release
 of hypre is available at

-   https://github.com/hypre-space/hypre/archive/v2.16.0.tar.gz
+   https://github.com/hypre-space/hypre/archive/v2.20.0.tar.gz

 The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.
@@ -62,7 +62,7 @@ following package managers:

 We also recommend downloading and building the MFEM-based GLVis visualization
 tool which can be used to visualize the meshes and solution in MFEM's examples
-and miniapps. See http://glvis.org and http://mfem.org/building.
+and miniapps. See https://glvis.org and https://mfem.org/building.

 Quick start with GNU make
 =========================
@@ -344,6 +344,16 @@ MFEM_USE_SUPERLU = YES/NO
   SuperLURowLocMatrix a distributed CSR matrix class needed by SuperLU. When
   enabled, this option uses the SUPERLU_* library options, see below.

+MFEM_USE_SUPERLU5 = YES/NO
+   If SuperLU functionality is enabled, use the older 5.1.0 version rather than
+   the more recent 6+ versions.
+
+MFEM_USE_MUMPS = YES/NO
+   Enable MFEM functionality based on the MUMPS library. Currently, this
+   option adds the class MUMPSSolver (a parallel sparse direct solver). 
+   When enabled, this option uses the MUMPS_* library options, see
+   below.
+   
 MFEM_USE_STRUMPACK = YES/NO
   Enable MFEM functionality based on the STRUMPACK sparse direct solver and
   preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
@@ -356,6 +366,11 @@ MFEM_USE_GINKGO = YES/NO
   https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
   Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.

+MFEM_USE_AMGX = YES/NO
+   Enable MFEM functionality based on the AmgX multigrid library from NVIDIA.
+   Allows the user to use SparseMatrices and HypreParMatrices to solve linear
+   systems with the routines from the AmgX library.
+
 MFEM_USE_GNUTLS = YES/NO
   Enable secure socket support in class socketstream, using the auxiliary
   GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -486,6 +501,13 @@ MFEM_USE_CEED = YES/NO
   library for performant high-order operator evaluation developed by the Center
   for Efficient Exascale Discretizations in the Exascale Computing Project.

+MFEM_USE_MKL_CPARDISO = YES/NO
+   Enables the interface to MKL CPardiso: the Intel MKL Parallel Direct Sparse
+   Solver for Clusters. Make sure to set the correct values for MKL_MPI_WRAPPER
+   and MKL_LIBRARY_SUBDIR as shown in defaults.mk. If you configure MFEM with
+   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
+   OpenMP capabilities are disabled at link time.
+
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -535,15 +557,19 @@ The specific libraries and their options are:
  Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
  Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
  Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
+  When MFEM_USE_CUDA is enabled, only SUNDIALS v5.4.0+ is supported.
  If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
  both MPI and hypre.
+  If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
+  for CUDA.
  URL: http://computation.llnl.gov/projects/sundials/sundials-software
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0.
+  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.

 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
  URL: http://trilinos.org/oldsite/packages/mesquite
  Options: MESQUITE_OPT, MESQUITE_LIB.
+  The Mesquite support is deprecated and will be removed in the future.

 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
  URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
@@ -553,11 +579,17 @@ The specific libraries and their options are:
 - SuperLU_DIST (optional), used when MFEM_USE_SUPERLU = YES. Note that
  SuperLU_DIST requires ParMETIS, which includes METIS 5 in its distribution.
  Both ParMETIS and the included METIS 5 should be built and installed in the
-  same location.
+  same location. If using SuperLU_Dist v5, set MFEM_USE_SUPERLU5=YES.
  URL: http://crd-legacy.lbl.gov/~xiaoye/SuperLU
  Options: SUPERLU_OPT, SUPERLU_LIB.
  Versions: SuperLU_DIST >= 5.1.0.

+- MUMPS (optional), used when MFEM_USE_MUMPS = YES. Note that MUMPS
+  requires LAPACK, SCALAPACK and a reordering package such as PORD or METIS.
+  URL: http://mumps.enseeiht.fr
+  Options: MUMPS_OPT, MUMPS_LIB.
+  Versions: MUMPS >= 5.1.1
+
 - STRUMPACK (optional), used when MFEM_USE_STRUMPACK = YES. Note that STRUMPACK
  requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
  includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
@@ -566,7 +598,7 @@ The specific libraries and their options are:
  2.0.0 or later.
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
-  Versions: STRUMPACK >= 3.0.0, requires HYPRE < 2.16.0.
+  Versions: STRUMPACK >= 3.0.0.

 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
  C++ compiler that supports the C++-11 standard. For additional requirements
@@ -574,6 +606,11 @@ The specific libraries and their options are:
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT (Not used), GINKGO_LIB.

+- AmgX (optional), used when MFEM_USE_AMGX = YES.
+  URL: https://github.com/NVIDIA/AMGX
+  Options: AMGX_OPT, AMGX_LIB.
+  Versions: AmgX >= 2.1, older versions may work too.
+
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
  GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
  one can get the library through the Homebrew package manager (http://brew.sh).
@@ -624,16 +661,18 @@ The specific libraries and their options are:

 - ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
  URL: https://adios2.readthedocs.io/
+  Versions: ADIOS >= 2.5.0.

 - PUMI (optional), used when MFEM_USE_PUMI = YES.
  URL: https://scorec.rpi.edu/pumi
       https://github.com/SCOREC/core
  Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI >= 2.2.3.
+  Versions: PUMI == 2.2.3.

 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
  Options: HIOP_OPT, HIOP_LIB.
+  Versions: HIOP >= 0.1.

 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
  built prior to the MFEM build, as follows: download gslib-1.0.5, untar it at
@@ -645,6 +684,11 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.5.

+- MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
+  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
+  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
+  Versions: Intel MKL >= 2020.
+
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
@@ -657,13 +701,13 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.0.9.
+  Versions: OCCA >= 1.1.0.

 - libCEED (optional), used when MFEM_USE_CEED = YES.
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED > 0.6, git-hash fe5822c.
+  Versions: libCEED >= 0.7.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -805,8 +849,10 @@ MFEM_TIMER_TYPE - Set automatically, can be overwritten.
 MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
+MFEM_USE_MUMPS
 MFEM_USE_STRUMPACK
 MFEM_USE_GINKGO
+MFEM_USE_AMGX
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
@@ -855,11 +901,12 @@ The CMake build system adds auto-detection for the following packages/libraries:

 - HYPRE
 - METIS - The option MFEM_USE_METIS_5 is auto-detected.
+ - ParMETIS
 - MESQUITE
 - SuiteSparse
 - SuperLUDist, STRUMPACK
- - ParMETIS
 - Ginkgo
+ - AMGX
 - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
 - NETCDF
 - MPFR
@@ -25,7 +25,8 @@ This project distributes the sources of several external software products with
 their own respective licenses which can be found in their code and attached
 license files. These software products and their licenses are as follows:

+* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
+* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
-* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
 * Zstr (general/zstr.hpp) -- MIT license
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 MFEM is a modular parallel C++ library for finite element methods. Its goal is
 to enable high-performance scalable finite element discretization research and
@@ -16,8 +16,8 @@ supercomputers.

 * Copyright and licensing information can be found in files LICENSE and NOTICE.

-* The best starting point for new users interested in MFEM's features is the
-  interactive documentation in examples/README.html.
+* The best starting point for new users interested in MFEM's features is to
+  review the examples and miniapps at https://mfem.org/examples.

 * Developers interested in contributing to the library, should read the
  instructions and documentation in the CONTRIBUTING.md file.
@@ -49,17 +49,17 @@ library. Nonlinear solvers (the Newton method), eigensolvers (LOBPCG), and
 several explicit and implicit Runge-Kutta time integrators are also available.

 MFEM supports MPI-based parallelism throughout the library, and can readily be
-used as a scalable unstructured finite element problem generator. As of version
-4.0, MFEM offers initial support for GPU acceleration, and programming models,
-such as CUDA, OCCA, RAJA and OpenMP. MFEM-based applications require minimal
-changes to switch from a serial to a high-performing MPI-parallel version of the
-code, where they can take advantage of the integrated linear solvers from the
-hypre library.  Comprehensive support for other external packages, e.g. PETSc
-and SUNDIALS is also included, giving access to many additional linear and
-nonlinear solvers, preconditioners, time integrators, etc.
+used as a scalable unstructured finite element problem generator. Starting with
+version 4.0, MFEM offers support for GPU acceleration, and programming models,
+such as CUDA, HIP, OCCA, RAJA and OpenMP. MFEM-based applications require
+minimal changes to switch from a serial to a highly-performant MPI-parallel
+version of the code, where they can take advantage of the integrated linear
+solvers from the hypre library. Comprehensive support for other external
+packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
+additional linear and nonlinear solvers, preconditioners, time integrators, etc.

 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
-as the OpenGL visualization tool GLVis which is available at http://glvis.org.
+as the OpenGL visualization tool GLVis which is available at https://glvis.org.

 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license. See LICENSE and NOTICE for details.
@@ -32,8 +32,11 @@ set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
 set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
+set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
+set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
+set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
@@ -74,6 +74,9 @@
 // [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 #cmakedefine MFEM_USE_LEGACY_OPENMP

+// Internal MFEM option: enable group/batch allocation for some small objects.
+#cmakedefine MFEM_USE_MEMALLOC
+
 // Enable MFEM functionality based on the Mesquite library.
 #cmakedefine MFEM_USE_MESQUITE

@@ -83,15 +86,18 @@
 // Enable MFEM functionality based on the SuperLU_DIST library.
 #cmakedefine MFEM_USE_SUPERLU

+// Enable MFEM functionality based on the MUMPS library.
+#cmakedefine MFEM_USE_MUMPS
+
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

-// Internal MFEM option: enable group/batch allocation for some small objects.
-#cmakedefine MFEM_USE_MEMALLOC
-
 // Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO

+// Enable MFEM functionality based on the AmgX library
+#cmakedefine MFEM_USE_AMGX
+
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS

@@ -156,4 +162,7 @@
 // library.
 #cmakedefine MFEM_USE_SIMMETRIX

+// Enable interface to the MKL CPardiso library.
+#cmakedefine MFEM_USE_MKL_CPARDISO
+
 #endif // MFEM_CONFIG_HEADER
@@ -38,7 +38,19 @@ if(NOT ADIOS2_FOUND)
  endif()

  find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 
+
+  # adios2 version 2.5.0
+  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
+
+  # adios2 version 2.6.0 and onwards
+  if(NOT ADIOS2_LIBRARY)
+	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
+	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
+	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
+	if(MFEM_USE_MPI)
+	  add_definitions(-DADIOS2_USE_MPI)
+	endif()
+  endif()

  include(FindPackageHandleStandardArgs)
  find_package_handle_standard_args(ADIOS2
@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - AMGX_FOUND
+#   - AMGX_LIBRARIES
+#   - AMGX_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
+mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
+  "Paths to headers required by AMGX." "Libraries required by AMGX.")
@@ -0,0 +1,32 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - MKL_CPARDISO_FOUND
+#   - MKL_CPARDISO_LIBRARIES
+#   - MKL_CPARDISO_INCLUDE_DIRS
+
+if(NOT MKL_MPI_WRAPPER_LIB)
+  message(FATAL_ERROR "MKL CPardiso enabled but no MKL MPI Wrapper lib specified")
+endif()
+
+if(NOT MKL_LIBRARY_DIR)
+  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
+  set(MKL_LIBRARY_DIR "lib")
+endif()
+
+include(MfemCmakeUtilities)
+mfem_find_package(MKL_CPARDISO MKL_CPARDISO
+    MKL_CPARDISO_DIR "include" mkl_cluster_sparse_solver.h ${MKL_LIBRARY_DIR} mkl_core
+  "Paths to headers required by MKL CPardiso." "Libraries required by MKL CPARDISO."
+  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
+  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential
+  ADD_COMPONENT MKL_MPI_WRAPPER "include" "" ${MKL_LIBRARY_DIR} ${MKL_MPI_WRAPPER_LIB})
@@ -0,0 +1,23 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Sets the following variables:
+#   - MUMPS_FOUND
+#   - MUMPS_INCLUDE_DIRS
+#   - MUMPS_LIBRARIES
+
+include(MfemCmakeUtilities)
+mfem_find_package(MUMPS MUMPS MUMPS_DIR
+  "include" dmumps_c.h "lib" dmumps
+  "Paths to headers required by MUMPS."
+  "Libraries required by MUMPS."
+  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
+  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
@@ -20,10 +20,14 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  "Paths to headers required by SUNDIALS." "Libraries required by SUNDIALS."
  ADD_COMPONENT NVector_Serial
    "include" nvector/nvector_serial.h "lib" sundials_nvecserial
-  ADD_COMPONENT NVector_Parallel
-    "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
+  ADD_COMPONENT NVector_Cuda
+    "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
  ADD_COMPONENT NVector_ParHyp
    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
+  ADD_COMPONENT NVector_Parallel
+    "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
+  ADD_COMPONENT NVector_MPIPlusX
+    "include" nvector/nvector_mpiplusx.h "lib" sundials_nvecmpiplusx
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
@@ -235,7 +235,7 @@ endfunction(mfem_find_component)
 #   code snippets. Additionally, a list of required/optional/alternative
 #   packages (given by ${Name}_REQUIRED_PACKAGES) are searched for and added to
 #   the ${Prefix}_INCLUDE_DIRS and ${Prefix}_LIBRARIES lists. The variable
-#   ${Name}_REQUIRED_LIBRARIES can be set to spcecify any additional libraries
+#   ${Name}_REQUIRED_LIBRARIES can be set to specify any additional libraries
 #   that are needed. This function defines the following CACHE variables:
 #
 #      ${Prefix}_FOUND
@@ -733,15 +733,16 @@ function(mfem_export_mk_files)
    set(shared_link_flag "-Wl,-rpath,")
  endif()

-  # Convert Boolean vars to YES/NO without writting the values to cache
+  # Convert Boolean vars to YES/NO without writing the values to cache
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
-      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
+      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
+      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
+      MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -819,7 +820,7 @@ function(mfem_export_mk_files)
      string(REGEX REPLACE "^SCOREC::" "" libname ${pumilib})
      string(FIND "${pumilib}" ".a" staticlib)
      string(FIND "${pumilib}" ".so" sharedlib)
-      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFUALT_PATH)
+      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFAULT_PATH)
      if (NOT "${sharedlib}" MATCHES "-1" OR
          NOT "${staticlib}" MATCHES "-1"   )
        set(MFEM_EXT_LIBS "${pumilib} ${MFEM_EXT_LIBS}")
@@ -42,9 +42,15 @@
 #ifdef MFEM_USE_SUPERLU
 #error Building with SuperLU_DIST (MFEM_USE_SUPERLU=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_MUMPS
+#error Building with MUMPS (MFEM_USE_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_MKL_CPARDISO
+#error Building with MKL CPARDISO (MFEM_USE_MKL_CPARDISO=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -93,6 +93,11 @@

 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
+// #define MFEM_USE_SUPERLU5
+
+// Enable MFEM functionality based on the MUMPS library.
+// #define MFEM_USE_MUMPS
+// #define MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@

 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
@@ -100,6 +105,9 @@
 // Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO

+// Enable MFEM functionality based on the AmgX library.
+// #define MFEM_USE_AMGX
+
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS

@@ -163,4 +171,7 @@
 // library.
 // #define MFEM_USE_SIMMETRIX

+// Enable interface to the MKL CPardiso library.
+// #define MFEM_USE_MKL_CPARDISO
+
 #endif // MFEM_CONFIG_HEADER
@@ -32,8 +32,11 @@ MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
+MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
+MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
+MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -52,6 +55,7 @@ MFEM_USE_CEED          = @MFEM_USE_CEED@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
+MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -33,8 +33,11 @@ option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
+option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
+option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
+option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -52,6 +55,7 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
+option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -88,7 +92,7 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
+# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
    "Path to the SUNDIALS library.")
@@ -109,12 +113,21 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
    "Additional packages required by ParMETIS.")

-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_5.1.0" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
    "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
 set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
    "Additional packages required by SuperLU_DIST.")

+set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
+    "Path to the MUMPS library.")
+# Packages required by MUMPS, depending on how it was compiled.
+set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
+    "Additional packages required by MUMPS.")    
+# If the MPI package does not find all required Fortran libraries:
+# set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
+#     "Additional libraries required by MUMPS.")    
+
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
    "Path to the STRUMPACK library.")
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
@@ -145,6 +158,8 @@ set(ScaLAPACK_TARGET_NAMES scalapack)

 set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")

+set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
+
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -180,6 +195,10 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
    "Packages that HiOp depends on.")

+set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
+set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
+set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
+
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
@@ -53,6 +53,8 @@ HIP_CXX = hipcc
 # gfx900, gfx1010, etc.
 HIP_ARCH = gfx900
 HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
+HIP_XCOMPILER =
+HIP_XLINKER   = -Wl,

 ifneq ($(NOTMAC),)
   AR      = ar
@@ -120,8 +122,11 @@ MFEM_USE_SUNDIALS      = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
+MFEM_USE_SUPERLU5      = NO
+MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
+MFEM_USE_AMGX          = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -140,6 +145,17 @@ MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
+MFEM_USE_MKL_CPARDISO  = NO
+
+# MPI library compile and link flags
+# These settings are used only when building MFEM with MPI + HIP
+ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
+   # We determine MPI_DIR assuming $(MPICXX) is in $(MPI_DIR)/bin
+   MPI_DIR := $(patsubst %/,%,$(dir $(shell which $(MPICXX))))
+   MPI_DIR := $(patsubst %/,%,$(dir $(MPI_DIR)))
+   MPI_OPT = -I$(MPI_DIR)/include
+   MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
+endif

 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -156,7 +172,7 @@ HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE

 # METIS library configuration
-ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)
+ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
   ifeq ($(MFEM_USE_METIS_5),NO)
     METIS_DIR = @MFEM_DIR@/../metis-4.0
     METIS_OPT =
@@ -189,15 +205,19 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
+# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
+SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
+SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
 -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
-   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
+   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
+endif
+ifeq ($(MFEM_USE_CUDA),YES)
+   SUNDIALS_LIB += -lsundials_nveccuda
 endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
@@ -216,9 +236,15 @@ SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
 -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)

 # SuperLU library configuration
-SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
-SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
-SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
+ifeq ($(MFEM_USE_SUPERLU5),YES)
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
+   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
+else
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
+   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
+endif

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
 # STRUMPACK >= v2.2.0)
@@ -227,12 +253,12 @@ SCOTCH_OPT = -I$(SCOTCH_DIR)/include
 SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\
 -lpthread

-# SCALAPACK library configuration (required by STRUMPACK)
+# SCALAPACK library configuration (required by STRUMPACK and MUMPS)
 SCALAPACK_DIR = @MFEM_DIR@/../scalapack-2.0.2
 SCALAPACK_OPT = -I$(SCALAPACK_DIR)/SRC
 SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)

-# MPI Fortran library, needed e.g. by STRUMPACK
+# MPI Fortran library, needed e.g. by STRUMPACK or MUMPS
 # MPICH:
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
@@ -240,6 +266,12 @@ MPI_FORTRAN_LIB = -lmpifort
 # Additional Fortan library:
 # MPI_FORTRAN_LIB += -lgfortran

+# MUMPS library configuration
+MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
+MUMPS_OPT = -I$(MUMPS_DIR)/include
+MUMPS_LIB = -Wl,-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
+ -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
@@ -251,7 +283,13 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
+ -lginkgo_omp -lginkgo_cuda -lginkgo_reference
+
+# AmgX library configuration
+AMGX_DIR = @MFEM_DIR@/../amgx
+AMGX_OPT = -I$(AMGX_DIR)/include
+AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -372,6 +410,15 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

+# MKL CPardiso library configuration
+MKL_CPARDISO_DIR ?=
+MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
+MKL_LIBRARY_SUBDIR ?= lib
+MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
+MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
+                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -0,0 +1,33 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "dmumps_c.h"
+#include <string>
+#include <iostream>
+#include <algorithm>
+
+// Macros to expand a macro as a string
+#define STR_EXPAND(s) #s
+#define STR(s) STR_EXPAND(s)
+
+int main()
+{
+#ifdef MUMPS_VERSION
+   const char *ptr = STR(MUMPS_VERSION);
+   std::string s(ptr);
+   s.erase(std::remove(s.begin(), s.end(), '"'), s.end());
+   s.erase(std::remove(s.begin(), s.end(), '.'), s.end());
+   std::cout << s << "\n";
+   return 0;
+#else
+   return -1;
+#endif
+}
@@ -42,6 +42,10 @@ GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
+MUMPS = $(MFEM_USE_MUMPS:NO=)
+GMV_CXX ?= $(MFEM_CXX)
+GMV = get_mumps_version
+GMV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -50,6 +54,13 @@ $(GHV).out: $(GHV)
 	./$(GHV) > $(GHV).out
 .INTERMEDIATE: $(GHV) $(GHV).out

+$(GMV): $(SRC)$(GMV).cpp
+	$(call mfem-info, Determining MUMPS version ...)
+	$(GMV_CXX) ${GMV_FLAGS} $(SRC)$(GMV).cpp -o $(GMV)
+$(GMV).out: $(GMV)
+	./$(GMV) > $(GMV).out
+.INTERMEDIATE: $(GMV) $(GMV).out
+
 get-hypre-version: $(GHV).out
 	$(eval MFEM_HYPRE_VERSION:=$(shell cat $(GHV).out))
 	$(if $(MFEM_HYPRE_VERSION),$(eval export MFEM_HYPRE_VERSION)\
@@ -62,10 +73,18 @@ check-smx:
 	$(call mfem-info, MFEM_USE_SIMMETRIX = $(MFEM_USE_SIMMETRIX))
 	$(eval export MFEM_USE_SIMMETRIX)

-header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx)
+get-mumps-version: $(GMV).out
+	$(eval MFEM_MUMPS_VERSION:=$(shell cat $(GMV).out))
+	$(if $(MFEM_MUMPS_VERSION),$(eval export MFEM_MUMPS_VERSION)\
+	$(info MUMPS version: $(MFEM_MUMPS_VERSION)),\
+	$(error Unable to determine MUMPS version))
+
+header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx,) \
+ $(if $(MUMPS),get-mumps-version,)
 	$(call mfem-info, Writing $(CONFIG_HPP) ...)
 	@set -- && \
-	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX); do \
+	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX) \
+	           $(if $(MUMPS),MFEM_MUMPS_VERSION); do \
 	   eval var=\$$$$def && \
 	   if [ "NO" != "$${var}" ]; then \
 	      set -- "$$@" -e "s|// \(#define $${def} \)|\1|" && \
@@ -48,6 +48,10 @@ groups_serial=(
   "miniapps/meshing"
   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
    mesh-optimizer.cpp minimal-surface.cpp"'
+'"convergence"
+   "Convergence tests:"
+   "tests/convergence"
+   "rates.cpp"'
 )
 # Parallel groups
 groups_parallel=(
@@ -78,6 +82,14 @@ groups_parallel=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
+'"convergence"
+   "Convergence tests:"
+   "tests/convergence"
+   "prates.cpp"'
+'"par-mesh-format"
+   "Parallel mesh tests:"
+   "tests/par-mesh-format"
+   "ex1p.cpp"'
 )
 # All groups serial + parallel runs mixed in the same group:
 groups_all=(
@@ -107,6 +119,14 @@ groups_all=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
+'"convergence"
+   "Convergence tests:"
+   "tests/convergence"
+   "{,p}rates.cpp"'
+'"par-mesh-format"
+   "Parallel mesh tests:"
+   "tests/par-mesh-format"
+   "ex1p.cpp"'
 )
 make_all="all"
 base_timeformat=$'real: %3Rs  user: %3Us  sys: %3Ss  %%cpu: %P'
@@ -380,10 +400,15 @@ function timed_run()
 # This function is used to execute the sample runs
 function go()
 {
-   local cmd=("$@")
+   # Strip leading and trailing spaces from $1 and store the result in cmd_line
+   shopt -s extglob
+   local cmd_line="${1##+( )}"
+   cmd_line="${cmd_line%%+( )}"
+   shopt -u extglob
+   eval local cmd=(${cmd_line})
   local res=""
   echo $sep
-   echo "<${group}>" "${cmd[@]}"
+   echo "<${group}>" "${cmd_line}"
   echo $sep
   if [ "${timing}" == "yes" ]; then
      timed_run "${cmd[@]}"
@@ -395,15 +420,15 @@ function go()
   else
      res="${red}FAILED${none}"
   fi
-   printf "[${res}] <${group}> ${cmd[*]}\n"
+   printf "[${res}] <${group}> ${cmd_line}\n"
   if [ "${timing}" == "yes" ]; then
      printf "Run time: %s\n" "${timer}"
      timer=(${timer})
      timer="${timer[1]}"
-      printf -v line "[$res](%8s) ${cmd[*]}" "$timer"
+      printf -v line "[$res](%8s) ${cmd_line}" "$timer"
      summary=("${summary[@]}" "$line")
   else
-      summary=("${summary[@]}" "[${res}] ${cmd[*]}")
+      summary=("${summary[@]}" "[${res}] ${cmd_line}")
   fi
   echo $sep
 }
@@ -438,7 +463,7 @@ function go_group()
      fi
      for run in "${runs[@]}"; do
         if [ "${run}" == "" ]; then continue; fi
-         eval go \${run_prefix} \${run} \${run_suffix} $output
+         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
      done
   done
   ${make} clean-exec
@@ -504,7 +529,7 @@ function echo_run()
 {
   echo "   $@"
   { echo "   $@"; echo "$sep";
-     "$@"
+     eval "$@"
     echo "$sep"; } >> "$echo_log" 2>&1
 }

@@ -524,6 +549,28 @@ function build_all()
   echo_run ${make} config ${mfem_config} || exit 1
   echo_run ${make} ${make_j} || exit 1
   echo_run ${make} ${make_all} ${make_j} || exit 1
+   # Build groups in directories other than the directories built by 'make all':
+   for group_params in "${groups[@]}"; do
+      eval params=(${group_params})
+      group_dir="${params[2]}"
+      case "$group_dir" in
+         (examples*|miniapps*)
+            # Built by 'make all'
+            ;;
+         (*)
+            if [ "${mfem_dir}" != "${mfem_build_dir}" ]; then
+               echo_run mkdir -p "${group_dir}" || exit 1
+               echo_run cd "${group_dir}" || exit 1
+               echo_run cp -af "${mfem_dir}/${group_dir}/makefile" . || exit 1
+            else
+               echo_run cd "${group_dir}" || exit 1
+            fi
+            echo_run ${make} clean || exit 1
+            echo_run ${make} MFEM_DIR="${mfem_dir}" ${make_j} || exit 1
+            echo_run cd "${mfem_build_dir}" || exit 1
+            ;;
+      esac
+   done
 }

 # Function that runs all sample runs, given by the array variable "groups".
@@ -16,7 +16,6 @@ if (DOXYGEN_FOUND)
  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CodeDocumentation.conf.in
    ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf @ONLY)

-
  add_custom_target(doc
    COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
    COMMAND echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
@@ -32,5 +31,4 @@ if (DOXYGEN_FOUND)
    COMMENT "Removing API documentation"
    VERBATIM)

-
 endif (DOXYGEN_FOUND)
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.1.1
+PROJECT_NUMBER         = v4.2.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "Finite element discretization library"
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.

-PROJECT_LOGO           = web/logo-small.png
+PROJECT_LOGO           = logo-small.png

 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -766,11 +766,14 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/mesh \
                         @MFEM_SOURCE_DIR@/fem \
                         @MFEM_SOURCE_DIR@/examples \
+                         @MFEM_SOURCE_DIR@/examples/amgx \
+                         @MFEM_SOURCE_DIR@/examples/ginkgo \
+                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/petsc \
                         @MFEM_SOURCE_DIR@/examples/pumi \
-                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/sundials \
-                         @MFEM_SOURCE_DIR@/miniapps/adjoint \			 
+                         @MFEM_SOURCE_DIR@/examples/superlu \
+                         @MFEM_SOURCE_DIR@/miniapps/adjoint \
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
@@ -2334,7 +2337,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.

-DOT_GRAPH_MAX_NODES    = 10
+DOT_GRAPH_MAX_NODES    = 50

 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the
@@ -62,7 +62,7 @@ namespace mfem {
 * - <a class="el" href="ex9p_8cpp_source.html">Example 9p</a>: parallel Discontinuous Galerkin (DG) time-dependent advection
 * - <a class="el" href="ex10_8cpp_source.html">Example 10</a>: time-dependent implicit nonlinear elasticity
 * - <a class="el" href="ex10p_8cpp_source.html">Example 10p</a>: parallel time-dependent implicit nonlinear elasticity
- * - <a class="el" href="examples_2ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
+ * - <a class="el" href="ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
 * - <a class="el" href="ex12p_8cpp_source.html">Example 12p</a>: parallel linear elasticity eigensolver
 * - <a class="el" href="ex13p_8cpp_source.html">Example 13p</a>: parallel Maxwell eigensolver
 * - <a class="el" href="ex14_8cpp_source.html">Example 14</a>: Discontinuous Galerkin (DG) for the Laplace problem
@@ -90,20 +90,25 @@ namespace mfem {
 * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
 * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM
 * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
+ * - <a class="el" href="ex27_8cpp_source.html">Example 27</a>: boundary conditions for the Laplace problem
+ * - <a class="el" href="ex27p_8cpp_source.html">Example 27p</a>: parallel boundary conditions for the Laplace problem
 *
- * <H4>SUNDIALS Examples</H4>
+ * <H4>AmgX Examples</H4>
 * - Variants of Examples
- *   <a class="el" href="sundials_2ex9_8cpp_source.html">9</a>,
- *   <a class="el" href="sundials_2ex9p_8cpp_source.html">9p</a>,
- *   <a class="el" href="sundials_2ex10_8cpp_source.html">10</a>,
- *   <a class="el" href="sundials_2ex10p_8cpp_source.html">10p</a>,
- *   <a class="el" href="sundials_2ex16_8cpp_source.html">16</a>,
- *   and
- *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>
- *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS classes\endlink
- * - CVODES adjoint examples:
- *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
- *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>
+ *   <a class="el" href="examples_2amgx_2ex1_8cpp_source.html">1</a> and
+ *   <a class="el" href="examples_2amgx_2ex1p_8cpp_source.html">1p</a>,
+ *   demonstrating the use of MFEM's \link amgxsolver.hpp AmgX integration\endlink.
+ *
+ * <H4>Ginkgo Examples</H4>
+ * - Variants of Example
+ *   <a class="el" href="examples_2ginkgo_2ex1_8cpp_source.html">1</a>,
+ *   demonstrating the use of MFEM's \link ginkgo.hpp Ginkgo integration\endlink.
+ *
+ * <H4>HiOp Examples</H4>
+ * - Variants of Examples
+ *   <a class="el" href="hiop_2ex9_8cpp_source.html">9</a> and
+ *   <a class="el" href="hiop_2ex9p_8cpp_source.html">9p</a>,
+ *   demonstrating the use of MFEM's \link hiop.hpp HiOp integration\endlink.
 *
 * <H4>PETSc Examples</H4>
 * - Variants of Examples
@@ -115,8 +120,8 @@ namespace mfem {
 *   <a class="el" href="petsc_2ex6p_8cpp_source.html">6p</a>,
 *   <a class="el" href="petsc_2ex9p_8cpp_source.html">9p</a>,
 *   and
- *   <a class="el" href="petsc_2ex10p_8cpp_source.html">10p</a>
- *   demonstrating the use of MFEM's \link petsc.hpp PETSc classes\endlink
+ *   <a class="el" href="petsc_2ex10p_8cpp_source.html">10p</a>,
+ *   demonstrating the use of MFEM's \link petsc.hpp PETSc integration\endlink.
 *
 * <H4>PUMI Examples</H4>
 * - Variants of Examples
@@ -124,19 +129,27 @@ namespace mfem {
 *   <a class="el" href="examples_2pumi_2ex1p_8cpp_source.html">1p</a>,
 *   <a class="el" href="pumi_2ex2_8cpp_source.html">2</a>,
 *   and
- *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>
- *   demonstrating the use of MFEM's \link pumi.hpp PUMI classes\endlink
+ *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>,
+ *   demonstrating the use of MFEM's \link pumi.hpp PUMI integration\endlink.
 *
- * <H4>HiOp Examples</H4>
+ * <H4>SUNDIALS Examples</H4>
 * - Variants of Examples
- *   <a class="el" href="hiop_2ex9_8cpp_source.html">9</a> and
- *   <a class="el" href="hiop_2ex9p_8cpp_source.html">9p</a>,
- *   demonstrating the use of MFEM's \link hiop.hpp HiOp classes\endlink
+ *   <a class="el" href="sundials_2ex9_8cpp_source.html">9</a>,
+ *   <a class="el" href="sundials_2ex9p_8cpp_source.html">9p</a>,
+ *   <a class="el" href="sundials_2ex10_8cpp_source.html">10</a>,
+ *   <a class="el" href="sundials_2ex10p_8cpp_source.html">10p</a>,
+ *   <a class="el" href="sundials_2ex16_8cpp_source.html">16</a>,
+ *   and
+ *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>,
+ *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS integration\endlink.
+ * - CVODES adjoint miniapps:
+ *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
+ *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>.
 *
- * <H4>Ginkgo Examples</H4>
+ * <H4>SuperLU Examples</H4>
 * - Variants of Example
- *   <a class="el" href="ginkgo_2ex1_8cpp_source.html">1</a>
- *   demonstrating the use of MFEM's \link ginkgo.hpp Ginkgo classes\endlink
+ *   <a class="el" href="examples_2superlu_2ex1p_8cpp_source.html">1p</a>,
+ *   demonstrating the use of MFEM's \link superlu.hpp SuperLU integration\endlink.
 *
 * <H3>Miniapps</H3>
 * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -149,6 +162,7 @@ namespace mfem {
 * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
 * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
 * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
+ * - <a class="el" href="polar-nc_8cpp_source.html">Polar NC</a>: generate polar non-conforming meshes
 * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
 * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
@@ -161,10 +175,11 @@ namespace mfem {
 * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
 * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
+ * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
- * See also the <a class="el" href="../../../examples/README.html">README</a> in the <code>examples/</code> directory
+ * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
 */

 }
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 This directory contains Doxygen configuration files for building HTML
 documentation of the MFEM sources (including the example codes and miniapps).
@@ -21,7 +21,7 @@ create the file CodeDocumentation.html, which can be viewed in any web browser.

 Alternatively, the (pre-build) documentation can be browsed directly online at

-   http://mfem.github.io/doxygen/html/index.html
+   https://mfem.github.io/doxygen/html/index.html


 Some building considerations:
@@ -13,7 +13,6 @@ SHELL = /bin/bash
 MFEM_DIR ?= ..
 DOXYGEN_CONF = CodeDocumentation.conf

-
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
@@ -1,208 +0,0 @@
-body {
-    padding-top: 70px;
-}
-
-/* hack to increase height of headers to make #anchors work with the top nav-bar,
-   (note: we only want this on pages with TOC, otherwise there are other problems) */
-div.col-md-9 h1[id]::before,
-div.col-md-9 h2[id]::before,
-div.col-md-9 h3[id]::before,
-div.col-md-9 h4[id]::before,
-div.col-md-9 h5[id]::before,
-div.col-md-9 h6[id]::before {
-    content: "";
-    display: block;
-    margin-top: -75px;
-    height: 75px;
-}
-
-img {
-    max-width: 100%;
-    height: auto;
-}
-
-pre {
-    overflow-x: auto;
-}
-
-pre code {
-    overflow-wrap: normal;
-    white-space: pre;
-    font-size: 13.5px;
-}
-
-label {
-    font-weight: normal;
-}
-
-ul.nav li.main {
-    font-weight: bold;
-}
-
-div.col-md-3 {
-    padding-right: 0;
-}
-
-div.col-md-9, div.col-md-12 {
-    padding-bottom: 100px;
-}
-
-div.source-links {
-    float: right;
-}
-
-img {
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    margin-top: 25px;
-    margin-bottom: 25px;
-}
-
-img.floatright {
-    display: block;
-    float: right;
-    margin-top: 0;
-    margin-left: 25px;
-    margin-bottom: 25px;
-}
-
-img.teamphoto {
-    box-shadow: 4px 4px 4px #999;
-}
-
-p.centered {
-    margin-left: auto;
-    margin-right: auto;
-    text-align: center;
-}
-
-/*label {
-    display: inline-block;
-    margin-bottom: 5px;
-    font-weight: normal;
-}*/
-
-/*
- * Side navigation
- *
- * Scrollspy and affixed enhanced navigation to highlight sections and secondary
- * sections of docs content.
- */
-
-/* By default it's not affixed in mobile views, so undo that */
-.bs-sidebar.affix {
-    position: static;
-}
-
-.bs-sidebar.well {
-    padding: 0;
-}
-
-/* First level of nav */
-.bs-sidenav {
-    /*margin-top: 10px;
-    margin-bottom: 10px;*/
-    padding-top:    10px;
-    padding-bottom: 10px;
-    border-radius: 5px;
-}
-
-/* All levels of nav */
-.bs-sidebar .nav > li > a {
-    display: block;
-    padding: 5px 20px;
-    z-index: 1;
-}
-.bs-sidebar .nav > li > a:hover,
-.bs-sidebar .nav > li > a:focus {
-    text-decoration: none;
-    border-right: 1px solid;
-}
-.bs-sidebar .nav > .active > a,
-.bs-sidebar .nav > .active:hover > a,
-.bs-sidebar .nav > .active:focus > a {
-    font-weight: bold;
-    background-color: transparent;
-    border-right: 1px solid;
-}
-
-/* Nav: second level (shown on .active) */
-.bs-sidebar .nav .nav {
-    display: none; /* Hide by default, but at >768px, show it */
-    margin-bottom: 8px;
-}
-.bs-sidebar .nav .nav > li > a {
-    padding-top:    3px;
-    padding-bottom: 3px;
-    padding-left: 30px;
-    font-size: 90%;
-}
-
-/* Show and affix the side nav when space allows it */
-@media (min-width: 992px) {
-    .bs-sidebar .nav > .active > ul {
-        display: block;
-    }
-    /* Widen the fixed sidebar */
-    .bs-sidebar.affix,
-    .bs-sidebar.affix-bottom {
-        width: 213px;
-    }
-    .bs-sidebar.affix {
-        position: fixed; /* Undo the static from mobile first approach */
-        top: 80px;
-    }
-    .bs-sidebar.affix-bottom {
-        position: absolute; /* Undo the static from mobile first approach */
-    }
-    .bs-sidebar.affix-bottom .bs-sidenav,
-    .bs-sidebar.affix .bs-sidenav {
-        margin-top: 0;
-        margin-bottom: 0;
-    }
-}
-@media (min-width: 1200px) {
-    /* Widen the fixed sidebar again */
-    .bs-sidebar.affix-bottom,
-    .bs-sidebar.affix {
-        width: 263px;
-    }
-}
-
-
-/* Sticky footer styles
-------------------------------------------------- */
-html {
-  position: relative;
-  min-height: 100%;
-}
-body {
-  /* Margin bottom by footer height */
-  margin-bottom: 50px;
-}
-.footer {
-  position: absolute;
-  bottom: 0;
-  width: 100%;
-  /* Set the fixed height of the footer here */
-  height: 50px;
-  background-color: #eeeeee;
-  overflow: hidden;
-}
-
-
-/* Custom page CSS
-------------------------------------------------- */
-/* Not required for template or sticky footer method. */
-
-.container .text-muted {
-  margin-top: 17px;
-  margin-bottom: 0;
-}
-.container .alignleft {
-  float: left;
-}
-.container .alignright {
-  float: right;
-}
@@ -117,16 +117,21 @@ if (MFEM_USE_SUPERLU)
    ${MPIEXEC_POSTFLAGS})
 endif()

-# Include the examples/sundials directory if SUNDIALS is enabled.
-if (MFEM_USE_SUNDIALS)
-  add_subdirectory(sundials)
+# Include the examples/amgx directory if AmgX is enabled
+if (MFEM_USE_AMGX)
+  add_subdirectory(amgx)
 endif()

-# Include the examples/sundials directory if SUNDIALS is enabled.
+# Include the examples/ginkgo directory if GINKGO is enabled.
 if (MFEM_USE_GINKGO)
  add_subdirectory(ginkgo)
 endif()

+# Include the examples/hiop directory if HiOp is enabled
+if (MFEM_USE_HIOP)
+  add_subdirectory(hiop)
+endif()
+
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -137,7 +142,12 @@ if (MFEM_USE_PUMI)
  add_subdirectory(pumi)
 endif()

-if (MFEM_USE_HIOP)
-  add_subdirectory(hiop)
+# Include the examples/sundials directory if SUNDIALS is enabled.
+if (MFEM_USE_SUNDIALS)
+  add_subdirectory(sundials)
 endif()

+# Include the examples/superlu directory if SUPERLU is enabled.
+if (MFEM_USE_SUPERLU)
+  add_subdirectory(superlu)
+endif()
@@ -0,0 +1,79 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(AMGX_EXAMPLES_SRCS)
+list(APPEND AMGX_EXAMPLES_SRCS
+  ex1.cpp
+  )
+
+if (MFEM_USE_MPI)
+  list(APPEND AMGX_EXAMPLES_SRCS
+    ex1p.cpp
+    )
+endif()
+
+set(AMGX_JSON_FILES amg_pcg.json multi_gs.json precon.json)
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add targets to copy *.json files from the source directory
+foreach(JSON_FILE ${AMGX_JSON_FILES})
+  add_custom_command(OUTPUT ${JSON_FILE}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/${JSON_FILE} ${JSON_FILE}
+    COMMENT "copy ${JSON_FILE}")
+endforeach()
+add_custom_target(copy_amgx_json_files DEPENDS ${AMGX_JSON_FILES}
+  COMMENT "Copying AMGX example json files ...")
+
+# Add "test_amgx" target, see below.
+add_custom_target(test_amgx
+  ${CMAKE_CTEST_COMMAND} -R amgx USES_TERMINAL)
+
+# Add one executable per cpp file, adding "amgx_" as prefix. Sets
+# "copy_amgx_json_files" as a prerequisite for the given examples. Also, sets
+# "test_amgx" as a target that depends on the given examples.
+set(PFX amgx_)
+add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
+
+# Testing.
+# The AMGX tests can be run separately using the target "test_amgx"
+# which builds the examples and runs:
+#   ctest -R amgx
+
+# Command line options for the tests.
+# Example 1/1p:
+set(EX1_TEST_OPTS)
+set(EX1P_TEST_OPTS)
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on NVIDIA's multigrid library AmgX.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_AMGX = YES", see the top-level INSTALL file for details (version
+2.1 of AmgX is recommended, though older versions may work too.
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.
@@ -0,0 +1,38 @@
+{
+    "config_version": 2,
+    "solver": {
+        "preconditioner": {
+            "print_grid_stats": 1,
+            "print_vis_data": 0,
+            "solver": "AMG",
+            "smoother": {
+                "scope": "jacobi",
+                "solver": "BLOCK_JACOBI",
+                "relaxation_factor": 0.7,
+                "monitor_residual": 0,
+                "print_solve_stats": 0
+            },
+            "print_solve_stats": 0,
+            "presweeps": 1,
+            "interpolator": "D2",
+            "max_row_sum" : 0.9,
+            "strength_threshold" : 0.25,
+            "max_iters": 2,
+            "monitor_residual": 0,
+            "store_res_history": 0,
+            "scope": "amg",
+            "max_levels": 100,
+            "cycle": "V",
+            "postsweeps": 1
+        },
+        "solver": "PCG",
+        "print_solve_stats": 1,
+        "obtain_timings": 1,
+        "max_iters": 100,
+        "monitor_residual": 1,
+        "convergence": "RELATIVE_MAX",
+        "scope": "main",
+        "tolerance": 1e-12,
+        "norm": "L2"
+    }
+}
--- a/Show More
+++ b/Show More