more temporary memory in tmop

use temp mem in tmop+pa

.codecov.yml

@@ -1,32 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  precision: 2
-  round: nearest
-  range: "0...100"
-  status:
-    patch:
-      default:
-        target: auto
-        threshold: 0%
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        informational: true
-        only_pulls: true
-    project:
-      default:
-        target: auto   # compares coverage to the previous base commit
-        threshold: 1%  # allows variations around the target
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        only_pulls: true
-
-github_checks:
-  annotations: false
-
-comment: false

.github/stale.yml

@@ -1,61 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before an Issue or Pull Request with the stale
-# label is closed.  Set to false to disable. If disabled, issues still need to
-# be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale.
-# Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set
-# to `[]` to disable
-exemptLabels:
-  - bug
-  - WIP
-  - ready-for-review
-  - in-review
-  - in-next
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  :warning: This issue or PR has been automatically marked as stale because it has not
-  had any activity in the last month. *If no activity occurs in the next week, it will
-  be automatically closed.* Thank you for your contributions.
-
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-# only: issues
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed

.github/workflows/builds-and-tests.yml

@@ -1,197 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# In this CI section, we build different variants of mfem and run test on them.
-name: builds-and-tests
-
-# Github actions can use the default "GITHUB_TOKEN". By default, this token
-# is set to have permissive access. However, this is not a good practice
-# security-wise. Here we use an external action, so we restrict the
-# permission to the minimum required.
-# When the 'permissions' is set, all the scopes not mentioned are set to the
-# most restrictive setting. So the following is enough.
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  MFEM_TOP_DIR: mfem
-
-# Note for future improvements:
-#
-# We cannot reuse cached dependencies and have to build them for each target
-# although they could be shared sometimes. That's because Github cache Action
-# has no read-only mode. But there is a PR ready for this
-# (https://github.com/actions/cache/pull/489)
-
-jobs:
-  builds-and-tests:
-    strategy:
-      matrix:
-        os: [ubuntu-18.04, macos-10.15]
-        target: [debug, optim]
-        mpi: [sequential, parallel]
-        build-system: [make]
-        # 'include' allows us to
-        # - add a variable without creating a new matrix dimension.
-        # - add a new combination ('build-system: cmake' case here)
-        #
-        # note: we will gather coverage info for any non-debug run except the
-        # CMake build.
-        include:
-          - target: debug
-            codecov: NO
-          - target: optim
-            codecov: YES
-          - os: ubuntu-18.04
-            target: optim
-            codecov: NO
-            mpi: parallel
-            build-system: cmake
-    name: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.build-system }}
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    # This external action allows to interrupt a workflow already running on
-    # the same branch to save resource
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
-    # Checkout MFEM in "mfem" subdirectory. Final path:
-    # /home/runner/work/mfem/mfem/mfem
-    # Note: Done now to access "install-hypre" and "install-metis" actions.
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        path: ${{ env.MFEM_TOP_DIR }}
-        # Fetch the complete history for codecov to access commits ID
-        fetch-depth: 0
-
-    # Only get MPI if defined for the job.
-    # TODO: It would be nice to have only one step, e.g. with a dedicated
-    # action, but I (@adrienbernede) don't see how at the moment.
-    - name: get MPI (Linux)
-      if: matrix.mpi == 'parallel' && matrix.os == 'ubuntu-18.04'
-      run: |
-        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-18.04'
-      run: |
-        sudo apt-get install lcov
-
-    - name: Set up Homebrew
-      if: ( matrix.mpi == 'parallel' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
-      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
-
-    - name: get MPI (MacOS)
-      if: matrix.mpi == 'parallel' && matrix.os == 'macos-10.15'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install openmpi
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: get MPI (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install lcov
-
-    # Get Hypre through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache hypre
-      id: hypre-cache
-      if: matrix.mpi == 'parallel'
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
-
-    - name: get hypre
-      if: matrix.mpi == 'parallel' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v1.0
-      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-
-    # Get Metis through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache metis
-      id: metis-cache
-      if: matrix.mpi == 'parallel'
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
-
-    - name: install metis
-      if: matrix.mpi == 'parallel' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v1.0
-      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-
-    # MFEM build and test
-    - name: build
-      uses: mfem/github-actions/build-mfem@v1.0
-      with:
-        os: ${{ matrix.os }}
-        target: ${{ matrix.target }}
-        codecov: ${{ matrix.codecov }}
-        mpi: ${{ matrix.mpi }}
-        build-system: ${{ matrix.build-system }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: ${{ env.MFEM_TOP_DIR }}
-
-    # Run checks (and only checks) on debug targets
-    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'debug'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }}  && make check
-
-    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make unittest
-
-    - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make test
-
-    - name: cmake unit tests
-      if: matrix.build-system == 'cmake'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure
-
-    # Code coverage (process and upload reports)
-    - name: codecov
-      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v1.0
-      with:
-        name: ${{ matrix.os }}-${{ matrix.mpi }}
-        project_dir: ${{ env.MFEM_TOP_DIR }}
-        directories: "fem general linalg mesh"

.github/workflows/mfem-analysis.yml

@@ -1,99 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: build-analysis
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  COVERAGE_ENV: mfem-coverage
-
-jobs:
-  gitignore:
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
-    - name: checkout MFEM
-      uses: actions/checkout@v2
-      with:
-        path: mfem
-
-    - name: Get MPI (Linux)
-      run: |
-        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: Cache Hypre Install
-      id: hypre-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
-
-    - name: Get Hypre
-      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@master
-      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-
-    - name: Cache Metis Install
-      id: metis-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
-
-    - name: Install Metis
-      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@master
-      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-
-    # MFEM build and test
-    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@master
-      with:
-        os: ${{ runner.os }}
-        target: optim
-        codecov: NO
-        mpi: parallel
-        build-system: make
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: mfem
-
-    - name: test (no clean)
-      run: |
-        cd mfem && make test-noclean
-
-    - name: gitignore
-      run: |
-        cd mfem/tests/scripts
-        ./runtest gitignore

.github/workflows/repo-check.yml

@@ -1,137 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: repo-check
-
-permissions:
-  actions: write
-
-on:
-  push:
-
-jobs:
-  file-headers-check:
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        path: mfem
-
-    - name: copyright check
-      id: copyright
-      run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt
-        then
-          echo "Please update the following files to Copyright (c) 2010-2021:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated copyright found."
-        fi
-      continue-on-error: true
-
-    - name: license check
-      id: license
-      run: |
-        cd mfem
-        if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt
-        then
-          echo "Please update the following files to the BSD-3 license:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No GNU GPL license found."
-        fi
-      continue-on-error: true
-
-    - name: release check
-      id: release
-      run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-        then
-          echo "Please update the following files to LLNL-CODE-806117:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated release number found."
-        fi
-      continue-on-error: true
-
-    - name: wrap-up
-      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
-      run: |
-        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-          echo "copyright check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-          echo "license check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-          echo "release check failed, unroll log for details"
-        fi
-        exit 1
-
-  code-style:
-    runs-on: ubuntu-16.04 # needed for astyle 2.05.1
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-
-    - name: get astyle
-      run: |
-        sudo apt-get install astyle=2.05.1-0ubuntu1
-
-    - name: style check
-      run: |
-        cd tests/scripts
-        ./runtest code-style
-
-  documentation:
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-
-    - name: get doxygen and graphviz
-      run: |
-        sudo apt-get install doxygen graphviz
-
-    - name: build documentation
-      run: |
-        cd tests/scripts
-        ./runtest documentation
-
-  branch-history:
-    if: github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: branch-history
-      run: |
-        git fetch origin master:master
-        git checkout -b gh-actions-branch-history
-        cd tests/scripts
-        ./runtest branch-history

.gitignore

@@ -44,8 +44,8 @@ doc/warnings.log
 
 # Example and miniapp binaries and outputs
 
-examples/ex[0-9]
-examples/ex[0-9]p
+examples/ex[1-9]
+examples/ex[1-9]p
 examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
@@ -66,7 +66,6 @@ examples/sol_u.*
 examples/sol_p.*
 examples/sol_r.*
 examples/sol_i.*
-examples/ex6p-checkpoint.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
@@ -102,33 +101,29 @@ examples/Example23*
 examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*
-examples/ex28_*
-examples/ex28p_*
-examples/flux.*
 
-examples/amgx/ex1
-examples/amgx/ex1p
-examples/amgx/.logamgx
-examples/amgx/refined.mesh
-examples/amgx/sol.gf
-examples/amgx/mesh.*
-examples/amgx/sol.*
+examples/sundials/ex9
+examples/sundials/ex1[06]
+examples/sundials/ex9p
+examples/sundials/ex1[06]p
 
-examples/gingko/ex1
-examples/gingko/refined.mesh
-examples/gingko/sol.gf
-examples/gingko/mesh.*
-examples/gingko/sol.*
-
-examples/hiop/ex9
-examples/hiop/ex9p
-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
+examples/sundials/ex9.mesh
+examples/sundials/ex9-mesh.*
+examples/sundials/ex9-init.*
+examples/sundials/ex9-final.*
+examples/sundials/Example9*
+examples/sundials/deformed.*
+examples/sundials/velocity.*
+examples/sundials/elastic_energy.*
+examples/sundials/ex16.mesh
+examples/sundials/ex16-mesh.*
+examples/sundials/ex16-init.*
+examples/sundials/ex16-final.*
+examples/sundials/Example16*
 
 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
+
 examples/petsc/mesh.*
 examples/petsc/sol.*
 examples/petsc/sol_p.*
@@ -146,51 +141,28 @@ examples/petsc/mode_*
 
 examples/pumi/ex1
 examples/pumi/ex[126]p
+
+examples/hiop/ex9.mesh
+examples/hiop/ex9-mesh.*
+examples/hiop/ex9-init.*
+examples/hiop/ex9-final.*
+
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
 examples/pumi/sol.*
 examples/pumi/displaced.mesh
 
-examples/sundials/ex9
-examples/sundials/ex1[06]
-examples/sundials/ex9p
-examples/sundials/ex1[06]p
-examples/sundials/ex9.mesh
-examples/sundials/ex9-mesh.*
-examples/sundials/ex9-init.*
-examples/sundials/ex9-final.*
-examples/sundials/Example9*
-examples/sundials/deformed.*
-examples/sundials/velocity.*
-examples/sundials/elastic_energy.*
-examples/sundials/ex16.mesh
-examples/sundials/ex16-mesh.*
-examples/sundials/ex16-init.*
-examples/sundials/ex16-final.*
-examples/sundials/Example16*
-
-examples/superlu/ex1p
-examples/superlu/mesh.*
-examples/superlu/sol.*
-
-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
+
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*
 
-miniapps/gslib/field-diff
-miniapps/gslib/field-interp
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid
@@ -203,7 +175,7 @@ miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
-miniapps/meshing/polar-nc
+
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -215,43 +187,14 @@ miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
-miniapps/meshing/polar-nc.mesh
-
-miniapps/mtop/parheat
-miniapps/mtop/ParHeat*
-miniapps/mtop/seqheat
-miniapps/mtop/SeqHeat*
-
-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_kovasznay_vs
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/refined.mesh
-miniapps/nurbs/mesh.*
-miniapps/nurbs/sol.*
-miniapps/nurbs/mode_*
-miniapps/nurbs/Example1*
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
+
 miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
 
-miniapps/shifted/distance
-miniapps/shifted/ParaViewDistance
-miniapps/shifted/diffusion
-miniapps/shifted/diffusion.mesh
-miniapps/shifted/diffusion.gf
-
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
@@ -265,6 +208,7 @@ miniapps/toys/rubik
 miniapps/toys/snake
 miniapps/toys/lissajous
 miniapps/toys/mondrian
+
 miniapps/toys/snake-init.mesh
 miniapps/toys/snake-user.mesh
 miniapps/toys/snake-joined.mesh
@@ -279,12 +223,29 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh
 
-miniapps/solvers/block-solvers
-miniapps/solvers/lor_solvers
-miniapps/solvers/plor_solvers
-miniapps/solvers/ParaView
-miniapps/solvers/mesh.*
-miniapps/solvers/sol.*
+miniapps/nurbs/nurbs_ex1
+miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/refined.mesh
+miniapps/nurbs/mesh.*
+miniapps/nurbs/sol.*
+miniapps/nurbs/mode_*
+miniapps/nurbs/Example1*
+
+miniapps/gslib/field-diff
+miniapps/gslib/findpts
+miniapps/gslib/pfindpts
+
+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output
+
+miniapps/adjoint/cvsRoberts_ASAi_dns
+miniapps/adjoint/adjoint_advection_diffusion
 
 # Unit test binary and outputs
 tests/unit/output_meshes
@@ -292,23 +253,20 @@ tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/tmop_pa_tests_*
-tests/unit/ptmop_pa_tests_*
-tests/unit/ceed_tests
+tests/unit/tmop_tests_*
+tests/unit/ptmop_tests_*
+tests/unit/cube.mesh
+tests/unit/star.mesh
+tests/unit/blade.mesh
+tests/unit/square01.mesh
+tests/unit/toroid-hex.mesh
+tests/unit/beam-hex-nurbs.mesh
+tests/unit/square-disc-nurbs.mesh
 
 # Test script output
 tests/scripts/*.err
 tests/scripts/*.out
 tests/scripts/*.msg
 
-# Other tests
-tests/convergence/rates
-tests/convergence/prates
-tests/par-mesh-format/ex1p
-
 # VPATH builds
 build-*/*
-
-# PETSc automated build
-petsc-build/*
-pkg.gitcommit

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -23,6 +23,10 @@
 # also means that the BUILD_ROOT directory will never be cleaned.
 # TODO: add a clean-up mechanism
 #
+# BUILD_PATH:
+# In BUILD_ROOT, we want to separate builds depending on the machine used,
+# typically because we build on a set of dependencies per machine.
+#
 # REBASELINE:
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
@@ -40,46 +44,42 @@
 # Directory used to place artifacts.
 
 variables:
-  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
+  BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  BUILD_PATH: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}/${PLAT}/${TOOLCHAIN}
   REBASELINE: "NO"
-  AUTOTEST: "NO"
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+  MFEM_ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
   TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
   TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
   ARTIFACTS_DIR: artifacts
 
 # The pipeline is divided into stages. Usually, these are also synchronization
 # points, however, we use "needs" keyword to express the DAG of jobs for more
 # efficiency.
 # - We use setup phase to download content outside of mfem directory.
-# - Allocate/Release is where quartz resources are allocated/released once for all.
-# - Build and Test is where we build and MFEM for multiple toolchains.
+# - Allocate is where quartz resources are allocated once for all.
+# - Libs is where dependencies are built.
+# - Build is where we build MFEM for multiple toolchains.
+# - Test is where we perform a first set of tests on MFEM build.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
+# - Deallocate releases quartz resources
 stages:
-  - q_allocate_resources
-  - q_build_and_test
-  - q_release_resources
-  - l_build_and_test
-  - c_build_and_test
   - setup
+  - allocate
+  - libs
+  - build
+  - test
+  - deallocate
   - baseline_check
-  - baseline_to_autotest
   - baseline_publish
 
-# The setup job in setup stage don't rely on MFEM git repo. It prepares a
+# The setup job in setup stage don’t rely on MFEM git repo. It prepares a
 # pipeline-wide working directory downloading/updating external repos.
 # TODO: updating tests and tpls is not necessary anymore since pipelines are
 # now using unique directories so repo are never shared with another pipeline.
 # This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
-setup:
-  tags:
-    - shell
-    - quartz
+.setup:
   stage: setup
   variables:
     GIT_STRATEGY: none
@@ -89,46 +89,105 @@ setup:
     - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
     - cd tpls && git pull && cd ..
     - cd tests && git pull && cd ..
-    - cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
-  needs: []
 
-.build_toss_3_x86_64_ib_script:
-  script:
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test
+# Share configurations (templates) for toolchain configuration.
+# Note: Any job using this template shouldn't redefine "before_script" key.
+.with_gcc_6_1_0:
+  variables:
+    TOOLCHAIN: gcc_6_1_0
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/6.1.0
 
-.build_toss_3_x86_64_ib_corona_script:
-  script:
-    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
+.with_gcc_4_9_3:
+  variables:
+    TOOLCHAIN: gcc_4_9_3
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/4.9.3
 
-# Lassen uses a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-# We use pdebug queue on lassen to speed-up the allocation.
-# However this would not be scalable to multiple builds.
-.build_blueos_3_ppc64le_ib_script:
+.if_srun_alloc_for_make: &if_srun_alloc_for_make |
+  if [[ "${MPIEXEC}" == "srun" ]]
+  then
+    export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+    [[ -n "JOB_ID" ]] && export JOB_ID="--jobid=${JOB_ID}"
+    RESOURCES="--extra-node-info=1:${MAKE_PAR}:1 --cpus-per-task=${MAKE_PAR} --ntasks=1"
+    # Allocation prefix:
+    export EXEC_PREFIX="srun ${JOB_ID} ${RESOURCES}"
+  fi
+
+# Build of dependencies and mfem:
+# if on quartz, uses a part of preallocated resources
+# Note: Those jobs don't rely on MFEM repository, hence the use of
+# "GIT_STRATEGY: none" preventing its clone.
+.build_hypre:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
   script:
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - make clean
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} hypre PREFIX=${BUILD_PATH}/hypre/src/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH}
+
+.build_metis:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
+  script:
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} ${METIS} PREFIX=${BUILD_PATH}/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH} CC=${CC} CXX=${CXX}
+
+# Building MFEM
+.build_mfem:
+  variables:
+    EXEC_PREFIX: ""
+  stage: build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - *if_srun_alloc_for_make
+    - make config CXX=${CXX} MPICXX=mpicxx MFEM_USE_MPI=${MFEM_USE_MPI} MFEM_DEBUG=${MFEM_DEBUG} MFEM_MPIEXEC="${EXEC_PREFIX}" MFEM_MPIEXEC_NP=${MPIEXEC_NP}
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} all
+
+# TODO: Could this phase use a parallel run?
+.sanitycheck_mfem:
+  stage: test
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - make test
 
 # Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
 # differentiates between the two tests.
 .baseline_script: &baseline_script |
   # locals
+  _glob_out=${BASELINE_TEST}.out
   _glob_err=${BASELINE_TEST}.err
   _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
   _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
   _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+
+  _out=${BASELINE_TEST}-${SYS_TYPE}.out
+  _ref=../${BASELINE_TEST}-${SYS_TYPE}.saved
+  _out_txt=${BASELINE_TEST}.txt
+  _diff=${BASELINE_TEST}-diff.txt
   # prepare
   cd ${BUILD_ROOT}
   ln -snf ${CI_PROJECT_DIR} mfem
   cd tests
   mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
   # run
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
   # post
   mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
   if [[ -s ${_glob_err} ]]
@@ -136,7 +195,7 @@ setup:
     echo "ERROR during ${BASELINE_TEST} execution";
     echo "Here is the ${_glob_err} file content";
     cat ${_glob_err}
-    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}.txt
     exit 1;
   elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
   then
@@ -146,20 +205,18 @@ setup:
   elif [[ -f ${_base_patch} ]]
   then
     echo "${BASELINE_TEST}: Differences found, patch generated"
-    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}.txt
   elif [[ -f ${_base_out} ]]
   then
     echo "${BASELINE_TEST}: Differences found, replacement file generated"
-    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}.txt
   fi
-  # _base_diff won't even exist if there is no difference.
+  # _base_diff won’t even exist if there is no difference.
   if [[ -f ${_base_diff} ]]
   then
     echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
     cat ${_base_diff}
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
-    # We create a .err file, because that's how we signal that there was a diff.
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}.txt
   fi
   if [[ ! -s ${_base_diff} ]]
   then
@@ -198,7 +255,7 @@ setup:
       - ${ARTIFACTS_DIR}
   allow_failure: true
 
-# This job can only be manually triggered on a pipeline for master branch, or if
+# This job can only be manually triggers on a pipeline for master branch, or if
 # the pipeline was triggered with REBASELINE="YES"
 .rebaseline_mfem:
   stage: baseline_publish
@@ -211,18 +268,19 @@ setup:
     - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
     - cd ${BUILD_ROOT}/tests
     - |
-      if [[ ! -f "${DIFF_FILE}" ]]
+      if [[ ! -f "${DIFF_FILE}.txt" ]]
       then
           echo "Nothing to be done: no relevant change in baseline"
           exit 0
-      elif [[ -f "${PATCH_FILE}" ]]
+      elif [[ -f "${PATCH_FILE}.txt" ]]
       then
+          mv ${PATCH_FILE}.txt ${PATCH_FILE}
           patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
-      elif [[ -f "${FULL_FILE}t" ]]
+      elif [[ -f "${FULL_FILE}.txt" ]]
       then
-          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
+          cp "${FULL_FILE}.txt" "./baseline-${SYS_TYPE}.saved"
       else
-          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+          echo "File missing: expected ${PATCH_FILE}.txt or ${FULL_FILE}.txt"
           exit 1
       fi
     - git add baseline-${SYS_TYPE}.saved
@@ -232,4 +290,3 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
   - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml

.gitlab/lassen.yml

@@ -1,33 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    - when: on_success
-
-# Spack helped builds
-# Generic lassen build job, extending build script
-.build_and_test_on_lassen:
-  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
-  stage: l_build_and_test
-  needs: []
-
-opt_mpi_cuda_xl_16_1_1_8:
-  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
-  extends: .build_and_test_on_lassen

.gitlab/quartz.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -15,151 +15,135 @@
   tags:
     - shell
     - quartz
-  rules:
-    # Don’t run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
-      when: never
-    # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
-      when: never
-    # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resources
-    - if: '$CI_JOB_NAME =~ /release_resources/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
+  variables:
+    PLAT: quartz
+    MPIEXEC: srun
+    MPIEXEC_NP: "-n"
+    MAKE_PAR: 6
+    BASELINE_PAR: 18
+
+# Setup
+setup_quartz:
+  extends: [.setup, .on_quartz]
 
 # Allocate
-q_allocate_resources:
+allocate_quartz:
   variables:
     GIT_STRATEGY: none
   extends: .on_quartz
-  stage: q_allocate_resources
+  stage: allocate
   script:
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      salloc --nodes=1 --extra-node-info=2:18:1 --partition=pdebug --time=30 --no-shell --job-name=${MFEM_ALLOC_NAME}
+      fi
   timeout: 6h
 
 # Release
-q_release_resources:
+deallocate_quartz:
   variables:
     GIT_STRATEGY: none
   extends: .on_quartz
-  stage: q_release_resources
+  stage: deallocate
   script:
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+      ([[ -n "${JOB_ID}" ]] && scancel ${JOB_ID})
+      fi
+  when: always
 
-# Release
-q_report_success:
+# Build external libraries (tpls)
+build_hypre_gcc_6_1_0_quartz:
+  extends: [.build_hypre, .with_gcc_6_1_0, .on_quartz]
+
+build_parmetis_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
   variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Can only run if all the quartz jobs passed"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
-    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
+    METIS: parmetis
 
-q_report_failure:
+build_metis4_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
   variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Runs if there was at least one failure on quartz"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
-    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
-    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
+    METIS: metis4
 
-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
-  stage: q_build_and_test
+build_metis5_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: metis5
+  allow_failure: true
 
 # Build MFEM
-debug_ser_gcc_4_9_3:
+build_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.build_mfem, .with_gcc_4_9_3, .on_quartz]
   variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
 
-debug_ser_gcc_6_1_0:
+build_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
 
-debug_par_gcc_6_1_0:
+build_mfem_debug_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
 
-opt_ser_gcc_6_1_0:
+build_mfem_opt_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"
 
-opt_par_gcc_6_1_0:
+build_mfem_opt_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
 
-opt_par_gcc_6_1_0_sundials:
-  variables:
-    SPEC: "%gcc@6.1.0 +sundials"
-  extends: .build_and_test_on_quartz
+### Tests
 
-opt_par_gcc_6_1_0_petsc:
+# Sanity check
+sanitycheck_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_4_9_3, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_4_9_3_quartz, allocate_quartz]
 
-opt_par_gcc_6_1_0_pumi:
+sanitycheck_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +pumi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_ser_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_opt_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_par_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_mfem_opt_par_gcc_6_1_0_quartz, allocate_quartz]
 
 # Baseline
 baselinecheck_mfem_intel_quartz:
   extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup]
-
-update_autotest:
-  extends: [.on_quartz]
-  needs: [baselinecheck_mfem_intel_quartz]
-  stage: baseline_to_autotest
-  script:
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
-    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-    - |
-      if [[ -f ${rundir}/*.err ]]
-      then
-        cp ${rundir}/*.err ${rundir}/autotest-email.html
-      fi
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
+  needs: [allocate_quartz]
 
 baselinepublish_mfem_quartz:
-  extends: [.on_quartz, .rebaseline_mfem]
+  extends: [.rebaseline_mfem, .on_quartz]
   needs: [baselinecheck_mfem_intel_quartz]

.travis.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

CHANGELOG

@@ -5,416 +5,138 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 
-Version 4.2.1 (development)
+Version 4.1.1 (development)
 ===========================
-- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
-
-- Memory management:
-  * Added method Device::SetMemoryTypes that can be used to change the default
-    host and device MemoryTypes before Device setup.
-  * In class MemoryManager, added methods GetDualMemoryType and
-    SetDualMemoryType; dual MemoryTypes are used to determine the second
-    MemoryType (host or device) when only one MemoryType is specified in methods
-    of class Memory.
-  * Added Memory constructor for setting both the host and device MemoryTypes.
-  * Switched the default behavior of device memory allocations so that they
-    are deferred until the device pointer is needed.
-  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
-    corresponding allocator that can be set with the method
-    MemoryManager::SetUmpireDevice2AllocatorName.
-  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
-
-- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
-
-- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
-
-- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
-
-- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of
-  the domain decomposition.
-
-- Introduced solver interface for linear problems with constraints, a few
-  concrete solvers that implement the interface, and a demonstration of their
-  use in Example 28(p), which solves an elasticity problem with zero normal
-  displacement (but allowed tangential displacement) on two boundaries.
-
-- Added high-order matrix-free auxiliary Maxwell solver for H(curl) problems,
-  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
-  Example 3p and linalg/auxiliary.?pp.
-
-- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
-
-- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
-
-- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
-
-- Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
-  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
-
-- Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
-  ESDIRK-32, and A-stable ESDIRK-33.
-
-- Introduced a new non-conforming mesh format that fixes known inconsistencies
-  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
-  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
-  be used to restart a parallel AMR computation. Example 6p has been extended to
-  demonstrate restarting from a previously saved checkpoint. Note that parallel
-  NC data files are compatible with serial code, e.g., can be viewed with serial
-  GLVis. Loading of legacy NC mesh files is still supported.
-
-- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
-- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
-- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
-- Generalized the Multigrid class to support non-geometric multigrid. The
-  previous functionality, based on FiniteElementSpaceHierarchy, is now available
-  in the derived class GeometricMultigrid.
-
-- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
-     and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
-- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
-- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
-- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
-- Added HIP support to the CMake build system.
-
-- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
-- Added support for reading VTK meshes in XML format.
-
-- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
-- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
-
-- Added new mesh quality metrics and improved the untangling capabilities of the
-  TMOP-based mesh optimization algorithms.
-
-- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
-- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
-- Changed the interface for the error estimator.
-
-- Implemented the Kelly error indicator for scalar-valued problems, supported
-  in serial and parallel builds.
-
-- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
-- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
-- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
-- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
-
-- Added support for creating refined meshes for all element types (e.g. by
-  splitting high-order elements into low-order refined elements), including
-  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
-  supports meshes with any element geometry.
-
-- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
-
-- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
-- Added a new, very simple example (ex0 and parallel version ex0p). This
-  example solves a simple Poisson problem using H1 elements (the same problem as
-  ex1), but is intended to be extremely simple and approachable for new users.
-
-- Meshes consisting of any type of elements (including mixed meshes) can be
-  converted to all-simplex meshes using Mesh::MakeSimplicial.
-
-- Several of the mesh constructors (creating Cartesian meshes, refined (LOR)
-  meshes, simplex meshes, etc.) are now available as "named constructors", e.g.
-  Mesh::MakeCartesian2D or Mesh::MakeRefined. The legacy constructors are marked
-  as deprecated.
-
-- Added support for creating periodic meshes with Mesh::MakePeriodic. The
-  requisite periodic vertex mappings can be created with
-  Mesh::CreatePeriodicVertexMapping.
-
-- Added support for transferring dual fields between high-order and low-order
-  refined finite element spaces using the transposed versions of the
-  L2ProjectionGridTransfer operators. This functionality is illustrated in the
-  lor-transfer miniapp.
-
-- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
-- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p and may not work in more general settings.
-
-- Added support for transferring fields (primary and dual) between high-order
-  and low-order refined H1 finite element spaces using the
-  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
-  through the lor-transfer miniapp when run with the -h1 option.
-
-- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
-
-- Added sample meshes in the `data` subdirectory showing the reference elements
-  of the six currently supported element types; ref-segment.mesh,
-  ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
-  ref-prism.mesh.
-
-- Added a high-order extension of the shifted boundary method to solve PDEs on
-  non body-fitted meshes. This is illustrated in the new Shifted Diffusion
-  miniapp, see miniapps/shifted/diffusion.cpp.
-
-- Added makefile rule to generate TAGS table for vi or Emacs users.
-
-libCEED integration improvements
---------------------------------
-- Refactor the libCEED integration
-
-- Add support for VectorCoefficient with libCEED backends.
-
-- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
-  libCEED backends.
-
-
-Version 4.2, released on October 30, 2020
-=========================================
-
-High-Performance Computing
---------------------------
-- Added support for explicit vectorization in the high-performance templated
-  code, which can now take advantage of specific classes on the following
-  architectures:
-  * x86 (SSE/AVX/AVX2/AVX512),
-  * Power8 & Power9 (VSX),
-  * BG/Q (QPX).
-  These are disabled by default, but can be enabled with MFEM_USE_SIMD=YES.
-  See the new file linalg/simd.hpp and the new directory linalg/simd.
-
-- Added an Element Assembly mode compatible with GPU device execution for H1 and
-  L2 spaces in the mass, convection, diffusion, transpose, and the face DG trace
-  integrators. See option '-ea' in Example 9. When enabled, this assembly level
-  stores independent dense matrices for the elements, and independent dense
-  matrices for the faces in the DG case.
-
-- Added a Full Assembly mode compatible with GPU device execution. This assembly
-  level builds on top of the Element Assembly kernels to compute a global sparse
-  matrix. All integrators supported by element assembly are also supported by
-  full assembly. See the '-fa' option in Example 9.
-
-- Optimized the AMD/HIP kernel support and enabled HIP support in the libCEED
-  integration. This is now available via the "ceed-hip" device backend.
-
-- Improved the libCEED integration to support:
-  * AssemblyLevel::NONE for Mass, Diffusion, VectorMass, and VectorDiffusion
-    Integrators. This level computes the full operator evaluation "on the fly".
-  * VectorMassIntegrator and VectorDiffusionIntegrator.
-  * All types of (scalar) Coefficients.
-
-- Added partial assembly / device support for:
-  * H(div) bilinear forms and VectorFEDivergenceIntegrator.
-  * BlockOperator, see the updated Example 5.
-  * Complex operators, see the updated Example 22.
-  * Chebyshev accelerated polynomial smoother.
-  * Convergent diagonal preconditioning on non-conforming adaptively refined
-    meshes, see Example 6/6p.
-
-- Added CUDA support for:
-  * Sparse matrix-vector multiplication with cuSPARSE,
-  * SUNDIALS ODE integrators, see updated SUNDIALS modification of Example 9/9p.
-
-Linear and nonlinear solvers
-----------------------------
-- Added a new solver class for simple integration with NVIDIA's multigrid
-  library, AmgX. The AmgX class is designed to work as a standalone solver or
-  preconditioner for existing MFEM solvers. It uses MFEM's sparse matrix format
-  for serial runs and the HypreParMatrix format for parallel runs. The new
-  solver may be configured to run with one GPU per MPI rank or with more MPI
-  ranks than GPUs. In the latter case, matrices and vectors are consolidated to
-  ranks communicating with the GPUs and the solution is then broadcasted.
-
-  Although CUDA is required to build, the AmgX support is compatible with the
-  MFEM CPU device configuration. The examples/amgx folder illustrates how to
-  integrate AmgX in existing MFEM applications. The AmgX solver class is
-  partially based on: "AmgXWrapper: An interface between PETSc and the NVIDIA
-  AmgX library", by Pi-Yueh Chuang and Lorena A. Barba, doi:10.21105/joss.00280.
-
-- Added initial support for geometric h- and p-multigrid preconditioners for
-  matrix-based and matrix-free discretizations with basic GPU capability, see
-  Example 26/26p.
-
-- Added support for the CVODES package in SUNDIALS which provides ODE solvers
-  with sensitivity analysis capabilities. See the CVODESSolver class and the new
-  adjoint miniapps in the miniapps/adjoint directory.
-
-- Added an interface to the MKL CPardiso solver, an MPI-parallel sparse direct
-  solver developed by Intel. See Example 11p for an illustration of its usage.
-
-- Added support for the SLEPc eigensolver package, https://slepc.upv.es.
-
-- Upgraded SuperLU interface to use SuperLU_DIST 6.3.1. Added a simple SuperLU
-  example in the new directory examples/superlu.
-
-- Extended the KINSOL (SUNDIALS) nonlinear solver interface to support the
-  Jacobian-free Newton-Krylov method. A usage example is shown in Example 10p.
-
-- Block arrays of parallel matrices can now be merged into a single parallel
-  matrix with the function HypreParMatrixFromBlocks. This could be useful for
-  solving block systems with parallel direct solvers such as STRUMPACK.
-
-- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
-- Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
-  preconditioner. The latter requires hypre version 2.19.0 or later.
-
-Discretization improvements
----------------------------
-- Extended GSLIB-FindPoints integration to support simplices and interpolation
-  of functions from L2, H(div) and H(curl) spaces.
-
-- Added support for computing asymptotic error estimates and convergence rates
-  for the whole de Rham sequence based on the new class ConvergenceStudy and new
-  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
-  the tests/convergence directory for sample usage.
-
-- Extended the GetValue and GetVectorValue methods of GridFunction to support
-  evaluation on boundary elements and, in the continuous field case, arbitrary
-  mesh edges and faces. This requires passing an ElementTransformation argument.
-
-- Added support for matrix-free interpolation and restriction operators between
-  continuous H1 finite element spaces of different order on the same mesh or
-  with the same order on uniformly refined meshes.
-
-- The Coefficient classes based on a C-function pointer (FunctionCoefficient,
-  VectorFunctionCoefficient and MatrixFunctionCoefficient) now use the more
-  general std::function class template. This allows the classes to be backward
-  compatible (i.e. they can still work with C-functions) and, in addition,
-  support any "callable", e.g. lambda functions.
-
-- Non-conforming meshes are now supported with block nonlinear forms. See the
-  updated Example 19/19p.
 
 Meshing improvements
 --------------------
 - The graph linear ordering library Gecko, previously an external dependency, is
   now included directly in MFEM. As a result, Mesh::GetGeckoElementOrdering is
   always available. The interface has also been improved, see for example the
-  Mesh Explorer miniapp.
+  mesh-explorer miniapp.
 
 - Added support for finite difference-based gradient and Hessian approximation
   in the TMOP mesh optimization algorithms. This improves the accuracy of the
   Hessian for r-adaptivity using discrete fields, and allows use of skewness
   and orientation based metrics.
 
-- Improved Gmsh reader (version 2.2), which now supports both high-order and
-  periodic meshes. Segments, triangles, quadrilaterals, and tetrahedra are
-  supported up to order 10. Wedges and hexahedra are supported up to order 9.
-  For sample periodic meshes, see the periodic*.msh files in the data directory.
-
-- Added support for construction of (serial) non-conforming meshes. Hanging
-  nodes can be marked with Mesh::AddVertexParents when building the mesh with
-  the "init" constructor. The usage is demonstrated in a new meshing miniapp,
-  Polar NC, which generates meshes that are non-conforming from the start.
-
 - Added support for r-adaptivity with more than one discrete field. This allows
-  the user to specify different discrete functions for controlling the size,
-  aspect-ratio, orientation, and skew of elements in the mesh.
+  the user to specify different discrete functions for controlling the
+  size, aspect-ratio, orientation, and skew of elements in the mesh.
 
-- Additional TMOP improvements:
-  * Capability for approximate tangential mesh relaxation.
-  * Support and examples for using TMOP on mixed meshes.
-  * Complete integrator action accounting for spatial derivatives of discrete
-    and analytic targets.
+- Added TMOP capability for approximate tangential mesh relaxation.
+
+- Added support for reading periodic meshes in Gmsh format (version 2.2). See
+  for example the periodic-annulus-sector and periodic-torus-sector files in
+  the data directory.
+
+- Added complete action of the TMOP Integrator to account for the spatial
+  derivatives of discrete and analytic targets.
+
+Performance improvements
+------------------------
+- Added support for explicit vectorization in the high-performance templated
+  code, which can now take advantage of specific intrinsics classes on the
+  following architectures:
+    - x86 (SSE/AVX/AVX2/AVX512),
+    - Power8 & Power9 (VSX),
+    - BG/Q (QPX).
+  These are disabled by default, and can be enabled with MFEM_USE_SIMD=YES.
+  See the new file linalg/simd.hpp and the new directory linalg/simd.
+
+Improved GPU capabilities
+-------------------------
+- Added support for Chebyshev accelerated polynomial smoother on GPU.
+- The TMOP mesh optimization algorithms were extended to GPU:
+   - QualityMetric #1, #2 and #7 are available in 2D, #302, #303 and #321 in 3D
+   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+   - Kernels for normalization and limiting have been added
+   - The AdvectorCG now also support AssemblyLevel::PARTIAL
+
+- Optimized AMD/HIP kernel support.
+
+- Added a Full Assembly mode compatible with Device kernel execution. This
+  assembly level builds on top of the current Element Assembly kernels to
+  compute a global sparse matrix. All integrators supported by element assembly
+  are also supported by full assembly. See the '-fa' option in Example 9.
+
+- Added support for BlockOperator on GPU. See the updated Example 5.
+
+Discretization improvements
+---------------------------
+- Added support for matrix-free interpolation and restriction operators between
+  continuous H1 finite element spaces of different order on the same mesh or
+  with the same order on uniformly refined meshes.
+
+- Added support for simplices in GSLIB-FindPoints.
+
+- Added support for H1 and L2 element matrix assembly in the mass, convection,
+  diffusion, transpose, and the face DG trace integrators. This is compatible
+  with GPU device execution and is illustrated in Example 9/9p, see the option
+  '-ea'. When enabled, this level of assembly stores independent dense matrices
+  for the elements, and independent dense matrices for the faces in the DG case.
+
+- Added new partial assembly kernels for H(div) bilinear forms, as well as
+  VectorFEDivergenceIntegrator.
+
+- Improved the documentation of the GridFunction GetValue and GetVectorValue
+  methods. Expanded the GetValue and GetVectorValue methods which accept an
+  ElementTransformation argument to support evaluation on boundary elements
+  and, in the continuous field case, arbitrary mesh edges and faces.
+
+- Added new coefficient and vector coefficient classes for QuadratureFunctions.
+  Additionally, new LinearForm integrators were also added which make use of
+  these new QuadratureFunction coefficient classes.
+
+- Added support face integrals on the boundaries of NURBS meshes.
+
+Linear and nonlinear solvers
+----------------------------
+- Added power method to iteratively estimate the largest eigenvalue and the
+  corresponding eigenvector of an operator.
+
+- Added initial support for h- and p-multigrid solvers and preconditioners for
+  matrix-based and matrix-free discretizations with basic GPU capability.
+
+- Added a new IterativeSolverMonitor class that allows to monitor the residual
+  and solution during the solving process of an IterativeSolver after every
+  iteration.
+
+- Added support for the CVODES package in SUNDIALS which provides ODE
+  solvers with sensitivity analysis capabilities. See the CVODESSolver
+  class and the new adjoint miniapps below.
+
+- Block arrays of parallel matrices can now be merged into a single parallel
+  matrix with the function HypreParMatrixFromBlocks. This could be useful for
+  solving block systems with parallel direct solvers such as STRUMPACK.
+
+- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
+  solver can work with non-SPD preconditioner B.
+
+- Added support for the SLEPc eigensolver package.
 
 New and updated examples and miniapps
 -------------------------------------
+- Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
+  Matched Layer (PML) for the simulation of electromagnetic wave propagation.
+  The example defines and solves several indefinite Maxwell problems.
+
+- Added a new Example 26/26p to demonstrate the construction of a matrix-free
+  geometric and p-multigrid preconditioner for the Laplace problem.
+
+- Added a new example, Example 27/27p, to demonstrate the enforcement of various
+  boundary conditions with the Laplace operator. The example shows the procedure
+  for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
+  and periodic boundary conditions with either H1 or DG discretizations.
+
 - Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
   equations of incompressible fluid dynamics. See the miniapps/navier directory
   for more details.
 
-- Added 10 new example codes:
-  * Example 25/25p demonstrates the use of a Perfectly Matched Layer (PML) for
-    electromagnetic wave propagation (indefinite Maxwell).
-  * Example 26/26p shows how to construct matrix-free geometric and p-multigrid
-    preconditioner for the Laplace problem.
-  * Example 27/27p demonstrates the enforcement of Dirichlet, Neumann, Robin,
-    and periodic boundary conditions with either H1 or DG Laplace problems.
-  * Versions of Example 1/1p in examples/amgx demonstrating the use of AmgX,
-    to solve the Laplace problem with AMG preconditioning on GPUs.
-  * A version of Example 11p in examples/petsc demonstrating the use of SLEPc,
-    to solve the Laplace eigenproblem with shift-and-invert transformation.
-  * A version of Example 1 in examples/superlu demonstrating the use of SuperLU
-    to solve the Laplace problem.
-
-- Added a new Field Interpolation miniapp in miniapps/gslib that demonstrates
-  transfer of grid functions between different meshes using GSLIB-FindPoints.
-
-- Added 2 miniapps in the new miniapps/adjoint directory demonstrating how to
+- Added a new miniapps/adjoint directory with two miniapps demonstrating how to
   solve adjoint problems in MFEM using the CVODES package in SUNDIALS. Both of
   these miniapps require the MFEM_USE_SUNDIALS configuration option.
   * The cvsRoberts_ASAi_dns miniapp solves a backward adjoint problem for a
@@ -422,31 +144,31 @@ New and updated examples and miniapps
   * The adjoint_advection_diffusion miniapp solves a backward adjoint problem
     for an advection diffusion PDE, evaluating adjoint quadratures in parallel.
 
-- Added 4 additional meshing miniapps:
-  * The Minimal Surface miniapp solves Plateau's problem: the Dirichlet problem
-    for the minimal surface equation.
-  * The Twist miniapp demonstrates how to stitch together opposite surfaces of a
-    mesh to create a topologically periodic mesh.
-  * The Trimmer miniapp trims away parts of a mesh based on element attributes.
-  * Polar NC shows the construction of polar non-conforming meshes.
+- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
+  equation with the shift-and-invert spectral transformation method.
 
-- Several examples and miniapps were updated to include:
-  * Full and element assembly support in Example 9/9p.
-  * Partial assembly with diagonal preconditioning in Examples 4/4p/5/5p/22/22p.
-  * Diagonal preconditioner in Example 6/6p for partial assembly with AMR.
-  * The option to plot a function in Mesh Explorer.
-  * A new test problem showing a mixed bilinear form for H1, H(curl), H(div) and
-    L2, with partial assembly support in Example 24/24p.
-  * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
+- Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
+  stitching together opposite surfaces of a mesh to create a topologically
+  periodic mesh.
 
-Data management and Visualization
----------------------------------
-- Added support for ADIOS2 for parallel I/O with ParaView visualization. See
-  Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection
-  provide the interface to generate ADIOS2 Binary Pack (BP4) directory datasets.
+- Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
+  the Dirichlet problem for the minimal surface equation.
 
-- Added VTU output of boundary elements and attributes and parallel VTU (PVTU)
-  output of parallel meshes for visualization using ParaView.
+- Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
+  preconditioning.
+
+- Added full assembly support in Example 9/9p.
+
+- Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
+  form for H1, H(curl), H(div) and L_2, with partial assembly support.
+
+- Added weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
+
+- Added a simple mesh editing miniapp, Trimmer, which trims away portions of a
+  mesh based on element attributes. Any newly exposed boundary elements are
+  assigned attribute numbers related to the trimmed element attributes.
+
+- Added device support in Example 5/5p.
 
 Improved testing
 ----------------
@@ -455,39 +177,17 @@ Improved testing
   triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
   and the updated CONTRIBUTING.md file.
 
-- Added additional testing for convergence, the parallel mesh I/O, and for the
-  libCEED integration in MFEM in the tests/ directory.
-
-- Upgraded the Catch unit test framework from version 1.6.1 to version 2.13.0.
-
 Miscellaneous
 -------------
-- Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
-  as DEBUG is sometimes used as a macro.
-
-- Added a new IterativeSolverMonitor class that allows to monitor the residual
-  and solution with an IterativeSolver after every iteration.
-
-- Added power method to iteratively estimate the largest eigenvalue and the
-  corresponding eigenvector of an operator.
-
-- Added support for face integrals on the boundaries of NURBS meshes.
-
-- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
-  solver can work with non-SPD preconditioner B.
-
-- Added new coefficient and vector coefficient classes for QuadratureFunctions,
-  with new LinearForm integrators which use them.
+- Added support for ADIOS2 for parallel I/O with ParaView visualization. The
+  classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
+  interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
+  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
+  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.
 
 - The integration order used in the ComputeLpError and ComputeElementLpError
   methods of class GridFunction has been increased.
 
-- Change the IntegrationRule inside VectorDiffusionIntegrator to use the same
-  quadrature as DiffusionIntegrator.
-
-- The README.html files previously included in several source directories have
-  been removed. Use the corresponding pages at mfem.org instead.
-
 - Various other simplifications, extensions, and bugfixes in the code.
 
 
@@ -820,7 +520,7 @@ Other meshing improvements
   follows precisely the paper:
 
      D. Arnold, A. Mukherjee, and L. Pouly, "Locally Adapted Tetrahedral Meshes
-     Using Bisection", SIAM J. Sci. Comput. 22 (2000), 431-448.
+     Using Bisection", SIAM J. Sci. Comput. 22 (2000), 431–448.
 
   This guarantees that the shape regularity of the elements will be preserved
   under refinement.
@@ -965,7 +665,7 @@ Discretization improvements
 
 - New specialized time integrators: symplectic integrators of orders 1-4 for
   systems of first order ODEs derived from a Hamiltonian and generalized-alpha
-  ODE solver for the filtered Navier-Stokes equations with stabilization. See
+  ODE solver for the filtered Navier–Stokes equations with stabilization. See
   classes SIASolver and GeneralizedAlphaSolver in linalg/ode.hpp.
 
 - Inherit finite element classes from the new base class TensorBasisElement,
@@ -1115,7 +815,7 @@ New and updated examples and miniapps
 
 - Added a new meshing miniapp, Shaper, that can be used to resolve complicated
   material interfaces by mesh refinement, e.g. as a tool for initial mesh
-  generation from prescribed "material()" function. Both conforming and
+  generation from prescribed "material()" function.  Both conforming and
   non-conforming (isotropic and anisotropic) refinements are supported.
 
 - Added a new meshing miniapp, Mesh Optimizer, that demonstrates the use of TMOP
@@ -1127,7 +827,7 @@ Discretization improvements
 ---------------------------
 - Added a FindPoints method of the Mesh and ParMesh classes that returns the
   elements that contain a given set of points, together with the coordinates of
-  the points in the reference space of the corresponding element. In parallel,
+  the points in the reference space of the corresponding element.  In parallel,
   if a point is shared by multiple processors, only one of them will mark that
   point as found. Note that the current implementation of this method is not
   optimal and/or 100% reliable. See the mesh-explorer miniapp for an example.
@@ -1454,8 +1154,8 @@ New and improved linear solvers
   which is a sparse direct solver for distributed memory architectures. As such
   it can only be enabled along with MFEM_USE_MPI. When MFEM is configured with
   MFEM_USE_SUPERLU, one also needs to alter the version of METIS, since SuperLU
-  requires ParMETIS (which comes packaged with a serial version of METIS). See
-  http://crd-legacy.lbl.gov/~xiaoye/SuperLU for SuperLU_DIST details.
+  requires ParMETIS (which comes packaged with a serial version of METIS).  See
+  http://http://crd-legacy.lbl.gov/~xiaoye/SuperLU for SuperLU_DIST details.
 
 - Added a wrapper for the KLU solver in SuiteSparse see
   http://faculty.cse.tamu.edu/davis/suitesparse.html for details of KLU.
@@ -1865,7 +1565,7 @@ New and updated examples
   (ADS) in hypre.
 
 - Modified Example 1 to use isoparametric discretization (use the FE space from
-  the mesh) including NURBS meshes and spaces. Updated Example 2 to support
+  the mesh) including NURBS meshes and spaces.  Updated Example 2 to support
   arbitrary order spaces. Updated all examples to work with NURBS meshes and
   spaces, as well as to not use projection onto discontinuous polynomial spaces
   for visualization (this is now handled directly in GLVis when necessary).
@@ -1957,7 +1657,7 @@ Version 1.1, released on Sep 13, 2010
 New MFEM format for general meshes
 ----------------------------------
 - New MFEM mesh v1.0 format with uniform structure for any dimension and support
-  for curved meshes including in 3D. Class Mesh will recognize and read the new
+  for curved meshes including in 3D.  Class Mesh will recognize and read the new
   format (in addition to all previously used formats) and Mesh::Print uses the
   new format by default. The old print function was renamed to Mesh::PrintXG.
 

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -9,16 +9,12 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 2.8.11)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO) 
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -54,7 +50,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.2.1)
+set(${PROJECT_NAME}_VERSION 4.1.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -91,46 +87,10 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 # Enable languages.
 enable_language(CXX)
 if (MFEM_USE_CUDA)
-   if (MFEM_USE_HIP)
-      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
-   endif()
    # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
    cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
-   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
-   if (NOT CMAKE_CUDA_HOST_COMPILER)
-      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
    enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO) 
-     set(CMAKE_CUDA_STANDARD 14)
-   endif()
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-   set(CMAKE_CUDA_EXTENSIONS OFF)
-   set(CUDA_FLAGS "--expt-extended-lambda")
-   if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
-   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
-      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
-      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
-         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
-      else()
-         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
-      endif()
-   else()
-      set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
-   endif()
    message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
-   if (CMAKE_VERSION VERSION_LESS 3.12.0)
-      # CMake versions 3.8 and 3.9 require this to work; 3.10 and 3.11 are not
-      # tested and may not actually need this (but should be ok to keep).
-      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
-      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
-   endif()
-   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
-       "CUDA flags set for MFEM" FORCE)
-   set(CUSPARSE_FOUND TRUE)
-   set(CUSPARSE_LIBRARIES "cusparse")
 endif()
 
 if (XSDK_ENABLE_C)
@@ -179,12 +139,6 @@ endif()
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
   set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
-  if (MFEM_MPIEXEC)
-    set(MPIEXEC ${MFEM_MPIEXEC})
-  endif()
-  if (MFEM_MPIEXEC_NP)
-    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
-  endif()
   # Parallel MFEM depends on hypre
   find_package(HYPRE REQUIRED)
   set(MFEM_HYPRE_VERSION ${HYPRE_VERSION})
@@ -201,7 +155,7 @@ if (MFEM_USE_MPI)
     endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU MUMPS PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -256,14 +210,12 @@ endif()
 
 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
-  set(SUNDIALS_COMPONENTS CVODES ARKODE KINSOL NVector_Serial)
-  if (MFEM_USE_MPI)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Parallel NVector_MPIPlusX)
+  if (NOT MFEM_USE_MPI)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
+  else()
+    find_package(SUNDIALS REQUIRED
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
   endif()
-  if (MFEM_USE_CUDA)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
-  endif()
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
 # Mesquite
@@ -280,15 +232,6 @@ if (MFEM_USE_SUPERLU)
   endif()
 endif()
 
-# MUMPS can only be enabled in parallel
-if (MFEM_USE_MUMPS)
-  if (MFEM_USE_MPI)
-    find_package(MUMPS REQUIRED mumps_common pord)
-  else()
-    message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
-  endif()
-endif()
-
 # STRUMPACK can only be enabled in parallel
 if (MFEM_USE_STRUMPACK)
   if (MFEM_USE_MPI)
@@ -322,10 +265,6 @@ if (MFEM_USE_CEED)
   find_package(libCEED REQUIRED)
 endif()
 
-if (MFEM_USE_AMGX)
-  find_package(AMGX REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -357,6 +296,22 @@ if (MFEM_USE_HIOP)
   # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()
 
+# CUDA
+if (MFEM_USE_CUDA)
+   set(CMAKE_CUDA_STANDARD 11)
+   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+   set(CMAKE_CUDA_EXTENSIONS OFF)
+   set(CMAKE_CUDA_FLAGS "-arch=${CUDA_ARCH} --expt-extended-lambda"
+       CACHE STRING "CUDA flags set for MFEM" FORCE)
+   if (MFEM_USE_MPI)
+      set(CUDA_CCBIN_COMPILER ${MPI_CXX_COMPILER})
+   else()
+      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
+   endif()
+   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
+   set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CUDA_CCBIN_COMPILER})
+endif()
+
 # OCCA
 if (MFEM_USE_OCCA)
    find_package(OCCA REQUIRED)
@@ -372,34 +327,11 @@ if (MFEM_USE_UMPIRE)
    find_package(UMPIRE REQUIRED)
 endif()
 
-# Caliper
-if (MFEM_USE_CALIPER)
-  find_package(Caliper REQUIRED)
-endif()
-
-# AMD HIP
-if (MFEM_USE_HIP)
-  find_package(HIP REQUIRED)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-    if (MFEM_USE_GINKGO)
-      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
-    endif()
-  endif()
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
 endif()
 
-if (MFEM_USE_MKL_CPARDISO)
-  if (MFEM_USE_MPI)
-   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
-  endif()
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -423,10 +355,9 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER)
+set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
+    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -464,9 +395,9 @@ foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
 endforeach()
 
 if (MFEM_USE_CUDA)
-  set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
-elseif(MFEM_USE_HIP)
-  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+   foreach(file IN LISTS SOURCES)
+      set_property(SOURCE ${file} PROPERTY LANGUAGE CUDA)
+   endforeach()
 endif()
 
 add_subdirectory(config)
@@ -485,7 +416,7 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
     "The MFEM install directory" FORCE)
 
 # Declaring the library
-mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
+add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 if (CMAKE_VERSION VERSION_GREATER 2.8.11)
   target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
@@ -529,8 +460,6 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
   endforeach()
 endif()
 
-set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")
-
 #-------------------------------------------------------------------------------
 # Examples, miniapps, and testing
 #-------------------------------------------------------------------------------
@@ -581,17 +510,16 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()
 
 # Add 'check' target - quick test
-set(MFEM_CHECK_TARGET_NAME ${MFEM_CUSTOM_TARGET_PREFIX}check)
 if (NOT MFEM_USE_MPI)
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
     ${CMAKE_CTEST_COMMAND} -R \"^ex1_ser\" -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1)
+  add_dependencies(check ex1)
 else()
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
     ${CMAKE_CTEST_COMMAND} -R \"^ex1p\" -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1p)
+  add_dependencies(check ex1p)
 endif()
 
 #-------------------------------------------------------------------------------
@@ -645,7 +573,7 @@ endif()
 if (MFEM_USE_CEED)
   install(DIRECTORY ${MFEM_SOURCE_DIRS}
     DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/*.h")
+    FILES_MATCHING PATTERN "fem/libceed/*.h")
 endif()
 
 # Install ${HEADERS}

CONTRIBUTING.md

@@ -1,12 +1,12 @@
 <p align="center">
-<a href="https://mfem.org/"><img alt="mfem" src="https://mfem.org/img/logo-300.png"></a>
+<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
 </p>
 
 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
 <a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
 
 
@@ -26,7 +26,7 @@ request (PR) toward the `mfem:master` branch to propose your contribution. If
 you are planning significant code changes or have questions, you may want to
 open an [issue](https://github.com/mfem/mfem/issues) before issuing a PR. In
 addition to technical contributions, we are also interested in your results and
-[simulation images](https://mfem.org/gallery/), which you can share via a pull
+[simulation images](http://mfem.org/gallery/), which you can share via a pull
 request in the [mfem/web](https://github.com/mfem/web) repo.
 
 See the [Quick Summary](#quick-summary) section for the main highlights of our
@@ -51,7 +51,7 @@ back to them before issuing pull requests:
 Contributing to MFEM requires knowledge of Git and, likely, finite elements. If
 you are new to Git, see the [GitHub learning
 resources](https://help.github.com/articles/git-and-github-learning-resources/).
-To learn more about the finite element method, see our [FEM page](https://mfem.org/fem).
+To learn more about the finite element method, see our [FEM page](http://mfem.org/fem).
 
 *By submitting a pull request, you are affirming the [Developer's Certificate of
 Origin](#developers-certificate-of-origin-11) at the end of this file.*
@@ -92,25 +92,21 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   └── cmake
-  │       └── ...
+  │       └── modules
   ├── data
   ├── doc
+  │   └── web
+  │       └── examples
   ├── examples
-  │   ├── amgx
-  │   ├── caliper
   │   ├── ginkgo
   │   ├── hiop
   │   ├── petsc
   │   ├── pumi
-  │   ├── sundials
-  |   └── superlu
+  │   └── sundials
   ├── fem
-  │   ├── ceed
-  │   ├── qinterp
-  │   └── tmop
+  │   └── libceed
   ├── general
   ├── linalg
-  │   └── simd
   ├── mesh
   ├── miniapps
   │   ├── adjoint
@@ -118,21 +114,15 @@ The MFEM source code has the following structure:
   │   ├── electromagnetics
   │   ├── gslib
   │   ├── meshing
-  │   ├── mtop
-  │   ├── navier
   │   ├── nurbs
   │   ├── performance
-  │   ├── shifted
-  │   ├── solvers
   │   ├── tools
   │   └── toys
   └── tests
-      ├── convergence
-      ├── gitlab
-      ├── par-mesh-format
       ├── scripts
-      └── unit
-          └── ...
+      ├── unit
+      │   ├── ...
+      └── ...
 ```
 
 #### Main directories and classes
@@ -142,24 +132,24 @@ classes implementing the finite element, mesh and linear algebra concepts
 respectively.
 
 - The main mesh classes are:
-  + [`Mesh`](https://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
-  + [`NCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
-  + [`Element`](https://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
-  + [`ElementTransformation`](https://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
+  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
 
 - The main finite element classes are:
-  + [`FiniteElement`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
-  + [`GridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
-  + [`BilinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
+  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
 
 - The main linear algebra classes and sources are
-  + [`Operator`](https://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
-  + [`DenseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
-  + Sparse [smoothers](https://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](https://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
 
 #### Parallel implementation
 
@@ -169,13 +159,13 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 
 - The main parallel classes are
-  + [`ParMesh`](https://mfem.github.io/doxygen/html/solvers_8hpp.html)
-  + [`ParNCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
-  + [`ParFiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
-  + [`ParGridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
-  + [`ParBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
-  + [`HypreParMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
-  + [`HypreSolver`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://mfem.github.io/doxygen/html/hypre_8hpp.html)
+  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
 
 #### GPU and general device support
 
@@ -184,10 +174,10 @@ backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.
 
 - The main device-relevant classes and sources are:
-  + [`Device`](https://mfem.github.io/doxygen/html/device_8hpp.html)
-  + [`MemoryManager`](https://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](https://mfem.github.io/doxygen/html/forall_8hpp.html) macro
-  + the [`cuda.hpp`](https://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](https://mfem.github.io/doxygen/html/occa_8hpp.html) files
+  + [`Device`](http://mfem.github.io/doxygen/html/device_8hpp.html)
+  + [`MemoryManager`](http://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
+  + the [`MFEM_FORALL`](http://mfem.github.io/doxygen/html/forall_8hpp.html) macro
+  + the [`cuda.hpp`](http://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](http://mfem.github.io/doxygen/html/occa_8hpp.html) files
 
 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -196,7 +186,7 @@ device/host memory manager.
   Makefile and the CMake build options.
 - The `doc/` directory contains configuration for the Doxygen code documentation
   that can either be built locally or browsed online at
-  https://mfem.github.io/doxygen/html/index.html.
+  http://mfem.github.io/doxygen/html/index.html.
 
 #### Examples and tests
 - `examples` and `miniapps` respectively gather simple and more fully-featured
@@ -205,7 +195,7 @@ device/host memory manager.
 - The `tests/` directory contains a unit test suite and will later contain more
   tests that run example codes.
 
-See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
+See also the [code overview](http://mfem.org/code-overview/) section on the MFEM
 website.
 
 ## GitHub Workflow
@@ -370,16 +360,16 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   `mfem:next`, see the [README](tests/scripts/README) file in that directory
   for more details.
 
-- Track the Travis CI, Github Actions and Appveyor [continuous integration](#automated-testing)
+- Track the Travis CI and Appveyor [continuous integration](#automated-testing)
   builds at the end of the PR. These should generally run clean, so address any
   errors as soon as possible. Please ask if you are unsure how to do that.
 
-- Note that some tests, such as the `branch-history` check in Travis and Github
-  Actions are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in Travis are
+  safeguards that are allowed to fail in certain cases.
 
 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in Travis and Github Actions enforce MFEM-specific rules which are
-  explained in the error messages and the `tests/scripts` directory.
+  checks in Travis enforce MFEM-specific rules which are explained in the
+  error messages and the `tests/scripts` directory.
 
 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
   one of the _LLNL developers_ for details.
@@ -429,7 +419,6 @@ Before a PR can be merged, it should satisfy the following:
       - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
       - [ ] Consider adding a new test for the new miniapp.
    - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
    - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
    - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
      - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
@@ -446,6 +435,7 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] Run `make unittest` to make sure all unit tests pass.
 - [ ] Run the tests in `tests/scripts`.
 - [ ] (LLNL only) After merging:
+   - [ ] Regenerate `README.html` files from companion documentation pull requests.
    - [ ] Update internal tests to include the new features.
 
 ### Master/Next Workflow
@@ -518,10 +508,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
     - [ ] `.travis.yml`
     - [ ] `.appveyor.yml`
-- [ ] Update the `CHANGELOG` to organize all release contributions
-- [ ] Review the whole source code once over
-- [ ] Ask MFEM-based applications to test the pre-release branch
-- [ ] Test on additional platforms and compilers
+- [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:
 
   ```
@@ -532,13 +519,14 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Recreate the `next` branch as described in previous section.
 - [ ] Update and push documentation  to `mfem/doxygen`.
 - [ ] Update URL shortlinks:
-    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
+    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. http://mfem.github.io/releases/mfem-3.1.tgz.
     - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
     - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
 - [ ] Update website in `mfem/web` repo:
     - Update version and shortlinks in `src/index.md` and `src/download.md`.
     -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.
 
+
 ## LLNL Workflow
 
 ### Mirroring on Bitbucket
@@ -569,9 +557,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   Computing systems. Gitlab pipeline status is reported in the corresponding
   GitHub pull request.
 
-- In Gitlab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
-
-- No change to the MFEM repo can be made on this instance.
+- No change can be made on this instance.
 
 ## Automated Testing
 
@@ -579,13 +565,12 @@ MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.
 
 ### Linux and Mac smoke tests
-We use Travis CI and Github Actions to drive the default tests on the `master`
-and `next` branches. See the `.travis` file and the logs at
+We use Travis CI to drive the default tests on the `master` and `next`
+branches. See the `.travis` file and the logs at
 [https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).
 
-Testing using Travis CI and Github Actions should be kept lightweight, as there
-is a time constraint on jobs. Two virtual machines are configured - Mac (OS X)
-and Linux.
+Testing using Travis CI should be kept lightweight, as there is a 50 minute time
+constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
 
 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

INSTALL

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
@@ -19,10 +19,10 @@ requires an MPI C++ compiler, as well as the following external libraries:
   http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
 
 The hypre dependency can be downloaded as a tarball from GitHub or from the
-project webpage https://www.llnl.gov/casc/hypre. For example, the 2.20.0 release
+project webpage https://www.llnl.gov/casc/hypre. For example, the 2.16.0 release
 of hypre is available at
 
-   https://github.com/hypre-space/hypre/archive/v2.20.0.tar.gz
+   https://github.com/hypre-space/hypre/archive/v2.16.0.tar.gz
 
 The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.
@@ -58,12 +58,11 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
 
 We also recommend downloading and building the MFEM-based GLVis visualization
 tool which can be used to visualize the meshes and solution in MFEM's examples
-and miniapps. See https://glvis.org and https://mfem.org/building.
+and miniapps. See http://glvis.org and http://mfem.org/building.
 
 Quick start with GNU make
 =========================
@@ -345,15 +344,6 @@ MFEM_USE_SUPERLU = YES/NO
    SuperLURowLocMatrix a distributed CSR matrix class needed by SuperLU. When
    enabled, this option uses the SUPERLU_* library options, see below.
 
-MFEM_USE_SUPERLU5 = YES/NO
-   If SuperLU functionality is enabled, use the older 5.1.0 version rather than
-   the more recent 6+ versions.
-
-MFEM_USE_MUMPS = YES/NO
-   Enable MFEM functionality based on the MUMPS library. Currently, this
-   option adds the class MUMPSSolver (a parallel sparse direct solver).
-   When enabled, this option uses the MUMPS_* library options, see below.
-
 MFEM_USE_STRUMPACK = YES/NO
    Enable MFEM functionality based on the STRUMPACK sparse direct solver and
    preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
@@ -366,11 +356,6 @@ MFEM_USE_GINKGO = YES/NO
    https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
    Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.
 
-MFEM_USE_AMGX = YES/NO
-   Enable MFEM functionality based on the AmgX multigrid library from NVIDIA.
-   Allows the user to use SparseMatrices and HypreParMatrices to solve linear
-   systems with the routines from the AmgX library.
-
 MFEM_USE_GNUTLS = YES/NO
    Enable secure socket support in class socketstream, using the auxiliary
    GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -460,8 +445,8 @@ MFEM_USE_UMPIRE = YES/NO
    memory devices like NUMA and GPUs.
 
 MFEM_USE_HIOP = YES/NO
-   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
-   HPC solver for nonlinear optimization problems.
+  Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
+  HPC solver for nonlinear optimization problems.
 
 MFEM_USE_CUDA = YES/NO
    Enables support for CUDA devices in MFEM. CUDA is a parallel computing
@@ -501,21 +486,6 @@ MFEM_USE_CEED = YES/NO
    library for performant high-order operator evaluation developed by the Center
    for Efficient Exascale Discretizations in the Exascale Computing Project.
 
-MFEM_USE_MKL_CPARDISO = YES/NO
-   Enables the interface to MKL CPardiso: the Intel MKL Parallel Direct Sparse
-   Solver for Clusters. Make sure to set the correct values for MKL_MPI_WRAPPER
-   and MKL_LIBRARY_SUBDIR as shown in defaults.mk. If you configure MFEM with
-   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
-   OpenMP capabilities are disabled at link time.
-
-MFEM_USE_CALIPER = YES/NO
-   Enables the interface to Caliper. Caliper is a library to integrate
-   performance profiling capabilities into applications. To use Caliper,
-   developers mark code regions of interest using either Caliper's annotation
-   API or their equivalent in MFEM. Applications can then enable performance
-   profiling at runtime with Caliper's configuration API. Alternatively, one
-   can configure Caliper through environment variables or config files.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -537,11 +507,9 @@ directory and use the string @MFEM_DIR@, e.g. HYPRE_OPT = -I@MFEM_DIR@/../hypre.
 The specific libraries and their options are:
 
 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
-  See also the "Specific options for hypre" section at the end of this file.
   URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b,
-            HYPRE >= 2.20.0 for '--enable-mixedint' support.
+  Versions: HYPRE >= 2.10.0b.
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -567,19 +535,15 @@ The specific libraries and their options are:
   Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
   Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
   Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
-  When MFEM_USE_CUDA is enabled, only SUNDIALS v5.4.0+ is supported.
   If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
   both MPI and hypre.
-  If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
-  for CUDA.
   URL: http://computation.llnl.gov/projects/sundials/sundials-software
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.
+  Versions: SUNDIALS >= 5.0.0.
 
 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
   URL: http://trilinos.org/oldsite/packages/mesquite
   Options: MESQUITE_OPT, MESQUITE_LIB.
-  The Mesquite support is deprecated and will be removed in the future.
 
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
@@ -589,17 +553,11 @@ The specific libraries and their options are:
 - SuperLU_DIST (optional), used when MFEM_USE_SUPERLU = YES. Note that
   SuperLU_DIST requires ParMETIS, which includes METIS 5 in its distribution.
   Both ParMETIS and the included METIS 5 should be built and installed in the
-  same location. If using SuperLU_Dist v5, set MFEM_USE_SUPERLU5=YES.
+  same location.
   URL: http://crd-legacy.lbl.gov/~xiaoye/SuperLU
   Options: SUPERLU_OPT, SUPERLU_LIB.
   Versions: SuperLU_DIST >= 5.1.0.
 
-- MUMPS (optional), used when MFEM_USE_MUMPS = YES. Note that MUMPS
-  requires LAPACK, SCALAPACK and a reordering package such as PORD or METIS.
-  URL: http://mumps.enseeiht.fr
-  Options: MUMPS_OPT, MUMPS_LIB.
-  Versions: MUMPS >= 5.1.1
-
 - STRUMPACK (optional), used when MFEM_USE_STRUMPACK = YES. Note that STRUMPACK
   requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
   includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
@@ -608,19 +566,13 @@ The specific libraries and their options are:
   2.0.0 or later.
   URL: http://portal.nersc.gov/project/sparse/strumpack
   Options: STRUMPACK_OPT, STRUMPACK_LIB.
-  Versions: STRUMPACK >= 3.0.0.
+  Versions: STRUMPACK >= 3.0.0, requires HYPRE < 2.16.0.
 
 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-14 standard. For additional requirements
-  and dependencies of specific modules, see the Ginkgo webpage below.
+  C++ compiler that supports the C++-11 standard. For additional requirements
+  and dependencies of specific modules see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0. 
-
-- AmgX (optional), used when MFEM_USE_AMGX = YES.
-  URL: https://github.com/NVIDIA/AMGX
-  Options: AMGX_OPT, AMGX_LIB.
-  Versions: AmgX >= 2.1, older versions may work too.
+  Options: GINKGO_OPT (Not used), GINKGO_LIB.
 
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
   GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
@@ -647,8 +599,7 @@ The specific libraries and their options are:
                 --with-shared-libraries=0
   URL: https://www.mcs.anl.gov/petsc
   Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA)
-            PETSc >= 3.15.0  (PETSc built with CUDA)
+  Versions: PETSc >= 3.8.0.
 
 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
   uses some of the PETSc options when compiled.
@@ -673,38 +624,31 @@ The specific libraries and their options are:
 
 - ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
   URL: https://adios2.readthedocs.io/
-  Versions: ADIOS >= 2.5.0.
 
 - PUMI (optional), used when MFEM_USE_PUMI = YES.
   URL: https://scorec.rpi.edu/pumi
        https://github.com/SCOREC/core
   Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI == 2.2.3.
+  Versions: PUMI >= 2.2.3.
 
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
   Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.
 
 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
-  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
-  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
+  built prior to the MFEM build, as follows: download gslib-1.0.5, untar it at
+  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.5 gslib".
   Build gslib in parallel or in serial based on the desired MFEM build: "make
   clean; make CC=mpicc" or "make clean; make CC=gcc MPI=0". Build MFEM with
   MFEM_USE_GSLIB=YES.
-  URL: https://github.com/gslib/gslib/archive/v1.0.7.tar.gz
+  URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
   Options: GSLIB_OPT, GSLIB_LIB.
-  Versions: GSLIB >= 1.0.7.
-
-- MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
-  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
-  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
-  Versions: Intel MKL >= 2020.
+  Versions: GSLIB >= 1.0.5.
 
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
   URL: https://developer.nvidia.com/cuda-toolkit
   Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
-  Versions: CUDA >= 10.1.168.
+  Versions: CUDA >= 9.1, older versions may work too.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
   URL: https://rocm.github.io/ROCmInstall.html
@@ -713,27 +657,21 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
   URL: https://libocca.org
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.1.0.
+  Versions: OCCA >= 1.0.9.
 
 - libCEED (optional), used when MFEM_USE_CEED = YES.
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.8.
+  Versions: libCEED >= 0.6, git-hash a970f63.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
+  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
   URL: https://github.com/LLNL/RAJA
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.13.0.
-
-- Caliper (optional), used when MFEM_USE_CALIPER = YES.
-  URL: https://github.com/LLNL/Caliper
-  Options: CALIPER_DIR
-  Versions: CALIPER >= 2.5.0, older versions may work too.
+  Versions: RAJA >= 0.10.0.
 
 - Umpire, used when MFEM_USE_UMPIRE = YES.
-  Umpire requires camp when the Umpire version is >= 3.0.0.
   URL: https://github.com/LLNL/Umpire
   Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
   Versions: Umpire >= 2.0.0.
@@ -867,10 +805,8 @@ MFEM_TIMER_TYPE - Set automatically, can be overwritten.
 MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
-MFEM_USE_MUMPS
 MFEM_USE_STRUMPACK
 MFEM_USE_GINKGO
-MFEM_USE_AMGX
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
@@ -883,7 +819,6 @@ MFEM_USE_CEED
 MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
-MFEM_USE_CALIPER
 
 The following options are CMake specific:
 
@@ -920,12 +855,11 @@ The CMake build system adds auto-detection for the following packages/libraries:
 
  - HYPRE
  - METIS - The option MFEM_USE_METIS_5 is auto-detected.
- - ParMETIS
  - MESQUITE
  - SuiteSparse
  - SuperLUDist, STRUMPACK
+ - ParMETIS
  - Ginkgo
- - AMGX
  - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
  - NETCDF
  - MPFR
@@ -937,7 +871,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - RAJA
  - UMPIRE
  - AXOM - Used when MFEM_USE_SIDRE is enabled
- - CALIPER
 
 The following built-in CMake packages are also used:
 
@@ -973,19 +906,3 @@ MFEM_MPIEXEC    = mpirun # default
 MFEM_MPIEXEC_NP = -np    # default
 MFEM_MPIEXEC    = srun   # example for platforms using SLURM
 MFEM_MPIEXEC_NP = -n     # example for platforms using SLURM
-
-
-Specific options for hypre
-==========================
-The hypre library has multiple options to define local and global index storage
-sizes. By default, all indices are stored as an architecture aware integer. For
-most platforms, this will be 32-bit. This limits the maximum number of global
-degrees of freedom in a vector or matrix to about 2 billion. In order to solve
-larger problems, there are two options:
-
-1. Building hypre with '--enable-bigint' defines the local and global indices to
-   be 64-bit. This is convenient, but requires more memory than necessary.
-
-2. Building hypre with '--enable-mixedint' defines the local indiced to be
-   32-bit, while using a 64-bit storage for global indices. This option is
-   currently tested only in ex1p, and may not work in more general settings.

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

NOTICE

@@ -25,9 +25,7 @@ This project distributes the sources of several external software products with
 their own respective licenses which can be found in their code and attached
 license files. These software products and their licenses are as follows:
 
-* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
-* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
-* TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
+* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
 * Zstr (general/zstr.hpp) -- MIT license

README

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 MFEM is a modular parallel C++ library for finite element methods. Its goal is
 to enable high-performance scalable finite element discretization research and
@@ -16,8 +16,8 @@ supercomputers.
 
 * Copyright and licensing information can be found in files LICENSE and NOTICE.
 
-* The best starting point for new users interested in MFEM's features is to
-  review the examples and miniapps at https://mfem.org/examples.
+* The best starting point for new users interested in MFEM's features is the
+  interactive documentation in examples/README.html.
 
 * Developers interested in contributing to the library, should read the
   instructions and documentation in the CONTRIBUTING.md file.
@@ -49,17 +49,17 @@ library. Nonlinear solvers (the Newton method), eigensolvers (LOBPCG), and
 several explicit and implicit Runge-Kutta time integrators are also available.
 
 MFEM supports MPI-based parallelism throughout the library, and can readily be
-used as a scalable unstructured finite element problem generator. Starting with
-version 4.0, MFEM offers support for GPU acceleration, and programming models,
-such as CUDA, HIP, OCCA, RAJA and OpenMP. MFEM-based applications require
-minimal changes to switch from a serial to a highly-performant MPI-parallel
-version of the code, where they can take advantage of the integrated linear
-solvers from the hypre library. Comprehensive support for other external
-packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
-additional linear and nonlinear solvers, preconditioners, time integrators, etc.
+used as a scalable unstructured finite element problem generator. As of version
+4.0, MFEM offers initial support for GPU acceleration, and programming models,
+such as CUDA, OCCA, RAJA and OpenMP. MFEM-based applications require minimal
+changes to switch from a serial to a high-performing MPI-parallel version of the
+code, where they can take advantage of the integrated linear solvers from the
+hypre library.  Comprehensive support for other external packages, e.g. PETSc
+and SUNDIALS is also included, giving access to many additional linear and
+nonlinear solvers, preconditioners, time integrators, etc.
 
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
-as the OpenGL visualization tool GLVis which is available at https://glvis.org.
+as the OpenGL visualization tool GLVis which is available at http://glvis.org.
 
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license. See LICENSE and NOTICE for details.

config/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/MFEMConfig.cmake.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -32,11 +32,8 @@ set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
 set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
-set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
-set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
-set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
@@ -53,7 +50,6 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -74,9 +74,6 @@
 // [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 #cmakedefine MFEM_USE_LEGACY_OPENMP
 
-// Internal MFEM option: enable group/batch allocation for some small objects.
-#cmakedefine MFEM_USE_MEMALLOC
-
 // Enable MFEM functionality based on the Mesquite library.
 #cmakedefine MFEM_USE_MESQUITE
 
@@ -86,18 +83,15 @@
 // Enable MFEM functionality based on the SuperLU_DIST library.
 #cmakedefine MFEM_USE_SUPERLU
 
-// Enable MFEM functionality based on the MUMPS library.
-#cmakedefine MFEM_USE_MUMPS
-
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK
 
+// Internal MFEM option: enable group/batch allocation for some small objects.
+#cmakedefine MFEM_USE_MEMALLOC
+
 // Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO
 
-// Enable MFEM functionality based on the AmgX library
-#cmakedefine MFEM_USE_AMGX
-
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS
 
@@ -132,10 +126,6 @@
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
 
-// Build the HIP-enabled version of the MFEM library.
-// Requires a HIP compiler (hipcc).
-#cmakedefine MFEM_USE_HIP
-
 // Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA
 
@@ -151,9 +141,6 @@
 // Enable MFEM functionality based on the ADIOS2 library
 #cmakedefine MFEM_USE_ADIOS2
 
-// Enable MFEM functionality based on the Caliper library
-#cmakedefine MFEM_USE_CALIPER
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -169,7 +156,4 @@
 // library.
 #cmakedefine MFEM_USE_SIMMETRIX
 
-// Enable interface to the MKL CPardiso library.
-#cmakedefine MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER

config/cmake/modules/FindADIOS2.cmake

@@ -38,19 +38,7 @@ if(NOT ADIOS2_FOUND)
   endif()
 
   find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-
-  # adios2 version 2.5.0
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
-
-  # adios2 version 2.6.0 and onwards
-  if(NOT ADIOS2_LIBRARY)
-	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
-	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
-	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
-	if(MFEM_USE_MPI)
-	  add_definitions(-DADIOS2_USE_MPI)
-	endif()
-  endif()
+  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 
 
   include(FindPackageHandleStandardArgs)
   find_package_handle_standard_args(ADIOS2

config/cmake/modules/FindAMGX.cmake

@@ -1,24 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - AMGX_FOUND
-#   - AMGX_LIBRARIES
-#   - AMGX_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas cublasLt nvToolsExt)
-mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
-  "Paths to headers required by AMGX." "Libraries required by AMGX.")
-# Make sure the library location is locked down
-foreach(lib ${AMGX_REQUIRED_LIBRARIES})
-  list(APPEND AMGX_LIBRARIES ${CUDA_TOOLKIT_ROOT_DIR}/lib64/lib${lib}${CMAKE_SHARED_LIBRARY_SUFFIX})
-endforeach()

config/cmake/modules/FindAxom.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCXXABIDemangle.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCaliper.cmake

@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - CALIPER_FOUND
-#   - CALIPER_LIBRARIES
-#   - CALIPER_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Caliper CALIPER CALIPER_DIR
-       	"include" "caliper/cali.h"
-       	"lib" "caliper"
-  	"Paths to headers required by Caliper."
-       	"Libraries required by Caliper.")

config/cmake/modules/FindConduit.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindGSLIB.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIOP.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIP.cmake

@@ -1,692 +0,0 @@
-###############################################################################
-# FindHIP.cmake
-###############################################################################
-include(CheckCXXCompilerFlag)
-###############################################################################
-# SET: Variable defaults
-###############################################################################
-# User defined flags
-set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
-set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
-set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
-set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
-mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
-
-set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
-list(REMOVE_DUPLICATES _hip_configuration_types)
-foreach(config ${_hip_configuration_types})
-    string(TOUPPER ${config} config_upper)
-    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
-    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
-    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
-    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
-    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
-endforeach()
-option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
-option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
-mark_as_advanced(HIP_HOST_COMPILATION_CPP)
-
-###############################################################################
-# FIND: HIP and associated helper binaries
-###############################################################################
-
-get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
-
-# HIP is supported on Linux only
-if(UNIX AND NOT APPLE AND NOT CYGWIN)
-    # Search for HIP installation
-    if(NOT HIP_ROOT_DIR)
-        # Search in user specified path first
-        find_path(
-            HIP_ROOT_DIR
-            NAMES bin/hipconfig
-            PATHS
-            "$ENV{ROCM_PATH}/hip"
-            ENV HIP_PATH
-            ${_IMPORT_PREFIX}
-            /opt/rocm/hip
-            DOC "HIP installed location"
-            NO_DEFAULT_PATH
-            )
-        if(NOT EXISTS ${HIP_ROOT_DIR})
-            if(HIP_FIND_REQUIRED)
-                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
-            elseif(NOT HIP_FIND_QUIETLY)
-                message("HIP_ROOT_DIR not found or specified")
-            endif()
-        endif()
-        # And push it back to the cache
-        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
-    endif()
-
-    # Find HIPCC executable
-    find_program(
-        HIP_HIPCC_EXECUTABLE
-        NAMES hipcc
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCC_EXECUTABLE hipcc)
-    endif()
-    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
-
-    # Find HIPCONFIG executable
-    find_program(
-        HIP_HIPCONFIG_EXECUTABLE
-        NAMES hipconfig
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCONFIG_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
-    endif()
-    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
-
-    # Find HIPCC_CMAKE_LINKER_HELPER executable
-    find_program(
-        HIP_HIPCC_CMAKE_LINKER_HELPER
-        NAMES hipcc_cmake_linker_helper
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
-        # Now search in default paths
-        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
-    endif()
-    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
-        # Compute the version
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
-            OUTPUT_VARIABLE _hip_version
-            ERROR_VARIABLE _hip_error
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE
-            )
-        if(NOT _hip_error)
-            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
-        else()
-            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
-        endif()
-        mark_as_advanced(HIP_VERSION)
-    endif()
-    if(HIP_VERSION)
-        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
-        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
-        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
-        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
-        set(HIP_VERSION_STRING "${HIP_VERSION}")
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
-        # Compute the platform
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
-            OUTPUT_VARIABLE _hip_platform
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
-        mark_as_advanced(HIP_PLATFORM)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
-        # Compute the compiler
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
-            OUTPUT_VARIABLE _hip_compiler
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
-        mark_as_advanced(HIP_COMPILER)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
-        # Compute the runtime
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
-            OUTPUT_VARIABLE _hip_runtime
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
-        mark_as_advanced(HIP_RUNTIME)
-    endif()
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(
-    HIP
-    REQUIRED_VARS
-    HIP_ROOT_DIR
-    HIP_HIPCC_EXECUTABLE
-    HIP_HIPCONFIG_EXECUTABLE
-    HIP_PLATFORM
-    HIP_COMPILER
-    HIP_RUNTIME
-    VERSION_VAR HIP_VERSION
-    )
-
-###############################################################################
-# Set HIP CMAKE Flags
-###############################################################################
-# Copy the invocation styles from CXX to HIP
-set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
-set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
-set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
-set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
-#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
-set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
-
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
-set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
-
-if("${HIP_COMPILER}" STREQUAL "nvcc")
-    # Set the CMake Flags to use the nvcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "hcc")
-    # Set the CMake Flags to use the hcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "clang")
-    #Number of parallel jobs by default is 1
-    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
-      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
-    endif()
-    #Add support for parallel build and link
-    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
-      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
-    endif()
-    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
-      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
-        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-      else()
-        message("clang compiler doesn't support parallel jobs")
-      endif()
-    endif()
-
-    # Set the CMake Flags to use the HIP-Clang Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-
-    if("${HIP_RUNTIME}" STREQUAL "rocclr")
-      if(TARGET host)
-        message(STATUS "host interface - found")
-        set(HIP_HOST_INTERFACE host)
-      endif()
-    endif()
-endif()
-
-###############################################################################
-# MACRO: Locate helper files
-###############################################################################
-macro(HIP_FIND_HELPER_FILE _name _extension)
-    set(_hip_full_name "${_name}.${_extension}")
-    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
-    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
-    if(NOT EXISTS "${HIP_${_name}}")
-        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
-        if(HIP_FIND_REQUIRED)
-            message(FATAL_ERROR "${error_message}")
-        else()
-            if(NOT HIP_FIND_QUIETLY)
-                message(STATUS "${error_message}")
-            endif()
-        endif()
-    endif()
-    # Set this variable as internal, so the user isn't bugged with it.
-    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
-endmacro()
-
-###############################################################################
-hip_find_helper_file(run_make2cmake cmake)
-hip_find_helper_file(run_hipcc cmake)
-###############################################################################
-
-###############################################################################
-# MACRO: Reset compiler flags
-###############################################################################
-macro(HIP_RESET_FLAGS)
-    unset(HIP_HIPCC_FLAGS)
-    unset(HIP_HCC_FLAGS)
-    unset(HIP_CLANG_FLAGS)
-    unset(HIP_NVCC_FLAGS)
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        unset(HIP_HIPCC_FLAGS_${config_upper})
-        unset(HIP_HCC_FLAGS_${config_upper})
-        unset(HIP_CLANG_FLAGS_${config_upper})
-        unset(HIP_NVCC_FLAGS_${config_upper})
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Separate the options from the sources
-###############################################################################
-macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
-    set(${_sources})
-    set(${_cmake_options})
-    set(${_hipcc_options})
-    set(${_hcc_options})
-    set(${_clang_options})
-    set(${_nvcc_options})
-    set(_hipcc_found_options FALSE)
-    set(_hcc_found_options FALSE)
-    set(_clang_found_options FALSE)
-    set(_nvcc_found_options FALSE)
-    foreach(arg ${ARGN})
-        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
-            set(_hipcc_found_options TRUE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options TRUE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options TRUE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options TRUE)
-        elseif(
-                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
-                "x${arg}" STREQUAL "xSTATIC" OR
-                "x${arg}" STREQUAL "xSHARED" OR
-                "x${arg}" STREQUAL "xMODULE"
-                )
-            list(APPEND ${_cmake_options} ${arg})
-        else()
-            if(_hipcc_found_options)
-                list(APPEND ${_hipcc_options} ${arg})
-            elseif(_hcc_found_options)
-                list(APPEND ${_hcc_options} ${arg})
-            elseif(_clang_found_options)
-                list(APPEND ${_clang_options} ${arg})
-            elseif(_nvcc_found_options)
-                list(APPEND ${_nvcc_options} ${arg})
-            else()
-                # Assume this is a file
-                list(APPEND ${_sources} ${arg})
-            endif()
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Add include directories to pass to the hipcc command
-###############################################################################
-set(HIP_HIPCC_INCLUDE_ARGS_USER "")
-macro(HIP_INCLUDE_DIRECTORIES)
-    foreach(dir ${ARGN})
-        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
-    endforeach()
-endmacro()
-
-###############################################################################
-# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
-###############################################################################
-function(HIP_COMPUTE_BUILD_PATH path build_path)
-    # Convert to cmake style paths
-    file(TO_CMAKE_PATH "${path}" bpath)
-    if(IS_ABSOLUTE "${bpath}")
-        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
-        if(_binary_dir_pos EQUAL 0)
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
-        else()
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
-        endif()
-    endif()
-
-    # Remove leading /
-    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
-    # Avoid absolute paths by removing ':'
-    string(REPLACE ":" "_" bpath "${bpath}")
-    # Avoid relative paths that go up the tree
-    string(REPLACE "../" "__/" bpath "${bpath}")
-    # Avoid spaces
-    string(REPLACE " " "_" bpath "${bpath}")
-    # Strip off the filename
-    get_filename_component(bpath "${bpath}" PATH)
-
-    set(${build_path} "${bpath}" PARENT_SCOPE)
-endfunction()
-
-###############################################################################
-# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
-###############################################################################
-macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
-    set(_hip_found_config)
-    foreach(arg ${ARGN})
-        # Determine if we are dealing with a per-configuration flag
-        foreach(config ${_hip_configuration_types})
-            string(TOUPPER ${config} config_upper)
-            if(arg STREQUAL "${config_upper}")
-                set(_hip_found_config _${arg})
-                # Clear arg to prevent it from being processed anymore
-                set(arg)
-            endif()
-        endforeach()
-        if(arg)
-            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Try and include dependency file if it exists
-###############################################################################
-macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
-    set(HIP_HIPCC_DEPEND)
-    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
-
-    # Create the dependency file if it doesn't exist
-    if(NOT EXISTS ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-    # Include the dependency file
-    include(${dependency_file})
-
-    # Verify the existence of all the included files
-    if(HIP_HIPCC_DEPEND)
-        foreach(f ${HIP_HIPCC_DEPEND})
-            if(NOT EXISTS ${f})
-                # If they aren't there, regenerate the file again
-                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-            endif()
-        endforeach()
-    else()
-        # No dependencies, so regenerate the file
-        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-    endif()
-
-    # Regenerate the dependency file if needed
-    if(HIP_HIPCC_DEPEND_REGENERATE)
-        set(HIP_HIPCC_DEPEND ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-endmacro()
-
-###############################################################################
-# MACRO: Prepare cmake commands for the target
-###############################################################################
-macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
-    set(_hip_flags "")
-    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
-    if(HIP_HOST_COMPILATION_CPP)
-        set(HIP_C_OR_CXX CXX)
-    else()
-        set(HIP_C_OR_CXX C)
-    endif()
-    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
-
-    # Initialize list of includes with those specified by the user. Append with
-    # ones specified to cmake directly.
-    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
-
-    # Add the include directories
-    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
-    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
-
-    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
-    list(REMOVE_DUPLICATES _hip_include_directories)
-    if(_hip_include_directories)
-        foreach(dir ${_hip_include_directories})
-            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
-        endforeach()
-    endif()
-
-    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
-
-    # Add the compile definitions
-    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
-    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
-
-    # Check if we are building shared library.
-    set(_hip_build_shared_libs FALSE)
-    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
-    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
-    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
-        set(_hip_build_shared_libs TRUE)
-    endif()
-    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
-    if(_hip_found_STATIC GREATER -1)
-        set(_hip_build_shared_libs FALSE)
-    endif()
-
-    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
-    if(_hip_build_shared_libs)
-        list(APPEND HIP_HCC_FLAGS "-fPIC")
-        list(APPEND HIP_CLANG_FLAGS "-fPIC")
-        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
-    endif()
-
-    # Set host compiler
-    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
-
-    # Set compiler flags
-    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
-    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
-    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
-    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
-    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
-        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
-        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
-        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
-        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
-    endforeach()
-
-    # Reset the output variable
-    set(_hip_generated_files "")
-    set(_hip_source_files "")
-
-    # Iterate over all arguments and create custom commands for all source files
-    foreach(file ${ARGN})
-        # Ignore any file marked as a HEADER_FILE_ONLY
-        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
-        # Allow per source file overrides of the format. Also allows compiling non .cu files.
-        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
-        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
-            set(host_flag FALSE)
-        else()
-            set(host_flag TRUE)
-        endif()
-
-        if(NOT host_flag)
-            # Determine output directory
-            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
-            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
-
-            get_filename_component(basename ${file} NAME)
-            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
-            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
-
-            # Set file names
-            set(generated_file "${generated_file_path}/${generated_file_basename}")
-            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
-            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
-            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
-
-            # Set properties for object files
-            set_source_files_properties("${generated_file}"
-                PROPERTIES
-                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
-                )
-
-            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
-            get_filename_component(file_path "${file}" PATH)
-            if(IS_ABSOLUTE "${file_path}")
-                set(source_file "${file}")
-            else()
-                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
-            endif()
-
-            # Bring in the dependencies
-            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
-
-            # Configure the build script
-            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
-            file(GENERATE
-                OUTPUT "${custom_target_script}"
-                INPUT "${custom_target_script_pregen}"
-                )
-            set(main_dep DEPENDS ${source_file})
-            if(CMAKE_GENERATOR MATCHES "Makefiles")
-                set(verbose_output "$(VERBOSE)")
-            elseif(HIP_VERBOSE_BUILD)
-                set(verbose_output ON)
-            else()
-                set(verbose_output OFF)
-            endif()
-
-            # Create up the comment string
-            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
-            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
-
-            # Build the generated file and dependency file
-            add_custom_command(
-                OUTPUT ${generated_file}
-                # These output files depend on the source_file and the contents of cmake_dependency_file
-                ${main_dep}
-                DEPENDS ${HIP_HIPCC_DEPEND}
-                DEPENDS ${custom_target_script}
-                # Make sure the output directory exists before trying to write to it.
-                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
-                COMMAND ${CMAKE_COMMAND} ARGS
-                -D verbose:BOOL=${verbose_output}
-                -D build_configuration:STRING=${_hip_build_configuration}
-                -D "generated_file:STRING=${generated_file}"
-                -P "${custom_target_script}"
-                WORKING_DIRECTORY "${hip_compile_output_dir}"
-                COMMENT "${hip_build_comment_string}"
-                )
-
-            # Make sure the build system knows the file is generated
-            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
-            list(APPEND _hip_generated_files ${generated_file})
-            list(APPEND _hip_source_files ${file})
-        endif()
-    endforeach()
-
-    # Set the return parameter
-    set(${_generated_files} ${_hip_generated_files})
-    set(${_source_files} ${_hip_source_files})
-endmacro()
-
-###############################################################################
-# HIP_ADD_EXECUTABLE
-###############################################################################
-macro(HIP_ADD_EXECUTABLE hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if("${HIP_COMPILER}" STREQUAL "hcc")
-        if("x${HCC_HOME}" STREQUAL "x")
-            if (DEFINED ENV{ROCM_PATH})
-                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
-            else()
-                set(HCC_HOME "/opt/rocm/hcc")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    elseif("${HIP_COMPILER}" STREQUAL "clang")
-        if("x${HIP_CLANG_PATH}" STREQUAL "x")
-            if(DEFINED ENV{HIP_CLANG_PATH})
-                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
-            elseif(DEFINED ENV{ROCM_PATH})
-                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
-            else()
-                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    else()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-###############################################################################
-# HIP_ADD_LIBRARY
-###############################################################################
-macro(HIP_ADD_LIBRARY hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_hipcc.cmake

@@ -1,182 +0,0 @@
-###############################################################################
-# Runs commands using HIPCC
-###############################################################################
-
-###############################################################################
-# This file runs the hipcc commands to produce the desired output file
-# along with the dependency file needed by CMake to compute dependencies.
-#
-# Input variables:
-#
-# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
-#                               ON : Describe each step
-# build_configuration:STRING=<> Build configuration. Defaults to Debug.
-# generated_file:STRING=<>      File to generate. Mandatory argument.
-
-if(NOT build_configuration)
-    set(build_configuration Debug)
-endif()
-if(NOT generated_file)
-    message(FATAL_ERROR "You must specify generated_file on the command line")
-endif()
-
-# Set these up as variables to make reading the generated file easier
-set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
-set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
-set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
-set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
-set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
-set(HCC_HOME "@HCC_HOME@") #path
-set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
-
-@HIP_HOST_FLAGS@
-@_HIP_HIPCC_FLAGS@
-@_HIP_HCC_FLAGS@
-@_HIP_CLANG_FLAGS@
-@_HIP_NVCC_FLAGS@
-#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
-set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
-
-set(cmake_dependency_file "@cmake_dependency_file@") # path
-set(source_file "@source_file@") # path
-set(host_flag "@host_flag@") # bool
-
-# Determine compiler and compiler flags
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
-if(NOT host_flag)
-    set(__CC ${HIP_HIPCC_EXECUTABLE})
-    if("${HIP_PLATFORM}" STREQUAL "amd")
-        if("${HIP_COMPILER}" STREQUAL "hcc")
-            if(NOT "x${HCC_HOME}" STREQUAL "x")
-                set(ENV{HCC_HOME} ${HCC_HOME})
-            endif()
-            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
-        elseif("${HIP_COMPILER}" STREQUAL "clang")
-            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
-                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
-            endif()
-            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
-            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
-        endif()
-    else()
-        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
-    endif()
-else()
-    set(__CC ${HIP_HOST_COMPILER})
-    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
-endif()
-set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
-
-# hip_execute_process - Executes a command with optional command echo and status message.
-#   status     - Status message to print if verbose is true
-#   command    - COMMAND argument from the usual execute_process argument structure
-#   ARGN       - Remaining arguments are the command with arguments
-#   HIP_result - Return value from running the command
-macro(hip_execute_process status command)
-    set(_command ${command})
-    if(NOT "x${_command}" STREQUAL "xCOMMAND")
-        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
-    endif()
-    if(verbose)
-        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
-        # Build command string to print
-        set(hip_execute_process_string)
-        foreach(arg ${ARGN})
-            # Escape quotes if any
-            string(REPLACE "\"" "\\\"" arg ${arg})
-            # Surround args with spaces with quotes
-            if(arg MATCHES " ")
-                list(APPEND hip_execute_process_string "\"${arg}\"")
-            else()
-                list(APPEND hip_execute_process_string ${arg})
-            endif()
-        endforeach()
-        # Echo the command
-        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
-    endif()
-    # Run the command
-    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
-endmacro()
-
-# Delete the target file
-hip_execute_process(
-    "Removing ${generated_file}"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-    )
-
-# Generate the dependency file
-hip_execute_process(
-    "Generating dependency file: ${cmake_dependency_file}.pre"
-    COMMAND "${__CC}"
-    -M
-    "${source_file}"
-    -o "${cmake_dependency_file}.pre"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the cmake readable dependency file to a temp file
-hip_execute_process(
-    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
-    COMMAND "${CMAKE_COMMAND}"
-    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
-    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
-    -D "verbose=${verbose}"
-    -P "${HIP_run_make2cmake}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Copy the file if it is different
-hip_execute_process(
-    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
-    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Delete the temporary file
-hip_execute_process(
-    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the output file
-hip_execute_process(
-    "Generating ${generated_file}"
-    COMMAND "${__CC}"
-    -c
-    "${source_file}"
-    -o "${generated_file}"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    # Make sure that we delete the output file
-    hip_execute_process(
-        "Removing ${generated_file}"
-        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-        )
-    message(FATAL_ERROR "Error generating file ${generated_file}")
-else()
-    if(verbose)
-        message("Generated ${generated_file} successfully.")
-    endif()
-endif()
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_make2cmake.cmake

@@ -1,50 +0,0 @@
-###############################################################################
-# Computes dependencies using HIPCC
-###############################################################################
-
-###############################################################################
-# This file converts dependency files generated using hipcc to a format that
-# cmake can understand.
-
-# Input variables:
-#
-# input_file:STRING=<> Dependency file to parse. Required argument
-# output_file:STRING=<> Output file to generate. Required argument
-
-if(NOT input_file OR NOT output_file)
-    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
-endif()
-
-file(READ ${input_file} depend_text)
-
-if (NOT "${depend_text}" STREQUAL "")
-    string(REPLACE " /" "\n/" depend_text ${depend_text})
-    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
-    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
-
-    set(dependency_list "")
-
-    foreach(file ${depend_text})
-        string(REGEX REPLACE "^ +" "" file ${file})
-        if(NOT EXISTS "${file}")
-            message(WARNING " Removing non-existent dependency file: ${file}")
-            set(file "")
-        endif()
-
-        if(NOT IS_DIRECTORY "${file}")
-            get_filename_component(file_absolute "${file}" ABSOLUTE)
-            list(APPEND dependency_list "${file_absolute}")
-        endif()
-    endforeach()
-endif()
-
-# Remove the duplicate entries and sort them.
-list(REMOVE_DUPLICATES dependency_list)
-list(SORT dependency_list)
-
-foreach(file ${dependency_list})
-    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
-endforeach()
-
-file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHYPRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLIBDL.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLibunwind.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -25,16 +25,16 @@ mfem_find_package(METIS METIS METIS_DIR "include;Lib" "metis.h"
 
 int main()
 {
-    idx_t n = 10;
-    idx_t nparts = 5;
-    idx_t edgecut;
-    idx_t* partitioning = new idx_t[10];
-    idx_t* I = partitioning,
+    int n = 10;
+    int nparts = 5;
+    int edgecut;
+    int* partitioning = new int[10];
+    int* I = partitioning,
        * J = partitioning;
 
-    idx_t ncon = 1;
+    int ncon = 1;
     int err;
-    idx_t options[40];
+    int options[40];
 
     METIS_SetDefaultOptions(options);
     options[10] = 1; // set METIS_OPTION_CONTIG

config/cmake/modules/FindMFEMBacktrace.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMKL_CPARDISO.cmake

@@ -1,32 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MKL_CPARDISO_FOUND
-#   - MKL_CPARDISO_LIBRARIES
-#   - MKL_CPARDISO_INCLUDE_DIRS
-
-if(NOT MKL_MPI_WRAPPER_LIB)
-  message(FATAL_ERROR "MKL CPardiso enabled but no MKL MPI Wrapper lib specified")
-endif()
-
-if(NOT MKL_LIBRARY_DIR)
-  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
-  set(MKL_LIBRARY_DIR "lib")
-endif()
-
-include(MfemCmakeUtilities)
-mfem_find_package(MKL_CPARDISO MKL_CPARDISO
-    MKL_CPARDISO_DIR "include" mkl_cluster_sparse_solver.h ${MKL_LIBRARY_DIR} mkl_core
-  "Paths to headers required by MKL CPardiso." "Libraries required by MKL CPARDISO."
-  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
-  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential
-  ADD_COMPONENT MKL_MPI_WRAPPER "include" "" ${MKL_LIBRARY_DIR} ${MKL_MPI_WRAPPER_LIB})

config/cmake/modules/FindMPFR.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMUMPS.cmake

@@ -1,23 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Sets the following variables:
-#   - MUMPS_FOUND
-#   - MUMPS_INCLUDE_DIRS
-#   - MUMPS_LIBRARIES
-
-include(MfemCmakeUtilities)
-mfem_find_package(MUMPS MUMPS MUMPS_DIR
-  "include" dmumps_c.h "lib" dmumps
-  "Paths to headers required by MUMPS."
-  "Libraries required by MUMPS."
-  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
-  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)

config/cmake/modules/FindMesquite.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindNetCDF.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -15,20 +15,5 @@
 #   - NETCDF_INCLUDE_DIRS
 
 include(MfemCmakeUtilities)
-
-# FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-# We need to guard against the case where HDF5 was already found but without
-# the HL extensions (in which case mfem_find_package will treat the package
-# as already having been found), so we reset the variable to force FindHDF5.cmake
-# to be called for a second time
-set(HDF5_FOUND OFF)
-enable_language(C) # FindHDF5.cmake uses the C compiler
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
   "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-# The HL extension libraries are in a separate variable and must precede
-# the "regular" hdf5 library, as hdf5_hl depends on hdf5
-# The netcdf library will always be the first element of NETCDF_LIBRARIES
-# and we need to insert after that library but before the hdf5 library, so
-# position 1 is used
-list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})

config/cmake/modules/FindOCCA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindPOSIXClocks.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindParMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindRAJA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSLEPc.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSTRUMPACK.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSUNDIALS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -20,14 +20,10 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
   "Paths to headers required by SUNDIALS." "Libraries required by SUNDIALS."
   ADD_COMPONENT NVector_Serial
     "include" nvector/nvector_serial.h "lib" sundials_nvecserial
-  ADD_COMPONENT NVector_Cuda
-    "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_ParHyp
-    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
   ADD_COMPONENT NVector_Parallel
     "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
-  ADD_COMPONENT NVector_MPIPlusX
-    "include" nvector/nvector_mpiplusx.h "lib" sundials_nvecmpiplusx
+  ADD_COMPONENT NVector_ParHyp
+    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
   ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
   ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
   ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode

config/cmake/modules/FindScotch.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuiteSparse.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuperLUDist.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindUMPIRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/Find_GnuTLS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindlibCEED.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -43,25 +43,6 @@ function(convert_filenames_to_full_paths NAMES)
   set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()
 
-# Wrapper for add_executable that calls the HIP wrapper if applicable
-macro(mfem_add_executable NAME)
-  if (MFEM_USE_HIP)
-    hip_add_executable(${NAME} ${ARGN})
-    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
-  else()
-    add_executable(${NAME} ${ARGN})
-  endif()
-endmacro()
-
-# Wrapper for add_library that calls the HIP wrapper if applicable
-macro(mfem_add_library NAME)
-  if (MFEM_USE_HIP)
-    hip_add_library(${NAME} ${ARGN})
-  else()
-    add_library(${NAME} ${ARGN})
-  endif()
-endmacro()
-
 # Simple shortcut to add_custom_target() with option to add the target to the
 # main target.
 function(add_mfem_target TARGET_NAME ADD_TO_ALL)
@@ -75,7 +56,7 @@ function(add_mfem_target TARGET_NAME ADD_TO_ALL)
 endfunction()
 
 # Add mfem examples
-macro(add_mfem_examples EXE_SRCS)
+function(add_mfem_examples EXE_SRCS)
   set(EXE_PREFIX "")
   set(EXE_PREREQUISITE "")
   set(EXE_NEEDED_BY "")
@@ -91,15 +72,13 @@ macro(add_mfem_examples EXE_SRCS)
   foreach(SRC_FILE IN LISTS ${EXE_SRCS})
     # If CUDA is enabled, tag source files to be compiled with nvcc.
     if (MFEM_USE_CUDA)
-      set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
-    elseif(MFEM_USE_HIP)
-      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+      set_property(SOURCE ${SRC_FILE} PROPERTY LANGUAGE CUDA)
     endif()
 
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
 
     string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
-    mfem_add_executable(${EXE_NAME} ${SRC_FILE})
+    add_executable(${EXE_NAME} ${SRC_FILE})
     add_dependencies(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${EXE_NAME})
     if (EXE_NEEDED_BY)
       add_dependencies(${EXE_NEEDED_BY} ${EXE_NAME})
@@ -128,10 +107,10 @@ macro(add_mfem_examples EXE_SRCS)
       endif()
     endif()
   endforeach(SRC_FILE)
-endmacro()
+endfunction()
 
 # A slightly more versatile function for adding miniapps to MFEM
-macro(add_mfem_miniapp MFEM_EXE_NAME)
+function(add_mfem_miniapp MFEM_EXE_NAME)
   # Parse the input arguments looking for the things we need
   set(POSSIBLE_ARGS "MAIN" "EXTRA_SOURCES" "EXTRA_HEADERS" "EXTRA_OPTIONS" "EXTRA_DEFINES" "LIBRARIES")
   set(CURRENT_ARG)
@@ -147,23 +126,14 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
 
   # If CUDA is enabled, tag source files to be compiled with nvcc.
   if (MFEM_USE_CUDA)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES LANGUAGE CUDA)
-    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
-      list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
-    else()
-      set(LIST_)
-      foreach(item IN LISTS EXTRA_OPTIONS_LIST)
-        list(APPEND LIST_ "-Xcompiler=${item}")
-      endforeach()
-      set(EXTRA_OPTIONS_LIST ${LIST_})
-    endif()
-  elseif(MFEM_USE_HIP)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+    set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
+      PROPERTY LANGUAGE CUDA)
+    list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
   endif()
 
   # Actually add the executable
-  mfem_add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
-      ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
+  add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
+    ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
   add_dependencies(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_EXE_NAME})
   add_dependencies(${MFEM_EXE_NAME} ${MFEM_EXEC_PREREQUISITES_TARGET_NAME})
 
@@ -216,7 +186,7 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
         LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
     endif()
   endif()
-endmacro()
+endfunction()
 
 
 # Auxiliary function, used in mfem_find_package().
@@ -257,7 +227,7 @@ endfunction(mfem_find_component)
 #   code snippets. Additionally, a list of required/optional/alternative
 #   packages (given by ${Name}_REQUIRED_PACKAGES) are searched for and added to
 #   the ${Prefix}_INCLUDE_DIRS and ${Prefix}_LIBRARIES lists. The variable
-#   ${Name}_REQUIRED_LIBRARIES can be set to specify any additional libraries
+#   ${Name}_REQUIRED_LIBRARIES can be set to spcecify any additional libraries
 #   that are needed. This function defines the following CACHE variables:
 #
 #      ${Prefix}_FOUND
@@ -755,16 +725,15 @@ function(mfem_export_mk_files)
     set(shared_link_flag "-Wl,-rpath,")
   endif()
 
-  # Convert Boolean vars to YES/NO without writing the values to cache
+  # Convert Boolean vars to YES/NO without writting the values to cache
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
-      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
-      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2)
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
+      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -776,13 +745,7 @@ function(mfem_export_mk_files)
   set(MFEM_CXX ${CMAKE_CXX_COMPILER})
   set(MFEM_HOST_CXX ${MFEM_CXX})
   set(MFEM_CPPFLAGS "")
-  get_target_property(cxx_std mfem CXX_STANDARD)
-  # For now, we ignore the setting of the CXX_EXTENSIONS property. If this
-  # property is set, then we need to use a variable like:
-  #    CMAKE_CXX11_EXTENSION_COMPILE_OPTION
-  set(cxx_std_flag ${CMAKE_CXX${cxx_std}_STANDARD_COMPILE_OPTION})
-  string(STRIP
-         "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
+  string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
          MFEM_CXXFLAGS)
   set(MFEM_TPLFLAGS "")
   foreach(dir ${MFEM_TPL_INCLUDE_DIRS})
@@ -848,7 +811,7 @@ function(mfem_export_mk_files)
       string(REGEX REPLACE "^SCOREC::" "" libname ${pumilib})
       string(FIND "${pumilib}" ".a" staticlib)
       string(FIND "${pumilib}" ".so" sharedlib)
-      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFAULT_PATH)
+      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFUALT_PATH)
       if (NOT "${sharedlib}" MATCHES "-1" OR
           NOT "${staticlib}" MATCHES "-1"   )
         set(MFEM_EXT_LIBS "${pumilib} ${MFEM_EXT_LIBS}")

config/config.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -34,12 +34,6 @@
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
-// On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
-// the following macro allows us to get M_PI and some needed functions, e.g.
-// posix_memalign(), strdup(), strerror_r().
-#ifdef __CYGWIN__
-#define _XOPEN_SOURCE 600
-#endif
 
 // Check dependencies:
 
@@ -48,15 +42,9 @@
 #ifdef MFEM_USE_SUPERLU
 #error Building with SuperLU_DIST (MFEM_USE_SUPERLU=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_MUMPS
-#error Building with MUMPS (MFEM_USE_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_MKL_CPARDISO
-#error Building with MKL CPARDISO (MFEM_USE_MKL_CPARDISO=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif

config/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -93,11 +93,6 @@
 
 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
-// #define MFEM_USE_SUPERLU5
-
-// Enable MFEM functionality based on the MUMPS library.
-// #define MFEM_USE_MUMPS
-// #define MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
@@ -105,9 +100,6 @@
 // Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO
 
-// Enable MFEM functionality based on the AmgX library.
-// #define MFEM_USE_AMGX
-
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS
 
@@ -158,9 +150,6 @@
 // Enable functionality based on the libCEED library.
 // #define MFEM_USE_CEED
 
-// Enable functionality based on the Caliper library.
-// #define MFEM_USE_CALIPER
-
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE
 
@@ -174,7 +163,4 @@
 // library.
 // #define MFEM_USE_SIMMETRIX
 
-// Enable interface to the MKL CPardiso library.
-// #define MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -32,11 +32,8 @@ MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
-MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
-MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
-MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -52,11 +49,9 @@ MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
 MFEM_USE_CEED          = @MFEM_USE_CEED@
-MFEM_USE_CALIPER       = @MFEM_USE_CALIPER@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
-MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -33,11 +33,8 @@ option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
-option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
-option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
-option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -55,13 +52,6 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
-option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
-option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
-
-# Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
-# set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
-# set(MFEM_MPIEXEC_NP "-np" CACHE STRING
-#     "Flag for setting the number of MPI tasks")
 
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 
@@ -82,9 +72,6 @@ option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 # Set the target CUDA architecture
 set(CUDA_ARCH "sm_60" CACHE STRING "Target CUDA architecture.")
 
-# Set the target HIP architecture
-set(HIP_ARCH "gfx900" CACHE STRING "Target HIP architecture.")
-
 set(MFEM_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 # The *_DIR paths below will be the first place searched for the corresponding
@@ -101,7 +88,7 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."
 
 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")
 
-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
 # and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
     "Path to the SUNDIALS library.")
@@ -122,21 +109,12 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
     "Additional packages required by ParMETIS.")
 
-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_5.1.0" CACHE PATH
     "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
 set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
     "Additional packages required by SuperLU_DIST.")
 
-set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
-    "Path to the MUMPS library.")
-# Packages required by MUMPS, depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")    
-# If the MPI package does not find all required Fortran libraries:
-# set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")    
-
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
     "Path to the STRUMPACK library.")
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
@@ -167,14 +145,13 @@ set(ScaLAPACK_TARGET_NAMES scalapack)
 
 set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")
 
-set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
 
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
-set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
+# May need to add "HDF5" as requirement.
+set(NetCDF_REQUIRED_PACKAGES "" CACHE STRING
     "Additional packages required by NetCDF.")
 
 set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
@@ -203,15 +180,10 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
     "Packages that HiOp depends on.")
 
-set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
-set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
-set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
-set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
 
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")

config/defaults.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -18,9 +18,6 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))
 
-ETAGS_BIN = $(shell command -v etags 2> /dev/null)
-EGREP_BIN = $(shell command -v egrep 2> /dev/null)
-
 CXX = g++
 MPICXX = mpicxx
 
@@ -56,8 +53,6 @@ HIP_CXX = hipcc
 # gfx900, gfx1010, etc.
 HIP_ARCH = gfx900
 HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
-HIP_XCOMPILER =
-HIP_XLINKER   = -Wl,
 
 ifneq ($(NOTMAC),)
    AR      = ar
@@ -125,11 +120,8 @@ MFEM_USE_SUNDIALS      = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
-MFEM_USE_SUPERLU5      = NO
-MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
-MFEM_USE_AMGX          = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -145,27 +137,16 @@ MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
-MFEM_USE_CALIPER       = NO
 MFEM_USE_UMPIRE        = NO
+MFEM_USE_CAMP          = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
-MFEM_USE_MKL_CPARDISO  = NO
-
-# MPI library compile and link flags
-# These settings are used only when building MFEM with MPI + HIP
-ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
-   # We determine MPI_DIR assuming $(MPICXX) is in $(MPI_DIR)/bin
-   MPI_DIR := $(patsubst %/,%,$(dir $(shell which $(MPICXX))))
-   MPI_DIR := $(patsubst %/,%,$(dir $(MPI_DIR)))
-   MPI_OPT = -I$(MPI_DIR)/include
-   MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
-endif
 
 # Compile and link options for zlib.
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
 ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz
-ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib
+ZLIB_RPATH = -Wl,-rpath,$(ZLIB_DIR)/lib
 
 LIBUNWIND_OPT = -g
 LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
@@ -176,7 +157,7 @@ HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
 
 # METIS library configuration
-ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
+ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)
    ifeq ($(MFEM_USE_METIS_5),NO)
      METIS_DIR = @MFEM_DIR@/../metis-4.0
      METIS_OPT =
@@ -209,19 +190,15 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt
 
 # SUNDIALS library configuration
-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
-SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
  -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
 
 ifeq ($(MFEM_USE_MPI),YES)
-   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
-endif
-ifeq ($(MFEM_USE_CUDA),YES)
-   SUNDIALS_LIB += -lsundials_nveccuda
+   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
 endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
@@ -235,22 +212,14 @@ MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
 SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include
-SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\
- -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\
- -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
+SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
+ -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd -lccolamd\
+ -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
 
 # SuperLU library configuration
-ifeq ($(MFEM_USE_SUPERLU5),YES)
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
-      -lsuperlu_dist_5.1.0
-else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
-      -lsuperlu_dist -lblas
-endif
+SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
+SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
+SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
 
 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
 # STRUMPACK >= v2.2.0)
@@ -259,12 +228,12 @@ SCOTCH_OPT = -I$(SCOTCH_DIR)/include
 SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\
  -lpthread
 
-# SCALAPACK library configuration (required by STRUMPACK and MUMPS)
+# SCALAPACK library configuration (required by STRUMPACK)
 SCALAPACK_DIR = @MFEM_DIR@/../scalapack-2.0.2
 SCALAPACK_OPT = -I$(SCALAPACK_DIR)/SRC
 SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 
-# MPI Fortran library, needed e.g. by STRUMPACK or MUMPS
+# MPI Fortran library, needed e.g. by STRUMPACK
 # MPICH:
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
@@ -272,12 +241,6 @@ MPI_FORTRAN_LIB = -lmpifort
 # Additional Fortan library:
 # MPI_FORTRAN_LIB += -lgfortran
 
-# MUMPS library configuration
-MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
-MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
- -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
-
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
@@ -288,28 +251,8 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_BUILD_TYPE=Release
-ifeq ($(MFEM_USE_GINKGO),YES)
-   BASE_FLAGS = -std=c++14
-endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a  $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
-GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
-ifeq ($(GINKGO_BUILD_TYPE),Debug)
-  ifneq (,$(ALL_GINKGO_LIBS_DEBUG))
-    GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG))
-  endif
-else
-endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK)
-
-# AmgX library configuration
-AMGX_DIR = @MFEM_DIR@/../amgx
-AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference
 
 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -319,8 +262,8 @@ GNUTLS_LIB = -lgnutls
 NETCDF_DIR = $(HOME)/local
 HDF5_DIR   = $(HOME)/local
 NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT)
-NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
- $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
+NETCDF_LIB = -Wl,-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
+ -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
  -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)
 
 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
@@ -332,10 +275,9 @@ PETSC_INC_VAR = PETSC_CC_INCLUDES
 PETSC_LIB_VAR = PETSC_EXTERNAL_LIB_BASIC
 ifeq ($(PETSC_FOUND),YES)
    PETSC_OPT := $(shell sed -n "s/$(PETSC_INC_VAR) = *//p" $(PETSC_VARS))
-   PETSC_DEP := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
-   PETSC_LIB = $(XLINKER)-rpath,$(abspath $(PETSC_DIR))/lib\
-      -L$(abspath $(PETSC_DIR))/lib -lpetsc\
-      $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP))
+   PETSC_LIB := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
+   PETSC_LIB := -Wl,-rpath,$(abspath $(PETSC_DIR))/lib\
+      -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
 endif
 
 SLEPC_DIR := $(MFEM_DIR)/../slepc
@@ -347,10 +289,9 @@ ifeq ($(SLEPC_FOUND),YES)
    SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
    # Some additional external libraries might be defined in this file
    -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
-   SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
-   SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
-      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\
-      $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP))
+   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
+   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
+      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
 endif
 
 # MPFR library configuration
@@ -361,7 +302,7 @@ MPFR_LIB = -lmpfr
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
 CONDUIT_LIB = \
-   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
    -lconduit -lconduit_relay -lconduit_blueprint  -ldl
 
 # Check if Conduit was built with hdf5 support, by looking
@@ -369,7 +310,7 @@ CONDUIT_LIB = \
 CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp
 ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER)))
    CONDUIT_OPT += -I$(HDF5_DIR)/include
-   CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   CONDUIT_LIB += -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
                   -lhdf5 $(ZLIB_LIB)
 endif
 
@@ -379,9 +320,9 @@ SIDRE_DIR = @MFEM_DIR@/../axom
 SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\
  -I$(HDF5_DIR)/include
 SIDRE_LIB = \
-   $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
-   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
-   $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   -Wl,-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
+   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl
 
 # PUMI
@@ -414,11 +355,6 @@ OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 
-# CALIPER library configuration
-CALIPER_DIR = @MFEM_DIR@/../caliper
-CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
-
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
 CEED_OPT = -I$(CEED_DIR)/include
@@ -430,24 +366,17 @@ RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
    RAJA_OPT += -I$(CUB_DIR)
 endif
-ifdef CAMP_DIR
-   RAJA_OPT += -I$(CAMP_DIR)/include
-endif
 RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA
 
 # UMPIRE library configuration
 UMPIRE_DIR = @MFEM_DIR@/../umpire
-UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
+UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
-# MKL CPardiso library configuration
-MKL_CPARDISO_DIR ?=
-MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
-MKL_LIBRARY_SUBDIR ?= lib
-MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
-MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
-   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+# CAMP library configuration
+CAMP_DIR = @MFEM_DIR@/../camp
+CAMP_OPT = -I$(CAMP_DIR)/include
+CAMP_LIB = -L$(CAMP_DIR)/lib
 
 # If YES, enable some informational messages
 VERBOSE = NO

config/get_hypre_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/get_mumps_version.cpp

@@ -1,33 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "dmumps_c.h"
-#include <string>
-#include <iostream>
-#include <algorithm>
-
-// Macros to expand a macro as a string
-#define STR_EXPAND(s) #s
-#define STR(s) STR_EXPAND(s)
-
-int main()
-{
-#ifdef MUMPS_VERSION
-   const char *ptr = STR(MUMPS_VERSION);
-   std::string s(ptr);
-   s.erase(std::remove(s.begin(), s.end(), '"'), s.end());
-   s.erase(std::remove(s.begin(), s.end(), '.'), s.end());
-   std::cout << s << "\n";
-   return 0;
-#else
-   return -1;
-#endif
-}

config/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -42,10 +42,6 @@ GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
-MUMPS = $(MFEM_USE_MUMPS:NO=)
-GMV_CXX ?= $(MFEM_CXX)
-GMV = get_mumps_version
-GMV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -54,13 +50,6 @@ $(GHV).out: $(GHV)
 	./$(GHV) > $(GHV).out
 .INTERMEDIATE: $(GHV) $(GHV).out
 
-$(GMV): $(SRC)$(GMV).cpp
-	$(call mfem-info, Determining MUMPS version ...)
-	$(GMV_CXX) ${GMV_FLAGS} $(SRC)$(GMV).cpp -o $(GMV)
-$(GMV).out: $(GMV)
-	./$(GMV) > $(GMV).out
-.INTERMEDIATE: $(GMV) $(GMV).out
-
 get-hypre-version: $(GHV).out
 	$(eval MFEM_HYPRE_VERSION:=$(shell cat $(GHV).out))
 	$(if $(MFEM_HYPRE_VERSION),$(eval export MFEM_HYPRE_VERSION)\
@@ -73,18 +62,10 @@ check-smx:
 	$(call mfem-info, MFEM_USE_SIMMETRIX = $(MFEM_USE_SIMMETRIX))
 	$(eval export MFEM_USE_SIMMETRIX)
 
-get-mumps-version: $(GMV).out
-	$(eval MFEM_MUMPS_VERSION:=$(shell cat $(GMV).out))
-	$(if $(MFEM_MUMPS_VERSION),$(eval export MFEM_MUMPS_VERSION)\
-	$(info MUMPS version: $(MFEM_MUMPS_VERSION)),\
-	$(error Unable to determine MUMPS version))
-
-header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx,) \
- $(if $(MUMPS),get-mumps-version,)
+header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx)
 	$(call mfem-info, Writing $(CONFIG_HPP) ...)
 	@set -- && \
-	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX) \
-	           $(if $(MUMPS),MFEM_MUMPS_VERSION); do \
+	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX); do \
 	   eval var=\$$$$def && \
 	   if [ "NO" != "$${var}" ]; then \
 	      set -- "$$@" -e "s|// \(#define $${def} \)|\1|" && \

config/sample-runs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -42,56 +42,12 @@ groups_serial=(
    "Performance miniapps:"
    "miniapps/performance"
    "ex1.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1.cpp"'
-'"ginkgo"
-   "Ginkgo examples:"
-   "examples/ginkgo"
-   "ex1.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1.cpp ex2.cpp"'
 #   ""'
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
    "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
     mesh-optimizer.cpp minimal-surface.cpp"'
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "cvsRoberts_ASAi_dns.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp "'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
-'"toys"
-   "Toys miniapps:"
-   "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "rates.cpp"'
 )
 # Parallel groups
 groups_parallel=(
@@ -112,26 +68,6 @@ groups_parallel=(
    "Performance miniapps:"
    "miniapps/performance"
    "ex1p.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1p.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1p.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9p.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1p.cpp ex6p.cpp"'
-'"superlu"
-   "Superlu examples:"
-   "examples/superlu"
-   "ex1p.cpp"'
 #   ""'
 '"meshing"
    "Meshing miniapps:"
@@ -142,42 +78,6 @@ groups_parallel=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "adjoint_advection_diffusion.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "pfindpts.cpp schwarz_ex1p.cpp"'
-'"navier"
-   "Navier miniapps:"
-   "miniapps/navier"
-   "navier_cht.cpp"'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-'"shifted"
-   "Shifted miniapps:"
-   "miniapps/shifted"
-   "distance.cpp"'
-'"solvers"
-   "Solvers miniapps:"
-   "miniapps/solvers"
-   "block-solvers.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-cd.cpp get-values.cpp load-dc.cpp"'
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "prates.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 # All groups serial + parallel runs mixed in the same group:
 groups_all=(
@@ -197,30 +97,6 @@ groups_all=(
    "Performance miniapps:"
    "miniapps/performance"
    "ex1{,p}.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1.cpp ex1p.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1.cpp ex1p.cpp"'
-'"ginkgo"
-   "Ginkgo examples:"
-   "examples/ginkgo"
-   "ex1.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9.cpp ex9p.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1.cpp ex1p.cpp ex2.cpp ex6p.cpp"'
-'"superlu"
-   "Superlu examples:"
-   "examples/superlu"
-   "ex1p.cpp"'
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
@@ -231,46 +107,6 @@ groups_all=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "adjoint_advection_diffusion.cpp cvsRoberts_ASAi_dns.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp schwarz_ex1p.cpp"'
-'"navier"
-   "Navier miniapps:"
-   "miniapps/navier"
-   "navier_cht.cpp"'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1.cpp nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-'"shifted"
-   "Shifted miniapps:"
-   "miniapps/shifted"
-   "distance.cpp"'
-'"solvers"
-   "Solvers miniapps:"
-   "miniapps/solvers"
-   "block-solvers.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
-'"toys"
-   "Toys miniapps:"
-   "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "{,p}rates.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 make_all="all"
 base_timeformat=$'real: %3Rs  user: %3Us  sys: %3Ss  %%cpu: %P'
@@ -544,15 +380,10 @@ function timed_run()
 # This function is used to execute the sample runs
 function go()
 {
-   # Strip leading and trailing spaces from $1 and store the result in cmd_line
-   shopt -s extglob
-   local cmd_line="${1##+( )}"
-   cmd_line="${cmd_line%%+( )}"
-   shopt -u extglob
-   eval local cmd=(${cmd_line})
+   local cmd=("$@")
    local res=""
    echo $sep
-   echo "<${group}>" "${cmd_line}"
+   echo "<${group}>" "${cmd[@]}"
    echo $sep
    if [ "${timing}" == "yes" ]; then
       timed_run "${cmd[@]}"
@@ -564,15 +395,15 @@ function go()
    else
       res="${red}FAILED${none}"
    fi
-   printf "[${res}] <${group}> ${cmd_line}\n"
+   printf "[${res}] <${group}> ${cmd[*]}\n"
    if [ "${timing}" == "yes" ]; then
       printf "Run time: %s\n" "${timer}"
       timer=(${timer})
       timer="${timer[1]}"
-      printf -v line "[$res](%8s) ${cmd_line}" "$timer"
+      printf -v line "[$res](%8s) ${cmd[*]}" "$timer"
       summary=("${summary[@]}" "$line")
    else
-      summary=("${summary[@]}" "[${res}] ${cmd_line}")
+      summary=("${summary[@]}" "[${res}] ${cmd[*]}")
    fi
    echo $sep
 }
@@ -588,8 +419,6 @@ function go_group()
       mkdir -p "${group_output_dir}" || exit 1
    fi
    for src in "$@"; do
-      ex_run_suffix=${run_suffix} && [[ $src =~ ex0p?\.cpp ]] \
-         && ex_run_suffix=""
       cd "${mfem_dir}/${group_dir}" || exit 1
       extract_sample_runs "${src}" || continue
       [ "${#runs[@]}" -eq 0 ] && continue
@@ -609,7 +438,7 @@ function go_group()
       fi
       for run in "${runs[@]}"; do
          if [ "${run}" == "" ]; then continue; fi
-         eval go \"\${run_prefix} \${run} \${ex_run_suffix}\" $output
+         eval go \${run_prefix} \${run} \${run_suffix} $output
       done
    done
    ${make} clean-exec
@@ -675,7 +504,7 @@ function echo_run()
 {
    echo "   $@"
    { echo "   $@"; echo "$sep";
-     eval "$@"
+     "$@"
      echo "$sep"; } >> "$echo_log" 2>&1
 }
 
@@ -695,28 +524,6 @@ function build_all()
    echo_run ${make} config ${mfem_config} || exit 1
    echo_run ${make} ${make_j} || exit 1
    echo_run ${make} ${make_all} ${make_j} || exit 1
-   # Build groups in directories other than the directories built by 'make all':
-   for group_params in "${groups[@]}"; do
-      eval params=(${group_params})
-      group_dir="${params[2]}"
-      case "$group_dir" in
-         (examples*|miniapps*)
-            # Built by 'make all'
-            ;;
-         (*)
-            if [ "${mfem_dir}" != "${mfem_build_dir}" ]; then
-               echo_run mkdir -p "${group_dir}" || exit 1
-               echo_run cd "${group_dir}" || exit 1
-               echo_run cp -af "${mfem_dir}/${group_dir}/makefile" . || exit 1
-            else
-               echo_run cd "${group_dir}" || exit 1
-            fi
-            echo_run ${make} clean || exit 1
-            echo_run ${make} MFEM_DIR="${mfem_dir}" ${make_j} || exit 1
-            echo_run cd "${mfem_build_dir}" || exit 1
-            ;;
-      esac
-   done
 }
 
 # Function that runs all sample runs, given by the array variable "groups".

config/tconfig.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/test.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -39,7 +39,7 @@ set -- $$($(1) $(SHELL) -c "$(2)" 2>&1); while [ "$$#" -gt 3 ]; do shift; done
 endef
 define TIMECMD.NOTGNU
 set -- $$($(1) -l $(SHELL) -c "{ $(2); } > /dev/null 2>&1" 2>&1; echo $$?); \
-set -- "$$1"s "$$(($$7/1024))"kB "$${!#}"
+set -- "$$1"s "$$(($$7/1024))"kB "$${60}"
 endef
 define TIMECMD.BASH
 TIMEFORMAT=$$'%3Rs'; \

data/amr-hex.mesh

@@ -1,258 +1,241 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1
 
-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
 # SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#
 
 dimension
 3
 
-# rank attr geom ref_type nodes/children
 elements
-137
--1 1 5 7 1 18 35 52 69 86 103 120
--1 1 5 7 2 3 4 5 6 7 8 17
-0 1 5 0 0 27 81 30 35 82 87 85
-0 1 5 0 27 8 28 81 82 36 83 87
-0 1 5 0 81 28 20 29 87 83 37 84
-0 1 5 0 30 81 29 11 85 87 84 38
-0 1 5 0 35 82 87 85 16 31 86 34
-0 1 5 0 82 36 83 87 31 21 32 86
--1 1 5 7 9 10 11 12 13 14 15 16
-0 1 5 0 87 88 179 91 92 180 185 183
-0 1 5 0 88 83 89 179 180 94 181 185
-0 1 5 0 179 89 37 90 185 181 98 182
-0 1 5 0 91 179 90 84 183 185 182 99
-0 1 5 0 92 180 185 183 86 93 184 97
-0 1 5 0 180 94 181 185 93 32 95 184
-0 1 5 0 185 181 98 182 184 95 26 96
-0 1 5 0 183 185 182 99 97 184 96 33
-0 1 5 0 85 87 84 38 34 86 33 24
--1 1 5 7 19 20 21 22 23 24 25 26
-0 1 5 0 8 39 100 28 36 101 105 83
-0 1 5 0 39 1 40 100 101 45 102 105
-0 1 5 0 100 40 9 41 105 102 46 103
-0 1 5 0 28 100 41 20 83 105 103 37
-0 1 5 0 36 101 105 83 21 42 104 32
-0 1 5 0 101 45 102 105 42 17 43 104
-0 1 5 0 105 102 46 103 104 43 22 44
--1 1 5 7 27 28 29 30 31 32 33 34
-0 1 5 0 83 106 186 89 94 187 191 181
-0 1 5 0 106 105 107 186 187 110 188 191
-0 1 5 0 186 107 103 108 191 188 112 189
-0 1 5 0 89 186 108 37 181 191 189 98
-0 1 5 0 94 187 191 181 32 109 190 95
-0 1 5 0 187 110 188 191 109 104 111 190
-0 1 5 0 191 188 112 189 190 111 44 113
-0 1 5 0 181 191 189 98 95 190 113 26
--1 1 5 7 36 37 38 39 40 49 50 51
-0 1 5 0 20 41 114 49 37 103 119 117
-0 1 5 0 41 9 47 114 103 46 115 119
-0 1 5 0 114 47 3 48 119 115 53 116
-0 1 5 0 49 114 48 10 117 119 116 54
--1 1 5 7 41 42 43 44 45 46 47 48
-0 1 5 0 37 108 192 122 98 189 197 195
-0 1 5 0 108 103 120 192 189 112 193 197
-0 1 5 0 192 120 119 121 197 193 126 194
-0 1 5 0 122 192 121 117 195 197 194 127
-0 1 5 0 98 189 197 195 26 113 196 125
-0 1 5 0 189 112 193 197 113 44 123 196
-0 1 5 0 197 193 126 194 196 123 118 124
-0 1 5 0 195 197 194 127 125 196 124 52
-0 1 5 0 103 46 115 119 44 22 50 118
-0 1 5 0 119 115 53 116 118 50 18 51
-0 1 5 0 117 119 116 54 52 118 51 23
--1 1 5 7 53 54 55 56 57 58 67 68
-0 1 5 0 11 29 128 56 38 84 132 130
-0 1 5 0 29 20 49 128 84 37 117 132
-0 1 5 0 128 49 10 55 132 117 54 129
-0 1 5 0 56 128 55 2 130 132 129 59
-0 1 5 0 38 84 132 130 24 33 131 58
--1 1 5 7 59 60 61 62 63 64 65 66
-0 1 5 0 84 90 198 133 99 182 202 200
-0 1 5 0 90 37 122 198 182 98 195 202
-0 1 5 0 198 122 117 134 202 195 127 199
-0 1 5 0 133 198 134 132 200 202 199 136
-0 1 5 0 99 182 202 200 33 96 201 135
-0 1 5 0 182 98 195 202 96 26 125 201
-0 1 5 0 202 195 127 199 201 125 52 137
-0 1 5 0 200 202 199 136 135 201 137 131
-0 1 5 0 132 117 54 129 131 52 23 57
-0 1 5 0 130 132 129 59 58 131 57 19
--1 1 5 7 70 71 72 81 82 83 84 85
-0 1 5 0 16 31 86 34 64 172 175 173
-0 1 5 0 31 21 32 86 172 65 164 175
--1 1 5 7 73 74 75 76 77 78 79 80
-0 1 5 0 86 93 184 97 176 219 222 220
-0 1 5 0 93 32 95 184 219 169 216 222
-0 1 5 0 184 95 26 96 222 216 150 203
-0 1 5 0 97 184 96 33 220 222 203 145
-0 1 5 0 176 219 222 220 175 177 221 178
-0 1 5 0 219 169 216 222 177 164 171 221
-0 1 5 0 222 216 150 203 221 171 66 147
-0 1 5 0 220 222 203 145 178 221 147 138
-0 1 5 0 34 86 33 24 173 175 138 67
-0 1 5 0 64 172 175 173 4 60 174 63
-0 1 5 0 172 65 164 175 60 12 61 174
-0 1 5 0 175 164 66 138 174 61 25 62
-0 1 5 0 173 175 138 67 63 174 62 15
--1 1 5 7 87 88 89 90 99 100 101 102
-0 1 5 0 21 42 104 32 65 162 166 164
-0 1 5 0 42 17 43 104 162 71 163 166
-0 1 5 0 104 43 22 44 166 163 72 152
--1 1 5 7 91 92 93 94 95 96 97 98
-0 1 5 0 32 109 190 95 169 214 218 216
-0 1 5 0 109 104 111 190 214 168 215 218
-0 1 5 0 190 111 44 113 218 215 160 209
-0 1 5 0 95 190 113 26 216 218 209 150
-0 1 5 0 169 214 218 216 164 167 217 171
-0 1 5 0 214 168 215 218 167 166 170 217
-0 1 5 0 218 215 160 209 217 170 152 157
-0 1 5 0 216 218 209 150 171 217 157 66
-0 1 5 0 65 162 166 164 12 68 165 61
-0 1 5 0 162 71 163 166 68 5 69 165
-0 1 5 0 166 163 72 152 165 69 13 70
-0 1 5 0 164 166 152 66 61 165 70 25
--1 1 5 7 104 113 114 115 116 117 118 119
--1 1 5 7 105 106 107 108 109 110 111 112
-0 1 5 0 26 113 196 125 150 209 213 204
-0 1 5 0 113 44 123 196 209 160 210 213
-0 1 5 0 196 123 118 124 213 210 161 211
-0 1 5 0 125 196 124 52 204 213 211 151
-0 1 5 0 150 209 213 204 66 157 212 148
-0 1 5 0 209 160 210 213 157 152 158 212
-0 1 5 0 213 210 161 211 212 158 156 159
-0 1 5 0 204 213 211 151 148 212 159 139
-0 1 5 0 44 22 50 118 152 72 153 156
-0 1 5 0 118 50 18 51 156 153 76 154
-0 1 5 0 52 118 51 23 139 156 154 77
-0 1 5 0 66 152 156 139 25 70 155 75
-0 1 5 0 152 72 153 156 70 13 73 155
-0 1 5 0 156 153 76 154 155 73 7 74
-0 1 5 0 139 156 154 77 75 155 74 14
--1 1 5 7 121 122 131 132 133 134 135 136
-0 1 5 0 24 33 131 58 67 138 143 141
--1 1 5 7 123 124 125 126 127 128 129 130
-0 1 5 0 33 96 201 135 145 203 208 206
-0 1 5 0 96 26 125 201 203 150 204 208
-0 1 5 0 201 125 52 137 208 204 151 205
-0 1 5 0 135 201 137 131 206 208 205 146
-0 1 5 0 145 203 208 206 138 147 207 144
-0 1 5 0 203 150 204 208 147 66 148 207
-0 1 5 0 208 204 151 205 207 148 139 149
-0 1 5 0 206 208 205 146 144 207 149 143
-0 1 5 0 131 52 23 57 143 139 77 140
-0 1 5 0 58 131 57 19 141 143 140 80
-0 1 5 0 67 138 143 141 15 62 142 79
-0 1 5 0 138 66 139 143 62 25 75 142
-0 1 5 0 143 139 77 140 142 75 14 78
-0 1 5 0 141 143 140 80 79 142 78 6
+120
+1 5 0 27 81 30 35 82 87 85
+1 5 30 81 29 11 85 87 84 38
+1 5 85 87 84 38 34 86 33 24
+1 5 35 82 87 85 16 31 86 34
+1 5 82 36 83 87 31 21 32 86
+1 5 180 94 181 185 93 32 95 184
+1 5 92 180 185 183 86 93 184 97
+1 5 183 185 182 99 97 184 96 33
+1 5 185 181 98 182 184 95 26 96
+1 5 179 89 37 90 185 181 98 182
+1 5 91 179 90 84 183 185 182 99
+1 5 87 88 179 91 92 180 185 183
+1 5 88 83 89 179 180 94 181 185
+1 5 81 28 20 29 87 83 37 84
+1 5 27 8 28 81 82 36 83 87
+1 5 8 39 100 28 36 101 105 83
+1 5 36 101 105 83 21 42 104 32
+1 5 101 45 102 105 42 17 43 104
+1 5 39 1 40 100 101 45 102 105
+1 5 100 40 9 41 105 102 46 103
+1 5 105 102 46 103 104 43 22 44
+1 5 186 107 103 108 191 188 112 189
+1 5 106 105 107 186 187 110 188 191
+1 5 187 110 188 191 109 104 111 190
+1 5 191 188 112 189 190 111 44 113
+1 5 181 191 189 98 95 190 113 26
+1 5 94 187 191 181 32 109 190 95
+1 5 83 106 186 89 94 187 191 181
+1 5 89 186 108 37 181 191 189 98
+1 5 28 100 41 20 83 105 103 37
+1 5 20 41 114 49 37 103 119 117
+1 5 37 108 192 122 98 189 197 195
+1 5 122 192 121 117 195 197 194 127
+1 5 195 197 194 127 125 196 124 52
+1 5 98 189 197 195 26 113 196 125
+1 5 189 112 193 197 113 44 123 196
+1 5 197 193 126 194 196 123 118 124
+1 5 192 120 119 121 197 193 126 194
+1 5 108 103 120 192 189 112 193 197
+1 5 103 46 115 119 44 22 50 118
+1 5 41 9 47 114 103 46 115 119
+1 5 114 47 3 48 119 115 53 116
+1 5 119 115 53 116 118 50 18 51
+1 5 117 119 116 54 52 118 51 23
+1 5 49 114 48 10 117 119 116 54
+1 5 128 49 10 55 132 117 54 129
+1 5 56 128 55 2 130 132 129 59
+1 5 11 29 128 56 38 84 132 130
+1 5 29 20 49 128 84 37 117 132
+1 5 84 90 198 133 99 182 202 200
+1 5 90 37 122 198 182 98 195 202
+1 5 198 122 117 134 202 195 127 199
+1 5 133 198 134 132 200 202 199 136
+1 5 200 202 199 136 135 201 137 131
+1 5 202 195 127 199 201 125 52 137
+1 5 182 98 195 202 96 26 125 201
+1 5 99 182 202 200 33 96 201 135
+1 5 38 84 132 130 24 33 131 58
+1 5 130 132 129 59 58 131 57 19
+1 5 132 117 54 129 131 52 23 57
+1 5 131 52 23 57 143 139 77 140
+1 5 58 131 57 19 141 143 140 80
+1 5 24 33 131 58 67 138 143 141
+1 5 33 96 201 135 145 203 208 206
+1 5 96 26 125 201 203 150 204 208
+1 5 201 125 52 137 208 204 151 205
+1 5 135 201 137 131 206 208 205 146
+1 5 206 208 205 146 144 207 149 143
+1 5 208 204 151 205 207 148 139 149
+1 5 203 150 204 208 147 66 148 207
+1 5 145 203 208 206 138 147 207 144
+1 5 138 66 139 143 62 25 75 142
+1 5 67 138 143 141 15 62 142 79
+1 5 141 143 140 80 79 142 78 6
+1 5 143 139 77 140 142 75 14 78
+1 5 139 156 154 77 75 155 74 14
+1 5 52 118 51 23 139 156 154 77
+1 5 118 50 18 51 156 153 76 154
+1 5 156 153 76 154 155 73 7 74
+1 5 152 72 153 156 70 13 73 155
+1 5 44 22 50 118 152 72 153 156
+1 5 209 160 210 213 157 152 158 212
+1 5 213 210 161 211 212 158 156 159
+1 5 196 123 118 124 213 210 161 211
+1 5 113 44 123 196 209 160 210 213
+1 5 26 113 196 125 150 209 213 204
+1 5 125 196 124 52 204 213 211 151
+1 5 204 213 211 151 148 212 159 139
+1 5 150 209 213 204 66 157 212 148
+1 5 66 152 156 139 25 70 155 75
+1 5 164 166 152 66 61 165 70 25
+1 5 216 218 209 150 171 217 157 66
+1 5 169 214 218 216 164 167 217 171
+1 5 32 109 190 95 169 214 218 216
+1 5 95 190 113 26 216 218 209 150
+1 5 190 111 44 113 218 215 160 209
+1 5 109 104 111 190 214 168 215 218
+1 5 214 168 215 218 167 166 170 217
+1 5 218 215 160 209 217 170 152 157
+1 5 104 43 22 44 166 163 72 152
+1 5 166 163 72 152 165 69 13 70
+1 5 162 71 163 166 68 5 69 165
+1 5 42 17 43 104 162 71 163 166
+1 5 21 42 104 32 65 162 166 164
+1 5 65 162 166 164 12 68 165 61
+1 5 172 65 164 175 60 12 61 174
+1 5 175 164 66 138 174 61 25 62
+1 5 219 169 216 222 177 164 171 221
+1 5 176 219 222 220 175 177 221 178
+1 5 220 222 203 145 178 221 147 138
+1 5 222 216 150 203 221 171 66 147
+1 5 184 95 26 96 222 216 150 203
+1 5 97 184 96 33 220 222 203 145
+1 5 86 93 184 97 176 219 222 220
+1 5 93 32 95 184 219 169 216 222
+1 5 31 21 32 86 172 65 164 175
+1 5 16 31 86 34 64 172 175 173
+1 5 34 86 33 24 173 175 138 67
+1 5 173 175 138 67 63 174 62 15
+1 5 64 172 175 173 4 60 174 63
 
-# attr geom nodes
 boundary
 96
 1 3 30 81 27 0
 2 3 0 27 82 35
 5 3 30 0 35 85
-1 3 81 28 8 27
-2 3 27 8 36 82
-1 3 29 20 28 81
 1 3 11 29 81 30
 5 3 11 30 85 38
+5 3 38 85 34 24
 2 3 35 82 31 16
 5 3 85 35 16 34
 2 3 82 36 21 31
-5 3 38 85 34 24
+1 3 29 20 28 81
+1 3 81 28 8 27
+2 3 27 8 36 82
 1 3 28 100 39 8
 2 3 8 39 101 36
+2 3 36 101 42 21
+2 3 101 45 17 42
+3 3 45 102 43 17
 1 3 100 40 1 39
 2 3 39 1 45 101
 3 3 1 40 102 45
 1 3 41 9 40 100
 3 3 40 9 46 102
-1 3 20 41 100 28
-2 3 36 101 42 21
-2 3 101 45 17 42
-3 3 45 102 43 17
 3 3 102 46 22 43
+1 3 20 41 100 28
 1 3 49 114 41 20
+3 3 46 115 50 22
 1 3 114 47 9 41
 3 3 9 47 115 46
 1 3 48 3 47 114
 3 3 47 3 53 115
 4 3 3 48 116 53
-1 3 10 48 114 49
-4 3 48 10 54 116
-3 3 46 115 50 22
 3 3 115 53 18 50
 4 3 53 116 51 18
 4 3 116 54 23 51
-1 3 56 128 29 11
-5 3 56 11 38 130
-1 3 128 49 20 29
+1 3 10 48 114 49
+4 3 48 10 54 116
 1 3 55 10 49 128
 4 3 10 55 129 54
 1 3 2 55 128 56
 4 3 55 2 59 129
 5 3 2 56 130 59
+1 3 56 128 29 11
+5 3 56 11 38 130
+1 3 128 49 20 29
 5 3 130 38 24 58
-4 3 54 129 57 23
 4 3 129 59 19 57
 5 3 59 130 58 19
-2 3 16 31 172 64
-5 3 34 16 64 173
-2 3 31 21 65 172
-5 3 24 34 173 67
-2 3 64 172 60 4
-5 3 173 64 4 63
-6 3 4 60 174 63
-2 3 172 65 12 60
-6 3 60 12 61 174
-6 3 174 61 25 62
-5 3 67 173 63 15
-6 3 63 174 62 15
-2 3 21 42 162 65
-2 3 42 17 71 162
-3 3 17 43 163 71
-3 3 43 22 72 163
-2 3 65 162 68 12
-6 3 12 68 165 61
-2 3 162 71 5 68
-3 3 71 163 69 5
-6 3 68 5 69 165
-3 3 163 72 13 69
-6 3 165 69 13 70
-6 3 61 165 70 25
-3 3 22 50 153 72
-3 3 50 18 76 153
-4 3 18 51 154 76
-4 3 51 23 77 154
-6 3 25 70 155 75
-3 3 72 153 73 13
-6 3 70 13 73 155
-3 3 153 76 7 73
-4 3 76 154 74 7
-6 3 155 73 7 74
-4 3 154 77 14 74
-6 3 75 155 74 14
-5 3 58 24 67 141
+4 3 54 129 57 23
 4 3 23 57 140 77
 4 3 57 19 80 140
 5 3 19 58 141 80
+5 3 58 24 67 141
+6 3 62 25 75 142
 5 3 141 67 15 79
 6 3 15 62 142 79
-6 3 62 25 75 142
-4 3 77 140 78 14
-6 3 142 75 14 78
 4 3 140 80 6 78
 5 3 80 141 79 6
 6 3 79 142 78 6
+4 3 77 140 78 14
+6 3 142 75 14 78
+4 3 154 77 14 74
+6 3 75 155 74 14
+4 3 51 23 77 154
+3 3 50 18 76 153
+4 3 18 51 154 76
+3 3 153 76 7 73
+4 3 76 154 74 7
+6 3 155 73 7 74
+3 3 72 153 73 13
+6 3 70 13 73 155
+3 3 22 50 153 72
+6 3 25 70 155 75
+6 3 61 165 70 25
+3 3 43 22 72 163
+3 3 163 72 13 69
+6 3 165 69 13 70
+2 3 162 71 5 68
+3 3 71 163 69 5
+6 3 68 5 69 165
+2 3 42 17 71 162
+3 3 17 43 163 71
+2 3 21 42 162 65
+2 3 65 162 68 12
+6 3 12 68 165 61
+2 3 172 65 12 60
+6 3 60 12 61 174
+6 3 174 61 25 62
+2 3 31 21 65 172
+2 3 16 31 172 64
+5 3 34 16 64 173
+5 3 24 34 173 67
+5 3 67 173 63 15
+6 3 63 174 62 15
+2 3 64 172 60 4
+5 3 173 64 4 63
+6 3 4 60 174 63
 
-# vert_id p1 p2
 vertex_parents
 215
 8 0 1
@@ -471,9 +454,28 @@ vertex_parents
 221 171 178
 222 203 219
 
-# top-level node coordinates
-coordinates
-8
+coarse_elements
+17
+7 11 12 9 10 6 5 8 7
+7 0 14 13 1 3 4 120 2
+7 27 22 21 28 26 23 24 25
+7 15 18 19 29 16 17 20 122
+7 31 38 37 32 34 35 36 33
+7 30 40 41 44 124 39 42 43
+7 49 50 51 52 56 55 54 53
+7 47 48 45 46 57 126 59 58
+7 113 114 111 112 108 107 110 109
+7 116 115 128 117 119 105 106 118
+7 93 96 95 94 92 97 98 91
+7 103 102 99 130 104 101 100 90
+7 85 84 83 86 88 81 82 87
+7 132 80 77 76 89 79 78 75
+7 63 64 65 66 70 69 68 67
+7 62 134 60 61 72 71 74 73
+7 121 123 125 127 129 131 133 135
+
+vertices
+223
 3
 0 0 0
 1 0 0
@@ -483,5 +485,218 @@ coordinates
 1 0 1
 0 1 1
 1 1 1
-
-mfem_mesh_end
+0.5 0 0
+1 0.5 0
+0.5 1 0
+0 0.5 0
+0.5 0 1
+1 0.5 1
+0.5 1 1
+0 0.5 1
+0 0 0.5
+1 0 0.5
+1 1 0.5
+0 1 0.5
+0.5 0.5 0
+0.5 0 0.5
+1 0.5 0.5
+0.5 1 0.5
+0 0.5 0.5
+0.5 0.5 1
+0.5 0.5 0.5
+0.25 0 0
+0.5 0.25 0
+0.25 0.5 0
+0 0.25 0
+0.25 0 0.5
+0.5 0.25 0.5
+0.25 0.5 0.5
+0 0.25 0.5
+0 0 0.25
+0.5 0 0.25
+0.5 0.5 0.25
+0 0.5 0.25
+0.75 0 0
+1 0.25 0
+0.75 0.5 0
+0.75 0 0.5
+1 0.25 0.5
+0.75 0.5 0.5
+1 0 0.25
+1 0.5 0.25
+1 0.75 0
+0.75 1 0
+0.5 0.75 0
+1 0.75 0.5
+0.75 1 0.5
+0.5 0.75 0.5
+1 1 0.25
+0.5 1 0.25
+0.25 1 0
+0 0.75 0
+0.25 1 0.5
+0 0.75 0.5
+0 1 0.25
+0.25 0 1
+0.5 0.25 1
+0.25 0.5 1
+0 0.25 1
+0 0 0.75
+0.5 0 0.75
+0.5 0.5 0.75
+0 0.5 0.75
+0.75 0 1
+1 0.25 1
+0.75 0.5 1
+1 0 0.75
+1 0.5 0.75
+1 0.75 1
+0.75 1 1
+0.5 0.75 1
+1 1 0.75
+0.5 1 0.75
+0.25 1 1
+0 0.75 1
+0 1 0.75
+0.25 0.25 0
+0.25 0 0.25
+0.5 0.25 0.25
+0.25 0.5 0.25
+0 0.25 0.25
+0.25 0.25 0.5
+0.25 0.25 0.25
+0.375 0.25 0.25
+0.5 0.375 0.25
+0.375 0.5 0.25
+0.25 0.375 0.25
+0.25 0.25 0.375
+0.375 0.25 0.5
+0.5 0.25 0.375
+0.5 0.375 0.5
+0.375 0.5 0.5
+0.25 0.375 0.5
+0.5 0.5 0.375
+0.25 0.5 0.375
+0.75 0.25 0
+0.75 0 0.25
+1 0.25 0.25
+0.75 0.5 0.25
+0.75 0.25 0.5
+0.75 0.25 0.25
+0.625 0.25 0.25
+0.75 0.375 0.25
+0.625 0.5 0.25
+0.625 0.25 0.5
+0.75 0.25 0.375
+0.75 0.375 0.5
+0.75 0.5 0.375
+0.625 0.5 0.5
+0.75 0.75 0
+1 0.75 0.25
+0.75 1 0.25
+0.5 0.75 0.25
+0.75 0.75 0.5
+0.75 0.75 0.25
+0.75 0.625 0.25
+0.625 0.75 0.25
+0.5 0.625 0.25
+0.75 0.625 0.5
+0.625 0.75 0.5
+0.5 0.625 0.5
+0.75 0.75 0.375
+0.5 0.75 0.375
+0.25 0.75 0
+0.25 1 0.25
+0 0.75 0.25
+0.25 0.75 0.5
+0.25 0.75 0.25
+0.25 0.625 0.25
+0.375 0.75 0.25
+0.25 0.625 0.5
+0.25 0.75 0.375
+0.375 0.75 0.5
+0.25 0.5 0.75
+0.5 0.75 0.75
+0.25 1 0.75
+0 0.75 0.75
+0.25 0.75 1
+0.25 0.75 0.75
+0.25 0.625 0.75
+0.25 0.5 0.625
+0.25 0.75 0.625
+0.375 0.5 0.75
+0.5 0.625 0.75
+0.375 0.75 0.75
+0.5 0.5 0.625
+0.5 0.75 0.625
+0.75 0.5 0.75
+1 0.75 0.75
+0.75 1 0.75
+0.75 0.75 1
+0.75 0.75 0.75
+0.625 0.5 0.75
+0.75 0.625 0.75
+0.625 0.75 0.75
+0.75 0.5 0.625
+0.75 0.75 0.625
+0.75 0 0.75
+1 0.25 0.75
+0.5 0.25 0.75
+0.75 0.25 1
+0.75 0.25 0.75
+0.625 0.25 0.75
+0.75 0.25 0.625
+0.5 0.25 0.625
+0.75 0.375 0.75
+0.5 0.375 0.75
+0.25 0 0.75
+0 0.25 0.75
+0.25 0.25 1
+0.25 0.25 0.75
+0.25 0.25 0.625
+0.375 0.25 0.75
+0.25 0.375 0.75
+0.375 0.375 0.25
+0.375 0.25 0.375
+0.5 0.375 0.375
+0.375 0.5 0.375
+0.25 0.375 0.375
+0.375 0.375 0.5
+0.375 0.375 0.375
+0.625 0.375 0.25
+0.625 0.25 0.375
+0.75 0.375 0.375
+0.625 0.5 0.375
+0.625 0.375 0.5
+0.625 0.375 0.375
+0.625 0.625 0.25
+0.75 0.625 0.375
+0.625 0.75 0.375
+0.5 0.625 0.375
+0.625 0.625 0.5
+0.625 0.625 0.375
+0.375 0.625 0.25
+0.375 0.75 0.375
+0.25 0.625 0.375
+0.375 0.625 0.5
+0.375 0.625 0.375
+0.375 0.5 0.625
+0.5 0.625 0.625
+0.375 0.75 0.625
+0.25 0.625 0.625
+0.375 0.625 0.75
+0.375 0.625 0.625
+0.625 0.5 0.625
+0.75 0.625 0.625
+0.625 0.75 0.625
+0.625 0.625 0.75
+0.625 0.625 0.625
+0.625 0.25 0.625
+0.75 0.375 0.625
+0.5 0.375 0.625
+0.625 0.375 0.75
+0.625 0.375 0.625
+0.375 0.25 0.625
+0.25 0.375 0.625
+0.375 0.375 0.75
+0.375 0.375 0.625

data/amr-quad.mesh

@@ -1,64 +1,56 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1
 
-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
 # SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#
 
 dimension
 2
 
-# rank attr geom ref_type nodes/children
 elements
-37
--1 1 3 3 1 10 19 28
--1 1 3 3 2 3 4 9
-0 1 3 0 0 9 21 12
-0 1 3 0 9 4 10 21
--1 1 3 3 5 6 7 8
-0 1 3 0 21 22 37 25
-0 1 3 0 22 10 23 37
-0 1 3 0 37 23 8 24
-0 1 3 0 25 37 24 11
-0 1 3 0 12 21 11 7
--1 1 3 3 11 12 13 14
-0 1 3 0 4 13 26 10
-0 1 3 0 13 1 14 26
-0 1 3 0 26 14 5 15
--1 1 3 3 15 16 17 18
-0 1 3 0 10 27 38 23
-0 1 3 0 27 26 28 38
-0 1 3 0 38 28 15 29
-0 1 3 0 23 38 29 8
--1 1 3 3 20 25 26 27
--1 1 3 3 21 22 23 24
-0 1 3 0 8 29 39 33
-0 1 3 0 29 15 31 39
-0 1 3 0 39 31 30 32
-0 1 3 0 33 39 32 18
-0 1 3 0 15 5 16 30
-0 1 3 0 30 16 3 17
-0 1 3 0 18 30 17 6
--1 1 3 3 29 30 35 36
-0 1 3 0 7 11 34 20
--1 1 3 3 31 32 33 34
-0 1 3 0 11 24 40 35
-0 1 3 0 24 8 33 40
-0 1 3 0 40 33 18 36
-0 1 3 0 35 40 36 34
-0 1 3 0 34 18 6 19
-0 1 3 0 20 34 19 2
+28
+1 3 0 9 21 12
+1 3 12 21 11 7
+1 3 21 22 37 25
+1 3 25 37 24 11
+1 3 37 23 8 24
+1 3 22 10 23 37
+1 3 9 4 10 21
+1 3 4 13 26 10
+1 3 13 1 14 26
+1 3 26 14 5 15
+1 3 38 28 15 29
+1 3 27 26 28 38
+1 3 10 27 38 23
+1 3 23 38 29 8
+1 3 8 29 39 33
+1 3 33 39 32 18
+1 3 39 31 30 32
+1 3 29 15 31 39
+1 3 15 5 16 30
+1 3 30 16 3 17
+1 3 18 30 17 6
+1 3 34 18 6 19
+1 3 40 33 18 36
+1 3 24 8 33 40
+1 3 11 24 40 35
+1 3 35 40 36 34
+1 3 7 11 34 20
+1 3 20 34 19 2
 
-# attr geom nodes
 boundary
 16
 1 1 0 9
 4 1 12 0
-1 1 9 4
 4 1 7 12
+1 1 9 4
 1 1 4 13
 1 1 13 1
 2 1 1 14
@@ -67,12 +59,11 @@ boundary
 2 1 16 3
 3 1 3 17
 3 1 17 6
-4 1 20 7
 3 1 6 19
+4 1 20 7
 3 1 19 2
 4 1 2 20
 
-# vert_id p1 p2
 vertex_parents
 37
 4 0 1
@@ -113,13 +104,59 @@ vertex_parents
 39 29 32
 40 24 36
 
-# top-level node coordinates
-coordinates
-4
+coarse_elements
+9
+3 2 5 4 3
+3 0 6 28 1
+3 12 11 10 13
+3 7 8 9 30
+3 14 17 16 15
+3 32 18 19 20
+3 24 23 22 25
+3 26 34 21 27
+3 29 31 33 35
+
+vertices
+41
 2
 0 0
 1 0
 0 1
 1 1
-
-mfem_mesh_end
+0.5 0
+1 0.5
+0.5 1
+0 0.5
+0.5 0.5
+0.25 0
+0.5 0.25
+0.25 0.5
+0 0.25
+0.75 0
+1 0.25
+0.75 0.5
+1 0.75
+0.75 1
+0.5 0.75
+0.25 1
+0 0.75
+0.25 0.25
+0.375 0.25
+0.5 0.375
+0.375 0.5
+0.25 0.375
+0.75 0.25
+0.625 0.25
+0.75 0.375
+0.625 0.5
+0.75 0.75
+0.75 0.625
+0.625 0.75
+0.5 0.625
+0.25 0.75
+0.25 0.625
+0.375 0.75
+0.375 0.375
+0.625 0.375
+0.625 0.625
+0.375 0.625

data/beam-quad-amr.mesh

@@ -1,43 +1,51 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1
 
-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
 # SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#
 
 dimension
 2
 
-# rank attr geom ref_type nodes/children
 elements
-20
-0 1 3 0 1 2 11 10
-0 1 3 0 2 3 12 11
-0 1 3 0 3 4 13 12
-0 2 3 0 4 5 14 13
-0 2 3 0 5 6 15 14
-0 2 3 0 6 7 16 15
-0 2 3 0 7 8 17 16
--1 1 3 3 8 13 14 15
--1 1 3 3 9 10 11 12
-0 1 3 0 0 23 30 26
-0 1 3 0 23 18 24 30
-0 1 3 0 30 24 22 25
-0 1 3 0 26 30 25 21
-0 1 3 0 18 1 19 22
-0 1 3 0 22 19 10 20
--1 1 3 3 16 17 18 19
-0 1 3 0 21 25 31 29
-0 1 3 0 25 22 27 31
-0 1 3 0 31 27 20 28
-0 1 3 0 29 31 28 9
+17
+1 3 0 23 30 26
+1 3 26 30 25 21
+1 3 30 24 22 25
+1 3 23 18 24 30
+1 3 18 1 19 22
+1 3 22 19 10 20
+1 3 31 27 20 28
+1 3 25 22 27 31
+1 3 21 25 31 29
+1 3 29 31 28 9
+1 3 1 2 11 10
+1 3 2 3 12 11
+1 3 3 4 13 12
+2 3 4 5 14 13
+2 3 5 6 15 14
+2 3 6 7 16 15
+2 3 7 8 17 16
 
-# attr geom nodes
 boundary
 25
+3 1 0 23
+1 1 26 0
+1 1 21 26
+3 1 23 18
+3 1 18 1
+3 1 10 20
+3 1 20 28
+1 1 29 21
+3 1 28 9
+1 1 9 29
 3 1 1 2
 3 1 11 10
 3 1 2 3
@@ -53,18 +61,7 @@ boundary
 3 1 7 8
 2 1 8 17
 3 1 17 16
-3 1 0 23
-1 1 26 0
-3 1 23 18
-1 1 21 26
-3 1 18 1
-3 1 10 20
-1 1 29 21
-3 1 20 28
-3 1 28 9
-1 1 9 29
 
-# vert_id p1 p2
 vertex_parents
 14
 18 0 1
@@ -82,21 +79,14 @@ vertex_parents
 30 23 25
 31 25 28
 
-# root element orientation
-root_state
-8
-1
-1
-1
-1
-1
-1
-0
-0
+coarse_elements
+3
+3 0 3 2 1
+3 8 7 6 9
+3 17 4 5 18
 
-# top-level node coordinates
-coordinates
-18
+vertices
+32
 2
 0 0
 1 0
@@ -116,5 +106,17 @@ coordinates
 6 1
 7 1
 8 1
-
-mfem_mesh_end
+0.5 0
+1 0.5
+0.5 1
+0 0.5
+0.5 0.5
+0.25 0
+0.5 0.25
+0.25 0.5
+0 0.25
+0.5 0.75
+0.25 1
+0 0.75
+0.25 0.25
+0.25 0.75

data/beam-wedge.vtk

@@ -31,14 +31,14 @@ POINTS 27 double
 7 0.5 1
 8 0.5 1
 CELLS 8 56
-6 0 18 9 1 19 10
-6 1 19 10 2 20 11
-6 2 20 11 3 21 12
-6 3 21 12 4 22 13
-6 4 22 13 5 23 14
-6 5 23 14 6 24 15
-6 6 24 15 7 25 16
-6 7 25 16 8 26 17
+6 0 9 18 1 10 19
+6 1 10 19 2 11 20
+6 2 11 20 3 12 21
+6 3 12 21 4 13 22
+6 4 13 22 5 14 23
+6 5 14 23 6 15 24
+6 6 15 24 7 16 25
+6 7 16 25 8 17 26
 CELL_TYPES 8
 13
 13

data/periodic-annulus-sector.geo

@@ -1,38 +1,13 @@
 SetFactory("OpenCASCADE");
 
-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
-
-// Set the geometry order (1, 2, ..., 9)
-order = 3;
-
-// Set the element type (3 - triangles, 4 - quadrilaterals)
-type = 3;
-
-// Number of radial elements
-nrad = 2;
-
-// Number of azimuthal elements on inner arc
-nazm1 = 3;
-
-// Number of azimuthal elements on outer arc
-nazm2 = 5;
-
-// Note: Using type = 4 with nazm1 != nazm2 can lead to mixed meshes
-//       containing both triangles and quadrilaterals.
-
-// Inner and outer radii
 R1 = 1.0;
 R2 = 2.0;
 
-// Angular size of the sector
-Phi = Pi/3.0;
-
 Point(1) = {0.0, 0, 0, 1.0};
 Point(2) = {R1, 0, 0, 1.0};
 Point(3) = {R2, 0, 0, 1.0};
-Point(4) = {R1*Cos(Phi), R1*Sin(Phi), 0, 1.0};
-Point(5) = {R2*Cos(Phi), R2*Sin(Phi), 0, 1.0};
+Point(4) = {R1*Cos(Pi/3), R1*Sin(Pi/3), 0, 1.0};
+Point(5) = {R2*Cos(Pi/3), R2*Sin(Pi/3), 0, 1.0};
 Line(1) = {2, 3};
 Line(2) = {4, 5};
 Circle(3) = {2, 1, 4};
@@ -40,23 +15,13 @@ Circle(4) = {3, 1, 5};
 Curve Loop(5) = {1, 4, -2, -3};
 Plane Surface(1) = {5};
 
-Transfinite Curve{1} = nrad+1;
-Transfinite Curve{2} = nrad+1;
-Transfinite Curve{3} = nazm1+1;
-Transfinite Curve{4} = nazm2+1;
-
-If (nazm1 == nazm2)
-   Transfinite Surface{1};
-EndIf
-
-If (type == 4)
-   Recombine Surface {1};
-EndIf
+Transfinite Curve{1} = 7;
+Transfinite Curve{2} = 7;
+Transfinite Curve{3} = 4;
+Transfinite Curve{4} = 10;
 
 // Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Phi};
-EndIf
+Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Pi/3};
 
 // Tag surfaces and volumes with positive integers
 Physical Curve(1) = {3};
@@ -65,22 +30,8 @@ Physical Curve(3) = {1};
 Physical Curve(4) = {2};
 Physical Surface(1) = {1};
 
-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
-
 // Generate 2D mesh
 Mesh 2;
-SetOrder order;
 Mesh.MshFileVersion = 2.2;
 
-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-annulus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("annulus-sector-t%01g-o%01g.msh", type, order);
-EndIf
+Save "periodic-annulus-sector.msh";

data/periodic-annulus-sector.msh

@@ -2,191 +2,184 @@ $MeshFormat
 2.2 0 8
 $EndMeshFormat
 $Nodes
-136
+55
 1 1 0 0
 2 2 0 0
 3 0.5000000000000001 0.8660254037844386 0
 4 1 1.732050807568877 0
-5 1.5 0 0
-6 1.166666666666667 0 0
-7 1.333333333333333 0 0
+5 1.166666666666667 0 0
+6 1.333333333333333 0 0
+7 1.5 0 0
 8 1.666666666666667 0 0
 9 1.833333333333333 0 0
-10 0.7500000000000002 1.299038105676658 0
-11 0.5833333333333335 1.010362971081845 0
-12 0.6666666666666667 1.154700538379251 0
+10 0.5833333333333335 1.010362971081845 0
+11 0.6666666666666667 1.154700538379251 0
+12 0.7500000000000002 1.299038105676658 0
 13 0.8333333333333335 1.443375672974064 0
 14 0.9166666666666669 1.587713240271471 0
 15 0.9396926207859085 0.3420201433256683 0
 16 0.7660444431189786 0.6427876096865386 0
-17 0.993238357741943 0.1160929141252301 0
-18 0.9730448705798238 0.2306158707424401 0
-19 0.8936326403234125 0.4487991802004617 0
-20 0.8354878114129367 0.5495089780708056 0
-21 0.6862416378687343 0.7273736415730481 0
-22 0.597158591702787 0.8021231927550432 0
-23 1.956295201467611 0.4158233816355181 0
-24 1.827090915285202 0.8134732861515996 0
-25 1.618033988749896 1.175570504584944 0
-26 1.338261212717719 1.486289650954786 0
-27 1.995128100519648 0.1395129474882505 0
-28 1.980536137483141 0.278346201920131 0
-29 1.922523391876638 0.551274711633998 0
-30 1.879385241571817 0.6840402866513373 0
-31 1.765895185717855 0.9389431255717802 0
-32 1.696096192312853 1.059838528466408 0
-33 1.532088886237958 1.285575219373077 0
-34 1.438679600677305 1.389316740917992 0
-35 1.231322950651319 1.576021507213442 0
-36 1.118385806941496 1.658075145110082 0
-37 1.162276263405681 0.6710405135499813 0
-38 1.248615852873337 1.079531485311822 0
-39 1.559209616901855 0.5415673055003691 0
-40 1.478306597054007 0.8535007117539289 0
-41 0.9210953433941653 0.9653302893212266 0
-42 1.296548225291847 0.3150268220262836 0
-43 1.055002035226811 1.358510675893086 0
-44 1.704005774249187 0.2344032256041583 0
-45 0.6403651144647218 0.8991270322967013 0
-46 0.7807302289294435 0.9322286608089638 0
-47 0.864063562262777 1.07656622810637 0
-48 0.8070317811313885 1.187802166891514 0
-49 0.7236984477980553 1.043464599594108 0
-50 1.432182741763949 0.1050089406754279 0
-51 1.364365483527898 0.2100178813508558 0
-52 1.197698816861231 0.2100178813508558 0
-53 1.098849408430616 0.1050089406754279 0
-54 1.265516075097282 0.1050089406754278 0
-55 0.8177280765440409 0.7503018362314346 0
-56 0.869411709969103 0.8578160627763305 0
-57 0.7348318576552288 0.8316090412164392 0
-58 1.177596357123201 0.3240245957927452 0
-59 1.058644488954555 0.3330223695592067 0
-60 1.087610484537871 0.2205785356313179 0
-61 1.267619707955123 0.7318605796179638 0
-62 1.372963152504565 0.7926806456859463 0
-63 1.40174301566045 0.9288443029398934 0
-64 1.325179434266893 1.004187894125858 0
-65 1.219835989717452 0.9433678280578752 0
-66 1.191056126561566 0.8072041708039283 0
-67 1.296399571111008 0.8680242368719107 0
-68 1.532241943619239 0.645545107584889 0
-69 1.505274270336623 0.7495229096694089 0
-70 1.294587381237739 0.6278827775334439 0
-71 1.426898499069797 0.5847250415169065 0
-72 1.399930825787181 0.6887028436014264 0
-73 1.139442349713613 1.041464419981624 0
-74 1.030268846553889 1.003397354651425 0
-75 1.001488983398004 0.8672336973974781 0
-76 1.081882623401843 0.7691371054737297 0
-77 1.110662486557728 0.9053007627276766 0
-78 1.207033584034403 0.5523692830420821 0
-79 1.251790904663125 0.4336980525341829 0
-80 1.384102022495183 0.3905403165176455 0
-81 1.471655819698519 0.4660538110090073 0
-82 1.339344701866461 0.5092115470255447 0
-83 1.73779714915742 0.7228379592678562 0
-84 1.648503383029637 0.6322026323841127 0
-85 1.691571478423774 0.4996526642120855 0
-86 1.823933339945692 0.4577380229238018 0
-87 1.787039416783436 0.5922941012619014 0
-88 1.308379426102925 1.350703595740465 0
-89 1.278497639488131 1.215117540526144 0
-90 1.371755231498856 1.111544491736196 0
-91 1.494894610124376 1.14355749816057 0
-92 1.4064614465962 1.25147448186763 0
-93 1.013887168325833 0.451693600067106 0
-94 1.088081715865757 0.5613670568085436 0
-95 1.03019898997678 0.6616228789288338 0
-96 0.8981217165478794 0.6522052443076862 0
-97 0.9637989050473432 0.5564495572737495 0
-98 1.432367408277627 0.2881522898855752 0
-99 1.568186591263407 0.2612777577448668 0
-100 1.655740388466743 0.3367912522362286 0
-101 1.607475002684299 0.4391792788682989 0
-102 1.519921205480963 0.3636657843769371 0
-103 1.184077913657828 1.17252454883891 0
-104 1.119539974442319 1.265517612365998 0
-105 1.010366471282595 1.2274505470358 0
-106 0.9657309073383804 1.096390418178513 0
-107 1.074904410498104 1.134457483508712 0
-108 1.901335258083062 0.07813440853471942 0
-109 1.802670516166124 0.1562688170694388 0
-110 1.636003849499458 0.156268817069439 0
-111 1.568001924749729 0.07813440853471942 0
-112 1.734668591416396 0.07813440853471944 0
-113 0.8516673450756037 1.318862295748801 0
-114 0.9533346901512071 1.338686485820944 0
-115 1.03666802348454 1.48302405311835 0
-116 1.01833401174227 1.607537430343614 0
-117 0.9350006784089369 1.463199863046207 0
-118 1.710829475874804 0.8268157613523761 0
-119 1.594568036464405 0.8401582365531526 0
-120 1.621535709747021 0.7361804344686326 0
-121 1.52488239428597 0.9608573093642675 0
-122 1.571458191517933 1.068213906974606 0
-123 1.448318812892413 1.036200900550232 0
-124 0.908699126206992 1.207626356963657 0
-125 1.500184666513678 0.1831433492101474 0
-126 1.646765991694905 0.9507607885973723 0
-127 0.9498053499729417 0.7597194708525821 0
-128 1.132839036494479 0.4426958263006443 0
-129 1.149421761057113 1.40110366758032 0
-130 1.243841486887416 1.443696659267553 0
-131 1.134903597606542 1.530869109405537 0
-132 1.872198725728137 0.3553499962917315 0
-133 1.788102249988661 0.2948766109479449 0
-134 1.893223337417325 0.2174207916708467 0
-135 1.213959700272622 1.308110604053232 0
-136 1.739836864206217 0.3972646375800152 0
+17 1.986476715483886 0.2321858282504602 0
+18 1.946089741159648 0.4612317414848793 0
+19 1.879385241571817 0.6840402866513365 0
+20 1.787265280646825 0.8975983604009234 0
+21 1.670975622825874 1.09901795614161 0
+22 1.532088886237958 1.285575219373077 0
+23 1.372483275737469 1.454747283146095 0
+24 1.194317183405575 1.604246385510085 0
+25 1.425989114816062 0.1915326920916892 0
+26 0.8788667344146573 1.13917645290495 0
+27 1.630372059110754 0.7154531062316609 0
+28 1.436395769298814 1.053728612482506 0
+29 1.081023776188756 0.6241293681829633 0
+30 1.168737372335971 1.428012728596308 0
+31 1.821063986059922 0.298149890497067 0
+32 1.234707097211386 0.3469796339295647 0
+33 1.377747393186519 0.6200150626754309 0
+34 1.457047681210906 0.3890895843559762 0
+35 0.917846726184522 0.8957978954532204 0
+36 1.218335619030348 0.9017812086952638 0
+37 1.066623110765233 1.061857005744772 0
+38 1.587029716281926 0.1355955181472859 0
+39 1.744445799211916 0.1441515753740107 0
+40 1.25 0.1443375672974065 0
+41 1.453660070628011 0.8435769396609902 0
+42 1.741367044061892 0.499612708014486 0
+43 1.30550638526547 1.257610469847477 0
+44 1.118213276932792 0.1666674689105279 0
+45 0.9109440214958271 1.306610291787315 0
+46 0.9970618258753989 1.438658589955562 0
+47 0.7499999999999998 1.010362971081845 0
+48 0.7034449005273667 0.8850673702175776 0
+49 1.605449512513618 0.9269067082200894 0
+50 1.561654019115059 0.5298592532912715 0
+51 1.229782222487711 1.096820457143683 0
+52 1.617066998712459 0.3090202662210922 0
+53 1.079645953234324 1.246963713711438 0
+54 1.877063966817811 0.1348974588243076 0
+55 1.055356609656722 1.558136350380461 0
 $EndNodes
 $Elements
-38
-1 26 2 3 1 1 5 6 7
-2 26 2 3 1 5 2 8 9
-3 26 2 4 2 3 10 11 12
-4 26 2 4 2 10 4 13 14
-5 26 2 1 3 1 15 17 18
-6 26 2 1 3 15 16 19 20
-7 26 2 1 3 16 3 21 22
-8 26 2 2 4 2 23 27 28
-9 26 2 2 4 23 24 29 30
-10 26 2 2 4 24 25 31 32
-11 26 2 2 4 25 26 33 34
-12 26 2 2 4 26 4 35 36
-13 21 2 1 1 3 41 10 45 46 47 48 12 11 49
-14 21 2 1 1 5 42 1 50 51 52 53 6 7 54
-15 21 2 1 1 16 41 3 55 56 46 45 22 21 57
-16 21 2 1 1 1 42 15 53 52 58 59 18 17 60
-17 21 2 1 1 37 40 38 61 62 63 64 65 66 67
-18 21 2 1 1 39 40 37 68 69 62 61 70 71 72
-19 21 2 1 1 38 41 37 73 74 75 76 66 65 77
-20 21 2 1 1 37 42 39 78 79 80 81 71 70 82
-21 21 2 1 1 24 39 23 83 84 85 86 29 30 87
-22 21 2 1 1 26 38 25 88 89 90 91 33 34 92
-23 21 2 1 1 15 37 16 93 94 95 96 20 19 97
-24 21 2 1 1 42 44 39 98 99 100 101 81 80 102
-25 21 2 1 1 38 43 41 103 104 105 106 74 73 107
-26 21 2 1 1 2 44 5 108 109 110 111 8 9 112
-27 21 2 1 1 10 43 4 113 114 115 116 14 13 117
-28 21 2 1 1 24 40 39 118 119 69 68 84 83 120
-29 21 2 1 1 38 40 25 64 63 121 122 91 90 123
-30 21 2 1 1 41 43 10 106 105 114 113 48 47 124
-31 21 2 1 1 5 44 42 111 110 99 98 51 50 125
-32 21 2 1 1 25 40 24 122 121 119 118 31 32 126
-33 21 2 1 1 37 41 16 76 75 56 55 96 95 127
-34 21 2 1 1 15 42 37 59 58 79 78 94 93 128
-35 21 2 1 1 4 43 26 116 115 129 130 35 36 131
-36 21 2 1 1 23 44 2 132 133 109 108 27 28 134
-37 21 2 1 1 26 43 38 130 129 104 103 89 88 135
-38 21 2 1 1 39 44 23 101 100 133 132 86 85 136
+108
+1 1 2 3 1 1 5
+2 1 2 3 1 5 6
+3 1 2 3 1 6 7
+4 1 2 3 1 7 8
+5 1 2 3 1 8 9
+6 1 2 3 1 9 2
+7 1 2 4 2 3 10
+8 1 2 4 2 10 11
+9 1 2 4 2 11 12
+10 1 2 4 2 12 13
+11 1 2 4 2 13 14
+12 1 2 4 2 14 4
+13 1 2 1 3 1 15
+14 1 2 1 3 15 16
+15 1 2 1 3 16 3
+16 1 2 2 4 2 17
+17 1 2 2 4 17 18
+18 1 2 2 4 18 19
+19 1 2 2 4 19 20
+20 1 2 2 4 20 21
+21 1 2 2 4 21 22
+22 1 2 2 4 22 23
+23 1 2 2 4 23 24
+24 1 2 2 4 24 4
+25 2 2 1 1 32 40 25
+26 2 2 1 1 25 34 32
+27 2 2 1 1 33 41 36
+28 2 2 1 1 38 52 25
+29 2 2 1 1 33 36 29
+30 2 2 1 1 26 47 35
+31 2 2 1 1 35 37 26
+32 2 2 1 1 25 52 34
+33 2 2 1 1 32 44 40
+34 2 2 1 1 15 32 29
+35 2 2 1 1 15 29 16
+36 2 2 1 1 36 41 28
+37 2 2 1 1 32 33 29
+38 2 2 1 1 50 52 42
+39 2 2 1 1 32 34 33
+40 2 2 1 1 42 52 31
+41 2 2 1 1 43 53 51
+42 2 2 1 1 27 41 33
+43 2 2 1 1 26 53 45
+44 2 2 1 1 18 31 17
+45 2 2 1 1 29 35 16
+46 2 2 1 1 29 36 35
+47 2 2 1 1 24 30 23
+48 2 2 1 1 30 53 43
+49 2 2 1 1 17 54 2
+50 2 2 1 1 4 55 24
+51 2 2 1 1 28 51 36
+52 2 2 1 1 47 48 35
+53 2 2 1 1 36 37 35
+54 2 2 1 1 37 53 26
+55 2 2 1 1 22 28 21
+56 2 2 1 1 20 27 19
+57 2 2 1 1 33 50 27
+58 2 2 1 1 15 44 32
+59 2 2 1 1 18 42 31
+60 2 2 1 1 30 43 23
+61 2 2 1 1 35 48 16
+62 2 2 1 1 31 54 17
+63 2 2 1 1 9 39 8
+64 2 2 1 1 8 38 7
+65 2 2 1 1 7 25 6
+66 2 2 1 1 22 43 28
+67 2 2 1 1 23 43 22
+68 2 2 1 1 39 54 31
+69 2 2 1 1 19 42 18
+70 2 2 1 1 24 55 30
+71 2 2 1 1 27 42 19
+72 2 2 1 1 13 46 14
+73 2 2 1 1 51 53 37
+74 2 2 1 1 39 52 38
+75 2 2 1 1 6 40 5
+76 2 2 1 1 34 52 50
+77 2 2 1 1 12 45 13
+78 2 2 1 1 30 55 46
+79 2 2 1 1 10 47 11
+80 2 2 1 1 8 39 38
+81 2 2 1 1 28 49 21
+82 2 2 1 1 7 38 25
+83 2 2 1 1 41 49 28
+84 2 2 1 1 20 49 27
+85 2 2 1 1 11 26 12
+86 2 2 1 1 27 49 41
+87 2 2 1 1 31 52 39
+88 2 2 1 1 25 40 6
+89 2 2 1 1 2 54 9
+90 2 2 1 1 14 55 4
+91 2 2 1 1 45 53 46
+92 2 2 1 1 45 46 13
+93 2 2 1 1 5 44 1
+94 2 2 1 1 21 49 20
+95 2 2 1 1 46 53 30
+96 2 2 1 1 3 48 10
+97 2 2 1 1 34 50 33
+98 2 2 1 1 36 51 37
+99 2 2 1 1 26 45 12
+100 2 2 1 1 11 47 26
+101 2 2 1 1 27 50 42
+102 2 2 1 1 40 44 5
+103 2 2 1 1 43 51 28
+104 2 2 1 1 10 48 47
+105 2 2 1 1 9 54 39
+106 2 2 1 1 46 55 14
+107 2 2 1 1 1 44 15
+108 2 2 1 1 16 48 3
 $EndElements
 $Periodic
 1
 1 1 2
 Affine 0.5000000000000001 0.8660254037844386 0 0 -0.8660254037844386 0.5000000000000001 0 0 0 0 1 0 0 0 0 1
-3
+7
+9 14
+6 11
+8 13
 5 10
-1 3
+7 12
 2 4
+1 3
 $EndPeriodic

data/periodic-cube.geo

@@ -1,38 +0,0 @@
-// 0 for tetrahedra, 1 for hexahedra
-tet_or_hex = 1;
-
-Point(1) = {0, 0, 0, 1.0};
-Point(2) = {1, 0, 0, 1.0};
-Point(3) = {1, 1, 0, 1.0};
-Point(4) = {0, 1, 0, 1.0};
-
-Characteristic Length {:} = 0.25;
-
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-
-Periodic Curve {1} = {-3};
-Periodic Curve {2} = {-4};
-
-Curve Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-Transfinite Surface {1};
-
-If (tet_or_hex == 1)
-   Recombine Surface {1};
-   out[] = Extrude {0, 0, 1} { Surface{1}; Layers{4}; Recombine; };
-Else
-   out[] = Extrude {0, 0, 1} { Surface{1}; Layers{4}; }
-EndIf
-
-Physical Volume(1) = {out[1]};
-Physical Surface(1) = {1,out[0],out[2],out[3],out[4],out[5]};
-
-Mesh 3;
-Mesh.MshFileVersion = 2.2;
-
-Periodic Surface {out[0]} = {1} Translate {0, 0, 1};
-Periodic Surface {out[4]} = {out[2]} Translate {0, 1, 0};
-Periodic Surface {out[3]} = {out[5]} Translate {1, 0, 0};

data/periodic-cube.msh

@@ -1,381 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-125
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0 0 1
-6 1 0 1
-7 1 1 1
-8 0 1 1
-9 0.2500000000010404 0 0
-10 0.5000000000020591 0 0
-11 0.7500000000003465 0 0
-12 1 0.2500000000010404 0
-13 1 0.5000000000020591 0
-14 1 0.7500000000003465 0
-15 0.7500000000003465 1 0
-16 0.5000000000020591 1 0
-17 0.2500000000010404 1 0
-18 0 0.7500000000003465 0
-19 0 0.5000000000020591 0
-20 0 0.2500000000010404 0
-21 0.2500000000010404 0 1
-22 0.5000000000020591 0 1
-23 0.7500000000003465 0 1
-24 1 0.2500000000010404 1
-25 1 0.5000000000020591 1
-26 1 0.7500000000003465 1
-27 0.7500000000003465 1 1
-28 0.5000000000020591 1 1
-29 0.2500000000010404 1 1
-30 0 0.7500000000003465 1
-31 0 0.5000000000020591 1
-32 0 0.2500000000010404 1
-33 0 0 0.25
-34 0 0 0.5
-35 0 0 0.75
-36 1 0 0.25
-37 1 0 0.5
-38 1 0 0.75
-39 1 1 0.25
-40 1 1 0.5
-41 1 1 0.75
-42 0 1 0.25
-43 0 1 0.5
-44 0 1 0.75
-45 0.2500000000010404 0.2500000000010404 0
-46 0.2500000000010404 0.5000000000020591 0
-47 0.2500000000010404 0.7500000000003464 0
-48 0.5000000000020591 0.2500000000010404 0
-49 0.5000000000020591 0.5000000000020591 0
-50 0.5000000000020591 0.7500000000003465 0
-51 0.7500000000003464 0.2500000000010404 0
-52 0.7500000000003467 0.5000000000020591 0
-53 0.7500000000003464 0.7500000000003466 0
-54 0.2500000000010404 0 0.25
-55 0.2500000000010404 0 0.5
-56 0.2500000000010404 0 0.75
-57 0.5000000000020591 0 0.25
-58 0.5000000000020591 0 0.5
-59 0.5000000000020591 0 0.75
-60 0.7500000000003465 0 0.25
-61 0.7500000000003465 0 0.5
-62 0.7500000000003465 0 0.75
-63 1 0.2500000000010404 0.25
-64 1 0.2500000000010404 0.5
-65 1 0.2500000000010404 0.75
-66 1 0.5000000000020591 0.25
-67 1 0.5000000000020591 0.5
-68 1 0.5000000000020591 0.75
-69 1 0.7500000000003465 0.25
-70 1 0.7500000000003465 0.5
-71 1 0.7500000000003465 0.75
-72 0.7500000000003465 1 0.25
-73 0.7500000000003465 1 0.5
-74 0.7500000000003465 1 0.75
-75 0.5000000000020591 1 0.25
-76 0.5000000000020591 1 0.5
-77 0.5000000000020591 1 0.75
-78 0.2500000000010404 1 0.25
-79 0.2500000000010404 1 0.5
-80 0.2500000000010404 1 0.75
-81 0 0.7500000000003465 0.25
-82 0 0.7500000000003465 0.5
-83 0 0.7500000000003465 0.75
-84 0 0.5000000000020591 0.25
-85 0 0.5000000000020591 0.5
-86 0 0.5000000000020591 0.75
-87 0 0.2500000000010404 0.25
-88 0 0.2500000000010404 0.5
-89 0 0.2500000000010404 0.75
-90 0.2500000000010404 0.2500000000010404 1
-91 0.2500000000010404 0.5000000000020591 1
-92 0.2500000000010404 0.7500000000003464 1
-93 0.5000000000020591 0.2500000000010404 1
-94 0.5000000000020591 0.5000000000020591 1
-95 0.5000000000020591 0.7500000000003465 1
-96 0.7500000000003464 0.2500000000010404 1
-97 0.7500000000003467 0.5000000000020591 1
-98 0.7500000000003464 0.7500000000003466 1
-99 0.2500000000010404 0.2500000000010404 0.25
-100 0.2500000000010404 0.2500000000010404 0.5
-101 0.2500000000010404 0.2500000000010404 0.75
-102 0.2500000000010404 0.5000000000020591 0.25
-103 0.2500000000010404 0.5000000000020591 0.5
-104 0.2500000000010404 0.5000000000020591 0.75
-105 0.2500000000010404 0.7500000000003464 0.25
-106 0.2500000000010404 0.7500000000003464 0.5
-107 0.2500000000010404 0.7500000000003464 0.75
-108 0.5000000000020591 0.2500000000010404 0.25
-109 0.5000000000020591 0.2500000000010404 0.5
-110 0.5000000000020591 0.2500000000010404 0.75
-111 0.5000000000020591 0.5000000000020591 0.25
-112 0.5000000000020591 0.5000000000020591 0.5
-113 0.5000000000020591 0.5000000000020591 0.75
-114 0.5000000000020591 0.7500000000003465 0.25
-115 0.5000000000020591 0.7500000000003465 0.5
-116 0.5000000000020591 0.7500000000003465 0.75
-117 0.7500000000003464 0.2500000000010404 0.25
-118 0.7500000000003464 0.2500000000010404 0.5
-119 0.7500000000003464 0.2500000000010404 0.75
-120 0.7500000000003467 0.5000000000020591 0.25
-121 0.7500000000003467 0.5000000000020591 0.5
-122 0.7500000000003467 0.5000000000020591 0.75
-123 0.7500000000003464 0.7500000000003466 0.25
-124 0.7500000000003464 0.7500000000003466 0.5
-125 0.7500000000003464 0.7500000000003466 0.75
-$EndNodes
-$Elements
-160
-1 3 2 1 1 1 9 45 20
-2 3 2 1 1 20 45 46 19
-3 3 2 1 1 19 46 47 18
-4 3 2 1 1 18 47 17 4
-5 3 2 1 1 9 10 48 45
-6 3 2 1 1 45 48 49 46
-7 3 2 1 1 46 49 50 47
-8 3 2 1 1 47 50 16 17
-9 3 2 1 1 10 11 51 48
-10 3 2 1 1 48 51 52 49
-11 3 2 1 1 49 52 53 50
-12 3 2 1 1 50 53 15 16
-13 3 2 1 1 11 2 12 51
-14 3 2 1 1 51 12 13 52
-15 3 2 1 1 52 13 14 53
-16 3 2 1 1 53 14 3 15
-17 3 2 1 13 1 9 54 33
-18 3 2 1 13 33 54 55 34
-19 3 2 1 13 34 55 56 35
-20 3 2 1 13 35 56 21 5
-21 3 2 1 13 9 10 57 54
-22 3 2 1 13 54 57 58 55
-23 3 2 1 13 55 58 59 56
-24 3 2 1 13 56 59 22 21
-25 3 2 1 13 10 11 60 57
-26 3 2 1 13 57 60 61 58
-27 3 2 1 13 58 61 62 59
-28 3 2 1 13 59 62 23 22
-29 3 2 1 13 11 2 36 60
-30 3 2 1 13 60 36 37 61
-31 3 2 1 13 61 37 38 62
-32 3 2 1 13 62 38 6 23
-33 3 2 1 17 2 12 63 36
-34 3 2 1 17 36 63 64 37
-35 3 2 1 17 37 64 65 38
-36 3 2 1 17 38 65 24 6
-37 3 2 1 17 12 13 66 63
-38 3 2 1 17 63 66 67 64
-39 3 2 1 17 64 67 68 65
-40 3 2 1 17 65 68 25 24
-41 3 2 1 17 13 14 69 66
-42 3 2 1 17 66 69 70 67
-43 3 2 1 17 67 70 71 68
-44 3 2 1 17 68 71 26 25
-45 3 2 1 17 14 3 39 69
-46 3 2 1 17 69 39 40 70
-47 3 2 1 17 70 40 41 71
-48 3 2 1 17 71 41 7 26
-49 3 2 1 21 3 15 72 39
-50 3 2 1 21 39 72 73 40
-51 3 2 1 21 40 73 74 41
-52 3 2 1 21 41 74 27 7
-53 3 2 1 21 15 16 75 72
-54 3 2 1 21 72 75 76 73
-55 3 2 1 21 73 76 77 74
-56 3 2 1 21 74 77 28 27
-57 3 2 1 21 16 17 78 75
-58 3 2 1 21 75 78 79 76
-59 3 2 1 21 76 79 80 77
-60 3 2 1 21 77 80 29 28
-61 3 2 1 21 17 4 42 78
-62 3 2 1 21 78 42 43 79
-63 3 2 1 21 79 43 44 80
-64 3 2 1 21 80 44 8 29
-65 3 2 1 25 4 18 81 42
-66 3 2 1 25 42 81 82 43
-67 3 2 1 25 43 82 83 44
-68 3 2 1 25 44 83 30 8
-69 3 2 1 25 18 19 84 81
-70 3 2 1 25 81 84 85 82
-71 3 2 1 25 82 85 86 83
-72 3 2 1 25 83 86 31 30
-73 3 2 1 25 19 20 87 84
-74 3 2 1 25 84 87 88 85
-75 3 2 1 25 85 88 89 86
-76 3 2 1 25 86 89 32 31
-77 3 2 1 25 20 1 33 87
-78 3 2 1 25 87 33 34 88
-79 3 2 1 25 88 34 35 89
-80 3 2 1 25 89 35 5 32
-81 3 2 1 26 5 21 90 32
-82 3 2 1 26 32 90 91 31
-83 3 2 1 26 31 91 92 30
-84 3 2 1 26 30 92 29 8
-85 3 2 1 26 21 22 93 90
-86 3 2 1 26 90 93 94 91
-87 3 2 1 26 91 94 95 92
-88 3 2 1 26 92 95 28 29
-89 3 2 1 26 22 23 96 93
-90 3 2 1 26 93 96 97 94
-91 3 2 1 26 94 97 98 95
-92 3 2 1 26 95 98 27 28
-93 3 2 1 26 23 6 24 96
-94 3 2 1 26 96 24 25 97
-95 3 2 1 26 97 25 26 98
-96 3 2 1 26 98 26 7 27
-97 5 2 1 1 1 9 45 20 33 54 99 87
-98 5 2 1 1 33 54 99 87 34 55 100 88
-99 5 2 1 1 34 55 100 88 35 56 101 89
-100 5 2 1 1 35 56 101 89 5 21 90 32
-101 5 2 1 1 20 45 46 19 87 99 102 84
-102 5 2 1 1 87 99 102 84 88 100 103 85
-103 5 2 1 1 88 100 103 85 89 101 104 86
-104 5 2 1 1 89 101 104 86 32 90 91 31
-105 5 2 1 1 19 46 47 18 84 102 105 81
-106 5 2 1 1 84 102 105 81 85 103 106 82
-107 5 2 1 1 85 103 106 82 86 104 107 83
-108 5 2 1 1 86 104 107 83 31 91 92 30
-109 5 2 1 1 18 47 17 4 81 105 78 42
-110 5 2 1 1 81 105 78 42 82 106 79 43
-111 5 2 1 1 82 106 79 43 83 107 80 44
-112 5 2 1 1 83 107 80 44 30 92 29 8
-113 5 2 1 1 9 10 48 45 54 57 108 99
-114 5 2 1 1 54 57 108 99 55 58 109 100
-115 5 2 1 1 55 58 109 100 56 59 110 101
-116 5 2 1 1 56 59 110 101 21 22 93 90
-117 5 2 1 1 45 48 49 46 99 108 111 102
-118 5 2 1 1 99 108 111 102 100 109 112 103
-119 5 2 1 1 100 109 112 103 101 110 113 104
-120 5 2 1 1 101 110 113 104 90 93 94 91
-121 5 2 1 1 46 49 50 47 102 111 114 105
-122 5 2 1 1 102 111 114 105 103 112 115 106
-123 5 2 1 1 103 112 115 106 104 113 116 107
-124 5 2 1 1 104 113 116 107 91 94 95 92
-125 5 2 1 1 47 50 16 17 105 114 75 78
-126 5 2 1 1 105 114 75 78 106 115 76 79
-127 5 2 1 1 106 115 76 79 107 116 77 80
-128 5 2 1 1 107 116 77 80 92 95 28 29
-129 5 2 1 1 10 11 51 48 57 60 117 108
-130 5 2 1 1 57 60 117 108 58 61 118 109
-131 5 2 1 1 58 61 118 109 59 62 119 110
-132 5 2 1 1 59 62 119 110 22 23 96 93
-133 5 2 1 1 48 51 52 49 108 117 120 111
-134 5 2 1 1 108 117 120 111 109 118 121 112
-135 5 2 1 1 109 118 121 112 110 119 122 113
-136 5 2 1 1 110 119 122 113 93 96 97 94
-137 5 2 1 1 49 52 53 50 111 120 123 114
-138 5 2 1 1 111 120 123 114 112 121 124 115
-139 5 2 1 1 112 121 124 115 113 122 125 116
-140 5 2 1 1 113 122 125 116 94 97 98 95
-141 5 2 1 1 50 53 15 16 114 123 72 75
-142 5 2 1 1 114 123 72 75 115 124 73 76
-143 5 2 1 1 115 124 73 76 116 125 74 77
-144 5 2 1 1 116 125 74 77 95 98 27 28
-145 5 2 1 1 11 2 12 51 60 36 63 117
-146 5 2 1 1 60 36 63 117 61 37 64 118
-147 5 2 1 1 61 37 64 118 62 38 65 119
-148 5 2 1 1 62 38 65 119 23 6 24 96
-149 5 2 1 1 51 12 13 52 117 63 66 120
-150 5 2 1 1 117 63 66 120 118 64 67 121
-151 5 2 1 1 118 64 67 121 119 65 68 122
-152 5 2 1 1 119 65 68 122 96 24 25 97
-153 5 2 1 1 52 13 14 53 120 66 69 123
-154 5 2 1 1 120 66 69 123 121 67 70 124
-155 5 2 1 1 121 67 70 124 122 68 71 125
-156 5 2 1 1 122 68 71 125 97 25 26 98
-157 5 2 1 1 53 14 3 15 123 69 39 72
-158 5 2 1 1 123 69 39 72 124 70 40 73
-159 5 2 1 1 124 70 40 73 125 71 41 74
-160 5 2 1 1 125 71 41 74 98 26 7 27
-$EndElements
-$Periodic
-3
-2 17 25
-Affine 1 0 0 1 0 1 0 0 0 0 1 0 0 0 0 1
-25
-2 1
-3 4
-6 5
-7 8
-63 87
-64 88
-65 89
-66 84
-67 85
-68 86
-69 81
-70 82
-71 83
-14 18
-24 32
-25 31
-26 30
-36 33
-37 34
-38 35
-39 42
-40 43
-41 44
-12 20
-13 19
-2 21 13
-Affine 1 0 0 0 0 1 0 1 0 0 1 0 0 0 0 1
-25
-3 2
-4 1
-7 6
-8 5
-15 11
-16 10
-17 9
-72 60
-73 61
-74 62
-75 57
-76 58
-77 59
-78 54
-79 55
-80 56
-27 23
-28 22
-29 21
-39 36
-40 37
-41 38
-42 33
-43 34
-44 35
-2 26 1
-Affine 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 1
-25
-5 1
-6 2
-7 3
-8 4
-90 45
-91 46
-92 47
-93 48
-94 49
-95 50
-96 51
-97 52
-98 53
-21 9
-22 10
-23 11
-24 12
-25 13
-26 14
-27 15
-28 16
-29 17
-30 18
-31 19
-32 20
-$EndPeriodic

data/periodic-square.geo

@@ -1,31 +0,0 @@
-// 0 for triangles, 1 for quads
-tri_or_quad = 1;
-
-Point(1) = {0, 0, 0, 1.0};
-Point(2) = {1, 0, 0, 1.0};
-Point(3) = {1, 1, 0, 1.0};
-Point(4) = {0, 1, 0, 1.0};
-
-Characteristic Length {:} = 0.25;
-
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-
-Periodic Line {3} = {-1};
-Periodic Line {2} = {-4};
-
-Curve Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-Transfinite Surface {1};
-
-If (tri_or_quad == 1)
-   Recombine Surface {1};
-EndIf
-
-Physical Surface(1) = {1};
-Physical Curve(1) = {1, 2, 3, 4};
-
-Mesh.MshFileVersion = 2.2;
-Mesh 2;

data/periodic-square.msh

@@ -1,83 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-25
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0.2499999999994121 0 0
-6 0.499999999998694 0 0
-7 0.7499999999993416 0 0
-8 1 0.2500000000010404 0
-9 1 0.5000000000020591 0
-10 1 0.7500000000003465 0
-11 0.7499999999993416 1 0
-12 0.4999999999986939 1 0
-13 0.249999999999412 1 0
-14 0 0.7500000000003465 0
-15 0 0.5000000000020591 0
-16 0 0.2500000000010404 0
-17 0.2499999999994121 0.2500000000010404 0
-18 0.249999999999412 0.5000000000020591 0
-19 0.249999999999412 0.7500000000003466 0
-20 0.4999999999986939 0.2500000000010404 0
-21 0.4999999999986939 0.5000000000020591 0
-22 0.4999999999986939 0.7500000000003466 0
-23 0.7499999999993416 0.2500000000010404 0
-24 0.7499999999993416 0.5000000000020591 0
-25 0.7499999999993416 0.7500000000003465 0
-$EndNodes
-$Elements
-32
-1 1 2 1 1 1 5
-2 1 2 1 1 5 6
-3 1 2 1 1 6 7
-4 1 2 1 1 7 2
-5 1 2 1 2 2 8
-6 1 2 1 2 8 9
-7 1 2 1 2 9 10
-8 1 2 1 2 10 3
-9 1 2 1 3 3 11
-10 1 2 1 3 11 12
-11 1 2 1 3 12 13
-12 1 2 1 3 13 4
-13 1 2 1 4 4 14
-14 1 2 1 4 14 15
-15 1 2 1 4 15 16
-16 1 2 1 4 16 1
-17 3 2 1 1 1 5 17 16
-18 3 2 1 1 16 17 18 15
-19 3 2 1 1 15 18 19 14
-20 3 2 1 1 14 19 13 4
-21 3 2 1 1 5 6 20 17
-22 3 2 1 1 17 20 21 18
-23 3 2 1 1 18 21 22 19
-24 3 2 1 1 19 22 12 13
-25 3 2 1 1 6 7 23 20
-26 3 2 1 1 20 23 24 21
-27 3 2 1 1 21 24 25 22
-28 3 2 1 1 22 25 11 12
-29 3 2 1 1 7 2 8 23
-30 3 2 1 1 23 8 9 24
-31 3 2 1 1 24 9 10 25
-32 3 2 1 1 25 10 3 11
-$EndElements
-$Periodic
-2
-1 2 4
-5
-8 16
-9 15
-10 14
-2 1
-3 4
-1 3 1
-5
-11 7
-12 6
-13 5
-3 2
-4 1
-$EndPeriodic

data/periodic-torus-sector.geo

@@ -1,141 +1,25 @@
-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
+SetFactory("OpenCASCADE");
 
-// Set the geometry order (1, 2, ..., 10 for tetrahedra or 9 for other types)
-order = 3;
+R = 1.5;
+r = 0.5;
 
-// Set the element type (4 - tetrahedra, 6 - wedges, 8 - hexahedra)
-type = 8;
+Torus(1) = {0,0,0, R, r, Pi/3};
 
-// Minor and major radii
-R1 = 1.0;
-R2 = 2.0;
+pts() = PointsOf{ Volume{1}; };
 
-// Side length of interior square
-A1 = 0.8;
-
-// Angular size of the sector
-Phi = Pi/3.0;
-
-// Number of azimuthal elements
-nazm = 3;
-
-// Number of elements around a quarter of the circle
-narc = 2;
-
-// Number of elements between surface and interior square
-nshl = 1;
-
-lc = 0.5;
-a1 = A1 / Sqrt(2.0);
-
-Point(1) = {R2+R1, 0, 0, lc};
-Point(2) = {R2, 0, R1, lc};
-Point(3) = {R2-R1, 0, 0, lc};
-Point(4) = {R2, 0, -R1, lc};
-Point(5) = {R2, 0, 0, lc};
-Point(6) = {R2+a1, 0, 0, lc};
-Point(7) = {R2, 0, a1, lc};
-Point(8) = {R2-a1, 0, 0, lc};
-Point(9) = {R2, 0, -a1, lc};
-
-Circle(1) = {1,5,2};
-Circle(2) = {2,5,3};
-Circle(3) = {3,5,4};
-Circle(4) = {4,5,1};
-
-Line(5) = {6,1};
-Line(6) = {7,2};
-Line(7) = {8,3};
-Line(8) = {9,4};
-
-Line(9) = {6, 7};
-Line(10) = {7, 8};
-Line(11) = {8, 9};
-Line(12) = {9, 6};
-
-Line Loop(101) = {1, -6, -9, 5};
-Line Loop(102) = {2, -7, -10, 6};
-Line Loop(103) = {3, -8, -11, 7};
-Line Loop(104) = {4, -5, -12, 8};
-Line Loop(105) = {9, 10, 11, 12};
-
-Plane Surface(201) = {101};
-Plane Surface(202) = {102};
-Plane Surface(203) = {103};
-Plane Surface(204) = {104};
-Plane Surface(205) = {105};
-
-Transfinite Curve{1} = narc+1;
-Transfinite Curve{2} = narc+1;
-Transfinite Curve{3} = narc+1;
-Transfinite Curve{4} = narc+1;
-
-Transfinite Curve{5} = nshl+1;
-Transfinite Curve{6} = nshl+1;
-Transfinite Curve{7} = nshl+1;
-Transfinite Curve{8} = nshl+1;
-
-Transfinite Curve{9} = narc+1;
-Transfinite Curve{10} = narc+1;
-Transfinite Curve{11} = narc+1;
-Transfinite Curve{12} = narc+1;
-
-If (type == 8)
-   Recombine Surface {201};
-   Recombine Surface {202};
-   Recombine Surface {203};
-   Recombine Surface {204};
-   Recombine Surface {205};
-
-   Transfinite Surface {201} = {1,2,7,6};
-   Transfinite Surface {202} = {2,3,8,7};
-   Transfinite Surface {203} = {3,4,9,8};
-   Transfinite Surface {204} = {4,1,6,9};
-   Transfinite Surface {205} = {6,7,8,9};
-EndIf
-
-If (type == 4)
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm};
-}
-Else
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm}; Recombine;
-}
-EndIf
+Characteristic Length{ pts() } = 0.25;
 
 // Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Surface{227} = {201} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{249} = {202} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{271} = {203} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{293} = {204} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{315} = {205} Rotate{{0,0,1}, {0,0,0}, Phi};
-EndIf
+Periodic Surface{3} = {2} Rotate{{0,0,1}, {0,0,0}, Pi/3};
 
 // Tag surfaces and volumes with positive integers
-Physical Surface(1) = {201,202,203,204,205};
-Physical Surface(2) = {227,249,271,293,315};
-Physical Surface(3) = {214,236,258,280};
-Physical Volume(1) = {1,2,3,4,5};
-
-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
+Physical Surface(1) = {1};
+Physical Surface(2) = {2};
+Physical Surface(3) = {3};
+Physical Volume(1) = {1};
 
 // Generate 3D mesh
 Mesh 3;
-SetOrder order;
+
 Mesh.MshFileVersion = 2.2;
-
-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-torus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("torus-sector-t%01g-o%01g.msh", type, order);
-EndIf
+Save "periodic-torus-sector.msh";

data/ref-cube.mesh

@@ -1,41 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 5 0 1 2 3 4 5 6 7
-
-boundary
-6
-1 3 3 2 1 0
-2 3 0 1 5 4
-3 3 1 2 6 5
-4 3 2 3 7 6
-5 3 3 0 4 7
-6 3 4 5 6 7
-
-vertices
-8
-3
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-0 0 1
-1 0 1
-1 1 1
-0 1 1

data/ref-prism.mesh

@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 6 0 1 2 3 4 5
-
-boundary
-5
-1 2 0 2 1
-2 2 3 4 5
-3 3 0 1 4 3
-4 3 1 2 5 4
-5 3 2 0 3 5
-
-vertices
-6
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1
-1 0 1
-0 1 1

data/ref-segment.mesh

@@ -1,31 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-1
-
-elements
-1
-1 1 0 1
-
-boundary
-2
-1 0 0
-2 0 1
-
-vertices
-2
-1
-0
-1

data/ref-square.mesh

@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 3 0 1 2 3
-
-boundary
-4
-1 1 0 1
-2 1 1 2
-3 1 2 3
-4 1 3 0
-
-vertices
-4
-2
-0 0
-1 0
-1 1
-0 1

data/ref-tetrahedron.mesh

@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 4 0 1 2 3
-
-boundary
-4
-1 2 1 2 3
-2 2 0 3 2
-3 2 0 1 3
-4 2 0 2 1
-
-vertices
-4
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1

data/ref-triangle.mesh

@@ -1,33 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 2 0 1 2
-
-boundary
-3
-1 1 0 1
-2 1 1 2
-3 1 2 0
-
-vertices
-3
-2
-0 0
-1 0
-0 1

data/square-mixed.mesh

@@ -1,118 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-20
-1 3 0 1 6 5
-1 3 1 2 7 6
-1 3 2 3 8 7
-1 3 3 4 9 8
-1 3 5 6 11 10
-1 2 6 7 11
-1 2 7 12 11
-1 2 7 8 13
-1 2 7 13 12
-1 3 8 9 14 13
-1 3 10 11 16 15
-1 2 11 12 17
-1 2 11 17 16
-1 2 12 13 17
-1 2 13 18 17
-1 3 13 14 19 18
-1 3 15 16 21 20
-1 3 16 17 22 21
-1 3 17 18 23 22
-1 3 18 19 24 23
-
-boundary
-16
-2 1 0 1
-2 1 1 2
-2 1 2 3
-2 1 3 4
-2 1 21 20
-2 1 22 21
-2 1 23 22
-2 1 24 23
-1 1 5 0
-1 1 10 5
-1 1 15 10
-1 1 20 15
-1 1 4 9
-1 1 9 14
-1 1 14 19
-1 1 19 24
-
-vertices
-25
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P1
-VDim: 2
-Ordering: 0
-
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0
-0
-0
-0
-0.25
-0.25
-0.25
-0.25
-0.25
-0.5
-0.5
-0.5
-0.5
-0.5
-0.75
-0.75
-0.75
-0.75
-0.75
-1
-1
-1
-1
-1

doc/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -16,6 +16,7 @@ if (DOXYGEN_FOUND)
   configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CodeDocumentation.conf.in
     ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf @ONLY)
 
+
   add_custom_target(doc
     COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
     COMMAND echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
@@ -31,4 +32,5 @@ if (DOXYGEN_FOUND)
     COMMENT "Removing API documentation"
     VERBATIM)
 
+
 endif (DOXYGEN_FOUND)

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.2.1
+PROJECT_NUMBER         = v4.1.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "Finite element discretization library"
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.
 
-PROJECT_LOGO           = logo-small.png
+PROJECT_LOGO           = web/logo-small.png
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -766,25 +766,18 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mesh \
                          @MFEM_SOURCE_DIR@/fem \
                          @MFEM_SOURCE_DIR@/examples \
-                         @MFEM_SOURCE_DIR@/examples/caliper \
-                         @MFEM_SOURCE_DIR@/examples/amgx \
-                         @MFEM_SOURCE_DIR@/examples/ginkgo \
-                         @MFEM_SOURCE_DIR@/examples/hiop \
                          @MFEM_SOURCE_DIR@/examples/petsc \
                          @MFEM_SOURCE_DIR@/examples/pumi \
+                         @MFEM_SOURCE_DIR@/examples/hiop \
                          @MFEM_SOURCE_DIR@/examples/sundials \
-                         @MFEM_SOURCE_DIR@/examples/superlu \
-                         @MFEM_SOURCE_DIR@/miniapps/adjoint \
+                         @MFEM_SOURCE_DIR@/miniapps/adjoint \			 
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/mtop \
                          @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
                          @MFEM_SOURCE_DIR@/miniapps/performance \
-                         @MFEM_SOURCE_DIR@/miniapps/shifted \
-                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
                          @MFEM_SOURCE_DIR@/miniapps/toys
 
@@ -823,9 +816,7 @@ RECURSIVE              = NO
 # run.
 
 EXCLUDE                = @MFEM_SOURCE_DIR@/config/_config.hpp \
-                         @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.h \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp
+                         @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -2343,7 +2334,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_GRAPH_MAX_NODES    = 50
+DOT_GRAPH_MAX_NODES    = 10
 
 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the

doc/CodeDocumentation.dox

@@ -42,10 +42,8 @@ namespace mfem {
  * - MFEM_FORALL macro in forall.hpp
  *
  * <H3>Example codes</H3>
- * - <a class="el" href="ex0_8cpp_source.html">Example 0</a>: simplest example, nodal H1 FEM for the Laplace problem
- * - <a class="el" href="ex0p_8cpp_source.html">Example 0p</a>: simplest parallel example, nodal H1 FEM for the Laplace problem
- * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem (same discretization as ex0 but with more sophisticated options)
- * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem (same discretization as ex0p but with more sophisticated options)
+ * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
  * - <a class="el" href="ex2_8cpp_source.html">Example 2</a>: vector FEM for linear elasticity
  * - <a class="el" href="ex2p_8cpp_source.html">Example 2p</a>: parallel vector FEM for linear elasticity
  * - <a class="el" href="ex3_8cpp_source.html">Example 3</a>: Nedelec H(curl) FEM for the definite Maxwell problem
@@ -64,7 +62,7 @@ namespace mfem {
  * - <a class="el" href="ex9p_8cpp_source.html">Example 9p</a>: parallel Discontinuous Galerkin (DG) time-dependent advection
  * - <a class="el" href="ex10_8cpp_source.html">Example 10</a>: time-dependent implicit nonlinear elasticity
  * - <a class="el" href="ex10p_8cpp_source.html">Example 10p</a>: parallel time-dependent implicit nonlinear elasticity
- * - <a class="el" href="ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
+ * - <a class="el" href="examples_2ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
  * - <a class="el" href="ex12p_8cpp_source.html">Example 12p</a>: parallel linear elasticity eigensolver
  * - <a class="el" href="ex13p_8cpp_source.html">Example 13p</a>: parallel Maxwell eigensolver
  * - <a class="el" href="ex14_8cpp_source.html">Example 14</a>: Discontinuous Galerkin (DG) for the Laplace problem
@@ -92,35 +90,20 @@ namespace mfem {
  * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
  * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM
  * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
- * - <a class="el" href="ex27_8cpp_source.html">Example 27</a>: boundary conditions for the Laplace problem
- * - <a class="el" href="ex27p_8cpp_source.html">Example 27p</a>: parallel boundary conditions for the Laplace problem
- * - <a class="el" href="ex28_8cpp_source.html">Example 28</a>: sliding contact in elasticity
- * - <a class="el" href="ex28p_8cpp_source.html">Example 28p</a>: parallel sliding contact in elasticity
- * - <a class="el" href="ex29_8cpp_source.html">Example 29</a>: Laplace solve on a 3D-embedded surface
- * - <a class="el" href="ex29p_8cpp_source.html">Example 29p</a>: parallel Laplace solve on a 3D-embedded surface
  *
- * <H4>AmgX Examples</H4>
+ * <H4>SUNDIALS Examples</H4>
  * - Variants of Examples
- *   <a class="el" href="examples_2amgx_2ex1_8cpp_source.html">1</a> and
- *   <a class="el" href="examples_2amgx_2ex1p_8cpp_source.html">1p</a>,
- *   demonstrating the use of MFEM's \link amgxsolver.hpp AmgX integration\endlink.
- *
- * <H4>Caliper Examples</H4>
- * - Variants of Example
- *   <a class="el" href="examples_2caliper_2ex1_8cpp_source.html">1</a> and
- *   <a class="el" href="examples_2caliper_2ex1p_8cpp_source.html">1p</a>,
- *   demonstrating the use of MFEM's \link annotation.hpp Ginkgo integration\endlink.
- *
- * <H4>Ginkgo Examples</H4>
- * - Variants of Example
- *   <a class="el" href="examples_2ginkgo_2ex1_8cpp_source.html">1</a>,
- *   demonstrating the use of MFEM's \link ginkgo.hpp Ginkgo integration\endlink.
- *
- * <H4>HiOp Examples</H4>
- * - Variants of Examples
- *   <a class="el" href="hiop_2ex9_8cpp_source.html">9</a> and
- *   <a class="el" href="hiop_2ex9p_8cpp_source.html">9p</a>,
- *   demonstrating the use of MFEM's \link hiop.hpp HiOp integration\endlink.
+ *   <a class="el" href="sundials_2ex9_8cpp_source.html">9</a>,
+ *   <a class="el" href="sundials_2ex9p_8cpp_source.html">9p</a>,
+ *   <a class="el" href="sundials_2ex10_8cpp_source.html">10</a>,
+ *   <a class="el" href="sundials_2ex10p_8cpp_source.html">10p</a>,
+ *   <a class="el" href="sundials_2ex16_8cpp_source.html">16</a>,
+ *   and
+ *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>
+ *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS classes\endlink
+ * - CVODES adjoint examples:
+ *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
+ *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>
  *
  * <H4>PETSc Examples</H4>
  * - Variants of Examples
@@ -132,8 +115,8 @@ namespace mfem {
  *   <a class="el" href="petsc_2ex6p_8cpp_source.html">6p</a>,
  *   <a class="el" href="petsc_2ex9p_8cpp_source.html">9p</a>,
  *   and
- *   <a class="el" href="petsc_2ex10p_8cpp_source.html">10p</a>,
- *   demonstrating the use of MFEM's \link petsc.hpp PETSc integration\endlink.
+ *   <a class="el" href="petsc_2ex10p_8cpp_source.html">10p</a>
+ *   demonstrating the use of MFEM's \link petsc.hpp PETSc classes\endlink
  *
  * <H4>PUMI Examples</H4>
  * - Variants of Examples
@@ -141,27 +124,19 @@ namespace mfem {
  *   <a class="el" href="examples_2pumi_2ex1p_8cpp_source.html">1p</a>,
  *   <a class="el" href="pumi_2ex2_8cpp_source.html">2</a>,
  *   and
- *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>,
- *   demonstrating the use of MFEM's \link pumi.hpp PUMI integration\endlink.
+ *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>
+ *   demonstrating the use of MFEM's \link pumi.hpp PUMI classes\endlink
  *
- * <H4>SUNDIALS Examples</H4>
+ * <H4>HiOp Examples</H4>
  * - Variants of Examples
- *   <a class="el" href="sundials_2ex9_8cpp_source.html">9</a>,
- *   <a class="el" href="sundials_2ex9p_8cpp_source.html">9p</a>,
- *   <a class="el" href="sundials_2ex10_8cpp_source.html">10</a>,
- *   <a class="el" href="sundials_2ex10p_8cpp_source.html">10p</a>,
- *   <a class="el" href="sundials_2ex16_8cpp_source.html">16</a>,
- *   and
- *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>,
- *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS integration\endlink.
- * - CVODES adjoint miniapps:
- *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
- *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>.
+ *   <a class="el" href="hiop_2ex9_8cpp_source.html">9</a> and
+ *   <a class="el" href="hiop_2ex9p_8cpp_source.html">9p</a>,
+ *   demonstrating the use of MFEM's \link hiop.hpp HiOp classes\endlink
  *
- * <H4>SuperLU Examples</H4>
+ * <H4>Ginkgo Examples</H4>
  * - Variants of Example
- *   <a class="el" href="examples_2superlu_2ex1p_8cpp_source.html">1p</a>,
- *   demonstrating the use of MFEM's \link superlu.hpp SuperLU integration\endlink.
+ *   <a class="el" href="ginkgo_2ex1_8cpp_source.html">1</a>
+ *   demonstrating the use of MFEM's \link ginkgo.hpp Ginkgo classes\endlink
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -169,12 +144,12 @@ namespace mfem {
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
  * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
- * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
+
+* - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
  * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
  * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
  * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="polar-nc_8cpp_source.html">Polar NC</a>: generate polar non-conforming meshes
  * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
  * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
  * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
@@ -187,15 +162,10 @@ namespace mfem {
  * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
  * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
  * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
- * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
- * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
- * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
- * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
- * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *
- * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
+ * See also the <a class="el" href="../../../examples/README.html">README</a> in the <code>examples/</code> directory
  */
 
 }

doc/README

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 This directory contains Doxygen configuration files for building HTML
 documentation of the MFEM sources (including the example codes and miniapps).
@@ -21,7 +21,7 @@ create the file CodeDocumentation.html, which can be viewed in any web browser.
 
 Alternatively, the (pre-build) documentation can be browsed directly online at
 
-   https://mfem.github.io/doxygen/html/index.html
+   http://mfem.github.io/doxygen/html/index.html
 
 
 Some building considerations:

doc/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -13,12 +13,13 @@ SHELL = /bin/bash
 MFEM_DIR ?= ..
 DOXYGEN_CONF = CodeDocumentation.conf
 
+
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
 	@doxygen $(DOXYGEN_CONF)
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
-	@cat warnings.log 1>&2
+	@cat warnings.log
 	@# Generate the log of undocumented methods
 	@( cat $(DOXYGEN_CONF) ; echo "GENERATE_HTML=NO" ; echo "EXTRACT_ALL=NO" ; echo "WARN_LOGFILE=undoc.log" ; echo "QUIET=YES" ) | doxygen - &> /dev/null
 

doc/web/base.css

@@ -0,0 +1,208 @@
+body {
+    padding-top: 70px;
+}
+
+/* hack to increase height of headers to make #anchors work with the top nav-bar,
+   (note: we only want this on pages with TOC, otherwise there are other problems) */
+div.col-md-9 h1[id]::before,
+div.col-md-9 h2[id]::before,
+div.col-md-9 h3[id]::before,
+div.col-md-9 h4[id]::before,
+div.col-md-9 h5[id]::before,
+div.col-md-9 h6[id]::before {
+    content: "";
+    display: block;
+    margin-top: -75px;
+    height: 75px;
+}
+
+img {
+    max-width: 100%;
+    height: auto;
+}
+
+pre {
+    overflow-x: auto;
+}
+
+pre code {
+    overflow-wrap: normal;
+    white-space: pre;
+    font-size: 13.5px;
+}
+
+label {
+    font-weight: normal;
+}
+
+ul.nav li.main {
+    font-weight: bold;
+}
+
+div.col-md-3 {
+    padding-right: 0;
+}
+
+div.col-md-9, div.col-md-12 {
+    padding-bottom: 100px;
+}
+
+div.source-links {
+    float: right;
+}
+
+img {
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    margin-top: 25px;
+    margin-bottom: 25px;
+}
+
+img.floatright {
+    display: block;
+    float: right;
+    margin-top: 0;
+    margin-left: 25px;
+    margin-bottom: 25px;
+}
+
+img.teamphoto {
+    box-shadow: 4px 4px 4px #999;
+}
+
+p.centered {
+    margin-left: auto;
+    margin-right: auto;
+    text-align: center;
+}
+
+/*label {
+    display: inline-block;
+    margin-bottom: 5px;
+    font-weight: normal;
+}*/
+
+/*
+ * Side navigation
+ *
+ * Scrollspy and affixed enhanced navigation to highlight sections and secondary
+ * sections of docs content.
+ */
+
+/* By default it's not affixed in mobile views, so undo that */
+.bs-sidebar.affix {
+    position: static;
+}
+
+.bs-sidebar.well {
+    padding: 0;
+}
+
+/* First level of nav */
+.bs-sidenav {
+    /*margin-top: 10px;
+    margin-bottom: 10px;*/
+    padding-top:    10px;
+    padding-bottom: 10px;
+    border-radius: 5px;
+}
+
+/* All levels of nav */
+.bs-sidebar .nav > li > a {
+    display: block;
+    padding: 5px 20px;
+    z-index: 1;
+}
+.bs-sidebar .nav > li > a:hover,
+.bs-sidebar .nav > li > a:focus {
+    text-decoration: none;
+    border-right: 1px solid;
+}
+.bs-sidebar .nav > .active > a,
+.bs-sidebar .nav > .active:hover > a,
+.bs-sidebar .nav > .active:focus > a {
+    font-weight: bold;
+    background-color: transparent;
+    border-right: 1px solid;
+}
+
+/* Nav: second level (shown on .active) */
+.bs-sidebar .nav .nav {
+    display: none; /* Hide by default, but at >768px, show it */
+    margin-bottom: 8px;
+}
+.bs-sidebar .nav .nav > li > a {
+    padding-top:    3px;
+    padding-bottom: 3px;
+    padding-left: 30px;
+    font-size: 90%;
+}
+
+/* Show and affix the side nav when space allows it */
+@media (min-width: 992px) {
+    .bs-sidebar .nav > .active > ul {
+        display: block;
+    }
+    /* Widen the fixed sidebar */
+    .bs-sidebar.affix,
+    .bs-sidebar.affix-bottom {
+        width: 213px;
+    }
+    .bs-sidebar.affix {
+        position: fixed; /* Undo the static from mobile first approach */
+        top: 80px;
+    }
+    .bs-sidebar.affix-bottom {
+        position: absolute; /* Undo the static from mobile first approach */
+    }
+    .bs-sidebar.affix-bottom .bs-sidenav,
+    .bs-sidebar.affix .bs-sidenav {
+        margin-top: 0;
+        margin-bottom: 0;
+    }
+}
+@media (min-width: 1200px) {
+    /* Widen the fixed sidebar again */
+    .bs-sidebar.affix-bottom,
+    .bs-sidebar.affix {
+        width: 263px;
+    }
+}
+
+
+/* Sticky footer styles
+-------------------------------------------------- */
+html {
+  position: relative;
+  min-height: 100%;
+}
+body {
+  /* Margin bottom by footer height */
+  margin-bottom: 50px;
+}
+.footer {
+  position: absolute;
+  bottom: 0;
+  width: 100%;
+  /* Set the fixed height of the footer here */
+  height: 50px;
+  background-color: #eeeeee;
+  overflow: hidden;
+}
+
+
+/* Custom page CSS
+-------------------------------------------------- */
+/* Not required for template or sticky footer method. */
+
+.container .text-muted {
+  margin-top: 17px;
+  margin-bottom: 0;
+}
+.container .alignleft {
+  float: left;
+}
+.container .alignright {
+  float: right;
+}