Href ex edits

.codecov.yml

@@ -1,32 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  precision: 2
-  round: nearest
-  range: "0...100"
-  status:
-    patch:
-      default:
-        target: auto
-        threshold: 0%
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        informational: true
-        only_pulls: true
-    project:
-      default:
-        target: auto   # compares coverage to the previous base commit
-        threshold: 1%  # allows variations around the target
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        only_pulls: true
-
-github_checks:
-  annotations: false
-
-comment: false

.github/workflows/builds-and-tests.yml

@@ -1,197 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# In this CI section, we build different variants of mfem and run test on them.
-name: builds-and-tests
-
-# Github actions can use the default "GITHUB_TOKEN". By default, this token
-# is set to have permissive access. However, this is not a good practice
-# security-wise. Here we use an external action, so we restrict the
-# permission to the minimum required.
-# When the 'permissions' is set, all the scopes not mentioned are set to the
-# most restrictive setting. So the following is enough.
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  MFEM_TOP_DIR: mfem
-
-# Note for future improvements:
-#
-# We cannot reuse cached dependencies and have to build them for each target
-# although they could be shared sometimes. That's because Github cache Action
-# has no read-only mode. But there is a PR ready for this
-# (https://github.com/actions/cache/pull/489)
-
-jobs:
-  builds-and-tests:
-    strategy:
-      matrix:
-        os: [ubuntu-18.04, macos-10.15]
-        target: [debug, optim]
-        mpi: [sequential, parallel]
-        build-system: [make]
-        # 'include' allows us to
-        # - add a variable without creating a new matrix dimension.
-        # - add a new combination ('build-system: cmake' case here)
-        #
-        # note: we will gather coverage info for any non-debug run except the
-        # CMake build.
-        include:
-          - target: debug
-            codecov: NO
-          - target: optim
-            codecov: YES
-          - os: ubuntu-18.04
-            target: optim
-            codecov: NO
-            mpi: parallel
-            build-system: cmake
-    name: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.build-system }}
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    # This external action allows to interrupt a workflow already running on
-    # the same branch to save resource
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
-    # Checkout MFEM in "mfem" subdirectory. Final path:
-    # /home/runner/work/mfem/mfem/mfem
-    # Note: Done now to access "install-hypre" and "install-metis" actions.
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        path: ${{ env.MFEM_TOP_DIR }}
-        # Fetch the complete history for codecov to access commits ID
-        fetch-depth: 0
-
-    # Only get MPI if defined for the job.
-    # TODO: It would be nice to have only one step, e.g. with a dedicated
-    # action, but I (@adrienbernede) don't see how at the moment.
-    - name: get MPI (Linux)
-      if: matrix.mpi == 'parallel' && matrix.os == 'ubuntu-18.04'
-      run: |
-        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-18.04'
-      run: |
-        sudo apt-get install lcov
-
-    - name: Set up Homebrew
-      if: ( matrix.mpi == 'parallel' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
-      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
-
-    - name: get MPI (MacOS)
-      if: matrix.mpi == 'parallel' && matrix.os == 'macos-10.15'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install openmpi
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: get MPI (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install lcov
-
-    # Get Hypre through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache hypre
-      id: hypre-cache
-      if: matrix.mpi == 'parallel'
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
-
-    - name: get hypre
-      if: matrix.mpi == 'parallel' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v1.0
-      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-
-    # Get Metis through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache metis
-      id: metis-cache
-      if: matrix.mpi == 'parallel'
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
-
-    - name: install metis
-      if: matrix.mpi == 'parallel' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v1.0
-      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-
-    # MFEM build and test
-    - name: build
-      uses: mfem/github-actions/build-mfem@v1.0
-      with:
-        os: ${{ matrix.os }}
-        target: ${{ matrix.target }}
-        codecov: ${{ matrix.codecov }}
-        mpi: ${{ matrix.mpi }}
-        build-system: ${{ matrix.build-system }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: ${{ env.MFEM_TOP_DIR }}
-
-    # Run checks (and only checks) on debug targets
-    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'debug'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }}  && make check
-
-    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make unittest
-
-    - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make test
-
-    - name: cmake unit tests
-      if: matrix.build-system == 'cmake'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure
-
-    # Code coverage (process and upload reports)
-    - name: codecov
-      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v1.0
-      with:
-        name: ${{ matrix.os }}-${{ matrix.mpi }}
-        project_dir: ${{ env.MFEM_TOP_DIR }}
-        directories: "fem general linalg mesh"

.github/workflows/mfem-analysis.yml

@@ -1,99 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: build-analysis
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  COVERAGE_ENV: mfem-coverage
-
-jobs:
-  gitignore:
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
-    - name: checkout MFEM
-      uses: actions/checkout@v2
-      with:
-        path: mfem
-
-    - name: Get MPI (Linux)
-      run: |
-        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: Cache Hypre Install
-      id: hypre-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
-
-    - name: Get Hypre
-      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@master
-      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-
-    - name: Cache Metis Install
-      id: metis-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
-
-    - name: Install Metis
-      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@master
-      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-
-    # MFEM build and test
-    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@master
-      with:
-        os: ${{ runner.os }}
-        target: optim
-        codecov: NO
-        mpi: parallel
-        build-system: make
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: mfem
-
-    - name: test (no clean)
-      run: |
-        cd mfem && make test-noclean
-
-    - name: gitignore
-      run: |
-        cd mfem/tests/scripts
-        ./runtest gitignore

.github/workflows/repo-check.yml

@@ -11,31 +11,28 @@
 
 name: repo-check
 
-permissions:
-  actions: write
-
 on:
   push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
 
 jobs:
-  file-headers-check:
-    runs-on: ubuntu-18.04
+  copyright-check:
+    runs-on: ubuntu-latest
 
     steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
     - name: checkout mfem
       uses: actions/checkout@v2
       with:
         path: mfem
 
     - name: copyright check
-      id: copyright
       run: |
         cd mfem
-        if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt
+        if git grep -l "^#.*\-2020" > matches.txt
         then
           echo "Please update the following files to Copyright (c) 2010-2021:"
           cat matches.txt
@@ -43,95 +40,3 @@ jobs:
         else
           echo "No outdated copyright found."
         fi
-      continue-on-error: true
-
-    - name: license check
-      id: license
-      run: |
-        cd mfem
-        if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt
-        then
-          echo "Please update the following files to the BSD-3 license:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No GNU GPL license found."
-        fi
-      continue-on-error: true
-
-    - name: release check
-      id: release
-      run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-        then
-          echo "Please update the following files to LLNL-CODE-806117:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated release number found."
-        fi
-      continue-on-error: true
-
-    - name: wrap-up
-      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
-      run: |
-        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-          echo "copyright check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-          echo "license check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-          echo "release check failed, unroll log for details"
-        fi
-        exit 1
-
-  code-style:
-    runs-on: ubuntu-16.04 # needed for astyle 2.05.1
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-
-    - name: get astyle
-      run: |
-        sudo apt-get install astyle=2.05.1-0ubuntu1
-
-    - name: style check
-      run: |
-        cd tests/scripts
-        ./runtest code-style
-
-  documentation:
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-
-    - name: get doxygen and graphviz
-      run: |
-        sudo apt-get install doxygen graphviz
-
-    - name: build documentation
-      run: |
-        cd tests/scripts
-        ./runtest documentation
-
-  branch-history:
-    if: github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: branch-history
-      run: |
-        git fetch origin master:master
-        git checkout -b gh-actions-branch-history
-        cd tests/scripts
-        ./runtest branch-history

.gitignore

@@ -44,8 +44,8 @@ doc/warnings.log
 
 # Example and miniapp binaries and outputs
 
-examples/ex[0-9]
-examples/ex[0-9]p
+examples/ex[1-9]
+examples/ex[1-9]p
 examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
@@ -102,9 +102,6 @@ examples/Example23*
 examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*
-examples/ex28_*
-examples/ex28p_*
-examples/flux.*
 
 examples/amgx/ex1
 examples/amgx/ex1p
@@ -217,11 +214,6 @@ miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh
 
-miniapps/mtop/parheat
-miniapps/mtop/ParHeat*
-miniapps/mtop/seqheat
-miniapps/mtop/SeqHeat*
-
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
 miniapps/navier/navier_kovasznay_vs
@@ -248,9 +240,6 @@ miniapps/performance/sol.*
 
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
-miniapps/shifted/diffusion
-miniapps/shifted/diffusion.mesh
-miniapps/shifted/diffusion.gf
 
 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -280,11 +269,6 @@ miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh
 
 miniapps/solvers/block-solvers
-miniapps/solvers/lor_solvers
-miniapps/solvers/plor_solvers
-miniapps/solvers/ParaView
-miniapps/solvers/mesh.*
-miniapps/solvers/sol.*
 
 # Unit test binary and outputs
 tests/unit/output_meshes
@@ -292,8 +276,6 @@ tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/tmop_pa_tests_*
-tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
 
 # Test script output
@@ -308,7 +290,3 @@ tests/par-mesh-format/ex1p
 
 # VPATH builds
 build-*/*
-
-# PETSc automated build
-petsc-build/*
-pkg.gitcommit

.gitlab-ci.yml

@@ -40,14 +40,11 @@
 # Directory used to place artifacts.
 
 variables:
-  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
+  BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
   REBASELINE: "NO"
-  AUTOTEST: "NO"
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
   TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
   TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
   ARTIFACTS_DIR: artifacts
 
 # The pipeline is divided into stages. Usually, these are also synchronization
@@ -67,7 +64,6 @@ stages:
   - c_build_and_test
   - setup
   - baseline_check
-  - baseline_to_autotest
   - baseline_publish
 
 # The setup job in setup stage don't rely on MFEM git repo. It prepares a
@@ -89,9 +85,6 @@ setup:
     - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
     - cd tpls && git pull && cd ..
     - cd tests && git pull && cd ..
-    - cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
   needs: []
 
 .build_toss_3_x86_64_ib_script:
@@ -106,22 +99,26 @@ setup:
   script:
     - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
 
-# Lassen uses a different job scheduler (spectrum lsf) that does not
+# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
 # allow pre-allocation the same way slurm does.
-# We use pdebug queue on lassen to speed-up the allocation.
-# However this would not be scalable to multiple builds.
 .build_blueos_3_ppc64le_ib_script:
   script:
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
+    - lalloc 1 -W 15 tests/gitlab/build_and_test
 
 # Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
 # differentiates between the two tests.
 .baseline_script: &baseline_script |
   # locals
+  _glob_out=${BASELINE_TEST}.out
   _glob_err=${BASELINE_TEST}.err
   _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
   _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
   _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+
+  _out=${BASELINE_TEST}-${SYS_TYPE}.out
+  _ref=../${BASELINE_TEST}-${SYS_TYPE}.saved
+  _out_txt=${BASELINE_TEST}.txt
+  _diff=${BASELINE_TEST}-diff.txt
   # prepare
   cd ${BUILD_ROOT}
   ln -snf ${CI_PROJECT_DIR} mfem
@@ -136,7 +133,7 @@ setup:
     echo "ERROR during ${BASELINE_TEST} execution";
     echo "Here is the ${_glob_err} file content";
     cat ${_glob_err}
-    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}.txt
     exit 1;
   elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
   then
@@ -146,20 +143,18 @@ setup:
   elif [[ -f ${_base_patch} ]]
   then
     echo "${BASELINE_TEST}: Differences found, patch generated"
-    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}.txt
   elif [[ -f ${_base_out} ]]
   then
     echo "${BASELINE_TEST}: Differences found, replacement file generated"
-    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}.txt
   fi
   # _base_diff won't even exist if there is no difference.
   if [[ -f ${_base_diff} ]]
   then
     echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
     cat ${_base_diff}
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
-    # We create a .err file, because that's how we signal that there was a diff.
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}.txt
   fi
   if [[ ! -s ${_base_diff} ]]
   then
@@ -198,7 +193,7 @@ setup:
       - ${ARTIFACTS_DIR}
   allow_failure: true
 
-# This job can only be manually triggered on a pipeline for master branch, or if
+# This job can only be manually triggers on a pipeline for master branch, or if
 # the pipeline was triggered with REBASELINE="YES"
 .rebaseline_mfem:
   stage: baseline_publish
@@ -211,18 +206,19 @@ setup:
     - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
     - cd ${BUILD_ROOT}/tests
     - |
-      if [[ ! -f "${DIFF_FILE}" ]]
+      if [[ ! -f "${DIFF_FILE}.txt" ]]
       then
           echo "Nothing to be done: no relevant change in baseline"
           exit 0
-      elif [[ -f "${PATCH_FILE}" ]]
+      elif [[ -f "${PATCH_FILE}.txt" ]]
       then
+          mv ${PATCH_FILE}.txt ${PATCH_FILE}
           patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
-      elif [[ -f "${FULL_FILE}t" ]]
+      elif [[ -f "${FULL_FILE}.txt" ]]
       then
-          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
+          cp "${FULL_FILE}.txt" "./baseline-${SYS_TYPE}.saved"
       else
-          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+          echo "File missing: expected ${PATCH_FILE}.txt or ${FULL_FILE}.txt"
           exit 1
       fi
     - git add baseline-${SYS_TYPE}.saved
@@ -232,4 +228,4 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
   - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml
+    #  - local: .gitlab/lassen.yml

.gitlab/lassen.yml

@@ -15,19 +15,43 @@
   tags:
     - shell
     - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    - when: on_success
-
-# Spack helped builds
-# Generic lassen build job, extending build script
-.build_and_test_on_lassen:
-  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
-  stage: l_build_and_test
-  needs: []
-
-opt_mpi_cuda_xl_16_1_1_8:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
-  extends: .build_and_test_on_lassen
+    PLAT: lassen
+
+# Build MFEM
+build_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
+
+build_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70
+
+# Sanity check
+sanitycheck_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 15 -q pdebug make -j test
+
+sanitycheck_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_debug_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 30 -q pdebug make -j test

.gitlab/quartz.yml

@@ -16,25 +16,10 @@
     - shell
     - quartz
   rules:
-    # Don’t run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"' #run except if ...
       when: never
-    # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
-      when: never
-    # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resources
     - if: '$CI_JOB_NAME =~ /release_resources/'
       when: always
-    # Default is to run if previous stage succeeded
     - when: on_success
 
 # Allocate
@@ -57,38 +42,6 @@ q_release_resources:
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
     - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
 
-# Release
-q_report_success:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Can only run if all the quartz jobs passed"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
-    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
-q_report_failure:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Runs if there was at least one failure on quartz"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
-    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
-    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
 # Spack helped builds
 # Generic quartz build job, extending build script
 .build_and_test_on_quartz:
@@ -141,25 +94,6 @@ baselinecheck_mfem_intel_quartz:
   extends: [.baselinecheck_mfem, .on_quartz]
   needs: [setup]
 
-update_autotest:
-  extends: [.on_quartz]
-  needs: [baselinecheck_mfem_intel_quartz]
-  stage: baseline_to_autotest
-  script:
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
-    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-    - |
-      if [[ -f ${rundir}/*.err ]]
-      then
-        cp ${rundir}/*.err ${rundir}/autotest-email.html
-      fi
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
 baselinepublish_mfem_quartz:
   extends: [.on_quartz, .rebaseline_mfem]
   needs: [baselinecheck_mfem_intel_quartz]

CHANGELOG

@@ -10,43 +10,6 @@
 
 Version 4.2.1 (development)
 ===========================
-- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
-
-- Memory management:
-  * Added method Device::SetMemoryTypes that can be used to change the default
-    host and device MemoryTypes before Device setup.
-  * In class MemoryManager, added methods GetDualMemoryType and
-    SetDualMemoryType; dual MemoryTypes are used to determine the second
-    MemoryType (host or device) when only one MemoryType is specified in methods
-    of class Memory.
-  * Added Memory constructor for setting both the host and device MemoryTypes.
-  * Switched the default behavior of device memory allocations so that they
-    are deferred until the device pointer is needed.
-  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
-    corresponding allocator that can be set with the method
-    MemoryManager::SetUmpireDevice2AllocatorName.
-  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
-
-- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
-
-- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
-
-- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
-
-- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of
-  the domain decomposition.
-
-- Introduced solver interface for linear problems with constraints, a few
-  concrete solvers that implement the interface, and a demonstration of their
-  use in Example 28(p), which solves an elasticity problem with zero normal
-  displacement (but allowed tangential displacement) on two boundaries.
 
 - Added high-order matrix-free auxiliary Maxwell solver for H(curl) problems,
   as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
@@ -80,9 +43,6 @@ Version 4.2.1 (development)
   NC data files are compatible with serial code, e.g., can be viewed with serial
   GLVis. Loading of legacy NC mesh files is still supported.
 
-- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
 - Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
   help identify elements with poor mesh quality.
 
@@ -95,21 +55,6 @@ Version 4.2.1 (development)
 
 - Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
 
-- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
-     and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
-- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
-- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
 - Implemented a filter method for the Navier miniapp to stabilize highly
   turbulent flows in direct numerical simulation.
 
@@ -132,15 +77,9 @@ Version 4.2.1 (development)
 - Added convective and skew-symmetric integrators for the nonlinear term in the
   Navier-Stokes equations.
 
-- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
 - Changed the interface for the error estimator.
 
-- Implemented the Kelly error indicator for scalar-valued problems, supported
-  in serial and parallel builds.
+- Implemented the parallel Kelly error indicator for scalar-valued problems.
 
 - Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
   efficient evaluation of symmetric matrix coefficients. This replaces the now
@@ -175,70 +114,13 @@ Version 4.2.1 (development)
 
 - Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
 
-- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
-- Added a new, very simple example (ex0 and parallel version ex0p). This
-  example solves a simple Poisson problem using H1 elements (the same problem as
-  ex1), but is intended to be extremely simple and approachable for new users.
-
-- Meshes consisting of any type of elements (including mixed meshes) can be
-  converted to all-simplex meshes using Mesh::MakeSimplicial.
-
-- Several of the mesh constructors (creating Cartesian meshes, refined (LOR)
-  meshes, simplex meshes, etc.) are now available as "named constructors", e.g.
-  Mesh::MakeCartesian2D or Mesh::MakeRefined. The legacy constructors are marked
-  as deprecated.
-
-- Added support for creating periodic meshes with Mesh::MakePeriodic. The
-  requisite periodic vertex mappings can be created with
-  Mesh::CreatePeriodicVertexMapping.
-
-- Added support for transferring dual fields between high-order and low-order
-  refined finite element spaces using the transposed versions of the
-  L2ProjectionGridTransfer operators. This functionality is illustrated in the
-  lor-transfer miniapp.
-
-- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
-- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p and may not work in more general settings.
-
-- Added support for transferring fields (primary and dual) between high-order
-  and low-order refined H1 finite element spaces using the
-  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
-  through the lor-transfer miniapp when run with the -h1 option.
-
-- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
-
-- Added sample meshes in the `data` subdirectory showing the reference elements
-  of the six currently supported element types; ref-segment.mesh,
-  ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
-  ref-prism.mesh.
-
-- Added a high-order extension of the shifted boundary method to solve PDEs on
-  non body-fitted meshes. This is illustrated in the new Shifted Diffusion
-  miniapp, see miniapps/shifted/diffusion.cpp.
-
-- Added makefile rule to generate TAGS table for vi or Emacs users.
-
 libCEED integration improvements
 --------------------------------
 - Refactor the libCEED integration
 
 - Add support for VectorCoefficient with libCEED backends.
 
-- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
-  libCEED backends.
+- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
 
 
 Version 4.2, released on October 30, 2020
@@ -326,9 +208,6 @@ Linear and nonlinear solvers
   matrix with the function HypreParMatrixFromBlocks. This could be useful for
   solving block systems with parallel direct solvers such as STRUMPACK.
 
-- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
   preconditioner. The latter requires hypre version 2.19.0 or later.
 

CMakeLists.txt

@@ -16,9 +16,6 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO) 
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -102,9 +99,6 @@ if (MFEM_USE_CUDA)
    endif()
    enable_language(CUDA)
    set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO) 
-     set(CMAKE_CUDA_STANDARD 14)
-   endif()
    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
    set(CMAKE_CUDA_EXTENSIONS OFF)
    set(CUDA_FLAGS "--expt-extended-lambda")
@@ -179,12 +173,6 @@ endif()
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
   set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
-  if (MFEM_MPIEXEC)
-    set(MPIEXEC ${MFEM_MPIEXEC})
-  endif()
-  if (MFEM_MPIEXEC_NP)
-    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
-  endif()
   # Parallel MFEM depends on hypre
   find_package(HYPRE REQUIRED)
   set(MFEM_HYPRE_VERSION ${HYPRE_VERSION})
@@ -372,20 +360,12 @@ if (MFEM_USE_UMPIRE)
    find_package(UMPIRE REQUIRED)
 endif()
 
-# Caliper
-if (MFEM_USE_CALIPER)
-  find_package(Caliper REQUIRED)
-endif()
-
 # AMD HIP
 if (MFEM_USE_HIP)
   find_package(HIP REQUIRED)
   if (HIP_ARCH)
     message(STATUS "Using HIP architecture: ${HIP_ARCH}")
     list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-    if (MFEM_USE_GINKGO)
-      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
-    endif()
   endif()
 endif()
 
@@ -423,10 +403,10 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
+    SLEPC MESQUITE SuperLUDist MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER)
+    CUSPARSE MKL_CPARDISO AMGX)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -529,8 +509,6 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
   endforeach()
 endif()
 
-set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")
-
 #-------------------------------------------------------------------------------
 # Examples, miniapps, and testing
 #-------------------------------------------------------------------------------
@@ -581,17 +559,16 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()
 
 # Add 'check' target - quick test
-set(MFEM_CHECK_TARGET_NAME ${MFEM_CUSTOM_TARGET_PREFIX}check)
 if (NOT MFEM_USE_MPI)
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
     ${CMAKE_CTEST_COMMAND} -R \"^ex1_ser\" -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1)
+  add_dependencies(check ex1)
 else()
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
     ${CMAKE_CTEST_COMMAND} -R \"^ex1p\" -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1p)
+  add_dependencies(check ex1p)
 endif()
 
 #-------------------------------------------------------------------------------

CONTRIBUTING.md

@@ -97,7 +97,6 @@ The MFEM source code has the following structure:
   ├── doc
   ├── examples
   │   ├── amgx
-  │   ├── caliper
   │   ├── ginkgo
   │   ├── hiop
   │   ├── petsc
@@ -105,9 +104,7 @@ The MFEM source code has the following structure:
   │   ├── sundials
   |   └── superlu
   ├── fem
-  │   ├── ceed
-  │   ├── qinterp
-  │   └── tmop
+  │   └── ceed
   ├── general
   ├── linalg
   │   └── simd
@@ -118,7 +115,6 @@ The MFEM source code has the following structure:
   │   ├── electromagnetics
   │   ├── gslib
   │   ├── meshing
-  │   ├── mtop
   │   ├── navier
   │   ├── nurbs
   │   ├── performance
@@ -370,16 +366,16 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   `mfem:next`, see the [README](tests/scripts/README) file in that directory
   for more details.
 
-- Track the Travis CI, Github Actions and Appveyor [continuous integration](#automated-testing)
+- Track the Travis CI and Appveyor [continuous integration](#automated-testing)
   builds at the end of the PR. These should generally run clean, so address any
   errors as soon as possible. Please ask if you are unsure how to do that.
 
-- Note that some tests, such as the `branch-history` check in Travis and Github
-  Actions are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in Travis are
+  safeguards that are allowed to fail in certain cases.
 
 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in Travis and Github Actions enforce MFEM-specific rules which are
-  explained in the error messages and the `tests/scripts` directory.
+  checks in Travis enforce MFEM-specific rules which are explained in the
+  error messages and the `tests/scripts` directory.
 
 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
   one of the _LLNL developers_ for details.
@@ -429,7 +425,6 @@ Before a PR can be merged, it should satisfy the following:
       - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
       - [ ] Consider adding a new test for the new miniapp.
    - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
    - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
    - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
      - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
@@ -579,13 +574,12 @@ MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.
 
 ### Linux and Mac smoke tests
-We use Travis CI and Github Actions to drive the default tests on the `master`
-and `next` branches. See the `.travis` file and the logs at
+We use Travis CI to drive the default tests on the `master` and `next`
+branches. See the `.travis` file and the logs at
 [https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).
 
-Testing using Travis CI and Github Actions should be kept lightweight, as there
-is a time constraint on jobs. Two virtual machines are configured - Mac (OS X)
-and Linux.
+Testing using Travis CI should be kept lightweight, as there is a 50 minute time
+constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
 
 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

INSTALL

@@ -58,7 +58,6 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
 
 We also recommend downloading and building the MFEM-based GLVis visualization
@@ -351,9 +350,10 @@ MFEM_USE_SUPERLU5 = YES/NO
 
 MFEM_USE_MUMPS = YES/NO
    Enable MFEM functionality based on the MUMPS library. Currently, this
-   option adds the class MUMPSSolver (a parallel sparse direct solver).
-   When enabled, this option uses the MUMPS_* library options, see below.
-
+   option adds the class MUMPSSolver (a parallel sparse direct solver). 
+   When enabled, this option uses the MUMPS_* library options, see
+   below.
+   
 MFEM_USE_STRUMPACK = YES/NO
    Enable MFEM functionality based on the STRUMPACK sparse direct solver and
    preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
@@ -460,8 +460,8 @@ MFEM_USE_UMPIRE = YES/NO
    memory devices like NUMA and GPUs.
 
 MFEM_USE_HIOP = YES/NO
-   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
-   HPC solver for nonlinear optimization problems.
+  Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
+  HPC solver for nonlinear optimization problems.
 
 MFEM_USE_CUDA = YES/NO
    Enables support for CUDA devices in MFEM. CUDA is a parallel computing
@@ -508,14 +508,6 @@ MFEM_USE_MKL_CPARDISO = YES/NO
    MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
    OpenMP capabilities are disabled at link time.
 
-MFEM_USE_CALIPER = YES/NO
-   Enables the interface to Caliper. Caliper is a library to integrate
-   performance profiling capabilities into applications. To use Caliper,
-   developers mark code regions of interest using either Caliper's annotation
-   API or their equivalent in MFEM. Applications can then enable performance
-   profiling at runtime with Caliper's configuration API. Alternatively, one
-   can configure Caliper through environment variables or config files.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -537,11 +529,9 @@ directory and use the string @MFEM_DIR@, e.g. HYPRE_OPT = -I@MFEM_DIR@/../hypre.
 The specific libraries and their options are:
 
 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
-  See also the "Specific options for hypre" section at the end of this file.
   URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b,
-            HYPRE >= 2.20.0 for '--enable-mixedint' support.
+  Versions: HYPRE >= 2.10.0b.
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -611,11 +601,10 @@ The specific libraries and their options are:
   Versions: STRUMPACK >= 3.0.0.
 
 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-14 standard. For additional requirements
-  and dependencies of specific modules, see the Ginkgo webpage below.
+  C++ compiler that supports the C++-11 standard. For additional requirements
+  and dependencies of specific modules see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0. 
+  Options: GINKGO_OPT (Not used), GINKGO_LIB.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX
@@ -647,8 +636,7 @@ The specific libraries and their options are:
                 --with-shared-libraries=0
   URL: https://www.mcs.anl.gov/petsc
   Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA)
-            PETSc >= 3.15.0  (PETSc built with CUDA)
+  Versions: PETSc >= 3.8.0.
 
 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
   uses some of the PETSc options when compiled.
@@ -684,17 +672,17 @@ The specific libraries and their options are:
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
   Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.
+  Versions: HIOP >= 0.1.
 
 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
-  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
-  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
+  built prior to the MFEM build, as follows: download gslib-1.0.5, untar it at
+  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.5 gslib".
   Build gslib in parallel or in serial based on the desired MFEM build: "make
   clean; make CC=mpicc" or "make clean; make CC=gcc MPI=0". Build MFEM with
   MFEM_USE_GSLIB=YES.
-  URL: https://github.com/gslib/gslib/archive/v1.0.7.tar.gz
+  URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
   Options: GSLIB_OPT, GSLIB_LIB.
-  Versions: GSLIB >= 1.0.7.
+  Versions: GSLIB >= 1.0.5.
 
 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
   URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
@@ -719,7 +707,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.8.
+  Versions: libCEED >= 0.7.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
@@ -727,13 +715,7 @@ The specific libraries and their options are:
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
   Versions: RAJA >= 0.13.0.
 
-- Caliper (optional), used when MFEM_USE_CALIPER = YES.
-  URL: https://github.com/LLNL/Caliper
-  Options: CALIPER_DIR
-  Versions: CALIPER >= 2.5.0, older versions may work too.
-
 - Umpire, used when MFEM_USE_UMPIRE = YES.
-  Umpire requires camp when the Umpire version is >= 3.0.0.
   URL: https://github.com/LLNL/Umpire
   Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
   Versions: Umpire >= 2.0.0.
@@ -883,7 +865,6 @@ MFEM_USE_CEED
 MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
-MFEM_USE_CALIPER
 
 The following options are CMake specific:
 
@@ -937,7 +918,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - RAJA
  - UMPIRE
  - AXOM - Used when MFEM_USE_SIDRE is enabled
- - CALIPER
 
 The following built-in CMake packages are also used:
 
@@ -973,19 +953,3 @@ MFEM_MPIEXEC    = mpirun # default
 MFEM_MPIEXEC_NP = -np    # default
 MFEM_MPIEXEC    = srun   # example for platforms using SLURM
 MFEM_MPIEXEC_NP = -n     # example for platforms using SLURM
-
-
-Specific options for hypre
-==========================
-The hypre library has multiple options to define local and global index storage
-sizes. By default, all indices are stored as an architecture aware integer. For
-most platforms, this will be 32-bit. This limits the maximum number of global
-degrees of freedom in a vector or matrix to about 2 billion. In order to solve
-larger problems, there are two options:
-
-1. Building hypre with '--enable-bigint' defines the local and global indices to
-   be 64-bit. This is convenient, but requires more memory than necessary.
-
-2. Building hypre with '--enable-mixedint' defines the local indiced to be
-   32-bit, while using a 64-bit storage for global indices. This option is
-   currently tested only in ex1p, and may not work in more general settings.

config/cmake/MFEMConfig.cmake.in

@@ -53,7 +53,6 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -151,9 +151,6 @@
 // Enable MFEM functionality based on the ADIOS2 library
 #cmakedefine MFEM_USE_ADIOS2
 
-// Enable MFEM functionality based on the Caliper library
-#cmakedefine MFEM_USE_CALIPER
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.

config/cmake/modules/FindAMGX.cmake

@@ -15,10 +15,6 @@
 #   - AMGX_INCLUDE_DIRS
 
 include(MfemCmakeUtilities)
-set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas cublasLt nvToolsExt)
+set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
 mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
   "Paths to headers required by AMGX." "Libraries required by AMGX.")
-# Make sure the library location is locked down
-foreach(lib ${AMGX_REQUIRED_LIBRARIES})
-  list(APPEND AMGX_LIBRARIES ${CUDA_TOOLKIT_ROOT_DIR}/lib64/lib${lib}${CMAKE_SHARED_LIBRARY_SUFFIX})
-endforeach()

config/cmake/modules/FindCaliper.cmake

@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - CALIPER_FOUND
-#   - CALIPER_LIBRARIES
-#   - CALIPER_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Caliper CALIPER CALIPER_DIR
-       	"include" "caliper/cali.h"
-       	"lib" "caliper"
-  	"Paths to headers required by Caliper."
-       	"Libraries required by Caliper.")

config/cmake/modules/FindNetCDF.cmake

@@ -15,20 +15,5 @@
 #   - NETCDF_INCLUDE_DIRS
 
 include(MfemCmakeUtilities)
-
-# FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-# We need to guard against the case where HDF5 was already found but without
-# the HL extensions (in which case mfem_find_package will treat the package
-# as already having been found), so we reset the variable to force FindHDF5.cmake
-# to be called for a second time
-set(HDF5_FOUND OFF)
-enable_language(C) # FindHDF5.cmake uses the C compiler
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
   "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-# The HL extension libraries are in a separate variable and must precede
-# the "regular" hdf5 library, as hdf5_hl depends on hdf5
-# The netcdf library will always be the first element of NETCDF_LIBRARIES
-# and we need to insert after that library but before the hdf5 library, so
-# position 1 is used
-list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -47,7 +47,6 @@ endfunction()
 macro(mfem_add_executable NAME)
   if (MFEM_USE_HIP)
     hip_add_executable(${NAME} ${ARGN})
-    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
   else()
     add_executable(${NAME} ${ARGN})
   endif()

config/config.hpp.in

@@ -158,9 +158,6 @@
 // Enable functionality based on the libCEED library.
 // #define MFEM_USE_CEED
 
-// Enable functionality based on the Caliper library.
-// #define MFEM_USE_CALIPER
-
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE
 

config/config.mk.in

@@ -52,7 +52,6 @@ MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
 MFEM_USE_CEED          = @MFEM_USE_CEED@
-MFEM_USE_CALIPER       = @MFEM_USE_CALIPER@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@

config/defaults.cmake

@@ -55,14 +55,8 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
-option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
 
-# Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
-# set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
-# set(MFEM_MPIEXEC_NP "-np" CACHE STRING
-#     "Flag for setting the number of MPI tasks")
-
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 
 # Allow a user to disable testing, examples, and/or miniapps at CONFIGURE TIME
@@ -174,7 +168,8 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
 
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
-set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
+# May need to add "HDF5" as requirement.
+set(NetCDF_REQUIRED_PACKAGES "" CACHE STRING
     "Additional packages required by NetCDF.")
 
 set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
@@ -211,7 +206,6 @@ set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
-set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
 
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")

config/defaults.mk

@@ -18,9 +18,6 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))
 
-ETAGS_BIN = $(shell command -v etags 2> /dev/null)
-EGREP_BIN = $(shell command -v egrep 2> /dev/null)
-
 CXX = g++
 MPICXX = mpicxx
 
@@ -145,7 +142,6 @@ MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
-MFEM_USE_CALIPER       = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
@@ -288,23 +284,9 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_BUILD_TYPE=Release
-ifeq ($(MFEM_USE_GINKGO),YES)
-   BASE_FLAGS = -std=c++14
-endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a  $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
-GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
-ifeq ($(GINKGO_BUILD_TYPE),Debug)
-  ifneq (,$(ALL_GINKGO_LIBS_DEBUG))
-    GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG))
-  endif
-else
-endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK)
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
+ -lginkgo_omp -lginkgo_cuda -lginkgo_reference
 
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
@@ -414,11 +396,6 @@ OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 
-# CALIPER library configuration
-CALIPER_DIR = @MFEM_DIR@/../caliper
-CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
-
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
 CEED_OPT = -I$(CEED_DIR)/include
@@ -430,14 +407,11 @@ RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
    RAJA_OPT += -I$(CUB_DIR)
 endif
-ifdef CAMP_DIR
-   RAJA_OPT += -I$(CAMP_DIR)/include
-endif
 RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA
 
 # UMPIRE library configuration
 UMPIRE_DIR = @MFEM_DIR@/../umpire
-UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
+UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
 # MKL CPardiso library configuration

config/sample-runs.sh

@@ -42,52 +42,12 @@ groups_serial=(
    "Performance miniapps:"
    "miniapps/performance"
    "ex1.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1.cpp"'
-'"ginkgo"
-   "Ginkgo examples:"
-   "examples/ginkgo"
-   "ex1.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1.cpp ex2.cpp"'
 #   ""'
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
    "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
     mesh-optimizer.cpp minimal-surface.cpp"'
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "cvsRoberts_ASAi_dns.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp "'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
-'"toys"
-   "Toys miniapps:"
-   "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -112,26 +72,6 @@ groups_parallel=(
    "Performance miniapps:"
    "miniapps/performance"
    "ex1p.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1p.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1p.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9p.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1p.cpp ex6p.cpp"'
-'"superlu"
-   "Superlu examples:"
-   "examples/superlu"
-   "ex1p.cpp"'
 #   ""'
 '"meshing"
    "Meshing miniapps:"
@@ -142,34 +82,6 @@ groups_parallel=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "adjoint_advection_diffusion.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "pfindpts.cpp schwarz_ex1p.cpp"'
-'"navier"
-   "Navier miniapps:"
-   "miniapps/navier"
-   "navier_cht.cpp"'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-'"shifted"
-   "Shifted miniapps:"
-   "miniapps/shifted"
-   "distance.cpp"'
-'"solvers"
-   "Solvers miniapps:"
-   "miniapps/solvers"
-   "block-solvers.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-cd.cpp get-values.cpp load-dc.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -197,30 +109,6 @@ groups_all=(
    "Performance miniapps:"
    "miniapps/performance"
    "ex1{,p}.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1.cpp ex1p.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1.cpp ex1p.cpp"'
-'"ginkgo"
-   "Ginkgo examples:"
-   "examples/ginkgo"
-   "ex1.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9.cpp ex9p.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1.cpp ex1p.cpp ex2.cpp ex6p.cpp"'
-'"superlu"
-   "Superlu examples:"
-   "examples/superlu"
-   "ex1p.cpp"'
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
@@ -231,38 +119,6 @@ groups_all=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "adjoint_advection_diffusion.cpp cvsRoberts_ASAi_dns.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp schwarz_ex1p.cpp"'
-'"navier"
-   "Navier miniapps:"
-   "miniapps/navier"
-   "navier_cht.cpp"'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1.cpp nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-'"shifted"
-   "Shifted miniapps:"
-   "miniapps/shifted"
-   "distance.cpp"'
-'"solvers"
-   "Solvers miniapps:"
-   "miniapps/solvers"
-   "block-solvers.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
-'"toys"
-   "Toys miniapps:"
-   "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -588,8 +444,6 @@ function go_group()
       mkdir -p "${group_output_dir}" || exit 1
    fi
    for src in "$@"; do
-      ex_run_suffix=${run_suffix} && [[ $src =~ ex0p?\.cpp ]] \
-         && ex_run_suffix=""
       cd "${mfem_dir}/${group_dir}" || exit 1
       extract_sample_runs "${src}" || continue
       [ "${#runs[@]}" -eq 0 ] && continue
@@ -609,7 +463,7 @@ function go_group()
       fi
       for run in "${runs[@]}"; do
          if [ "${run}" == "" ]; then continue; fi
-         eval go \"\${run_prefix} \${run} \${ex_run_suffix}\" $output
+         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
       done
    done
    ${make} clean-exec

config/test.mk

@@ -39,7 +39,7 @@ set -- $$($(1) $(SHELL) -c "$(2)" 2>&1); while [ "$$#" -gt 3 ]; do shift; done
 endef
 define TIMECMD.NOTGNU
 set -- $$($(1) -l $(SHELL) -c "{ $(2); } > /dev/null 2>&1" 2>&1; echo $$?); \
-set -- "$$1"s "$$(($$7/1024))"kB "$${!#}"
+set -- "$$1"s "$$(($$7/1024))"kB "$${60}"
 endef
 define TIMECMD.BASH
 TIMEFORMAT=$$'%3Rs'; \

data/amr-hex.mesh

@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0
 
 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4

data/amr-quad.mesh

@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0
 
 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4

data/beam-quad-amr.mesh

@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0
 
 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4

data/fichera-amr.mesh

@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0
 
 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4

data/inline-oneTri.mesh

@@ -0,0 +1,7 @@
+MFEM INLINE mesh v1.0
+
+type = tri
+nx = 1
+ny = 1
+sx = 3.14
+sy = 3.14

data/periodic-cube.geo

@@ -1,38 +0,0 @@
-// 0 for tetrahedra, 1 for hexahedra
-tet_or_hex = 1;
-
-Point(1) = {0, 0, 0, 1.0};
-Point(2) = {1, 0, 0, 1.0};
-Point(3) = {1, 1, 0, 1.0};
-Point(4) = {0, 1, 0, 1.0};
-
-Characteristic Length {:} = 0.25;
-
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-
-Periodic Curve {1} = {-3};
-Periodic Curve {2} = {-4};
-
-Curve Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-Transfinite Surface {1};
-
-If (tet_or_hex == 1)
-   Recombine Surface {1};
-   out[] = Extrude {0, 0, 1} { Surface{1}; Layers{4}; Recombine; };
-Else
-   out[] = Extrude {0, 0, 1} { Surface{1}; Layers{4}; }
-EndIf
-
-Physical Volume(1) = {out[1]};
-Physical Surface(1) = {1,out[0],out[2],out[3],out[4],out[5]};
-
-Mesh 3;
-Mesh.MshFileVersion = 2.2;
-
-Periodic Surface {out[0]} = {1} Translate {0, 0, 1};
-Periodic Surface {out[4]} = {out[2]} Translate {0, 1, 0};
-Periodic Surface {out[3]} = {out[5]} Translate {1, 0, 0};

data/periodic-cube.msh

@@ -1,381 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-125
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0 0 1
-6 1 0 1
-7 1 1 1
-8 0 1 1
-9 0.2500000000010404 0 0
-10 0.5000000000020591 0 0
-11 0.7500000000003465 0 0
-12 1 0.2500000000010404 0
-13 1 0.5000000000020591 0
-14 1 0.7500000000003465 0
-15 0.7500000000003465 1 0
-16 0.5000000000020591 1 0
-17 0.2500000000010404 1 0
-18 0 0.7500000000003465 0
-19 0 0.5000000000020591 0
-20 0 0.2500000000010404 0
-21 0.2500000000010404 0 1
-22 0.5000000000020591 0 1
-23 0.7500000000003465 0 1
-24 1 0.2500000000010404 1
-25 1 0.5000000000020591 1
-26 1 0.7500000000003465 1
-27 0.7500000000003465 1 1
-28 0.5000000000020591 1 1
-29 0.2500000000010404 1 1
-30 0 0.7500000000003465 1
-31 0 0.5000000000020591 1
-32 0 0.2500000000010404 1
-33 0 0 0.25
-34 0 0 0.5
-35 0 0 0.75
-36 1 0 0.25
-37 1 0 0.5
-38 1 0 0.75
-39 1 1 0.25
-40 1 1 0.5
-41 1 1 0.75
-42 0 1 0.25
-43 0 1 0.5
-44 0 1 0.75
-45 0.2500000000010404 0.2500000000010404 0
-46 0.2500000000010404 0.5000000000020591 0
-47 0.2500000000010404 0.7500000000003464 0
-48 0.5000000000020591 0.2500000000010404 0
-49 0.5000000000020591 0.5000000000020591 0
-50 0.5000000000020591 0.7500000000003465 0
-51 0.7500000000003464 0.2500000000010404 0
-52 0.7500000000003467 0.5000000000020591 0
-53 0.7500000000003464 0.7500000000003466 0
-54 0.2500000000010404 0 0.25
-55 0.2500000000010404 0 0.5
-56 0.2500000000010404 0 0.75
-57 0.5000000000020591 0 0.25
-58 0.5000000000020591 0 0.5
-59 0.5000000000020591 0 0.75
-60 0.7500000000003465 0 0.25
-61 0.7500000000003465 0 0.5
-62 0.7500000000003465 0 0.75
-63 1 0.2500000000010404 0.25
-64 1 0.2500000000010404 0.5
-65 1 0.2500000000010404 0.75
-66 1 0.5000000000020591 0.25
-67 1 0.5000000000020591 0.5
-68 1 0.5000000000020591 0.75
-69 1 0.7500000000003465 0.25
-70 1 0.7500000000003465 0.5
-71 1 0.7500000000003465 0.75
-72 0.7500000000003465 1 0.25
-73 0.7500000000003465 1 0.5
-74 0.7500000000003465 1 0.75
-75 0.5000000000020591 1 0.25
-76 0.5000000000020591 1 0.5
-77 0.5000000000020591 1 0.75
-78 0.2500000000010404 1 0.25
-79 0.2500000000010404 1 0.5
-80 0.2500000000010404 1 0.75
-81 0 0.7500000000003465 0.25
-82 0 0.7500000000003465 0.5
-83 0 0.7500000000003465 0.75
-84 0 0.5000000000020591 0.25
-85 0 0.5000000000020591 0.5
-86 0 0.5000000000020591 0.75
-87 0 0.2500000000010404 0.25
-88 0 0.2500000000010404 0.5
-89 0 0.2500000000010404 0.75
-90 0.2500000000010404 0.2500000000010404 1
-91 0.2500000000010404 0.5000000000020591 1
-92 0.2500000000010404 0.7500000000003464 1
-93 0.5000000000020591 0.2500000000010404 1
-94 0.5000000000020591 0.5000000000020591 1
-95 0.5000000000020591 0.7500000000003465 1
-96 0.7500000000003464 0.2500000000010404 1
-97 0.7500000000003467 0.5000000000020591 1
-98 0.7500000000003464 0.7500000000003466 1
-99 0.2500000000010404 0.2500000000010404 0.25
-100 0.2500000000010404 0.2500000000010404 0.5
-101 0.2500000000010404 0.2500000000010404 0.75
-102 0.2500000000010404 0.5000000000020591 0.25
-103 0.2500000000010404 0.5000000000020591 0.5
-104 0.2500000000010404 0.5000000000020591 0.75
-105 0.2500000000010404 0.7500000000003464 0.25
-106 0.2500000000010404 0.7500000000003464 0.5
-107 0.2500000000010404 0.7500000000003464 0.75
-108 0.5000000000020591 0.2500000000010404 0.25
-109 0.5000000000020591 0.2500000000010404 0.5
-110 0.5000000000020591 0.2500000000010404 0.75
-111 0.5000000000020591 0.5000000000020591 0.25
-112 0.5000000000020591 0.5000000000020591 0.5
-113 0.5000000000020591 0.5000000000020591 0.75
-114 0.5000000000020591 0.7500000000003465 0.25
-115 0.5000000000020591 0.7500000000003465 0.5
-116 0.5000000000020591 0.7500000000003465 0.75
-117 0.7500000000003464 0.2500000000010404 0.25
-118 0.7500000000003464 0.2500000000010404 0.5
-119 0.7500000000003464 0.2500000000010404 0.75
-120 0.7500000000003467 0.5000000000020591 0.25
-121 0.7500000000003467 0.5000000000020591 0.5
-122 0.7500000000003467 0.5000000000020591 0.75
-123 0.7500000000003464 0.7500000000003466 0.25
-124 0.7500000000003464 0.7500000000003466 0.5
-125 0.7500000000003464 0.7500000000003466 0.75
-$EndNodes
-$Elements
-160
-1 3 2 1 1 1 9 45 20
-2 3 2 1 1 20 45 46 19
-3 3 2 1 1 19 46 47 18
-4 3 2 1 1 18 47 17 4
-5 3 2 1 1 9 10 48 45
-6 3 2 1 1 45 48 49 46
-7 3 2 1 1 46 49 50 47
-8 3 2 1 1 47 50 16 17
-9 3 2 1 1 10 11 51 48
-10 3 2 1 1 48 51 52 49
-11 3 2 1 1 49 52 53 50
-12 3 2 1 1 50 53 15 16
-13 3 2 1 1 11 2 12 51
-14 3 2 1 1 51 12 13 52
-15 3 2 1 1 52 13 14 53
-16 3 2 1 1 53 14 3 15
-17 3 2 1 13 1 9 54 33
-18 3 2 1 13 33 54 55 34
-19 3 2 1 13 34 55 56 35
-20 3 2 1 13 35 56 21 5
-21 3 2 1 13 9 10 57 54
-22 3 2 1 13 54 57 58 55
-23 3 2 1 13 55 58 59 56
-24 3 2 1 13 56 59 22 21
-25 3 2 1 13 10 11 60 57
-26 3 2 1 13 57 60 61 58
-27 3 2 1 13 58 61 62 59
-28 3 2 1 13 59 62 23 22
-29 3 2 1 13 11 2 36 60
-30 3 2 1 13 60 36 37 61
-31 3 2 1 13 61 37 38 62
-32 3 2 1 13 62 38 6 23
-33 3 2 1 17 2 12 63 36
-34 3 2 1 17 36 63 64 37
-35 3 2 1 17 37 64 65 38
-36 3 2 1 17 38 65 24 6
-37 3 2 1 17 12 13 66 63
-38 3 2 1 17 63 66 67 64
-39 3 2 1 17 64 67 68 65
-40 3 2 1 17 65 68 25 24
-41 3 2 1 17 13 14 69 66
-42 3 2 1 17 66 69 70 67
-43 3 2 1 17 67 70 71 68
-44 3 2 1 17 68 71 26 25
-45 3 2 1 17 14 3 39 69
-46 3 2 1 17 69 39 40 70
-47 3 2 1 17 70 40 41 71
-48 3 2 1 17 71 41 7 26
-49 3 2 1 21 3 15 72 39
-50 3 2 1 21 39 72 73 40
-51 3 2 1 21 40 73 74 41
-52 3 2 1 21 41 74 27 7
-53 3 2 1 21 15 16 75 72
-54 3 2 1 21 72 75 76 73
-55 3 2 1 21 73 76 77 74
-56 3 2 1 21 74 77 28 27
-57 3 2 1 21 16 17 78 75
-58 3 2 1 21 75 78 79 76
-59 3 2 1 21 76 79 80 77
-60 3 2 1 21 77 80 29 28
-61 3 2 1 21 17 4 42 78
-62 3 2 1 21 78 42 43 79
-63 3 2 1 21 79 43 44 80
-64 3 2 1 21 80 44 8 29
-65 3 2 1 25 4 18 81 42
-66 3 2 1 25 42 81 82 43
-67 3 2 1 25 43 82 83 44
-68 3 2 1 25 44 83 30 8
-69 3 2 1 25 18 19 84 81
-70 3 2 1 25 81 84 85 82
-71 3 2 1 25 82 85 86 83
-72 3 2 1 25 83 86 31 30
-73 3 2 1 25 19 20 87 84
-74 3 2 1 25 84 87 88 85
-75 3 2 1 25 85 88 89 86
-76 3 2 1 25 86 89 32 31
-77 3 2 1 25 20 1 33 87
-78 3 2 1 25 87 33 34 88
-79 3 2 1 25 88 34 35 89
-80 3 2 1 25 89 35 5 32
-81 3 2 1 26 5 21 90 32
-82 3 2 1 26 32 90 91 31
-83 3 2 1 26 31 91 92 30
-84 3 2 1 26 30 92 29 8
-85 3 2 1 26 21 22 93 90
-86 3 2 1 26 90 93 94 91
-87 3 2 1 26 91 94 95 92
-88 3 2 1 26 92 95 28 29
-89 3 2 1 26 22 23 96 93
-90 3 2 1 26 93 96 97 94
-91 3 2 1 26 94 97 98 95
-92 3 2 1 26 95 98 27 28
-93 3 2 1 26 23 6 24 96
-94 3 2 1 26 96 24 25 97
-95 3 2 1 26 97 25 26 98
-96 3 2 1 26 98 26 7 27
-97 5 2 1 1 1 9 45 20 33 54 99 87
-98 5 2 1 1 33 54 99 87 34 55 100 88
-99 5 2 1 1 34 55 100 88 35 56 101 89
-100 5 2 1 1 35 56 101 89 5 21 90 32
-101 5 2 1 1 20 45 46 19 87 99 102 84
-102 5 2 1 1 87 99 102 84 88 100 103 85
-103 5 2 1 1 88 100 103 85 89 101 104 86
-104 5 2 1 1 89 101 104 86 32 90 91 31
-105 5 2 1 1 19 46 47 18 84 102 105 81
-106 5 2 1 1 84 102 105 81 85 103 106 82
-107 5 2 1 1 85 103 106 82 86 104 107 83
-108 5 2 1 1 86 104 107 83 31 91 92 30
-109 5 2 1 1 18 47 17 4 81 105 78 42
-110 5 2 1 1 81 105 78 42 82 106 79 43
-111 5 2 1 1 82 106 79 43 83 107 80 44
-112 5 2 1 1 83 107 80 44 30 92 29 8
-113 5 2 1 1 9 10 48 45 54 57 108 99
-114 5 2 1 1 54 57 108 99 55 58 109 100
-115 5 2 1 1 55 58 109 100 56 59 110 101
-116 5 2 1 1 56 59 110 101 21 22 93 90
-117 5 2 1 1 45 48 49 46 99 108 111 102
-118 5 2 1 1 99 108 111 102 100 109 112 103
-119 5 2 1 1 100 109 112 103 101 110 113 104
-120 5 2 1 1 101 110 113 104 90 93 94 91
-121 5 2 1 1 46 49 50 47 102 111 114 105
-122 5 2 1 1 102 111 114 105 103 112 115 106
-123 5 2 1 1 103 112 115 106 104 113 116 107
-124 5 2 1 1 104 113 116 107 91 94 95 92
-125 5 2 1 1 47 50 16 17 105 114 75 78
-126 5 2 1 1 105 114 75 78 106 115 76 79
-127 5 2 1 1 106 115 76 79 107 116 77 80
-128 5 2 1 1 107 116 77 80 92 95 28 29
-129 5 2 1 1 10 11 51 48 57 60 117 108
-130 5 2 1 1 57 60 117 108 58 61 118 109
-131 5 2 1 1 58 61 118 109 59 62 119 110
-132 5 2 1 1 59 62 119 110 22 23 96 93
-133 5 2 1 1 48 51 52 49 108 117 120 111
-134 5 2 1 1 108 117 120 111 109 118 121 112
-135 5 2 1 1 109 118 121 112 110 119 122 113
-136 5 2 1 1 110 119 122 113 93 96 97 94
-137 5 2 1 1 49 52 53 50 111 120 123 114
-138 5 2 1 1 111 120 123 114 112 121 124 115
-139 5 2 1 1 112 121 124 115 113 122 125 116
-140 5 2 1 1 113 122 125 116 94 97 98 95
-141 5 2 1 1 50 53 15 16 114 123 72 75
-142 5 2 1 1 114 123 72 75 115 124 73 76
-143 5 2 1 1 115 124 73 76 116 125 74 77
-144 5 2 1 1 116 125 74 77 95 98 27 28
-145 5 2 1 1 11 2 12 51 60 36 63 117
-146 5 2 1 1 60 36 63 117 61 37 64 118
-147 5 2 1 1 61 37 64 118 62 38 65 119
-148 5 2 1 1 62 38 65 119 23 6 24 96
-149 5 2 1 1 51 12 13 52 117 63 66 120
-150 5 2 1 1 117 63 66 120 118 64 67 121
-151 5 2 1 1 118 64 67 121 119 65 68 122
-152 5 2 1 1 119 65 68 122 96 24 25 97
-153 5 2 1 1 52 13 14 53 120 66 69 123
-154 5 2 1 1 120 66 69 123 121 67 70 124
-155 5 2 1 1 121 67 70 124 122 68 71 125
-156 5 2 1 1 122 68 71 125 97 25 26 98
-157 5 2 1 1 53 14 3 15 123 69 39 72
-158 5 2 1 1 123 69 39 72 124 70 40 73
-159 5 2 1 1 124 70 40 73 125 71 41 74
-160 5 2 1 1 125 71 41 74 98 26 7 27
-$EndElements
-$Periodic
-3
-2 17 25
-Affine 1 0 0 1 0 1 0 0 0 0 1 0 0 0 0 1
-25
-2 1
-3 4
-6 5
-7 8
-63 87
-64 88
-65 89
-66 84
-67 85
-68 86
-69 81
-70 82
-71 83
-14 18
-24 32
-25 31
-26 30
-36 33
-37 34
-38 35
-39 42
-40 43
-41 44
-12 20
-13 19
-2 21 13
-Affine 1 0 0 0 0 1 0 1 0 0 1 0 0 0 0 1
-25
-3 2
-4 1
-7 6
-8 5
-15 11
-16 10
-17 9
-72 60
-73 61
-74 62
-75 57
-76 58
-77 59
-78 54
-79 55
-80 56
-27 23
-28 22
-29 21
-39 36
-40 37
-41 38
-42 33
-43 34
-44 35
-2 26 1
-Affine 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 1
-25
-5 1
-6 2
-7 3
-8 4
-90 45
-91 46
-92 47
-93 48
-94 49
-95 50
-96 51
-97 52
-98 53
-21 9
-22 10
-23 11
-24 12
-25 13
-26 14
-27 15
-28 16
-29 17
-30 18
-31 19
-32 20
-$EndPeriodic

data/periodic-square.geo

@@ -1,31 +0,0 @@
-// 0 for triangles, 1 for quads
-tri_or_quad = 1;
-
-Point(1) = {0, 0, 0, 1.0};
-Point(2) = {1, 0, 0, 1.0};
-Point(3) = {1, 1, 0, 1.0};
-Point(4) = {0, 1, 0, 1.0};
-
-Characteristic Length {:} = 0.25;
-
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-
-Periodic Line {3} = {-1};
-Periodic Line {2} = {-4};
-
-Curve Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-Transfinite Surface {1};
-
-If (tri_or_quad == 1)
-   Recombine Surface {1};
-EndIf
-
-Physical Surface(1) = {1};
-Physical Curve(1) = {1, 2, 3, 4};
-
-Mesh.MshFileVersion = 2.2;
-Mesh 2;

data/periodic-square.msh

@@ -1,83 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-25
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0.2499999999994121 0 0
-6 0.499999999998694 0 0
-7 0.7499999999993416 0 0
-8 1 0.2500000000010404 0
-9 1 0.5000000000020591 0
-10 1 0.7500000000003465 0
-11 0.7499999999993416 1 0
-12 0.4999999999986939 1 0
-13 0.249999999999412 1 0
-14 0 0.7500000000003465 0
-15 0 0.5000000000020591 0
-16 0 0.2500000000010404 0
-17 0.2499999999994121 0.2500000000010404 0
-18 0.249999999999412 0.5000000000020591 0
-19 0.249999999999412 0.7500000000003466 0
-20 0.4999999999986939 0.2500000000010404 0
-21 0.4999999999986939 0.5000000000020591 0
-22 0.4999999999986939 0.7500000000003466 0
-23 0.7499999999993416 0.2500000000010404 0
-24 0.7499999999993416 0.5000000000020591 0
-25 0.7499999999993416 0.7500000000003465 0
-$EndNodes
-$Elements
-32
-1 1 2 1 1 1 5
-2 1 2 1 1 5 6
-3 1 2 1 1 6 7
-4 1 2 1 1 7 2
-5 1 2 1 2 2 8
-6 1 2 1 2 8 9
-7 1 2 1 2 9 10
-8 1 2 1 2 10 3
-9 1 2 1 3 3 11
-10 1 2 1 3 11 12
-11 1 2 1 3 12 13
-12 1 2 1 3 13 4
-13 1 2 1 4 4 14
-14 1 2 1 4 14 15
-15 1 2 1 4 15 16
-16 1 2 1 4 16 1
-17 3 2 1 1 1 5 17 16
-18 3 2 1 1 16 17 18 15
-19 3 2 1 1 15 18 19 14
-20 3 2 1 1 14 19 13 4
-21 3 2 1 1 5 6 20 17
-22 3 2 1 1 17 20 21 18
-23 3 2 1 1 18 21 22 19
-24 3 2 1 1 19 22 12 13
-25 3 2 1 1 6 7 23 20
-26 3 2 1 1 20 23 24 21
-27 3 2 1 1 21 24 25 22
-28 3 2 1 1 22 25 11 12
-29 3 2 1 1 7 2 8 23
-30 3 2 1 1 23 8 9 24
-31 3 2 1 1 24 9 10 25
-32 3 2 1 1 25 10 3 11
-$EndElements
-$Periodic
-2
-1 2 4
-5
-8 16
-9 15
-10 14
-2 1
-3 4
-1 3 1
-5
-11 7
-12 6
-13 5
-3 2
-4 1
-$EndPeriodic

data/ref-cube.mesh

@@ -1,41 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 5 0 1 2 3 4 5 6 7
-
-boundary
-6
-1 3 3 2 1 0
-2 3 0 1 5 4
-3 3 1 2 6 5
-4 3 2 3 7 6
-5 3 3 0 4 7
-6 3 4 5 6 7
-
-vertices
-8
-3
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-0 0 1
-1 0 1
-1 1 1
-0 1 1

data/ref-prism.mesh

@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 6 0 1 2 3 4 5
-
-boundary
-5
-1 2 0 2 1
-2 2 3 4 5
-3 3 0 1 4 3
-4 3 1 2 5 4
-5 3 2 0 3 5
-
-vertices
-6
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1
-1 0 1
-0 1 1

data/ref-segment.mesh

@@ -1,31 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-1
-
-elements
-1
-1 1 0 1
-
-boundary
-2
-1 0 0
-2 0 1
-
-vertices
-2
-1
-0
-1

data/ref-square.mesh

@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 3 0 1 2 3
-
-boundary
-4
-1 1 0 1
-2 1 1 2
-3 1 2 3
-4 1 3 0
-
-vertices
-4
-2
-0 0
-1 0
-1 1
-0 1

data/ref-tetrahedron.mesh

@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 4 0 1 2 3
-
-boundary
-4
-1 2 1 2 3
-2 2 0 3 2
-3 2 0 1 3
-4 2 0 2 1
-
-vertices
-4
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1

data/ref-triangle.mesh

@@ -1,33 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 2 0 1 2
-
-boundary
-3
-1 1 0 1
-2 1 1 2
-3 1 2 0
-
-vertices
-3
-2
-0 0
-1 0
-0 1

doc/CodeDocumentation.conf.in

@@ -766,7 +766,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mesh \
                          @MFEM_SOURCE_DIR@/fem \
                          @MFEM_SOURCE_DIR@/examples \
-                         @MFEM_SOURCE_DIR@/examples/caliper \
                          @MFEM_SOURCE_DIR@/examples/amgx \
                          @MFEM_SOURCE_DIR@/examples/ginkgo \
                          @MFEM_SOURCE_DIR@/examples/hiop \
@@ -779,7 +778,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/mtop \
                          @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
                          @MFEM_SOURCE_DIR@/miniapps/performance \

doc/CodeDocumentation.dox

@@ -42,10 +42,8 @@ namespace mfem {
  * - MFEM_FORALL macro in forall.hpp
  *
  * <H3>Example codes</H3>
- * - <a class="el" href="ex0_8cpp_source.html">Example 0</a>: simplest example, nodal H1 FEM for the Laplace problem
- * - <a class="el" href="ex0p_8cpp_source.html">Example 0p</a>: simplest parallel example, nodal H1 FEM for the Laplace problem
- * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem (same discretization as ex0 but with more sophisticated options)
- * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem (same discretization as ex0p but with more sophisticated options)
+ * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
  * - <a class="el" href="ex2_8cpp_source.html">Example 2</a>: vector FEM for linear elasticity
  * - <a class="el" href="ex2p_8cpp_source.html">Example 2p</a>: parallel vector FEM for linear elasticity
  * - <a class="el" href="ex3_8cpp_source.html">Example 3</a>: Nedelec H(curl) FEM for the definite Maxwell problem
@@ -94,10 +92,6 @@ namespace mfem {
  * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
  * - <a class="el" href="ex27_8cpp_source.html">Example 27</a>: boundary conditions for the Laplace problem
  * - <a class="el" href="ex27p_8cpp_source.html">Example 27p</a>: parallel boundary conditions for the Laplace problem
- * - <a class="el" href="ex28_8cpp_source.html">Example 28</a>: sliding contact in elasticity
- * - <a class="el" href="ex28p_8cpp_source.html">Example 28p</a>: parallel sliding contact in elasticity
- * - <a class="el" href="ex29_8cpp_source.html">Example 29</a>: Laplace solve on a 3D-embedded surface
- * - <a class="el" href="ex29p_8cpp_source.html">Example 29p</a>: parallel Laplace solve on a 3D-embedded surface
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -105,12 +99,6 @@ namespace mfem {
  *   <a class="el" href="examples_2amgx_2ex1p_8cpp_source.html">1p</a>,
  *   demonstrating the use of MFEM's \link amgxsolver.hpp AmgX integration\endlink.
  *
- * <H4>Caliper Examples</H4>
- * - Variants of Example
- *   <a class="el" href="examples_2caliper_2ex1_8cpp_source.html">1</a> and
- *   <a class="el" href="examples_2caliper_2ex1p_8cpp_source.html">1p</a>,
- *   demonstrating the use of MFEM's \link annotation.hpp Ginkgo integration\endlink.
- *
  * <H4>Ginkgo Examples</H4>
  * - Variants of Example
  *   <a class="el" href="examples_2ginkgo_2ex1_8cpp_source.html">1</a>,
@@ -189,9 +177,7 @@ namespace mfem {
  * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
  * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
  * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
- * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
  * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
- * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *

examples/CMakeLists.txt

@@ -10,7 +10,6 @@
 # CONTRIBUTING.md for details.
 
 list(APPEND ALL_EXE_SRCS
-  ex0.cpp
   ex1.cpp
   ex2.cpp
   ex3.cpp
@@ -35,13 +34,10 @@ list(APPEND ALL_EXE_SRCS
   ex25.cpp
   ex26.cpp
   ex27.cpp
-  ex28.cpp
-  ex29.cpp
   )
 
 if (MFEM_USE_MPI)
   list(APPEND ALL_EXE_SRCS
-    ex0p.cpp
     ex1p.cpp
     ex2p.cpp
     ex3p.cpp
@@ -68,8 +64,6 @@ if (MFEM_USE_MPI)
     ex25p.cpp
     ex26p.cpp
     ex27p.cpp
-    ex28p.cpp
-    ex29p.cpp
     )
 endif()
 
@@ -84,9 +78,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
-    set(THIS_TEST_OPTIONS "-no-vis")
-  endif()
+  set(THIS_TEST_OPTIONS "-no-vis")
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
   elseif(${TEST_NAME} MATCHES "ex15p*")
@@ -155,10 +147,6 @@ if (MFEM_USE_SUNDIALS)
   add_subdirectory(sundials)
 endif()
 
-if(MFEM_USE_CALIPER)
-  add_subdirectory(caliper)
-endif()
-
 # Include the examples/superlu directory if SUPERLU is enabled.
 if (MFEM_USE_SUPERLU)
   add_subdirectory(superlu)

examples/amgx/ex1p.cpp

@@ -157,7 +157,7 @@ int main(int argc, char *argv[])
       delete_fec = true;
    }
    ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   HYPRE_Int size = fespace.GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;

examples/caliper/CMakeLists.txt

@@ -1,51 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(CALIPER_EXAMPLES_SRCS)
-
-list(APPEND CALIPER_EXE_SRCS
-    ex1.cpp
-    )
-
-if (MFEM_USE_MPI)
-  list(APPEND CALIPER_EXE_SRCS
-      ex1p.cpp
-      )
-endif()
-
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add one executable per cpp file
-set(PREFIX caliper_)
-add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})
-
-# Add a test for each example
-foreach(SRC_FILE ${CALIPER_EXE_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
-
-

examples/caliper/ex1.cpp

@@ -1,270 +0,0 @@
-//                                MFEM Example 1
-//                             Caliper Modification
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/periodic-annulus-sector.msh
-//               ex1 -m ../data/periodic-torus-sector.msh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
-//
-// Description: This example is a copy of Example 1 instrumented with the
-//              Caliper performance profilinh library. Any option supported by
-//              the Caliper ConfigManager can be passed to the code using a
-//              configuration string after -p or --caliper flag. For more
-//              information, see the Caliper documentation.
-//
-// Examples: ex1 --caliper runtime-report
-//           ex1 --caliper runtime-report,mem.highwatermark
-//
-// The first run will return the default report. The second run will also output
-// the memory high-water mark and time spent in MPI routines.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // Define Caliper ConfigManager
-   cali::ConfigManager mgr;
-   // Caliper instrumentation
-   MFEM_PERF_FUNCTION;
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   const char* cali_config = "runtime-report";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&cali_config, "-p", "--caliper",
-                  "Caliper configuration string.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // Caliper configuration
-   mgr.add(cali_config);
-   mgr.start();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (mesh.GetNodes())
-   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   MFEM_PERF_BEGIN("Set up the linear form");
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-   MFEM_PERF_END("Set up the linear form");
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   MFEM_PERF_BEGIN("Set up the bilinear form");
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-   MFEM_PERF_END("Set up the bilinear form");
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   MFEM_PERF_BEGIN("Solve A X=B");
-   if (!pa)
-   {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
-#endif
-   }
-   else // Jacobi preconditioning in partial assembly mode
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-   MFEM_PERF_END("Solve A X=B");
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   MFEM_PERF_BEGIN("Save the results");
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-   MFEM_PERF_END("Save the results");
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   // Flush output
-   mgr.flush();
-
-   return 0;
-}

examples/caliper/ex1p.cpp

@@ -1,298 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                             Caliper Modification
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//
-// Description: This example is a copy of Example 1 instrumented with the
-//              Caliper performance profilinh library. Any option supported by
-//              the Caliper ConfigManager can be passed to the code using a
-//              configuration string after -p or --caliper flag. For more
-//              information, see the Caliper documentation.
-//
-// Examples:  mpirun -np 4 ex1p --caliper runtime-report
-//            mpirun -np 4 ex1p --caliper runtime-report,mem.highwatermark,mpi-report
-//
-// The first run will return the default report. The second run will also output
-// the memory high-water mark and time spent in MPI routines.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-   // Define Caliper ConfigManager
-   cali::ConfigManager mgr;
-   // Caliper instrumentation
-   MFEM_PERF_FUNCTION;
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   const char* cali_config = "runtime-report";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&cali_config, "-p", "--caliper",
-                  "Caliper configuration string.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // Caliper configuration
-   mgr.add(cali_config);
-   mgr.start();
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   MFEM_PERF_BEGIN("Set up the linear form");
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-   MFEM_PERF_END("Set up the linear form");
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   MFEM_PERF_BEGIN("Set up the bilinear form");
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-   MFEM_PERF_END("Set up the bilinear form");
-   // 13. Solve the linear system A X = B.
-   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use Jacobi smoothing, for now.
-   MFEM_PERF_BEGIN("Solve A X = B");
-   Solver *prec = NULL;
-   if (pa)
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
-   MFEM_PERF_END("Solve A X = B");
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   MFEM_PERF_BEGIN("Save the results");
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-   MFEM_PERF_END("Save the results");
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   // Flush output before MPI_finalize
-   mgr.flush();
-   MPI_Finalize();
-
-   return 0;
-}

examples/caliper/makefile

@@ -1,76 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/caliper,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
--include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-# Testing: Specific execution options
-ex1-test-seq: ex1
-	@$(call mfem-test,$<,, Caliper serial example)
-ex1p-test-par: ex1p
-	@$(call mfem-test,$<, $(RUN_MPI), Caliper parallel example)
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec $(SUBDIRS_CLEAN)
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*

examples/ex0.cpp

@@ -1,81 +0,0 @@
-//                                MFEM Example 0
-//
-// Compile with: make ex0
-//
-// Sample runs:  ex0
-//               ex0 -m ../data/fichera.mesh
-//               ex0 -m ../data/square-disc.mesh -o 2
-//
-// Description: This example code demonstrates the most basic usage of MFEM to
-//              define a simple finite element discretization of the Laplace
-//              problem -Delta u = 1 with zero Dirichlet boundary conditions.
-//              General 2D/3D mesh files and finite element polynomial degrees
-//              can be specified by command line options.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   mesh.UniformRefinement();
-
-   // 3. Define a finite element space on the mesh. Here we use H1 continuous
-   //    high-order Lagrange finite elements of the given order.
-   H1_FECollection fec(order, mesh.Dimension());
-   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of unknowns: " << fespace.GetTrueVSize() << endl;
-
-   // 4. Extract the list of all the boundary DOFs. These will be marked as
-   //    Dirichlet in order to enforce zero boundary conditions.
-   Array<int> boundary_dofs;
-   fespace.GetBoundaryTrueDofs(boundary_dofs);
-
-   // 5. Define the solution x as a finite element grid function in fespace. Set
-   //    the initial guess to zero, which also sets the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 6. Set up the linear form b(.) corresponding to the right-hand side.
-   ConstantCoefficient one(1.0);
-   LinearForm b(&fespace);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 7. Set up the bilinear form a(.,.) corresponding to the -Delta operator.
-   BilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator);
-   a.Assemble();
-
-   // 8. Form the linear system A X = B. This includes eliminating boundary
-   //    conditions, applying AMR constraints, and other transformations.
-   SparseMatrix A;
-   Vector B, X;
-   a.FormLinearSystem(boundary_dofs, x, b, A, X, B);
-
-   // 9. Solve the system using PCG with symmetric Gauss-Seidel preconditioner.
-   GSSmoother M(A);
-   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
-
-   // 10. Recover the solution x as a grid function and save to file. The output
-   //     can be viewed using GLVis as follows: "glvis -m mesh.mesh -g sol.gf"
-   a.RecoverFEMSolution(X, b, x);
-   x.Save("sol.gf");
-   mesh.Save("mesh.mesh");
-
-   return 0;
-}

examples/ex0p.cpp

@@ -1,96 +0,0 @@
-//                       MFEM Example 0 - Parallel Version
-//
-// Compile with: make ex0p
-//
-// Sample runs:  mpirun -np 4 ex0p
-//               mpirun -np 4 ex0p -m ../data/fichera.mesh
-//               mpirun -np 4 ex0p -m ../data/square-disc.mesh -o 2
-//
-// Description: This example code demonstrates the most basic parallel usage of
-//              MFEM to define a simple finite element discretization of the
-//              Laplace problem -Delta u = 1 with zero Dirichlet boundary
-//              conditions. General 2D/3D serial mesh files and finite element
-//              polynomial degrees can be specified by command line options.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI
-   MPI_Session mpi(argc, argv);
-
-   // 2. Parse command line options
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   // 3. Read the serial mesh from the given mesh file.
-   Mesh serial_mesh(mesh_file);
-
-   // 4. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh once in parallel to increase the resolution.
-   ParMesh mesh(MPI_COMM_WORLD, serial_mesh);
-   serial_mesh.Clear(); // the serial mesh is no longer needed
-   mesh.UniformRefinement();
-
-   // 5. Define a finite element space on the mesh. Here we use H1 continuous
-   //    high-order Lagrange finite elements of the given order.
-   H1_FECollection fec(order, mesh.Dimension());
-   ParFiniteElementSpace fespace(&mesh, &fec);
-   HYPRE_BigInt total_num_dofs = fespace.GlobalTrueVSize();
-   if (mpi.Root()) { cout << "Number of unknowns: " << total_num_dofs << endl; }
-
-   // 6. Extract the list of all the boundary DOFs. These will be marked as
-   //    Dirichlet in order to enforce zero boundary conditions.
-   Array<int> boundary_dofs;
-   fespace.GetBoundaryTrueDofs(boundary_dofs);
-
-   // 7. Define the solution x as a finite element grid function in fespace. Set
-   //    the initial guess to zero, which also sets the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 8. Set up the linear form b(.) corresponding to the right-hand side.
-   ConstantCoefficient one(1.0);
-   ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 9. Set up the bilinear form a(.,.) corresponding to the -Delta operator.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator);
-   a.Assemble();
-
-   // 10. Form the linear system A X = B. This includes eliminating boundary
-   //     conditions, applying AMR constraints, parallel assembly, etc.
-   HypreParMatrix A;
-   Vector B, X;
-   a.FormLinearSystem(boundary_dofs, x, b, A, X, B);
-
-   // 11. Solve the system using PCG with hypre's BoomerAMG preconditioner.
-   HypreBoomerAMG M(A);
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(A);
-   cg.Mult(B, X);
-
-   // 12. Recover the solution x as a grid function and save to file. The output
-   //     can be viewed using GLVis as follows: "glvis -np <np> -m mesh -g sol"
-   a.RecoverFEMSolution(X, b, x);
-   x.Save("sol");
-   mesh.Save("mesh");
-
-   return 0;
-}

examples/ex1.cpp

@@ -35,7 +35,6 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cpu -o 4 -a
 //             * ex1 -pa -d ceed-cuda
 //             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
@@ -74,7 +73,6 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   bool algebraic_ceed = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -88,10 +86,6 @@ int main(int argc, char *argv[])
                   "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
-#ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
-                  "Use algebraic Ceed solver");
-#endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -213,20 +207,12 @@ int main(int argc, char *argv[])
       umf_solver.Mult(B, X);
 #endif
    }
-   else
+   else // Jacobi preconditioning in partial assembly mode
    {
       if (UsesTensorBasis(fespace))
       {
-         if (algebraic_ceed)
-         {
-            ceed::AlgebraicSolver M(a, ess_tdof_list);
-            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }
-         else
-         {
-            OperatorJacobiSmoother M(a, ess_tdof_list);
-            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
       }
       else
       {

examples/ex10p.cpp

@@ -293,7 +293,7 @@ int main(int argc, char *argv[])
    H1_FECollection fe_coll(order, dim);
    ParFiniteElementSpace fespace(pmesh, &fe_coll, dim);
 
-   HYPRE_BigInt glob_size = fespace.GlobalTrueVSize();
+   HYPRE_Int glob_size = fespace.GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of velocity/deformation unknowns: " << glob_size << endl;

examples/ex11p.cpp

@@ -174,7 +174,7 @@ int main(int argc, char *argv[])
       fec = new H1_FECollection(order = 1, dim);
    }
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of unknowns: " << size << endl;

examples/ex12p.cpp

@@ -165,7 +165,7 @@ int main(int argc, char *argv[])
       fec = new H1_FECollection(order, dim);
       fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
    }
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of unknowns: " << size << endl

examples/ex13p.cpp

@@ -116,7 +116,7 @@ int main(int argc, char *argv[])
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of unknowns: " << size << endl;

examples/ex14p.cpp

@@ -166,7 +166,7 @@ int main(int argc, char *argv[])
    //    use discontinuous finite elements of the specified order >= 0.
    FiniteElementCollection *fec = new DG_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of unknowns: " << size << endl;

examples/ex15.cpp

@@ -19,14 +19,6 @@
 //               ex15 -m ../data/square-disc.mesh
 //               ex15 -m ../data/escher.mesh -r 2 -tf 0.3
 //
-//               Kelly estimator:
-//
-//               ex15 -est 1 -e 0.0001
-//               ex15 -est 1 -o 1 -y 0.4
-//               ex15 -est 1 -o 4 -y 0.1
-//               ex15 -est 1 -n 5
-//               ex15 -est 1 -p 1 -n 3
-//
 // Description:  Building on Example 6, this example demonstrates dynamic AMR.
 //               The mesh is adapted to a time-dependent solution by refinement
 //               as well as by derefinement. For simplicity, the solution is
@@ -36,10 +28,10 @@
 //               At each outer iteration the right hand side function is changed
 //               to mimic a time dependent problem.  Within each inner iteration
 //               the problem is solved on a sequence of meshes which are locally
-//               refined according to a simple ZZ or Kelly error estimator.  At
-//               the end of the inner iteration the error estimates are also
-//               used to identify any elements which may be over-refined and a
-//               single derefinement step is performed.
+//               refined according to a simple ZZ error estimator.  At the end
+//               of the inner iteration the error estimates are also used to
+//               identify any elements which may be over-refined and a single
+//               derefinement step is performed.
 //
 //               The example demonstrates MFEM's capability to refine and
 //               derefine nonconforming meshes, in 2D and 3D, and on linear,
@@ -86,7 +78,6 @@ int main(int argc, char *argv[])
    int nc_limit = 3;         // maximum level of hanging nodes
    bool visualization = true;
    bool visit = false;
-   int which_estimator = 0;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -107,9 +98,6 @@ int main(int argc, char *argv[])
                   "Maximum level of hanging nodes.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
-   args.AddOption(&which_estimator, "-est", "--estimator",
-                  "Which estimator to use: "
-                  "0 = ZZ, 1 = Kelly. Defaults to ZZ.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -198,38 +186,19 @@ int main(int argc, char *argv[])
    visit_dc.RegisterField("solution", &x);
    int vis_cycle = 0;
 
-   // 9. As in Example 6, we set up an estimator that will be used to obtain
-   //    element error indicators. The integrator needs to provide the method
-   //    ComputeElementFlux. The smoothed flux space is a vector valued H1 (ZZ)
-   //    or L2 (Kelly) space here.
-   L2_FECollection flux_fec(order, dim);
-   ErrorEstimator* estimator{nullptr};
-
-   switch (which_estimator)
-   {
-      case 1:
-      {
-         auto flux_fes = new FiniteElementSpace(&mesh, &flux_fec, sdim);
-         estimator = new KellyErrorEstimator(*integ, x, flux_fes);
-         break;
-      }
-
-      default:
-         std::cout << "Unknown estimator. Falling back to ZZ." << std::endl;
-      case 0:
-      {
-         auto flux_fes = new FiniteElementSpace(&mesh, &fec, sdim);
-         estimator = new ZienkiewiczZhuEstimator(*integ, x, flux_fes);
-         break;
-      }
-   }
+   // 9. As in Example 6, we set up a Zienkiewicz-Zhu estimator that will be
+   //    used to obtain element error indicators. The integrator needs to
+   //    provide the method ComputeElementFlux. The smoothed flux space is a
+   //    vector valued H1 space here.
+   FiniteElementSpace flux_fespace(&mesh, &fec, sdim);
+   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
 
    // 10. As in Example 6, we also need a refiner. This time the refinement
    //     strategy is based on a fixed threshold that is applied locally to each
    //     element. The global threshold is turned off by setting the total error
    //     fraction to zero. We also enforce a maximum refinement ratio between
    //     adjacent elements.
-   ThresholdRefiner refiner(*estimator);
+   ThresholdRefiner refiner(estimator);
    refiner.SetTotalErrorFraction(0.0); // use purely local threshold
    refiner.SetLocalErrorGoal(max_elem_error);
    refiner.PreferConformingRefinement();
@@ -238,7 +207,7 @@ int main(int argc, char *argv[])
    // 11. A derefiner selects groups of elements that can be coarsened to form
    //     a larger element. A conservative enough threshold needs to be set to
    //     prevent derefining elements that would immediately be refined again.
-   ThresholdDerefiner derefiner(*estimator);
+   ThresholdDerefiner derefiner(estimator);
    derefiner.SetThreshold(hysteresis * max_elem_error);
    derefiner.SetNCLimit(nc_limit);
 
@@ -339,8 +308,6 @@ int main(int argc, char *argv[])
       b.Update();
    }
 
-   delete estimator;
-
    return 0;
 }
 

examples/ex15p.cpp

@@ -223,14 +223,13 @@ int main(int argc, char *argv[])
    visit_dc.RegisterField("solution", &x);
    int vis_cycle = 0;
 
-   // 10. As in Example 6p, we set up an estimator that will be used to obtain
-   //     element error indicators. The integrator needs to provide the method
-   //     ComputeElementFlux. We supply an L2 space for the discontinuous flux
-   //     and an H(div) space for the smoothed flux.
+   // 10. As in Example 6p, we set up a Zienkiewicz-Zhu estimator that will be
+   //     used to obtain element error indicators. The integrator needs to
+   //     provide the method ComputeElementFlux. We supply an L2 space for the
+   //     discontinuous flux and an H(div) space for the smoothed flux.
    L2_FECollection flux_fec(order, dim);
    RT_FECollection smooth_flux_fec(order-1, dim);
-   ErrorEstimator* estimator{nullptr};
-
+   ErrorEstimator* estimator;
    switch (which_estimator)
    {
       case 1:
@@ -249,7 +248,7 @@ int main(int argc, char *argv[])
       default:
          if (myid == 0)
          {
-            std::cout << "Unknown estimator. Falling back to L2ZZ." << std::endl;
+            std::cout << "Unkown estimator. Falling back to L2ZZ." << std::endl;
          }
       case 0:
       {
@@ -301,7 +300,7 @@ int main(int argc, char *argv[])
       //     time step resolved to the prescribed tolerance in each element.
       for (int ref_it = 1; ; ref_it++)
       {
-         HYPRE_BigInt global_dofs = fespace.GlobalTrueVSize();
+         HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
          if (myid == 0)
          {
             cout << "Iteration: " << ref_it << ", number of unknowns: "

examples/ex16p.cpp

@@ -213,7 +213,7 @@ int main(int argc, char *argv[])
    H1_FECollection fe_coll(order, dim);
    ParFiniteElementSpace fespace(pmesh, &fe_coll);
 
-   HYPRE_BigInt fe_size = fespace.GlobalTrueVSize();
+   int fe_size = fespace.GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of temperature unknowns: " << fe_size << endl;

examples/ex17p.cpp

@@ -190,7 +190,7 @@ int main(int argc, char *argv[])
    DG_FECollection fec(order, dim, BasisType::GaussLobatto);
    ParFiniteElementSpace fespace(&pmesh, &fec, dim, Ordering::byVDIM);
 
-   HYPRE_BigInt glob_size = fespace.GlobalTrueVSize();
+   HYPRE_Int glob_size = fespace.GlobalTrueVSize();
    if (mpi.Root())
    {
       cout << "Number of finite element unknowns: " << glob_size

examples/ex18.hpp

@@ -35,8 +35,8 @@ private:
    void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;
 
 public:
-   FE_Evolution(FiniteElementSpace &vfes_,
-                Operator &A_, SparseMatrix &Aflux_);
+   FE_Evolution(FiniteElementSpace &_vfes,
+                Operator &_A, SparseMatrix &_Aflux);
 
    virtual void Mult(const Vector &x, Vector &y) const;
 
@@ -99,13 +99,13 @@ public:
 };
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(FiniteElementSpace &vfes_,
-                           Operator &A_, SparseMatrix &Aflux_)
-   : TimeDependentOperator(A_.Height()),
-     dim(vfes_.GetFE(0)->GetDim()),
-     vfes(vfes_),
-     A(A_),
-     Aflux(Aflux_),
+FE_Evolution::FE_Evolution(FiniteElementSpace &_vfes,
+                           Operator &_A, SparseMatrix &_Aflux)
+   : TimeDependentOperator(_A.Height()),
+     dim(_vfes.GetFE(0)->GetDim()),
+     vfes(_vfes),
+     A(_A),
+     Aflux(_Aflux),
      Me_inv(vfes.GetFE(0)->GetDof(), vfes.GetFE(0)->GetDof(), vfes.GetNE()),
      state(num_equation),
      f(num_equation, dim),

examples/ex18p.cpp

@@ -171,7 +171,7 @@ int main(int argc, char *argv[])
    // This example depends on this ordering of the space.
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES, "");
 
-   HYPRE_BigInt glob_size = vfes.GlobalTrueVSize();
+   HYPRE_Int glob_size = vfes.GlobalTrueVSize();
    if (mpi.Root()) { cout << "Number of unknowns: " << glob_size << endl; }
 
    // 8. Define the initial conditions, save the corresponding mesh and grid

examples/ex19p.cpp

@@ -285,8 +285,8 @@ int main(int argc, char *argv[])
    spaces[0] = &R_space;
    spaces[1] = &W_space;
 
-   HYPRE_BigInt glob_R_size = R_space.GlobalTrueVSize();
-   HYPRE_BigInt glob_W_size = W_space.GlobalTrueVSize();
+   HYPRE_Int glob_R_size = R_space.GlobalTrueVSize();
+   HYPRE_Int glob_W_size = W_space.GlobalTrueVSize();
 
    // 8. Define the Dirichlet conditions (set to boundary attribute 1 and 2)
    Array<Array<int> *> ess_bdr(2);

examples/ex1p.cpp

@@ -32,7 +32,6 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
 //             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
@@ -63,9 +62,10 @@ using namespace mfem;
 int main(int argc, char *argv[])
 {
    // 1. Initialize MPI.
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 
    // 2. Parse command-line options.
    const char *mesh_file = "../data/star.mesh";
@@ -74,7 +74,6 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   bool algebraic_ceed = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -88,10 +87,6 @@ int main(int argc, char *argv[])
                   "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
-#ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
-                  "Use algebraic Ceed solver");
-#endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -102,6 +97,7 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
+      MPI_Finalize();
       return 1;
    }
    if (myid == 0)
@@ -171,7 +167,7 @@ int main(int argc, char *argv[])
       delete_fec = true;
    }
    ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   HYPRE_Int size = fespace.GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;
@@ -229,14 +225,7 @@ int main(int argc, char *argv[])
    {
       if (UsesTensorBasis(fespace))
       {
-         if (algebraic_ceed)
-         {
-            prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
-         }
-         else
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
+         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
       }
    }
    else
@@ -288,6 +277,7 @@ int main(int argc, char *argv[])
    {
       delete fec;
    }
+   MPI_Finalize();
 
    return 0;
 }

examples/ex20.cpp

@@ -3,10 +3,6 @@
 // Compile with: make ex20
 //
 // Sample runs:  ex20
-//               ex20 -p 1 -o 1 -n 120 -dt 0.1
-//               ex20 -p 1 -o 2 -n 60 -dt 0.2
-//               ex20 -p 1 -o 3 -n 40 -dt 0.3
-//               ex20 -p 1 -o 4 -n 30 -dt 0.4
 //
 // Description: This example demonstrates the use of the variable order,
 //              symplectic ODE integration algorithm.  Symplectic integration
@@ -235,7 +231,6 @@ int main(int argc, char *argv[])
    // 9. Finalize the GLVis output
    if (visualization)
    {
-      mesh.FinalizeQuadMesh(1);
       H1_FECollection fec(order = 1, 2);
       FiniteElementSpace fespace(&mesh, &fec);
       GridFunction energy(&fespace);

examples/ex20p.cpp

@@ -3,10 +3,6 @@
 // Compile with: make ex20p
 //
 // Sample runs:  mpirun -np 4 ex20p
-//               mpirun -np 4 ex20p -p 1 -o 1 -n 120 -dt 0.1
-//               mpirun -np 4 ex20p -p 1 -o 2 -n 60 -dt 0.2
-//               mpirun -np 4 ex20p -p 1 -o 3 -n 40 -dt 0.3
-//               mpirun -np 4 ex20p -p 1 -o 4 -n 30 -dt 0.4
 //
 // Description: This example demonstrates the use of the variable order,
 //              symplectic ODE integration algorithm.  Symplectic integration
@@ -172,7 +168,7 @@ int main(int argc, char *argv[])
    }
 
    // 6. Create a Mesh for visualization in phase space
-   int nverts = (visualization) ? 2*num_procs*(nsteps+1) : 0;
+   int nverts = (visualization) ? (num_procs+1)*(nsteps+1) : 0;
    int nelems = (visualization) ? (nsteps * num_procs) : 0;
    Mesh mesh(2, nverts, nelems, 0, 3);
 
@@ -194,9 +190,9 @@ int main(int argc, char *argv[])
 
          if (visualization)
          {
+            mesh.AddVertex(x0);
             for (int j = 0; j < num_procs; j++)
             {
-               mesh.AddVertex(x0);
                x1[0] = q(0);
                x1[1] = p(0);
                x1[2] = 0.0;
@@ -220,17 +216,17 @@ int main(int argc, char *argv[])
       if (visualization)
       {
          x0[2] = t;
+         mesh.AddVertex(x0);
          for (int j = 0; j < num_procs; j++)
          {
-            mesh.AddVertex(x0);
             x1[0] = q(0);
             x1[1] = p(0);
             x1[2] = t;
             mesh.AddVertex(x1);
-            v[0] = 2 * num_procs * i + 2 * j;
-            v[1] = 2 * num_procs * (i + 1) + 2 * j;
-            v[2] = 2 * num_procs * (i + 1) + 2 * j + 1;
-            v[3] = 2 * num_procs * i + 2 * j + 1;
+            v[0] = (num_procs + 1) * i;
+            v[1] = (num_procs + 1) * (i + 1);
+            v[2] = (num_procs + 1) * (i + 1) + j + 1;
+            v[3] = (num_procs + 1) * i + j + 1;
             mesh.AddQuad(v);
             part[num_procs * i + j] = j;
          }

examples/ex21p.cpp

@@ -211,7 +211,7 @@ int main(int argc, char *argv[])
    const int max_amr_itr = 20;
    for (int it = 0; it <= max_amr_itr; it++)
    {
-      HYPRE_BigInt global_dofs = fespace.GlobalTrueVSize();
+      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
       if (myid == 0)
       {
          cout << "\nAMR iteration " << it << endl;

examples/ex22p.cpp

@@ -213,7 +213,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;

examples/ex24p.cpp

@@ -167,8 +167,8 @@ int main(int argc, char *argv[])
    ParFiniteElementSpace trial_fes(pmesh, trial_fec);
    ParFiniteElementSpace test_fes(pmesh, test_fec);
 
-   HYPRE_BigInt trial_size = trial_fes.GlobalTrueVSize();
-   HYPRE_BigInt test_size = test_fes.GlobalTrueVSize();
+   HYPRE_Int trial_size = trial_fes.GlobalTrueVSize();
+   HYPRE_Int test_size = test_fes.GlobalTrueVSize();
 
    if (myid == 0)
    {

examples/ex25p.cpp

@@ -326,7 +326,7 @@ int main(int argc, char *argv[])
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;

examples/ex26p.cpp

@@ -224,7 +224,7 @@ int main(int argc, char *argv[])
       fespaces->AddOrderRefinedLevel(collections.Last());
    }
 
-   HYPRE_BigInt size = fespaces->GetFinestFESpace().GlobalTrueVSize();
+   HYPRE_Int size = fespaces->GetFinestFESpace().GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;

examples/ex27p.cpp

@@ -176,7 +176,7 @@ int main(int argc, char *argv[])
       h1 ? (FiniteElementCollection*)new H1_FECollection(order, dim) :
       (FiniteElementCollection*)new DG_FECollection(order, dim);
    ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   HYPRE_Int size = fespace.GlobalTrueVSize();
    mfem::out << "Number of finite element unknowns: " << size << endl;
 
    // 6. Create "marker arrays" to define the portions of boundary associated

examples/ex28.cpp

@@ -1,267 +0,0 @@
-//                                MFEM Example 28
-//
-// Compile with: make ex28
-//
-// Sample runs:  ex28
-//               ex28 --visit-datafiles
-//               ex28 --order 2
-//
-// Description:  Demonstrates a sliding boundary condition in an elasticity
-//               problem. A trapezoid, roughly as pictured below, is pushed
-//               from the right into a rigid notch. Normal displacement is
-//               restricted, but tangential movement is allowed, so the
-//               trapezoid compresses into the notch.
-//
-//                                       /-------+
-//               normal constrained --->/        | <--- boundary force (2)
-//               boundary (4)          /---------+
-//                                          ^
-//                                          |
-//                                normal constrained boundary (1)
-//
-//               This example demonstrates the use of the ConstrainedSolver
-//               framework.
-//
-//               We recommend viewing Example 2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <set>
-
-using namespace std;
-using namespace mfem;
-
-// Return a mesh with a single element with vertices (0, 0), (1, 0), (1, 1),
-// (offset, 1) to demonstrate boundary conditions on a surface that is not
-// axis-aligned.
-Mesh * build_trapezoid_mesh(double offset)
-{
-   MFEM_VERIFY(offset < 0.9, "offset is too large!");
-
-   const int dimension = 2;
-   const int nvt = 4; // vertices
-   const int nbe = 4; // num boundary elements
-   Mesh * mesh = new Mesh(dimension, nvt, 1, nbe);
-
-   // vertices
-   double vc[dimension];
-   vc[0] = 0.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = offset; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-
-   // element
-   Array<int> vert(4);
-   vert[0] = 0; vert[1] = 1; vert[2] = 3; vert[3] = 2;
-   mesh->AddQuad(vert, 1);
-
-   // boundary
-   Array<int> sv(2);
-   sv[0] = 0; sv[1] = 1;
-   mesh->AddBdrSegment(sv, 1);
-   sv[0] = 1; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 2);
-   sv[0] = 2; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 3);
-   sv[0] = 0; sv[1] = 2;
-   mesh->AddBdrSegment(sv, 4);
-
-   mesh->FinalizeQuadMesh(1, 0, true);
-
-   return mesh;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int order = 1;
-   bool visualization = 1;
-   double offset = 0.3;
-   bool visit = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&offset, "--offset", "--offset",
-                  "How much to offset the trapezoid.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Build a trapezoidal mesh with a single quadrilateral element, where
-   //    'offset' determines how far off it is from a rectangle.
-   Mesh *mesh = build_trapezoid_mesh(offset);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 1,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use vector finite
-   //    elements, i.e. dim copies of a scalar finite element space. The vector
-   //    dimension is specified by the last argument of the FiniteElementSpace
-   //    constructor.
-   FiniteElementCollection *fec = new H1_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec, dim);
-   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
-        << endl;
-   cout << "Assembling matrix and r.h.s... " << flush;
-
-   // 5. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, there are no essential boundary
-   //    conditions in the usual sense, but we leave the machinery here for
-   //    users to modify if they wish.
-   Array<int> ess_tdof_list, ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 0;
-   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system. In this case, b_i equals the boundary integral
-   //    of f*phi_i where f represents a "push" force on the right side of the
-   //    trapezoid.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-   {
-      Vector push_force(mesh->bdr_attributes.Max());
-      push_force = 0.0;
-      push_force(1) = -5.0e-2; // index 1 attribute 2
-      f.Set(0, new PWConstCoefficient(push_force));
-   }
-   LinearForm *b = new LinearForm(fespace);
-   b->AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 8. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the linear elasticity integrator with piece-wise
-   //    constants coefficient lambda and mu. We use constant coefficients,
-   //    but see ex2 for how to set up piecewise constant coefficients based
-   //    on attribute.
-   Vector lambda(mesh->attributes.Max());
-   lambda = 1.0;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(mesh->attributes.Max());
-   mu = 1.0;
-   PWConstCoefficient mu_func(mu);
-
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
-
-   // 9. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   a->Assemble();
-
-   SparseMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-   cout << "done." << endl;
-   cout << "Size of linear system: " << A.Height() << endl;
-
-   // 10. Set up constraint matrix to constrain normal displacement (but
-   //     allow tangential displacement) on specified boundaries.
-   Array<int> constraint_atts(2);
-   constraint_atts[0] = 1;  // attribute 1 bottom
-   constraint_atts[1] = 4;  // attribute 4 left side
-   Array<int> lagrange_rowstarts;
-   SparseMatrix* local_constraints =
-      BuildNormalConstraints(*fespace, constraint_atts, lagrange_rowstarts);
-
-   // 11. Define and apply an iterative solver for the constrained system
-   //     in saddle-point form with a Gauss-Seidel smoother for the
-   //     displacement block.
-   GSSmoother M(A);
-   SchurConstrainedSolver * solver =
-      new SchurConstrainedSolver(A, *local_constraints, M);
-   solver->SetRelTol(1e-5);
-   solver->SetMaxIter(2000);
-   solver->SetPrintLevel(1);
-   solver->Mult(B, X);
-
-   // 12. Recover the solution as a finite element grid function. Move the
-   //     mesh to reflect the displacement of the elastic body being
-   //     simulated, for purposes of output.
-   a->RecoverFEMSolution(X, *b, x);
-   mesh->SetNodalFESpace(fespace);
-   GridFunction *nodes = mesh->GetNodes();
-   *nodes += x;
-
-   // 13. Save the refined mesh and the solution in VisIt format.
-   if (visit)
-   {
-      VisItDataCollection visit_dc("ex28", mesh);
-      visit_dc.SetLevelsOfDetail(4);
-      visit_dc.RegisterField("displacement", &x);
-      visit_dc.Save();
-   }
-
-   // 14. Save the displaced mesh and the inverted solution (which gives the
-   //     backward displacements to the original grid). This output can be
-   //     viewed later using GLVis: "glvis -m displaced.mesh -g sol.gf".
-   {
-      x *= -1; // sign convention for GLVis displacements
-      ofstream mesh_ofs("displaced.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-      ofstream sol_ofs("sol.gf");
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the above data by socket to a GLVis server.  Use the "n" and "b"
-   //     keys in GLVis to visualize the displacements.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   delete local_constraints;
-   delete solver;
-   delete a;
-   delete b;
-   if (fec)
-   {
-      delete fespace;
-      delete fec;
-   }
-   delete mesh;
-
-   return 0;
-}

examples/ex28p.cpp

@@ -1,366 +0,0 @@
-//                       MFEM Example 28 - Parallel Version
-//
-// Compile with: make ex28p
-//
-// Sample runs:  ex28p
-//               ex28p --visit-datafiles
-//               ex28p --order 4
-//               ex28p --penalty 1e+5
-//
-//               mpirun -np 4 ex28p
-//               mpirun -np 4 ex28p --penalty 1e+5
-//
-// Description:  Demonstrates a sliding boundary condition in an elasticity
-//               problem. A trapezoid, roughly as pictured below, is pushed
-//               from the right into a rigid notch. Normal displacement is
-//               restricted, but tangential movement is allowed, so the
-//               trapezoid compresses into the notch.
-//
-//                                       /-------+
-//               normal constrained --->/        | <--- boundary force (2)
-//               boundary (4)          /---------+
-//                                          ^
-//                                          |
-//                                normal constrained boundary (1)
-//
-//               This example demonstrates the use of the ConstrainedSolver
-//               framework.
-//
-//               We recommend viewing Example 2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Return a mesh with a single element with vertices (0, 0), (1, 0), (1, 1),
-// (offset, 1) to demonstrate boundary conditions on a surface that is not
-// axis-aligned.
-Mesh * build_trapezoid_mesh(double offset)
-{
-   MFEM_VERIFY(offset < 0.9, "offset is too large!");
-
-   const int dimension = 2;
-   const int nvt = 4; // vertices
-   const int nbe = 4; // num boundary elements
-   Mesh * mesh = new Mesh(dimension, nvt, 1, nbe);
-
-   // vertices
-   double vc[dimension];
-   vc[0] = 0.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = offset; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-
-   // element
-   Array<int> vert(4);
-   vert[0] = 0; vert[1] = 1; vert[2] = 3; vert[3] = 2;
-   mesh->AddQuad(vert, 1);
-
-   // boundary
-   Array<int> sv(2);
-   sv[0] = 0; sv[1] = 1;
-   mesh->AddBdrSegment(sv, 1);
-   sv[0] = 1; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 2);
-   sv[0] = 2; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 3);
-   sv[0] = 0; sv[1] = 2;
-   mesh->AddBdrSegment(sv, 4);
-
-   mesh->FinalizeQuadMesh(1, 0, true);
-
-   return mesh;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   int order = 1;
-   bool visualization = 1;
-   bool reorder_space = false;
-   double offset = 0.3;
-   bool visit = false;
-   double penalty = 0.0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
-                  "Use byNODES ordering of vector space instead of byVDIM");
-   args.AddOption(&offset, "--offset", "--offset",
-                  "How much to offset the trapezoid.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&penalty, "-p", "--penalty",
-                  "Penalty parameter; 0 means use elimination solver.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Build a trapezoidal mesh with a single quadrilateral element, where
-   //    'offset' determines how far off it is from a rectangle.
-   Mesh *mesh = build_trapezoid_mesh(offset);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 1;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use vector finite elements, i.e. dim copies of a scalar finite element
-   //    space. We use the ordering by vector dimension (the last argument of
-   //    the FiniteElementSpace constructor) which is expected in the systems
-   //    version of BoomerAMG preconditioner. For NURBS meshes, we use the
-   //    (degree elevated) NURBS space associated with the mesh nodes.
-   FiniteElementCollection *fec;
-   ParFiniteElementSpace *fespace;
-   const bool use_nodal_fespace = pmesh->NURBSext;
-   if (use_nodal_fespace)
-   {
-      fec = NULL;
-      fespace = (ParFiniteElementSpace *)pmesh->GetNodes()->FESpace();
-   }
-   else
-   {
-      fec = new H1_FECollection(order, dim);
-      if (reorder_space)
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
-      }
-      else
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
-      }
-   }
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl
-           << "Assembling matrix and r.h.s... " << flush;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, there are no essential boundary
-   //    conditions in the usual sense, but we leave the machinery here for
-   //    users to modify if they wish.
-   Array<int> ess_tdof_list, ess_bdr(pmesh->bdr_attributes.Max());
-   ess_bdr = 0;
-   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system. In this case, b_i equals the
-   //    boundary integral of f*phi_i where f represents a "pull down" force on
-   //    the Neumann part of the boundary and phi_i are the basis functions in
-   //    the finite element fespace. The force is defined by the object f, which
-   //    is a vector of Coefficient objects. The fact that f is non-zero on
-   //    boundary attribute 2 is indicated by the use of piece-wise constants
-   //    coefficient for its last component.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-
-   // 9. Put a leftward force on the right side of the trapezoid
-   {
-      Vector push_force(pmesh->bdr_attributes.Max());
-      push_force = 0.0;
-      push_force(1) = -5.0e-2; // index 1 attribute 2
-      f.Set(0, new PWConstCoefficient(push_force));
-   }
-
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
-   b->Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the linear elasticity integrator with piece-wise
-   //     constants coefficient lambda and mu. We use constant coefficients,
-   //     but see ex2 for how to set up piecewise constant coefficients based
-   //     on attribute.
-   Vector lambda(pmesh->attributes.Max());
-   lambda = 1.0;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(pmesh->attributes.Max());
-   mu = 1.0;
-   PWConstCoefficient mu_func(mu);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, etc.
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-   if (myid == 0)
-   {
-      cout << "done." << endl;
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 13. Set up constraint matrix to constrain normal displacement (but
-   //     allow tangential displacement) on specified boundaries.
-   Array<int> constraint_atts(2);
-   constraint_atts[0] = 1;  // attribute 1 bottom
-   constraint_atts[1] = 4;  // attribute 4 left side
-   Array<int> constraint_rowstarts;
-   SparseMatrix* local_constraints =
-      ParBuildNormalConstraints(*fespace, constraint_atts,
-                                constraint_rowstarts);
-
-   // 14. Define and apply a parallel PCG solver for the constrained system
-   //     where the normal boundary constraints have been separately eliminated
-   //     from the system.
-   ConstrainedSolver * solver;
-   if (penalty == 0.0)
-   {
-      solver = new EliminationCGSolver(A, *local_constraints,
-                                       constraint_rowstarts, dim,
-                                       reorder_space);
-   }
-   else
-   {
-      solver = new PenaltyPCGSolver(A, *local_constraints, penalty,
-                                    dim, reorder_space);
-   }
-
-   solver->SetRelTol(1e-8);
-   solver->SetMaxIter(500);
-   solver->SetPrintLevel(1);
-   solver->Mult(B, X);
-
-   // 15. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
-   //     space. This means that we define the mesh elements through a fespace
-   //     based transformation of the reference element.  This allows us to save
-   //     the displaced mesh as a curved mesh when using high-order finite
-   //     element displacement field. We assume that the initial mesh (read from
-   //     the file) is not higher order curved mesh compared to the chosen FE
-   //     space.
-   if (!use_nodal_fespace)
-   {
-      pmesh->SetNodalFESpace(fespace);
-   }
-
-   GridFunction *nodes = pmesh->GetNodes();
-   *nodes += x;
-
-   // 17. Save the refined mesh and the solution in VisIt format.
-   if (visit)
-   {
-      VisItDataCollection visit_dc(MPI_COMM_WORLD, "ex28p", pmesh);
-      visit_dc.SetLevelsOfDetail(4);
-      visit_dc.RegisterField("displacement", &x);
-      visit_dc.Save();
-   }
-
-   // 18. Save in parallel the displaced mesh and the inverted solution (which
-   //     gives the backward displacements to the original grid). This output
-   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      x *= -1; // sign convention for GLVis displacements
-
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 19. Send the above data by socket to a GLVis server.  Use the "n" and "b"
-   //     keys in GLVis to visualize the displacements.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 20. Free the used memory.
-   delete local_constraints;
-   delete solver;
-   delete a;
-   delete b;
-   if (fec)
-   {
-      delete fespace;
-      delete fec;
-   }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}

examples/ex29.cpp

@@ -1,350 +0,0 @@
-//                                MFEM Example 29
-//
-// Compile with: make ex29
-//
-// Sample runs:  ex29
-//               ex29 -r 2 -sc
-//               ex29 -mt 3 -o 4 -sc
-//               ex29 -mt 3 -r 2 -o 4 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               finite element discretization of a PDE on a 2 dimensional
-//               surface embedded in a 3 dimensional domain. In this case we
-//               solve the Laplace problem -Div(sigma Grad u) = 1, with
-//               homogeneous Dirichlet boundary conditions, where sigma is an
-//               anisotropic diffusion constant defined as a 3x3 matrix
-//               coefficient.
-//
-//               This example demonstrates the use of finite element integrators
-//               on 2D domains with 3D coefficients.
-//
-//               We recommend viewing examples 1 and 7 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-Mesh * GetMesh(int type);
-
-void trans(const Vector &x, Vector &r);
-
-void sigmaFunc(const Vector &x, DenseMatrix &s);
-
-double uExact(const Vector &x)
-{
-   return (0.25 * (2.0 + x[0]) - x[2]) * (x[2] + 0.25 * (2.0 + x[0]));
-}
-
-void duExact(const Vector &x, Vector &du)
-{
-   du.SetSize(3);
-   du[0] = 0.125 * (2.0 + x[0]) * x[1] * x[1];
-   du[1] = -0.125 * (2.0 + x[0]) * x[0] * x[1];
-   du[2] = -2.0 * x[2];
-}
-
-void fluxExact(const Vector &x, Vector &f)
-{
-   f.SetSize(3);
-
-   DenseMatrix s(3);
-   sigmaFunc(x, s);
-
-   Vector du(3);
-   duExact(x, du);
-
-   s.Mult(du, f);
-   f *= -1.0;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int order = 3;
-   int mesh_type = 4; // Default to Quadrilateral mesh
-   int mesh_order = 3;
-   int ref_levels = 0;
-   bool static_cond = false;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_type, "-mt", "--mesh-type",
-                  "Mesh type: 3 - Triangular, 4 - Quadrilateral.");
-   args.AddOption(&mesh_order, "-mo", "--mesh-order",
-                  "Geometric order of the curved mesh.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.ParseCheck();
-
-   // 2. Construct a quadrilateral or triangular mesh with the topology of a
-   //    cylindrical surface.
-   Mesh *mesh = GetMesh(mesh_type);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement.
-   for (int l = 0; l < ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 4. Transform the mesh so that it has a more interesting geometry.
-   mesh->SetCurvature(mesh_order);
-   mesh->Transform(trans);
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order.
-   H1_FECollection fec(order, dim);
-   FiniteElementSpace fespace(mesh, &fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   MatrixFunctionCoefficient sigma(3, sigmaFunc);
-   BilinearFormIntegrator *integ = new DiffusionIntegrator(sigma);
-   a.AddDomainIntegrator(integ);
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   //     Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-   GSSmoother M((SparseMatrix&)(*A));
-   PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Compute error in the solution and its flux
-   FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
-
-   cout << "|u - u_h|_2 = " << err << endl;
-
-   FiniteElementSpace flux_fespace(mesh, &fec, 3);
-   GridFunction flux(&flux_fespace);
-   x.ComputeFlux(*integ, flux); flux *= -1.0;
-
-   VectorFunctionCoefficient fluxCoef(3, fluxExact);
-   double flux_err = flux.ComputeL2Error(fluxCoef);
-
-   cout << "|f - f_h|_2 = " << flux_err << endl;
-
-   // 14. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x
-               << "window_title 'Solution'\n" << flush;
-
-      socketstream flux_sock(vishost, visport);
-      flux_sock.precision(8);
-      flux_sock << "solution\n" << *mesh << flux
-                << "keys vvv\n"
-                << "window_geometry 402 0 400 350\n"
-                << "window_title 'Flux'\n"  << flush;
-   }
-
-   // 16. Free the used memory.
-   delete mesh;
-
-   return 0;
-}
-
-// Defines a mesh consisting of four flat rectangular surfaces connected to form
-// a loop.
-Mesh * GetMesh(int type)
-{
-   Mesh * mesh = NULL;
-
-   if (type == 3)
-   {
-      mesh = new Mesh(2, 12, 16, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-      mesh->AddVertex( 0.0, -1.0, 0.5);
-      mesh->AddVertex( 1.0,  0.0, 0.5);
-      mesh->AddVertex( 0.0,  1.0, 0.5);
-      mesh->AddVertex(-1.0,  0.0, 0.5);
-
-      mesh->AddTriangle(0, 1, 8);
-      mesh->AddTriangle(1, 5, 8);
-      mesh->AddTriangle(5, 4, 8);
-      mesh->AddTriangle(4, 0, 8);
-      mesh->AddTriangle(1, 2, 9);
-      mesh->AddTriangle(2, 6, 9);
-      mesh->AddTriangle(6, 5, 9);
-      mesh->AddTriangle(5, 1, 9);
-      mesh->AddTriangle(2, 3, 10);
-      mesh->AddTriangle(3, 7, 10);
-      mesh->AddTriangle(7, 6, 10);
-      mesh->AddTriangle(6, 2, 10);
-      mesh->AddTriangle(3, 0, 11);
-      mesh->AddTriangle(0, 4, 11);
-      mesh->AddTriangle(4, 7, 11);
-      mesh->AddTriangle(7, 3, 11);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else if (type == 4)
-   {
-      mesh = new Mesh(2, 8, 4, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-
-      mesh->AddQuad(0, 1, 5, 4);
-      mesh->AddQuad(1, 2, 6, 5);
-      mesh->AddQuad(2, 3, 7, 6);
-      mesh->AddQuad(3, 0, 4, 7);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else
-   {
-      MFEM_ABORT("Unrecognized mesh type " << type << "!");
-   }
-   mesh->FinalizeTopology();
-
-   return mesh;
-}
-
-// Transforms the four-sided loop into a curved cylinder with skewed top and
-// base.
-void trans(const Vector &x, Vector &r)
-{
-   r.SetSize(3);
-
-   double tol = 1e-6;
-   double theta = 0.0;
-   if (fabs(x[1] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (x[0] - 2.0);
-   }
-   else if (fabs(x[0] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * x[1];
-   }
-   else if (fabs(x[1] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (2.0 - x[0]);
-   }
-   else if (fabs(x[0] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (4.0 - x[1]);
-   }
-   else
-   {
-      cout << "side not recognized "
-           << x[0] << " " << x[1] << " " << x[2] << endl;
-   }
-
-   r[0] = cos(theta);
-   r[1] = sin(theta);
-   r[2] = 0.25 * (2.0 * x[2] - 1.0) * (r[0] + 2.0);
-}
-
-// Anisotropic diffusion coefficient
-void sigmaFunc(const Vector &x, DenseMatrix &s)
-{
-   s.SetSize(3);
-   double a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
-   s(0,0) = 0.5 + x[0] * x[0] * (8.0 / a - 0.5);
-   s(0,1) = x[0] * x[1] * (8.0 / a - 0.5);
-   s(0,2) = 0.0;
-   s(1,0) = s(0,1);
-   s(1,1) = 0.5 * x[0] * x[0] + 8.0 * x[1] * x[1] / a;
-   s(1,2) = 0.0;
-   s(2,0) = 0.0;
-   s(2,1) = 0.0;
-   s(2,2) = a / 32.0;
-}

examples/ex29p.cpp

@@ -1,391 +0,0 @@
-//                      MFEM Example 29 - Parallel Version
-//
-// Compile with: make ex29p
-//
-// Sample runs:  mpirun -np 4 ex29p
-//               mpirun -np 4 ex29p -sc
-//               mpirun -np 4 ex29p -mt 3 -o 3 -sc
-//               mpirun -np 4 ex29p -mt 3 -rs 1 -o 4 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               finite element discretization of a PDE on a 2 dimensional
-//               surface embedded in a 3 dimensional domain. In this case we
-//               solve the Laplace problem -Div(sigma Grad u) = 1, with
-//               homogeneous Dirichlet boundary conditions, where sigma is an
-//               anisotropic diffusion constant defined as a 3x3 matrix
-//               coefficient.
-//
-//               This example demonstrates the use of finite element integrators
-//               on 2D domains with 3D coefficients.
-//
-//               We recommend viewing examples 1 and 7 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-Mesh * GetMesh(int type);
-
-void trans(const Vector &x, Vector &r);
-
-void sigmaFunc(const Vector &x, DenseMatrix &s);
-
-double uExact(const Vector &x)
-{
-   return (0.25 * (2.0 + x[0]) - x[2]) * (x[2] + 0.25 * (2.0 + x[0]));
-}
-
-void duExact(const Vector &x, Vector &du)
-{
-   du.SetSize(3);
-   du[0] = 0.125 * (2.0 + x[0]) * x[1] * x[1];
-   du[1] = -0.125 * (2.0 + x[0]) * x[0] * x[1];
-   du[2] = -2.0 * x[2];
-}
-
-void fluxExact(const Vector &x, Vector &f)
-{
-   f.SetSize(3);
-
-   DenseMatrix s(3);
-   sigmaFunc(x, s);
-
-   Vector du(3);
-   duExact(x, du);
-
-   s.Mult(du, f);
-   f *= -1.0;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   MPI_Session mpi(argc, argv);
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
-
-   // 2. Parse command-line options.
-   int order = 3;
-   int mesh_type = 4; // Default to Quadrilateral mesh
-   int mesh_order = 3;
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   bool static_cond = false;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_type, "-mt", "--mesh-type",
-                  "Mesh type: 3 - Triangular, 4 - Quadrilateral.");
-   args.AddOption(&mesh_order, "-mo", "--mesh-order",
-                  "Geometric order of the curved mesh.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.ParseCheck();
-
-   // 3. Construct a quadrilateral or triangular mesh with the topology of a
-   //    cylindrical surface.
-   Mesh *mesh = GetMesh(mesh_type);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ser_ref_levels' of uniform refinement.
-   for (int l = 0; l < ser_ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int l = 0; l < par_ref_levels; l++)
-   {
-      pmesh.UniformRefinement();
-   }
-
-   // 6. Transform the mesh so that it has a more interesting geometry.
-   pmesh.SetCurvature(mesh_order);
-   pmesh.Transform(trans);
-
-   // 7. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-   HYPRE_Int total_num_dofs = fespace.GlobalTrueVSize();
-   if (mpi.Root()) { cout << "Number of unknowns: " << total_num_dofs << endl; }
-
-   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
-   ParBilinearForm a(&fespace);
-   MatrixFunctionCoefficient sigma(3, sigmaFunc);
-   BilinearFormIntegrator *integ = new DiffusionIntegrator(sigma);
-   a.AddDomainIntegrator(integ);
-
-   // 12. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: "
-           << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
-   }
-
-   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
-   //     preconditioner from hypre.
-   HypreBoomerAMG *amg = new HypreBoomerAMG;
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(*amg);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete amg;
-
-   // 14. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Compute error in the solution and its flux
-   FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
-
-   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }
-
-   ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
-   ParGridFunction flux(&flux_fespace);
-   x.ComputeFlux(*integ, flux); flux *= -1.0;
-
-   VectorFunctionCoefficient fluxCoef(3, fluxExact);
-   double flux_err = flux.ComputeL2Error(fluxCoef);
-
-   if (myid == 0) { cout << "|f - f_h|_2 = " << flux_err << endl; }
-
-   // 16. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name, flux_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-      flux_name << "flux." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-
-      ofstream flux_ofs(flux_name.str().c_str());
-      flux_ofs.precision(8);
-      flux.Save(flux_ofs);
-   }
-
-   // 17. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x
-               << "window_title 'Solution'\n" << flush;
-
-      socketstream flux_sock(vishost, visport);
-      flux_sock << "parallel " << num_procs << " " << myid << "\n";
-      flux_sock.precision(8);
-      flux_sock << "solution\n" << pmesh << flux
-                << "keys vvv\n"
-                << "window_geometry 402 0 400 350\n"
-                << "window_title 'Flux'\n"  << flush;
-   }
-
-   return 0;
-}
-
-// Defines a mesh consisting of four flat rectangular surfaces connected to form
-// a loop.
-Mesh * GetMesh(int type)
-{
-   Mesh * mesh = NULL;
-
-   if (type == 3)
-   {
-      mesh = new Mesh(2, 12, 16, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-      mesh->AddVertex( 0.0, -1.0, 0.5);
-      mesh->AddVertex( 1.0,  0.0, 0.5);
-      mesh->AddVertex( 0.0,  1.0, 0.5);
-      mesh->AddVertex(-1.0,  0.0, 0.5);
-
-      mesh->AddTriangle(0, 1, 8);
-      mesh->AddTriangle(1, 5, 8);
-      mesh->AddTriangle(5, 4, 8);
-      mesh->AddTriangle(4, 0, 8);
-      mesh->AddTriangle(1, 2, 9);
-      mesh->AddTriangle(2, 6, 9);
-      mesh->AddTriangle(6, 5, 9);
-      mesh->AddTriangle(5, 1, 9);
-      mesh->AddTriangle(2, 3, 10);
-      mesh->AddTriangle(3, 7, 10);
-      mesh->AddTriangle(7, 6, 10);
-      mesh->AddTriangle(6, 2, 10);
-      mesh->AddTriangle(3, 0, 11);
-      mesh->AddTriangle(0, 4, 11);
-      mesh->AddTriangle(4, 7, 11);
-      mesh->AddTriangle(7, 3, 11);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else if (type == 4)
-   {
-      mesh = new Mesh(2, 8, 4, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-
-      mesh->AddQuad(0, 1, 5, 4);
-      mesh->AddQuad(1, 2, 6, 5);
-      mesh->AddQuad(2, 3, 7, 6);
-      mesh->AddQuad(3, 0, 4, 7);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else
-   {
-      MFEM_ABORT("Unrecognized mesh type " << type << "!");
-   }
-   mesh->FinalizeTopology();
-
-   return mesh;
-}
-
-// Transforms the four-sided loop into a curved cylinder with skewed top and
-// base.
-void trans(const Vector &x, Vector &r)
-{
-   r.SetSize(3);
-
-   double tol = 1e-6;
-   double theta = 0.0;
-   if (fabs(x[1] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (x[0] - 2.0);
-   }
-   else if (fabs(x[0] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * x[1];
-   }
-   else if (fabs(x[1] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (2.0 - x[0]);
-   }
-   else if (fabs(x[0] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (4.0 - x[1]);
-   }
-   else
-   {
-      cerr << "side not recognized "
-           << x[0] << " " << x[1] << " " << x[2] << endl;
-   }
-
-   r[0] = cos(theta);
-   r[1] = sin(theta);
-   r[2] = 0.25 * (2.0 * x[2] - 1.0) * (r[0] + 2.0);
-}
-
-// Anisotropic diffusion coefficient
-void sigmaFunc(const Vector &x, DenseMatrix &s)
-{
-   s.SetSize(3);
-   double a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
-   s(0,0) = 0.5 + x[0] * x[0] * (8.0 / a - 0.5);
-   s(0,1) = x[0] * x[1] * (8.0 / a - 0.5);
-   s(0,2) = 0.0;
-   s(1,0) = s(0,1);
-   s(1,1) = 0.5 * x[0] * x[0] + 8.0 * x[1] * x[1] / a;
-   s(1,2) = 0.0;
-   s(2,0) = 0.0;
-   s(2,1) = 0.0;
-   s(2,2) = a / 32.0;
-}

examples/ex2p.cpp

@@ -168,7 +168,7 @@ int main(int argc, char *argv[])
          fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
       }
    }
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl

examples/ex3p.cpp

@@ -156,7 +156,7 @@ int main(int argc, char *argv[])
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;

examples/ex4p.cpp

@@ -153,7 +153,7 @@ int main(int argc, char *argv[])
    //    use the Raviart-Thomas finite elements of the specified order.
    FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;

examples/ex5p.cpp

@@ -155,8 +155,8 @@ int main(int argc, char *argv[])
    ParFiniteElementSpace *R_space = new ParFiniteElementSpace(pmesh, hdiv_coll);
    ParFiniteElementSpace *W_space = new ParFiniteElementSpace(pmesh, l2_coll);
 
-   HYPRE_BigInt dimR = R_space->GlobalTrueVSize();
-   HYPRE_BigInt dimW = W_space->GlobalTrueVSize();
+   HYPRE_Int dimR = R_space->GlobalTrueVSize();
+   HYPRE_Int dimW = W_space->GlobalTrueVSize();
 
    if (verbose)
    {

examples/ex6.cpp

@@ -14,7 +14,6 @@
 //               ex6 -m ../data/star-surf.mesh -o 2
 //               ex6 -m ../data/square-disc-surf.mesh -o 2
 //               ex6 -m ../data/amr-quad.mesh
-//               ex6 -m ../data/inline-segment.mesh -o 1 -md 100
 //
 // Device sample runs:
 //               ex6 -pa -d cuda
@@ -54,7 +53,6 @@ int main(int argc, char *argv[])
    int order = 1;
    bool pa = false;
    const char *device_config = "cpu";
-   int max_dofs = 50000;
    bool visualization = true;
 
    OptionsParser args(argc, argv);
@@ -66,8 +64,6 @@ int main(int argc, char *argv[])
                   "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
-   args.AddOption(&max_dofs, "-md", "--max-dofs",
-                  "Stop after reaching this many degrees of freedom.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -164,6 +160,7 @@ int main(int argc, char *argv[])
 
    // 12. The main AMR loop. In each iteration we solve the problem on the
    //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 50000;
    for (int it = 0; ; it++)
    {
       int cdofs = fespace.GetTrueVSize();

examples/ex6p.cpp

@@ -2,20 +2,19 @@
 //
 // Compile with: make ex6p
 //
-// Sample runs:  mpirun -np 4 ex6p -m ../data/star-hilbert.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -rm 1 -o 1
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -rm 1 -o 2 -h1
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2 -cs
+// Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
+//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
+//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -rm 2 -o 2
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2 -cs
 //               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
 //               mpirun -np 4 ex6p -m ../data/pipe-nurbs.mesh
 //               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -rm 2 -o 2
-//               mpirun -np 4 ex6p -m ../data/inline-segment.mesh -o 1 -md 200
+//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
 //               mpirun -np 4 ex6p --restart
 //
@@ -63,10 +62,8 @@ int main(int argc, char *argv[])
    int order = 1;
    bool pa = false;
    const char *device_config = "cpu";
-   bool nc_simplices = true;
-   int reorder_mesh = 0;
+   bool nc_simplices = false;
    int max_dofs = 100000;
-   bool smooth_rt = true;
    bool restart = false;
    bool visualization = true;
 
@@ -79,17 +76,12 @@ int main(int argc, char *argv[])
                   "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
-   args.AddOption(&reorder_mesh, "-rm", "--reorder-mesh",
-                  "Reorder elements of the coarse mesh to improve "
-                  "dynamic partitioning: 0=none, 1=hilbert, 2=gecko.");
    args.AddOption(&nc_simplices, "-ns", "--nonconforming-simplices",
                   "-cs", "--conforming-simplices",
                   "For simplicial meshes, enable/disable nonconforming"
                   " refinement");
    args.AddOption(&max_dofs, "-md", "--max-dofs",
                   "Stop after reaching this many degrees of freedom.");
-   args.AddOption(&smooth_rt, "-rt", "--smooth-rt", "-h1", "--smooth-h1",
-                  "Represent the smooth flux in RT or vector H1 space.");
    args.AddOption(&restart, "-res", "--restart", "-no-res", "--no-restart",
                   "Restart computation from the last checkpoint.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -123,49 +115,23 @@ int main(int argc, char *argv[])
       //    surface and volume meshes with the same code.
       Mesh mesh(mesh_file, 1, 1);
 
-      // 5. A NURBS mesh cannot be refined locally so we refine it uniformly
-      //    and project it to a standard curvilinear mesh of order 2.
+      // 5. Refine the serial mesh on all processors to increase the resolution.
+      //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
+      //    sure that the mesh is non-conforming.
       if (mesh.NURBSext)
       {
          mesh.UniformRefinement();
          mesh.SetCurvature(2);
       }
-
-      // 6. MFEM supports dynamic partitioning (load balancing) of parallel non-
-      //    conforming meshes based on space-filling curve (SFC) partitioning.
-      //    SFC partitioning is extremely fast and scales to hundreds of
-      //    thousands of processors, but requires the coarse mesh to be ordered,
-      //    ideally as a sequence of face-neighbors. The mesh may already be
-      //    ordered (like star-hilbert.mesh) or we can order it here. Ordering
-      //    type 1 is a fast spatial sort of the mesh, type 2 is a high quality
-      //    optimization algorithm suitable for ordering general unstructured
-      //    meshes.
-      if (reorder_mesh)
-      {
-         Array<int> ordering;
-         switch (reorder_mesh)
-         {
-            case 1: mesh.GetHilbertElementOrdering(ordering); break;
-            case 2: mesh.GetGeckoElementOrdering(ordering); break;
-            default: MFEM_ABORT("Unknown mesh reodering type " << reorder_mesh);
-         }
-         mesh.ReorderElements(ordering);
-      }
-
-      // 7. Make sure the mesh is in the non-conforming mode to enable local
-      //    refinement of quadrilaterals/hexahedra, and the above partitioning
-      //    algorithm. Simplices can be refined either in conforming or in non-
-      //    conforming mode. The conforming mode however does not support
-      //    dynamic partitioning.
       mesh.EnsureNCMesh(nc_simplices);
 
-      // 8. Define a parallel mesh by partitioning the serial mesh.
+      // 6. Define a parallel mesh by partitioning the serial mesh.
       //    Once the parallel mesh is defined, the serial mesh can be deleted.
       pmesh = new ParMesh(MPI_COMM_WORLD, mesh);
    }
    else
    {
-      // 9. We can also restart the computation by loading the mesh from a
+      // 7. We can also restart the computation by loading the mesh from a
       //    previously saved check-point.
       string fname(MakeParFilename("ex6p-checkpoint.", myid));
       ifstream ifs(fname);
@@ -181,14 +147,14 @@ int main(int argc, char *argv[])
    Array<int> ess_bdr(pmesh->bdr_attributes.Max());
    ess_bdr = 1;
 
-   // 10. Define a finite element space on the mesh. The polynomial order is
-   //     one (linear) by default, but this can be changed on the command line.
+   // 8. Define a finite element space on the mesh. The polynomial order is
+   //    one (linear) by default, but this can be changed on the command line.
    H1_FECollection fec(order, dim);
    ParFiniteElementSpace fespace(pmesh, &fec);
 
-   // 11. As in Example 1p, we set up bilinear and linear forms corresponding to
-   //     the Laplace problem -\Delta u = 1. We don't assemble the discrete
-   //     problem yet, this will be done in the main loop.
+   // 9. As in Example 1p, we set up bilinear and linear forms corresponding to
+   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
+   //    problem yet, this will be done in the main loop.
    ParBilinearForm a(&fespace);
    if (pa)
    {
@@ -203,12 +169,12 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
 
-   // 12. The solution vector x and the associated finite element grid function
+   // 10. The solution vector x and the associated finite element grid function
    //     will be maintained over the AMR iterations. We initialize it to zero.
    ParGridFunction x(&fespace);
    x = 0;
 
-   // 13. Connect to GLVis.
+   // 11. Connect to GLVis.
    char vishost[] = "localhost";
    int  visport   = 19916;
 
@@ -230,59 +196,51 @@ int main(int argc, char *argv[])
       sout.precision(8);
    }
 
-   // 14. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   // 12. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
    //     with L2 projection in the smoothing step to better handle hanging
    //     nodes and parallel partitioning. We need to supply a space for the
-   //     discontinuous flux (L2) and a space for the smoothed flux.
+   //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
+   //     used here).
    L2_FECollection flux_fec(order, dim);
    ParFiniteElementSpace flux_fes(pmesh, &flux_fec, sdim);
-   FiniteElementCollection *smooth_flux_fec = NULL;
-   ParFiniteElementSpace *smooth_flux_fes = NULL;
-   if (smooth_rt && dim > 1)
-   {
-      // Use an H(div) space for the smoothed flux (this is the default).
-      smooth_flux_fec = new RT_FECollection(order-1, dim);
-      smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec, 1);
-   }
-   else
-   {
-      // Another possible option for the smoothed flux space: H1^dim space
-      smooth_flux_fec = new H1_FECollection(order, dim);
-      smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec, dim);
-   }
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, *smooth_flux_fes);
+   RT_FECollection smooth_flux_fec(order-1, dim);
+   ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec);
+   // Another possible option for the smoothed flux space:
+   // H1_FECollection smooth_flux_fec(order, dim);
+   // ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec, dim);
+   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
 
-   // 15. A refiner selects and refines elements based on a refinement strategy.
+   // 13. A refiner selects and refines elements based on a refinement strategy.
    //     The strategy here is to refine elements with errors larger than a
    //     fraction of the maximum element error. Other strategies are possible.
    //     The refiner will call the given error estimator.
    ThresholdRefiner refiner(estimator);
    refiner.SetTotalErrorFraction(0.7);
 
-   // 16. The main AMR loop. In each iteration we solve the problem on the
+   // 14. The main AMR loop. In each iteration we solve the problem on the
    //     current mesh, visualize the solution, and refine the mesh.
    for (int it = 0; ; it++)
    {
-      HYPRE_BigInt global_dofs = fespace.GlobalTrueVSize();
+      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
       if (myid == 0)
       {
          cout << "\nAMR iteration " << it << endl;
          cout << "Number of unknowns: " << global_dofs << endl;
       }
 
-      // 17. Assemble the right-hand side and determine the list of true
+      // 15. Assemble the right-hand side and determine the list of true
       //     (i.e. parallel conforming) essential boundary dofs.
       Array<int> ess_tdof_list;
       fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
       b.Assemble();
 
-      // 18. Assemble the stiffness matrix. Note that MFEM doesn't care at this
+      // 16. Assemble the stiffness matrix. Note that MFEM doesn't care at this
       //     point that the mesh is nonconforming and parallel.  The FE space is
       //     considered 'cut' along hanging edges/faces, and also across
       //     processor boundaries.
       a.Assemble();
 
-      // 19. Create the parallel linear system: eliminate boundary conditions.
+      // 17. Create the parallel linear system: eliminate boundary conditions.
       //     The system will be solved for true (unconstrained/unique) DOFs only.
       OperatorPtr A;
       Vector B, X;
@@ -290,7 +248,7 @@ int main(int argc, char *argv[])
       const int copy_interior = 1;
       a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
 
-      // 20. Solve the linear system A X = B.
+      // 18. Solve the linear system A X = B.
       //     * With full assembly, use the BoomerAMG preconditioner from hypre.
       //     * With partial assembly, use a diagonal preconditioner.
       Solver *M = NULL;
@@ -313,12 +271,12 @@ int main(int argc, char *argv[])
       cg.Mult(B, X);
       delete M;
 
-      // 21. Switch back to the host and extract the parallel grid function
+      // 19. Switch back to the host and extract the parallel grid function
       //     corresponding to the finite element approximation X. This is the
       //     local solution on each processor.
       a.RecoverFEMSolution(X, b, x);
 
-      // 22. Send the solution by socket to a GLVis server.
+      // 20. Send the solution by socket to a GLVis server.
       if (visualization)
       {
          sout << "parallel " << num_procs << " " << myid << "\n";
@@ -334,7 +292,7 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 23. Call the refiner to modify the mesh. The refiner calls the error
+      // 21. Call the refiner to modify the mesh. The refiner calls the error
       //     estimator to obtain element errors, then it selects elements to be
       //     refined and finally it modifies the mesh. The Stop() method can be
       //     used to determine if a stopping criterion was met.
@@ -348,7 +306,7 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 24. Update the finite element space (recalculate the number of DOFs,
+      // 22. Update the finite element space (recalculate the number of DOFs,
       //     etc.) and create a grid function update matrix. Apply the matrix
       //     to any GridFunctions over the space. In this case, the update
       //     matrix is an interpolation matrix so the updated GridFunction will
@@ -356,7 +314,7 @@ int main(int argc, char *argv[])
       fespace.Update();
       x.Update();
 
-      // 25. Load balance the mesh, and update the space and solution. Currently
+      // 23. Load balance the mesh, and update the space and solution. Currently
       //     available only for nonconforming meshes.
       if (pmesh->Nonconforming())
       {
@@ -368,12 +326,12 @@ int main(int argc, char *argv[])
          x.Update();
       }
 
-      // 26. Inform also the bilinear and linear forms that the space has
+      // 24. Inform also the bilinear and linear forms that the space has
       //     changed.
       a.Update();
       b.Update();
 
-      // 27. Save the current state of the mesh every 5 iterations. The
+      // 25. Save the current state of the mesh every 5 iterations. The
       //     computation can be restarted from this point. Note that unlike in
       //     visualization, we need to use the 'ParPrint' method to save all
       //     internal parallel data structures.
@@ -390,8 +348,6 @@ int main(int argc, char *argv[])
       }
    }
 
-   delete smooth_flux_fes;
-   delete smooth_flux_fec;
    delete pmesh;
 
    MPI_Finalize();

examples/ex7p.cpp

@@ -221,7 +221,7 @@ int main(int argc, char *argv[])
    // 5. Define a finite element space on the mesh. Here we use isoparametric
    //    finite elements -- the same as the mesh nodes.
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of unknowns: " << size << endl;

examples/ex8p.cpp

@@ -145,9 +145,9 @@ int main(int argc, char *argv[])
    xhat_space = new ParFiniteElementSpace(pmesh, xhat_fec);
    test_space = new ParFiniteElementSpace(pmesh, test_fec);
 
-   HYPRE_BigInt glob_true_s0     =   x0_space->GlobalTrueVSize();
-   HYPRE_BigInt glob_true_s1     = xhat_space->GlobalTrueVSize();
-   HYPRE_BigInt glob_true_s_test = test_space->GlobalTrueVSize();
+   HYPRE_Int glob_true_s0     =   x0_space->GlobalTrueVSize();
+   HYPRE_Int glob_true_s1     = xhat_space->GlobalTrueVSize();
+   HYPRE_Int glob_true_s_test = test_space->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "\nNumber of Unknowns:\n"

examples/ex9.cpp

@@ -16,8 +16,6 @@
 //    ex9 -m ../data/disc-nurbs.mesh -p 2 -r 3 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-square.mesh -p 3 -r 4 -dt 0.0025 -tf 9 -vs 20
 //    ex9 -m ../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8
-//    ex9 -m ../data/periodic-square.msh -p 0 -r 2 -dt 0.005 -tf 2
-//    ex9 -m ../data/periodic-cube.msh -p 0 -r 1 -o 2 -tf 2
 //
 // Device sample runs:
 //    ex9 -pa
@@ -131,7 +129,7 @@ private:
    mutable Vector z;
 
 public:
-   FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);
+   FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b);
 
    virtual void Mult(const Vector &x, Vector &y) const;
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
@@ -448,8 +446,8 @@ int main(int argc, char *argv[])
 
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
+FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
    Array<int> ess_tdof_list;
    if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)

examples/ex9p.cpp

@@ -16,8 +16,6 @@
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 2 -rp 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 3 -rp 2 -dt 0.0025 -tf 9 -vs 20
 //    mpirun -np 4 ex9p -m ../data/periodic-cube.mesh -p 0 -o 2 -rp 1 -dt 0.01 -tf 8
-//    mpirun -np 4 ex9p -m ../data/periodic-square.msh -p 0 -rs 2 -dt 0.005 -tf 2
-//    mpirun -np 4 ex9p -m ../data/periodic-cube.msh -p 0 -rs 1 -o 2 -tf 2
 //    mpirun -np 3 ex9p -m ../data/amr-hex.mesh -p 1 -rs 1 -rp 0 -dt 0.005 -tf 0.5
 //
 // Device sample runs:
@@ -219,7 +217,7 @@ private:
    mutable Vector z;
 
 public:
-   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_,
+   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b,
                 PrecType prec_type);
 
    virtual void Mult(const Vector &x, Vector &y) const;
@@ -391,7 +389,7 @@ int main(int argc, char *argv[])
    DG_FECollection fec(order, dim, BasisType::GaussLobatto);
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
-   HYPRE_BigInt global_vSize = fes->GlobalTrueVSize();
+   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
    if (mpi.Root())
    {
       cout << "Number of unknowns: " << global_vSize << endl;
@@ -651,38 +649,38 @@ int main(int argc, char *argv[])
 
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
-                           const Vector &b_, PrecType prec_type)
-   : TimeDependentOperator(M_.Height()), b(b_),
-     M_solver(M_.ParFESpace()->GetComm()),
-     z(M_.Height())
+FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
+                           const Vector &_b, PrecType prec_type)
+   : TimeDependentOperator(_M.Height()), b(_b),
+     M_solver(_M.ParFESpace()->GetComm()),
+     z(_M.Height())
 {
-   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
    {
-      M.Reset(M_.ParallelAssemble(), true);
-      K.Reset(K_.ParallelAssemble(), true);
+      M.Reset(_M.ParallelAssemble(), true);
+      K.Reset(_K.ParallelAssemble(), true);
    }
    else
    {
-      M.Reset(&M_, false);
-      K.Reset(&K_, false);
+      M.Reset(&_M, false);
+      K.Reset(&_K, false);
    }
 
    M_solver.SetOperator(*M);
 
    Array<int> ess_tdof_list;
-   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
    {
       HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
       HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
       HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
       M_prec = hypre_prec;
 
-      dg_solver = new DG_Solver(M_mat, K_mat, *M_.FESpace(), prec_type);
+      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace(), prec_type);
    }
    else
    {
-      M_prec = new OperatorJacobiSmoother(M_, ess_tdof_list);
+      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
       dg_solver = NULL;
    }
 

examples/ginkgo/ex1.cpp

@@ -69,8 +69,7 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   int solver_config = 0;
-   int print_lvl = 1;
+   bool use_ginkgo_solver= true;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -87,14 +86,9 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&solver_config, "-s", "--solver-config",
-                  "Solver and preconditioner combination: \n\t"
-                  "   0 - Ginkgo solver and Ginkgo preconditioner, \n\t"
-                  "   1 - Ginkgo solver and MFEM preconditioner, \n\t"
-                  "   2 - MFEM solver and Ginkgo preconditioner, \n\t"
-                  "   3 - MFEM solver and MFEM preconditioner.");
-   args.AddOption(&print_lvl, "-pl", "--print-level",
-                  "Print level for iterative solver (1 prints every iteration).");
+   args.AddOption(&use_ginkgo_solver, "-gko", "--use_gko_solver", "-no-gko",
+                  "--no-gko-solver",
+                  "Solve using ginkgo.");
    args.Parse();
    if (!args.Good())
    {
@@ -197,123 +191,39 @@ int main(int argc, char *argv[])
    // 11. Solve the linear system A X = B.
    if (!pa)
    {
-      switch (solver_config)
+      if (use_ginkgo_solver)
       {
-         // Solve the linear system with CG + IC from Ginkgo
-         case 0:
-         {
-            cout << "Using Ginkgo solver + preconditioner...\n";
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner ginkgo_precond(exec, "paric", 30);
-            Ginkgo::CGSolver ginkgo_solver(exec, ginkgo_precond);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Solve the linear system with CG from Ginkgo + MFEM preconditioner
-         case 1:
-         {
-            cout << "Using Ginkgo solver + MFEM preconditioner...\n";
-            Ginkgo::GinkgoExecutor exec(device);
-            //Create MFEM preconditioner and wrap it for Ginkgo's use.
-            DSmoother M((SparseMatrix&)(*A));
-            Ginkgo::MFEMPreconditioner gko_M(exec, M);
-            Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Ginkgo IC preconditioner +  MFEM CG solver
-         case 2:
-         {
-            cout << "Using MFEM solver + Ginkgo preconditioner...\n";
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner M(exec, "paric", 30);
-            M.SetOperator(*(A.Ptr()));  // Generate the preconditioner for the matrix A.
-            PCG(*A, M, B, X, print_lvl, 400, 1e-12, 0.0);
-            break;
-         }
-
-         // MFEM solver + MFEM preconditioner
-         case 3:
-         {
-            cout << "Using MFEM solver + MFEM preconditioner...\n";
-            // Use a simple Jacobi preconditioner with PCG.
-            DSmoother M((SparseMatrix&)(*A));
-            PCG(*A, M, B, X, print_lvl, 400, 1e-12, 0.0);
-            break;
-         }
-      } // End switch on solver_config
+#ifdef MFEM_USE_GINKGO
+         // Solve the linear system with CG + ILU from Ginkgo.
+         std::string executor = "reference";
+         auto exec = gko::ReferenceExecutor::create();
+         auto ilu_precond =
+            gko::preconditioner::Ilu<gko::solver::LowerTrs<>,
+            gko::solver::UpperTrs<>, false>::build()
+            .on(exec);
+         GinkgoWrappers::CGSolver ginkgo_solver(executor, 1, 2000, 1e-12, 0.0,
+                                                ilu_precond.release() );
+         ginkgo_solver.solve(&((SparseMatrix&)(*A)), X, B);
+#endif
+      }
+      else
+      {
+#ifndef MFEM_USE_SUITESPARSE
+         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+         GSSmoother M((SparseMatrix&)(*A));
+         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+         UMFPackSolver umf_solver;
+         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         umf_solver.SetOperator(*A);
+         umf_solver.Mult(B, X);
+#endif
+      }
    }
-   // Partial assembly mode.  Cannot use Ginkgo preconditioners, but can use Ginkgo
-   // solvers.
-   else
+   else // No preconditioning for now in partial assembly mode.
    {
-      if (UsesTensorBasis(*fespace))
-      {
-         // Use Jacobi preconditioning in partial assembly mode.
-         OperatorJacobiSmoother M(*a, ess_tdof_list);
-         switch (solver_config)
-         {
-            // No Ginkgo preconditioners work with matrix-free; error
-            case 0:
-            {
-               cout << "Using Ginkgo solver + preconditioner...\n";
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use Ginkgo solver with MFEM preconditioner
-            case 1:
-            {
-               cout << "Using Ginkgo solver + MFEM preconditioner...\n";
-               Ginkgo::GinkgoExecutor exec(device);
-               // Wrap MFEM preconditioner for Ginkgo's use.
-               Ginkgo::MFEMPreconditioner gko_M(exec, M);
-               Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
-               ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(1e-12);
-               ginkgo_solver.SetAbsTol(0.0);
-               ginkgo_solver.SetMaxIter(400);
-               ginkgo_solver.SetOperator(*(A.Ptr()));
-               ginkgo_solver.Mult(B, X);
-               break;
-            }
-
-            // No Ginkgo preconditioners work with matrix-free; error
-            case 2:
-            {
-               cout << "Using MFEM solver + Ginkgo preconditioner...\n";
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use MFEM solver and preconditioner
-            case 3:
-            {
-               cout << "Using MFEM solver + MFEM preconditioner...\n";
-               PCG(*A, M, B, X, print_lvl, 400, 1e-12, 0.0);
-               break;
-            }
-         } // End switch on solver_config
-      }
-      else // CG with no preconditioning
-      {
-         cout << "Using MFEM solver + no preconditioner...\n";
-         CG(*A, B, X, print_lvl, 400, 1e-12, 0.0);
-      }
+      CG(*A, B, X, 1, 2000, 1e-12, 0.0);
    }
 
    // 12. Recover the solution as a finite element grid function.

examples/hiop/ex9.cpp

@@ -201,11 +201,11 @@ private:
    Vector &M_rowsums;
 
 public:
-   FE_Evolution(SparseMatrix &M_, SparseMatrix &K_, const Vector &b_,
-                BilinearForm &bf_, Vector &M_rs);
+   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b,
+                BilinearForm &_bf, Vector &M_rs);
 
-   void SetTimeStep(double dt_) { dt = dt_; }
-   void SetK(SparseMatrix &K_) { K = K_; }
+   void SetTimeStep(double _dt) { dt = _dt; }
+   void SetK(SparseMatrix &_K) { K = _K; }
    virtual void Mult(const Vector &x, Vector &y) const;
 
    virtual ~FE_Evolution() { }
@@ -216,7 +216,7 @@ int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
    problem = 0;
-   optimizer_type = 2;
+   optimizer_type = 1;
    const char *mesh_file = "../../data/periodic-hexagon.mesh";
    int ref_levels = 2;
    int order = 3;
@@ -474,11 +474,11 @@ int main(int argc, char *argv[])
 
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(SparseMatrix &M_, SparseMatrix &K_,
-                           const Vector &b_, BilinearForm &bf_, Vector &M_rs)
-   : TimeDependentOperator(M_.Size()),
-     M(M_), K(K_), b(b_), M_prec(), M_solver(), z(M_.Size()),
-     bf(bf_), M_rowsums(M_rs)
+FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K,
+                           const Vector &_b, BilinearForm &_bf, Vector &M_rs)
+   : TimeDependentOperator(_M.Size()),
+     M(_M), K(_K), b(_b), M_prec(), M_solver(), z(_M.Size()),
+     bf(_bf), M_rowsums(M_rs)
 {
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(M);

examples/hiop/ex9p.cpp

@@ -226,11 +226,11 @@ private:
    Vector &M_rowsums;
 
 public:
-   FE_Evolution(HypreParMatrix &M_, HypreParMatrix &K_,
-                const Vector &b_, ParBilinearForm &pbf_, Vector &M_rs);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
+                const Vector &_b, ParBilinearForm &_pbf, Vector &M_rs);
 
-   void SetTimeStep(double dt_) { dt = dt_; }
-   void SetK(HypreParMatrix &K_) { K = K_; }
+   void SetTimeStep(double _dt) { dt = _dt; }
+   void SetK(HypreParMatrix &_K) { K = _K; }
    virtual void Mult(const Vector &x, Vector &y) const;
 
    virtual ~FE_Evolution() { }
@@ -247,7 +247,7 @@ int main(int argc, char *argv[])
 
    // 2. Parse command-line options.
    problem = 0;
-   optimizer_type = 2;
+   optimizer_type = 1;
    const char *mesh_file = "../../data/periodic-hexagon.mesh";
    int ser_ref_levels = 2;
    int par_ref_levels = 0;
@@ -358,7 +358,7 @@ int main(int argc, char *argv[])
    DG_FECollection fec(order, dim, BasisType::Positive);
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
-   HYPRE_BigInt global_vSize = fes->GlobalTrueVSize();
+   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of unknowns: " << global_vSize << endl;
@@ -576,12 +576,12 @@ int main(int argc, char *argv[])
 
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(HypreParMatrix &M_, HypreParMatrix &K_,
-                           const Vector &b_, ParBilinearForm &pbf_,
+FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
+                           const Vector &_b, ParBilinearForm &_pbf,
                            Vector &M_rs)
-   : TimeDependentOperator(M_.Height()),
-     M(M_), K(K_), b(b_), M_solver(M.GetComm()), z(M_.Height()),
-     pbf(pbf_), M_rowsums(M_rs)
+   : TimeDependentOperator(_M.Height()),
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height()),
+     pbf(_pbf), M_rowsums(M_rs)
 {
    M_prec.SetType(HypreSmoother::Jacobi);
    M_solver.SetPreconditioner(M_prec);

examples/hiop/makefile

@@ -50,13 +50,6 @@ endif
 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)
 
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 
 # Generate an error message if the MFEM library is not built and exit

examples/href_exs/href_exs.cpp

@@ -0,0 +1,471 @@
+// h-refinement examples
+//
+// Compile with: make href_exs
+//
+// Sample runs:  
+//
+// Description: Make h-refinement data sets for neural net
+//              comparison
+
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact smooth analytic solution for convergence study
+double u_exact(const Vector &);
+void u_grad_exact(const Vector &, Vector &);
+double u_exact_2(const Vector &);
+void u_grad_exact_2(const Vector &, Vector &);
+double u_exact_3(const Vector &);
+void u_grad_exact_3(const Vector &, Vector &);
+
+void convergenceStudy(const char *mesh_file, int num_ref, int &order,
+                      double &l2_err_prev, double &h1_err_prev, bool &visualization, 
+                      int &exact, int &solvePDE, bool static_cond)
+{
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the mesh num_ref times
+
+
+    for (int l = 1; l < num_ref+1; l++)
+    {
+        mesh->UniformRefinement();
+    }
+
+
+   // 5. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order.
+
+   FiniteElementCollection *fec;
+   if (order == 1)
+   {
+      fec = new H1_FECollection(1, 2);
+   }
+   else if (order > 1)
+   {
+      fec = new H1_FECollection(order, 2);
+   }
+   else if (order < 0)
+   {
+      // fec = new H1_FECollection(-order, 2, BasisType::Positive);
+      fec = new H1_FECollection(-order, 2);
+   }
+   else
+   {
+     cout << "Error - something went wrong in processing order input." << endl;
+     fec = NULL;
+   }
+
+   // Set exact solution
+
+   // exact == 1 case:
+   FunctionCoefficient *u1 = new FunctionCoefficient(u_exact);
+   VectorFunctionCoefficient *(u1_grad) = new VectorFunctionCoefficient(dim, u_grad_exact);
+   
+   // exact == 2 case:
+   FunctionCoefficient *u2 = new FunctionCoefficient(u_exact_2);
+   VectorFunctionCoefficient *u2_grad = new VectorFunctionCoefficient(dim, u_grad_exact_2);
+
+   // exact == 3 case:
+   FunctionCoefficient *u3 = new FunctionCoefficient(u_exact_3);
+   VectorFunctionCoefficient *u3_grad = new VectorFunctionCoefficient(dim, u_grad_exact_3);
+
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+
+   // this variable may not be right for more general meshes than lattices?
+   int gotNdofs = fespace->GetNDofs();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh->bdr_attributes.Max());
+   ess_bdr = 1;
+
+
+   if (solvePDE==1)
+   {
+      if (mesh->bdr_attributes.Size())
+      {
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   // For L2 Projection:
+   // Do not get boundary dofs
+
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+
+   LinearForm *b = new LinearForm(fespace);
+
+   if (solvePDE==1)
+   {
+      ConstantCoefficient zero(0.0);
+      b->AddDomainIntegrator(new DomainLFIntegrator(zero));
+   }
+   else // L2 Projection
+   {
+      if (exact == 1)
+      {
+         b->AddDomainIntegrator(new DomainLFIntegrator(*u1));
+      }
+      else if (exact == 2) 
+      {
+         b->AddDomainIntegrator(new DomainLFIntegrator(*u2));
+      }
+      else // exact == 3
+      {
+         b->AddDomainIntegrator(new DomainLFIntegrator(*u3));
+      }
+   }
+
+   b->Assemble();
+   
+   // 8. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x=0.0;
+
+   // 9. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+
+   // DiffusionIntegrator *my_diff_integrator = new DiffusionIntegrator;
+   // MassIntegrator *my_mass_integrator = new MassIntegrator;
+
+   if (solvePDE==1)
+   {   
+      if (exact == 1)
+      {
+         x.ProjectBdrCoefficient(*u1, ess_bdr);
+      }
+      else if (exact == 2)
+      {
+         x.ProjectBdrCoefficient(*u2, ess_bdr);
+      }
+      else if (exact == 3)
+      {
+         x.ProjectBdrCoefficient(*u3, ess_bdr);
+      }
+      a->AddDomainIntegrator(new DiffusionIntegrator);
+   }
+   else // L2 Projection
+   {
+      a->AddDomainIntegrator(new MassIntegrator);
+   }
+   
+   // 10. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+   
+   // cout << "Size of linear system: " << A->Height() << endl;
+  
+   // 11. Solve the linear system A X = B.
+
+   GSSmoother M((SparseMatrix&)(*A));
+   X = 0.0;
+   PCG(*A, M, B, X, 0, 200, 1e-24, 0.0);
+   
+   // 12. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+
+
+   // 13. Save the refined mesh and the solution. This output can be viewed later
+   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+
+   // 14. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+   
+
+
+   // Compute and print the L^2 and H^1 norms of the error.
+   ConstantCoefficient one(1.0);
+   
+   double l2_err = 0;
+   double h1_err = 0;
+
+   if (exact == 1)
+   {
+      l2_err = x.ComputeL2Error(*u1);
+      h1_err = x.ComputeH1Error(u1, u1_grad, &one, 1.0, 1);
+   }
+   else if (exact == 2)
+   {
+      l2_err = x.ComputeL2Error(*u2);
+      h1_err = x.ComputeH1Error(u2, u2_grad, &one, 1.0, 1);
+   }
+   else if (exact == 3)
+   {
+      l2_err = x.ComputeL2Error(*u3);
+      h1_err = x.ComputeH1Error(u3, u3_grad, &one, 1.0, 1);
+   }
+
+   double l2_rate, h1_rate;
+
+   if (num_ref != 0)
+   {
+      l2_rate = -log(l2_err/l2_err_prev) / log(2);
+      h1_rate = -log(h1_err/h1_err_prev) / log(2);
+   }
+   else
+   {
+      l2_rate = 0.0;
+      h1_rate = 0.0;
+   }
+
+   int one_over_h = mesh->GetNE();
+   one_over_h = sqrt(one_over_h);
+
+   cout << setw(16) << gotNdofs << setw(16) << one_over_h << setw(
+           16) << l2_err << setw( 16) << l2_rate;
+   cout << setw(16) << h1_err << setw(16) << h1_rate << endl;
+
+   l2_err_prev = l2_err;
+   h1_err_prev = h1_err;
+
+
+   // Save the data to pass to neural network
+   cout << "Mesh vertex array = " << endl;
+   cout <<  mesh->GetVertex(0) << endl;
+
+   // 15. Free the used memory.
+   // delete pcg;
+   // delete amg;
+   // delete my_diff_integrator;
+   // delete my_mass_integrator;
+   delete a; 
+   delete b;
+   delete fespace;
+   delete u1;
+   delete u1_grad;
+   delete u2;
+   delete u2_grad;
+   delete u3;
+   delete u3_grad;
+   delete fec;
+   delete mesh;
+
+   return;
+}
+
+double u_exact(const Vector &x)
+{
+   return(x(0)+x(1));
+}
+
+void u_grad_exact(const Vector &x, Vector &u)
+{
+   u(0) = 1;
+   u(1) = 1;
+}
+
+double u_exact_2(const Vector &x)
+{
+   return sin(x(1))*exp(x(0));
+}
+
+void u_grad_exact_2(const Vector &x, Vector &u)
+{
+   u(0) = sin(x(1))*exp(x(0));
+   u(1) = cos(x(1))*exp(x(0));
+}
+
+double u_exact_3(const Vector &x)
+{
+    // return (x(0)*x(0) + (0.5)*x(1)*x(1));
+    int m=10;
+    double total = sin(x(0))* pow( sin(  x(0)*x(0) / M_PI  ), 2*m  );
+    total += sin(x(1))* pow( sin( 2*x(1)*x(1) / M_PI  ), 2*m  );
+    total *= -1;
+    return total;
+}
+
+void u_grad_exact_3(const Vector &x, Vector &u)
+{
+    // presumes m=10
+    u(0)  = - (40.0 * x(0) * cos( x(0)*x(0)/M_PI ) * sin(x(0)) * pow(sin( x(0)*x(0)/M_PI ),19)) / M_PI;
+    u(0) += - cos(x(0))*pow(sin( x(0)*x(0)/M_PI ),20);
+    u(1)  = - (80.0 * x(1) * cos( 2.0 * x(1)*x(1)/M_PI ) * sin(x(1)) * pow(sin( 2.0 * x(1)*x(1)/M_PI ),19)   ) / M_PI;
+    u(1) += - cos(x(1))*pow(sin( 2.0 * x(1)*x(1)/M_PI ),20);
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int total_refinements = 0;
+
+   // const char *mesh_file = "../../data/twoSquare.mesh";
+   // const char *mesh_file = "../../data/star-q3.mesh";
+   const char *mesh_file = "../../data/inline-oneTri.mesh";
+   int order = 1;
+   bool static_cond = false;
+   const char *device_config = "cpu";
+   bool visualization = false;
+   int exact = 3;
+   int solvePDE = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&total_refinements, "-r", "--refine",
+                  "Number of refinements to do.");   
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   //args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+   //             "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&exact, "-e", "--exact", 
+                  "Choice of exact solution. 1=constant 1; 2=sin(x)e^y; 3=michalewicz.");
+   args.AddOption(&solvePDE, "-L", "--L2Project",
+                  "Solve a PDE (1) or do L2 Projection (2)");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (order != 1)
+   {
+      cout << "Only allowing order 1 triangle elements for now." << endl;
+      return 1;
+   }
+   else if (order == 1)
+   {
+      cout << "Using H1 triangular elements of order " << order << "." << endl;
+   }
+
+   if (solvePDE == 1)
+   {
+      cout << "Approximating solution to Laplace problem with ";
+      if (exact == 1)
+      {
+        cout << "exact solution u(x,y)=x+y" << endl;
+      }
+      else if (exact == 2)
+      {
+         cout << "exact solution u(x,y)=sin(y)e^x" << endl;
+      }
+      else if (exact == 3)
+      {
+         cout << endl << "Michalewicz is not harmonic. Use option -L 2 to do L2 projection instead."
+              << endl;
+         return 1;
+      }
+      else
+      {
+         cout << endl << "*** Wrong usage of exact solution parameter (-e)"
+              << endl;
+         return 1;
+      }
+   }
+   else if (solvePDE == 2)
+   {
+      cout << "Doing L^2 projection of basis with right hand side ";
+      if (exact == 1)
+      {
+        cout << "u(x,y)=x+y" << endl;
+      }
+      else if (exact == 2)
+      {
+         cout << "u(x,y)=sin(y)e^x" << endl;
+      }
+      else if (exact == 3)
+      {
+         cout << "u(x,y)=michalewicz function" << endl;
+      }
+      else
+      {
+         cout << endl << "*** Wrong usage of exact solution parameter (-e)"
+              << endl;
+         return 1;
+      }
+   }
+   else
+   {
+      cout << "Wrong usage of solve vs. L2 Projection option -L."
+           << endl;
+      return 1;
+   }
+
+
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // Set output options and print header
+   cout.precision(4);
+
+   cout << "----------------------------------------------------------------------------------------"
+        << endl;
+   cout << left << setw(16) << "DOFs "<< setw(16) <<"1/h "<< setw(
+           16) << "L^2 error "<< setw(16);
+   cout << "L^2 rate "<< setw(16) << "H^1 error "<< setw(16) << "H^1 rate" << endl;
+   cout << "----------------------------------------------------------------------------------------"
+        << endl;
+
+   double l2_err_prev = 0.0;
+   double h1_err_prev = 0.0;
+
+   // 3. Read the mesh from the given mesh file.
+   // Run last round with vis, if desired.
+
+   // can use this as a max DoF tolerance:  (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+
+   // Loop over number of refinements for convergence study
+
+    bool noVisYet = false;
+    for (int i = 0; i < (total_refinements); i++)
+    {
+        convergenceStudy(mesh_file, i, order, l2_err_prev, h1_err_prev, noVisYet, 
+        exact, solvePDE, static_cond);
+    }
+
+    convergenceStudy(mesh_file, total_refinements, order, l2_err_prev, h1_err_prev, visualization, 
+            exact, solvePDE, static_cond);
+
+    return 0;
+}

examples/makefile

@@ -21,11 +21,11 @@ CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
-  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29
-PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
-  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p
+SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
+  ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27
+PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
+  ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p ex25p\
+  ex26p ex27p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -54,9 +54,6 @@ endif
 ifeq ($(MFEM_USE_SUPERLU),YES)
    SUBDIRS += superlu
 endif
-ifeq ($(MFEM_USE_CALIPER),YES)
-   SUBDIRS += caliper
-endif
 
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
@@ -101,10 +98,6 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<,, Serial example)
 
 # Testing: Specific execution options
-ex0-test-seq: ex0
-	@$(call mfem-test,$<,, Serial example,,1)
-ex0p-test-par: ex0p
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,,1)
 ex1-test-seq: ex1
 	@$(call mfem-test,$<,, Serial example)
 ex1p-test-par: ex1p
@@ -155,7 +148,7 @@ clean-exec:
 	@rm -rf Example5* Example9* Example15* Example16* Example23* ParaView
 	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
-	@rm -f deformed.* velocity.* elastic_energy.* mode_* flux.*
+	@rm -f deformed.* velocity.* elastic_energy.* mode_*
 	@rm -f ex5-p-*.bp ex9-p-*.bp ex12-p-*.bp ex16-p-*.bp
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*
 	@rm -f vortex-mesh.* vortex.mesh vortex-?-init.* vortex-?-final.*
@@ -164,4 +157,3 @@ clean-exec:
 	@rm -f ex21*.mesh ex21*.sol ex21p_*.*
 	@rm -f ex23.mesh ex23-*.gf
 	@rm -f ex25.mesh ex25-*.gf ex25p-*.*
-	@rm -rf ex28_* ex28p_*

examples/maxwell-solver-dev/ParDST/DofMapsDST.cpp

@@ -1,960 +0,0 @@
-#include "DofMapsDST.hpp"
-
-double testcoeff(const Vector & x)
-{
-   return sin(3*M_PI*(x.Sum()));
-}
-
-int get_rank(int tdof, std::vector<int> & tdof_offsets)
-{
-   int size = tdof_offsets.size();
-   if (size == 1) { return 0; }
-   std::vector<int>::iterator up;
-   up=std::upper_bound(tdof_offsets.begin(), tdof_offsets.end(),tdof); // 
-   return std::distance(tdof_offsets.begin(),up)-1;
-}
-
-void ComputeTdofOffsets(const MPI_Comm & comm, const ParFiniteElementSpace * pfes, 
-                        std::vector<int> & tdof_offsets)
-{
-   int num_procs;
-   MPI_Comm_size(comm, &num_procs);
-   tdof_offsets.resize(num_procs);
-   int mytoffset = pfes->GetMyTDofOffset();
-   MPI_Allgather(&mytoffset,1,MPI_INT,&tdof_offsets[0],1,MPI_INT,comm);
-}
-
-void GetSubdomainijk(int ip, const Array<int> nxyz, Array<int> & ijk)
-{
-   ijk.SetSize(3);
-   ijk[2] = ip/(nxyz[0]*nxyz[1]);
-   ijk[1] = (ip-ijk[2]*nxyz[0]*nxyz[1])/nxyz[0];
-   ijk[0] = (ip-ijk[2]*nxyz[0]*nxyz[1])%nxyz[0];
-}
-void GetDirectionijk(int id, Array<int> & ijk)
-{
-   ijk.SetSize(3);
-   int n = 3;
-   ijk[2] = id/(n*n) - 1;
-   ijk[1] = (id-(ijk[2]+1)*n*n)/n - 1;
-   ijk[0] = (id-(ijk[2]+1)*n*n)%n - 1;
-}
-
-int GetSubdomainId(const Array<int> nxyz, Array<int> & ijk)
-{
-   int dim=ijk.Size();
-   int k = (dim==2)? 0 : ijk[2];
-   return k*nxyz[1]*nxyz[0] + ijk[1]*nxyz[0] + ijk[0];
-}
-
-int GetDirectionId(const Array<int> & ijk)
-{
-   int n = 3;
-   int dim = ijk.Size();
-   int k = (dim == 2) ? -1 : ijk[2];
-   return (k+1)*n*n + (ijk[1]+1)*n + ijk[0]+1; 
-}
-
-void DofMaps::Init()
-{
-   comm = pfes->GetComm();
-   MPI_Comm_size(comm, &num_procs);
-   MPI_Comm_rank(comm, &myid);
-
-   dim = pfes->GetParMesh()->Dimension();
-   ComputeTdofOffsets(comm, pfes, tdof_offsets);
-   myelemoffset = part->myelem_offset;
-   mytoffset = pfes->GetMyTDofOffset();
-   subdomain_rank = part->subdomain_rank;
-   nrsubdomains = part->nrsubdomains;
-   nxyz.SetSize(3);
-   for (int i = 0; i<3; i++) { nxyz[i] = part->nxyz[i]; }
-
-   //compute sign factors for tdofs
-   int lsize = pfes->GetVSize();
-   int tsize = pfes->GetTrueVSize();
-   tdof_sign.SetSize(tsize); 
-   for (int i = 0; i<lsize; i++)
-   {
-      int j = pfes->GetGlobalTDofNumber(i);
-      if (j<mytoffset || j>=mytoffset+tsize) continue;
-      tdof_sign[j-mytoffset] = pfes->GetDofSign(i);
-   }
-}
-
-DofMaps::DofMaps(ParFiniteElementSpace *pfes_, ParMeshPartition * part_, bool CompFlag_)
-: pfes(pfes_), part(part_), CompFlag(CompFlag_)
-{
-   Init();
-   Setup();
-}
-
-void DofMaps::Setup()
-{
-   // Setup the local FiniteElementSpaces
-   const FiniteElementCollection * fec = pfes->FEColl();
-   fes.SetSize(nrsubdomains);
-   for (int i = 0; i<nrsubdomains; i++)
-   {
-      fes[i] = nullptr; // initialize with null on all procs
-      if (myid == subdomain_rank[i])
-      {
-         fes[i] = new FiniteElementSpace(part->subdomain_mesh[i],fec);
-      }
-   }
-   // cout << "Computing Overlap Tdofs" << endl;
-   SubdomainToSubdomainMapsSetup();
-   // TestSubdomainToSubdomainMaps();
-
-   SubdomainToGlobalMapsSetup();
-   // TestSubdomainToGlobalMaps();
-}
-
-void DofMaps::SubdomainToSubdomainMapsSetup()
-{
-   ComputeOvlpElems();
-   ComputeOvlpTdofs();
-}
-
-void DofMaps::AddElementToOvlpLists(int l, int iel, 
-                            const Array<bool> & neg, const Array<bool> & pos)
-{
-   int kbeg = (dim == 2) ? 0 : -1;
-   int kend = (dim == 2) ? 0 :  1;
-   Array<int> dijk(3);
-   for (int k = kbeg; k<=kend; k++)
-   {
-      if (dim == 3)
-      {
-         if (k == -1 && !neg[2]) continue;   
-         if (k ==  1 && !pos[2]) continue;   
-      }   
-
-      for (int j = -1; j<=1; j++)
-      {
-         if (j== -1 && !neg[1]) continue;   
-         if (j==  1 && !pos[1]) continue;   
-         for (int i = -1; i<=1; i++)
-         {
-            // cases to skip 
-            if (i==-1 && !neg[0]) continue;   
-            if (i== 1 && !pos[0]) continue;   
-
-            if (i==0 && j==0 && k == 0) continue;
-            dijk[0] = i; dijk[1] = j; dijk[2] = (dim==2)?-1 : k;  
-            int DirId = GetDirectionId(dijk);
-            OvlpElems[l][DirId].Append(iel);
-         }
-      }
-   }
-}
-
-void DofMaps::ComputeOvlpElems()
-{
-   // first compute the element in the overlaps
-   OvlpElems.resize(nrsubdomains);
-   int nlayers = 2*part->OvlpNlayers; 
-   // loop through subdomains
-   for (int l = 0; l<nrsubdomains; l++)
-   {
-      if (myid == subdomain_rank[l])
-      {
-         Array<int> ijk;
-         GetSubdomainijk(l,nxyz,ijk);
-         Mesh * mesh = part->subdomain_mesh[l];
-         OvlpElems[l].resize(pow(3,dim)); 
-         Vector pmin, pmax;
-         mesh->GetBoundingBox(pmin,pmax);
-         double h = part->MeshSize;
-         // loop through the elements in the mesh and assign them to the 
-         // appropriate lists of overlaps
-         for (int iel=0; iel< mesh->GetNE(); iel++)
-         {
-            // Get element center
-            Vector center(dim);
-            int geom = mesh->GetElementBaseGeometry(iel);
-            ElementTransformation * tr = mesh->GetElementTransformation(iel);
-            tr->Transform(Geometries.GetCenter(geom),center);
-
-            Array<bool> pos(dim); pos = false;
-            Array<bool> neg(dim); neg = false;
-            // loop through dimensions   
-            for (int d=0;d<dim; d++)
-            {
-               if (ijk[d]>0 && center[d] < pmin[d]+h*nlayers)
-               {
-                  neg[d] = true;
-               }
-
-               if (ijk[d]<nxyz[d]-1 && center[d] > pmax[d]-h*nlayers) 
-               {
-                  pos[d] = true;
-               }
-            }
-            // Add the element to the appropriate lists
-            AddElementToOvlpLists(l,iel,neg,pos);        
-         }   
-      }
-   }
-}
-
-void DofMaps::ComputeOvlpTdofs()
-{
-   OvlpTDofs.resize(nrsubdomains);
-   int nrneighbors = pow(3,dim); // including its self
-
-   // loop through subdomains
-   for (int l = 0; l<nrsubdomains; l++)
-   {
-      if (myid != subdomain_rank[l]) continue;
-      int ntdofs = fes[l]->GetTrueVSize();
-      Array<int> tdof_marker(ntdofs); 
-      OvlpTDofs[l].resize(nrneighbors);
-      // loop through neighboring directions/neighbors
-      for (int d=0; d<nrneighbors; d++)
-      {
-         tdof_marker = 0;
-         Array<int> tdoflist;
-         // Get the direction
-         Array<int> dijk;
-         GetDirectionijk(l,dijk);
-         int nel = OvlpElems[l][d].Size();
-         Array<int>Elems = OvlpElems[l][d];
-         for (int iel = 0; iel<nel; ++iel)
-         {
-            int jel = Elems[iel];   
-            Array<int> ElemDofs;
-
-            fes[l]->GetElementDofs(jel,ElemDofs);
-            int ndof = ElemDofs.Size();
-            for (int i = 0; i<ndof; ++i)
-            {
-               int dof_ = ElemDofs[i];
-               int dof = (dof_ >= 0) ? dof_ : abs(dof_) - 1;
-               if (!tdof_marker[dof])
-               {
-                  tdoflist.Append(dof); // dofs of ip0 in ovlp
-                  tdof_marker[dof] = 1;
-               }
-            }
-         }
-         OvlpTDofs[l][d] = tdoflist; 
-         if (CompFlag)
-         {
-            for (int i=0; i<tdoflist.Size(); i++)
-            {
-               tdoflist[i] += fes[l]->GetTrueVSize();
-            }
-            OvlpTDofs[l][d].Append(tdoflist);
-         }
-      }
-   }
-}
-
-void DofMaps::PrintOvlpTdofs()
-{
-   int nrneighbors = pow(3,dim); // including its self
-   if (myid == 0)
-   {
-      for (int i = 0; i<nrsubdomains; i++)
-      {
-         if (myid != subdomain_rank[i]) continue;
-         Array<int> ijk;
-         GetSubdomainijk(i,nxyz,ijk);
-         cout << "subdomain = " ; ijk.Print();
-         cout << "myid = " << myid << endl;
-         cout << "ip   = " << i << endl;
-         for (int d = 0; d<nrneighbors; d++)
-         {
-            Array<int> dijk;
-            GetDirectionijk(d,dijk);
-            cout << "direction = " ; dijk.Print();
-
-            if (OvlpTDofs[i][d].Size())
-            {
-               cout << "OvlpTdofs = " ; 
-               OvlpTDofs[i][d].Print(cout,OvlpTDofs[i][d].Size() );
-            }
-         }
-      }
-   }
-}
-
-void DofMaps::TransferToNeighbors(const Array<int> & SubdomainIds, const Array<Vector *> & x,
-std::vector<std::vector<Vector * >> & OvlpSol)
-{
-   // 2D for now....
-   MFEM_VERIFY(SubdomainIds.Size() == x.Size(), "TransferToNeighbors: Size inconsistency");
-   int nrsendIds = SubdomainIds.Size();
-   int nrneighbors = pow(3,dim);
-   MPI_Request *recv_requests = new MPI_Request[nrsendIds*nrneighbors];
-   MPI_Request *send_requests = new MPI_Request[nrsendIds*nrneighbors];
-   MPI_Status  *recv_statuses = new MPI_Status[nrsendIds*nrneighbors];
-   MPI_Status  *send_statuses = new MPI_Status[nrsendIds*nrneighbors];
-   Array<Vector * > send_buffer(nrsendIds*nrneighbors);
-   Array<Vector * > recv_buffer(nrsendIds*nrneighbors);
-   int send_counter = 0;
-   int recv_counter = 0;
-   for (int is = 0; is<nrsendIds; is++)
-   {
-      int i0 = SubdomainIds[is];
-      Array<int> ijk;
-      GetSubdomainijk(i0,nxyz,ijk);
-      for (int d=0;d<nrneighbors; d++)
-      {
-         Array<int>directions;
-         GetDirectionijk(d,directions);
-
-         if (dim == 2 && directions[0] == 0 && directions[1] == 0) continue;
-         if (dim == 3 && directions[0] == 0 
-                      && directions[1] == 0 
-                      && directions[2] == 0) continue;
-         int i = ijk[0] + directions[0];
-         if (i<0 || i>=nxyz[0]) continue;
-         int j = ijk[1] + directions[1];
-         if (j<0 || j>=nxyz[1]) continue;
-         int k = (dim ==3 ) ? ijk[2] + directions[2] : 0;
-         if (k<0 || k>=nxyz[2]) continue;
-         Array<int>ijk1(3);
-         ijk1[0] = i;
-         ijk1[1] = j;
-         ijk1[2] = k;
-         int i1 = GetSubdomainId(nxyz,ijk1);
-         if (myid == subdomain_rank[i0])
-         {
-            Array<int> tdofs0 = OvlpTDofs[i0][d]; // map of dofs in the overlap
-            send_buffer[send_counter] = new Vector(tdofs0.Size());
-            x[is]->GetSubVector(tdofs0,*send_buffer[send_counter]);
-            // Destination rank
-            int dest = subdomain_rank[i1];
-            int tag = i0 * nrneighbors + d;
-
-            int count = tdofs0.Size();
-            MPI_Isend(send_buffer[send_counter]->GetData(),count,MPI_DOUBLE,dest,
-                      tag,comm,&send_requests[send_counter]);
-            send_counter++;
-
-         }
-         if (myid == subdomain_rank[i1])
-         {
-            Array<int> direction1(3); direction1 = -1;
-            for (int dd=0;dd<dim;dd++) 
-            { 
-               direction1[dd] = -directions[dd];
-            }
-            int d1 = GetDirectionId(direction1);
-
-            int count = OvlpTDofs[i1][d1].Size();
-            recv_buffer[recv_counter] = new Vector(count); 
-            int src = subdomain_rank[i0];
-            int tag = i0 * nrneighbors + d;
-            MPI_Irecv(recv_buffer[recv_counter]->GetData(), count,MPI_DOUBLE,src,
-                      tag,comm, &recv_requests[recv_counter]);
-            recv_counter++;
-         }
-      }   
-   }
-   MPI_Waitall(send_counter, send_requests, send_statuses);
-   MPI_Waitall(recv_counter, recv_requests, recv_statuses);
-
-   delete [] send_statuses;
-   delete [] send_requests;
-   delete [] recv_statuses;
-   delete [] recv_requests;
-
-   for (int i = 0; i<send_counter; i++)
-   {
-      delete send_buffer[i];
-   }
-   send_buffer.DeleteAll();
-
-
-   // Extract the transfered solutions
-   recv_counter = 0;
-   for (int is = 0; is<nrsendIds; is++)
-   {
-      int i0 = SubdomainIds[is];
-      Array<int> ijk;
-      GetSubdomainijk(i0,nxyz,ijk);
-      for (int d=0;d<nrneighbors; d++)
-      {
-         Array<int>directions;
-         GetDirectionijk(d,directions);
-         if (dim == 2 && directions[0] == 0 && directions[1] == 0) continue;
-         if (dim == 3 && directions[0] == 0 
-                      && directions[1] == 0 
-                      && directions[2] == 0) continue;
-         int i = ijk[0] + directions[0];
-         if (i<0 || i>=nxyz[0]) continue;
-         int j = ijk[1] + directions[1];
-         if (j<0 || j>=nxyz[1]) continue;
-         int k = (dim ==3 ) ? ijk[2] + directions[2] : 0;
-         if (k<0 || k>=nxyz[2]) continue;
-
-         Array<int>ijk1(3);
-         ijk1[0] = i;
-         ijk1[1] = j;
-         ijk1[2] = k;
-         int i1 = GetSubdomainId(nxyz,ijk1);
-         if (myid == subdomain_rank[i1])
-         {
-            Array<int> direction1(3); direction1 = -1;
-            for (int d=0;d<dim;d++) 
-            { 
-               direction1[d] = -directions[d];
-            }
-            int d1 = GetDirectionId(direction1);
-            Array<int> tdofs1 = OvlpTDofs[i1][d1];
-            if (!OvlpSol[i1][d1])
-            {
-               OvlpSol[i1][d1] = new Vector(2*fes[i1]->GetTrueVSize());
-            }
-            *OvlpSol[i1][d1] = 0.0;
-            OvlpSol[i1][d1]->SetSubVector(tdofs1,*recv_buffer[recv_counter]);
-            recv_counter++;
-         }
-      }   
-   }
-   for (int i = 0; i<recv_counter; i++)
-   {
-      delete recv_buffer[i];
-   }
-   recv_buffer.DeleteAll();
-}
-
-void DofMaps::TestSubdomainToSubdomainMaps()
-{
-      // testing inter-subdomain communication
-   FunctionCoefficient c1(testcoeff);
-   int nrsub = nrsubdomains;
-   Array<int> subdomain_ids(nrsub);
-   Array<Vector*> x(nrsub);
-   for (int i = 0; i<nrsub; i++)
-   {
-      x[i] = nullptr;
-      subdomain_ids[i] = i;
-      if (fes[i])
-      {
-         ComplexGridFunction gf(fes[i]);
-         gf = 0.0;
-         gf.ProjectCoefficient(c1,c1);
-         x[i] = new Vector(2*fes[i]->GetTrueVSize());
-         *x[i] = gf;
-      }
-   }
-
-   std::vector<std::vector<Vector * >> OvlpSol;
-
-   OvlpSol.resize(nrsubdomains);
-   int nrneighbors = pow(3,dim);
-   for (int ip = 0; ip<nrsubdomains; ip++)
-   {
-      if (myid == subdomain_rank[ip])
-      {
-         OvlpSol[ip].resize(nrneighbors);
-      }
-   }
-
-
-   TransferToNeighbors(subdomain_ids,x,OvlpSol);
-
-   string keys = "keys amrRljc\n";
-   for (int i0 = 0 ; i0< nrsubdomains; i0++)
-   {
-      if (fes[i0])
-      {
-         ComplexGridFunction gf0(fes[i0]);
-         for (int d = 0; d<nrneighbors; d++)
-         {
-            if(OvlpSol[i0][d])
-            {
-               Array<int>dijk;
-               GetDirectionijk(d,dijk);
-               Array<int>ijk;
-               GetSubdomainijk(i0,nxyz,ijk);
-               ostringstream oss;
-               oss << "myid: " << myid 
-                     << ", subdomain: (" << ijk[0] << "," << ijk[1] <<")"
-                     << ", direction: (" << dijk[0] << "," << dijk[1] <<")";
-
-               gf0 = 0.0;
-               gf0.real().SetVector(*OvlpSol[i0][d],0);
-               gf0.imag().SetVector(*OvlpSol[i0][d],fes[i0]->GetTrueVSize());
-               char vishost[] = "localhost";
-               int  visport   = 19916;
-               socketstream sol_sock(vishost, visport);
-               sol_sock.precision(8);
-               sol_sock << "solution\n" << *(part->subdomain_mesh[i0]) << gf0.real() 
-               << keys
-               << "window_title '" << oss.str() << "'" << flush;
-            }
-         }
-      }
-   }
-   for (int i = 0; i<nrsub; i++)
-   {
-      delete x[i];
-   }
-}
-
-void DofMaps::SubdomainToGlobalMapsSetup()
-{
-   // workspace for MPI_AlltoAll
-   send_count.SetSize(num_procs);  send_count = 0;
-   send_displ.SetSize(num_procs);  send_displ = 0; 
-   recv_count.SetSize(num_procs);  recv_count = 0;
-   recv_displ.SetSize(num_procs);  recv_displ = 0;
-
-   // 1. Communicate to the subdomain rank the list of tdofs 
-   // a. Compute send count
-   for (int ip = 0; ip<nrsubdomains; ++ip)
-   {
-      // avoid any communication if on subdomain rank
-      int nel = part->local_element_map[ip].Size();
-
-      for (int iel = 0; iel<nel; iel++)
-      {
-         int elem_idx = part->local_element_map[ip][iel] - myelemoffset;
-         // int ndofs = local_tdofs[ip].Size();
-         int ndofs = pfes->GetFE(elem_idx)->GetDof();
-
-         send_count[subdomain_rank[ip]] += 2 + ndofs; 
-      }
-   }
-   // b. Compute receive count
-   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
-   for (int k=0; k<num_procs-1; k++)
-   {
-      send_displ[k+1] = send_displ[k] + send_count[k];
-      recv_displ[k+1] = recv_displ[k] + recv_count[k];
-   }
-   sbuff_size = send_count.Sum();
-   rbuff_size = recv_count.Sum();
-   // c. Allocate and fill the send buffer
-   Array<int> sendbuf(sbuff_size);  sendbuf = 0;
-   Array<int> soffs(num_procs); soffs = 0;
-   for (int ip = 0; ip<nrsubdomains; ++ip)
-   {
-      int nel = part->local_element_map[ip].Size();
-      for (int iel = 0; iel<nel; iel++)
-      {
-         int elem_idx = part->local_element_map[ip][iel] - myelemoffset;
-         Array<int>ElemDofs;
-         pfes->GetElementDofs(elem_idx,ElemDofs);
-         int ndofs = ElemDofs.Size();
-
-         int j = send_displ[subdomain_rank[ip]] + soffs[subdomain_rank[ip]];
-         sendbuf[j] = ip;
-         sendbuf[j+1] = ndofs;
-
-         for (int k = 0; k < ndofs ; ++k)
-         {
-            int edof_ = ElemDofs[k];
-            int edof = (edof_ >= 0) ? edof_ : abs(edof_) - 1;
-            sendbuf[j+2+k] = pfes->GetGlobalTDofNumber(edof);
-         }
-         soffs[subdomain_rank[ip]] +=  2 + ndofs;
-      }
-   }
-
-   // d. Communication
-   Array<int> recvbuf(rbuff_size);
-   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_INT, recvbuf,
-                 recv_count, recv_displ, MPI_INT, comm);
-
-   // 3. Extract from recv_buffer
-   std::vector<Array<int>> global_tdofs(nrsubdomains);
-   int k=0;
-   while (k<rbuff_size)
-   {
-      int ip = recvbuf[k++]; 
-      int ndofs = recvbuf[k++];
-      for (int i = 0; i < ndofs; ++i)
-      {
-         global_tdofs[ip].Append(recvbuf[i+k]); 
-      }
-      k += ndofs;
-   }
-
-   SubdomainGTrueDofs.resize(nrsubdomains);
-   // 4. Construct SubdomainTdof to Global mesh tdof maps
-   for (int ip=0; ip<nrsubdomains; ++ip)
-   {
-      if (myid != subdomain_rank[ip]) continue;
-      int nrdof = fes[ip]->GetTrueVSize();
-
-      SubdomainGTrueDofs[ip].SetSize(nrdof);
-      int nel = part->element_map[ip].Size();
-      int k = 0;
-      for (int iel = 0; iel<nel; ++iel)
-      {
-         Array<int> elem_dofs;
-         fes[ip]->GetElementDofs(iel,elem_dofs);
-         int ndof = elem_dofs.Size();
-         for (int i = 0; i<ndof; ++i)
-         {
-            int edof_ = elem_dofs[i];
-            int edof = (edof_ >= 0) ? edof_ : abs(edof_) - 1;
-            // rearranging dofs from serial fespace to pfes ordering
-            SubdomainGTrueDofs[ip][edof] = global_tdofs[ip][k++];
-         }
-      }
-   }
-
-   // 5. Communicate SubdomainGTrueDofs to participating ranks 
-   send_count = 0;  send_displ = 0;
-   recv_count = 0;  recv_displ = 0;
-
-   for (int ip = 0; ip < nrsubdomains; ++ip)
-   {
-      if (myid != subdomain_rank[ip]) continue;
-      int ndofs = SubdomainGTrueDofs[ip].Size();
-      for (int i = 0; i<ndofs; ++i)
-      {
-         int tdof = SubdomainGTrueDofs[ip][i];
-         int rank = get_rank(tdof,tdof_offsets);
-         if (rank == subdomain_rank[ip]) continue; // <--------------
-         send_count[rank] += 2; // 1 for the dof and 1 for the ip that goes to 
-      }
-   }
-
-   // communicate so that recv_count is constructed
-   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
-   //
-   for (int k=0; k<num_procs-1; k++)
-   {
-      send_displ[k+1] = send_displ[k] + send_count[k];
-      recv_displ[k+1] = recv_displ[k] + recv_count[k];
-   }
-   sbuff_size = send_count.Sum();
-   rbuff_size = recv_count.Sum();
-
-   sendbuf.SetSize(sbuff_size);  
-   sendbuf = 0; soffs = 0;
-
-   for (int ip = 0; ip < nrsubdomains; ip++)
-   {
-      if (myid != subdomain_rank[ip]) continue;
-      int ndofs = SubdomainGTrueDofs[ip].Size();
-      // loop through dofs
-      for (int i = 0; i<ndofs; ++i)
-      {
-         int tdof = SubdomainGTrueDofs[ip][i];
-         int irank = get_rank(tdof,tdof_offsets);
-         if (irank == subdomain_rank[ip]) continue; // <--------------
-         int j = send_displ[irank] + soffs[irank];
-         sendbuf[j] = ip;
-         sendbuf[j+1] = SubdomainGTrueDofs[ip][i];
-         soffs[irank] +=  2 ;
-      }
-   }
-
-   recvbuf.SetSize(rbuff_size);
-   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_INT, recvbuf,
-                 recv_count, recv_displ, MPI_INT, comm);
-
-   // List of tdofs owned by the processor for subdomains not owned
-   SubdomainLTrueDofs.resize(nrsubdomains); 
-   for (int k=0; k<rbuff_size/2; k++)
-   {
-      int ip = recvbuf[2*k];
-      int tdof = recvbuf[2*k+1];
-      SubdomainLTrueDofs[ip].Append(tdof);
-   }    
-}
-
-// Restriction of global residual to subdomain residuals
-void DofMaps::GlobalToSubdomains(const Vector & y, Array<Vector*> & x)
-{
-   send_count = 0; send_displ = 0;
-   recv_count = 0; recv_displ = 0;
-
-   // Compute send_counts
-   int m = (CompFlag) ? 2 : 1 ;
-   for (int ip = 0; ip < nrsubdomains; ip++)
-   {
-      if (myid == subdomain_rank[ip]) continue;  // <---------------
-      int ndofs = SubdomainLTrueDofs[ip].Size();
-      send_count[subdomain_rank[ip]] += m * ndofs;
-   }
-
-    // communicate so that recv_count is constructed
-   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
-
-   for (int k=0; k<num_procs-1; k++)
-   {
-      send_displ[k+1] = send_displ[k] + send_count[k];
-      recv_displ[k+1] = recv_displ[k] + recv_count[k];
-   }
-   sbuff_size = send_count.Sum();
-   rbuff_size = recv_count.Sum();
-
-   Array<double> sendbuf(sbuff_size); sendbuf = 0;
-   Array<int> soffs(num_procs); soffs = 0;
-
-   for (int ip = 0; ip < nrsubdomains; ip++)
-   {
-      if (myid == subdomain_rank[ip]) continue;  // <---------------
-      int ndofs = SubdomainLTrueDofs[ip].Size();
-      for (int i = 0; i<ndofs; i++)
-      {
-         int tdof = SubdomainLTrueDofs[ip][i];
-         int j = send_displ[subdomain_rank[ip]] + soffs[subdomain_rank[ip]];
-         soffs[subdomain_rank[ip]] +=m;
-         int k = tdof - mytoffset;
-         // sendbuf[j] = y[k];
-         sendbuf[j] = tdof_sign[k]*y[k];
-         if (CompFlag)
-         {  // if complex valued 
-            int tsize = pfes->GetTrueVSize();
-            // sendbuf[j+1] = y[k+tsize];
-            sendbuf[j+1] = tdof_sign[k]*y[k+tsize];
-         }
-      }
-   }
-
-   // communication
-   Array<double> recvbuf(rbuff_size);
-   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_DOUBLE, recvbuf,
-                 recv_count, recv_displ, MPI_DOUBLE, comm);
-   Array<int> roffs(num_procs);
-   roffs = 0;
-   // Now each process will construct the res vector
-   x.SetSize(nrsubdomains);
-   for (int ip = 0; ip < nrsubdomains; ip++)
-   {
-      if (myid != subdomain_rank[ip]) continue;
-      int ndof = SubdomainGTrueDofs[ip].Size();
-      if (!x[ip]) x[ip] = new Vector(m*ndof); 
-      *x[ip] = 0.0;
-      // extract the data from receiv buffer
-      for (int i=0; i<ndof; i++)
-      {
-         // pick up the tdof and find its rank
-         int tdof = SubdomainGTrueDofs[ip][i];
-         int tdof_rank = get_rank(tdof,tdof_offsets);
-         if (tdof_rank != subdomain_rank[ip])  // <---------------
-         {
-            int k = recv_displ[tdof_rank] + roffs[tdof_rank];
-            roffs[tdof_rank] += m;
-            (*x[ip])[i] = recvbuf[k];
-            if (CompFlag)
-            {  
-               (*x[ip])[i+ndof] = recvbuf[k+1];
-            }
-         }
-         else
-         {
-            int k = tdof - mytoffset;
-            // (*x[ip])[i] = y[k];
-            (*x[ip])[i] = tdof_sign[k]*y[k];
-            if (CompFlag)
-            {  
-               int gtsize = pfes->GetTrueVSize();
-               (*x[ip])[i+ndof] = tdof_sign[k]*y[k+gtsize];
-            }
-         }
-      }
-   }
-}
-
-// Prolongation of subdomain solutions to the global solution
-void DofMaps::SubdomainsToGlobal(const Array<Vector*> & x, Vector & y)
-{
-   send_count = 0; send_displ = 0;
-   recv_count = 0; recv_displ = 0;
-
-   // Compute send_counts
-   int m = (CompFlag) ? 2 : 1 ;
-   for (int ip = 0; ip < nrsubdomains; ip++)
-   {
-      if (myid != subdomain_rank[ip]) continue;
-      int ndofs = SubdomainGTrueDofs[ip].Size();
-      for (int i=0; i<ndofs; i++)
-      {
-         // pick up the tdof and find its rank
-         int tdof = SubdomainGTrueDofs[ip][i];
-         int tdof_rank = get_rank(tdof,tdof_offsets);
-         if (tdof_rank == subdomain_rank[ip]) continue;
-         send_count[tdof_rank] +=m;
-      }
-   }
-
-   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
-
-   for (int k=0; k<num_procs-1; k++)
-   {
-      send_displ[k+1] = send_displ[k] + send_count[k];
-      recv_displ[k+1] = recv_displ[k] + recv_count[k];
-   }
-   sbuff_size = send_count.Sum();
-   rbuff_size = recv_count.Sum();
-
-   Array<double> sendbuf(sbuff_size); sendbuf = 0;
-   Array<int> soffs(num_procs); soffs = 0;
-
-   for (int ip = 0; ip < nrsubdomains; ip++)
-   {
-      if (myid != subdomain_rank[ip]) continue;
-      int ndofs = SubdomainGTrueDofs[ip].Size();
-      // loop through dofs
-      for (int i=0; i<ndofs; i++)
-      {
-         //  pick up the dof and find its tdof_rank
-         int tdof = SubdomainGTrueDofs[ip][i];
-         int tdof_rank = get_rank(tdof,tdof_offsets);
-         // offset
-         if (tdof_rank == subdomain_rank[ip]) continue;
-         int k = send_displ[tdof_rank] + soffs[tdof_rank];
-         soffs[tdof_rank] +=m;
-         sendbuf[k] = (*x[ip])[i];
-         if (CompFlag)
-         {
-            sendbuf[k+1] = (*x[ip])[i+ndofs];
-         }
-      }
-   }
-
-   Array<double> recvbuf(rbuff_size);
-   Array<int> roffs(num_procs); roffs = 0;
-   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_DOUBLE, recvbuf,
-                 recv_count, recv_displ, MPI_DOUBLE, comm);
-
-   for (int ip = 0; ip < nrsubdomains; ip++)
-   {
-      if (myid == subdomain_rank[ip]) 
-      {
-         int ndofs = SubdomainGTrueDofs[ip].Size();
-         for (int i = 0; i<ndofs; i++)
-         {
-            int tdof = SubdomainGTrueDofs[ip][i];
-            int k = tdof - mytoffset;
-            if (k<0 || k>=pfes->GetTrueVSize()) continue;
-            y[k] += tdof_sign[k] * (*x[ip])[i];
-            if (CompFlag)
-            {  
-               int gtsize = pfes->GetTrueVSize();
-               y[k+gtsize] += tdof_sign[k]*(*x[ip])[i+ndofs];
-            }
-         }
-      }
-      else
-      {
-         int ndofs = SubdomainLTrueDofs[ip].Size();
-         for (int i = 0; i<ndofs; i++)
-         {
-            int tdof = SubdomainLTrueDofs[ip][i];
-            int k = tdof - mytoffset;
-            int j = recv_displ[subdomain_rank[ip]] + roffs[subdomain_rank[ip]];
-            roffs[subdomain_rank[ip]] +=m;
-            y[k] += tdof_sign[k] * recvbuf[j];
-            if (CompFlag)
-            {
-               int tsize = pfes->GetTrueVSize();
-               y[k+tsize] += tdof_sign[k]*recvbuf[j+1];
-            }
-         }
-      }
-   }              
-}
-
-void DofMaps::TestSubdomainToGlobalMaps()
-{
-   cout << "Testing Subdomain To Global Maps" << endl;
-   FunctionCoefficient c1(testcoeff);
-   Array<Vector*> x(nrsubdomains);
-   Vector y(pfes->GetTrueVSize()); y = 0.0;
-   for (int i = 0 ; i<nrsubdomains; i++)
-   {
-      if (myid != subdomain_rank[i]) continue;
-      x[i] = new Vector(fes[i]->GetTrueVSize());
-      GridFunction gf(fes[i]);
-      gf = 0.0;
-
-      if (i==3) gf.ProjectCoefficient(c1);
-      *x[i] = gf;
-   }
-
-   SubdomainsToGlobal(x,y);
-
-   // cout << "1: myid = " <<  myid << ", y = "; y.Print();
-
-   string keys = (dim==2) ? "keys amrRljc\n": "keys m\n";
-   ParGridFunction pgf(pfes);
-
-   const Operator &P = *pfes->GetProlongationMatrix();
-   P.Mult(y, pgf);
-   
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock(vishost, visport);
-   sol_sock.precision(8);
-   sol_sock << "parallel " << num_procs << " " << myid << "\n"
-               << "solution\n" << *pfes->GetParMesh() << pgf 
-               << keys << flush;
-
-   ParGridFunction pgf1(pfes);
-   pgf1.ProjectCoefficient(c1);
-   Vector y1(pfes->GetTrueVSize());
-   const SparseMatrix * R = pfes->GetRestrictionMatrix();
-
-   R->Mult(pgf1,y1);
-   // P.MultTranspose(pgf1,y1);
-   Array<Vector*> x1;
-   GlobalToSubdomains(y1,x1);
-
-
-   // for (int i = 0 ; i<nrsubdomains; i++)
-   // {
-   //    if (myid != subdomain_rank[i]) continue;
-   //    ostringstream mesh_name;
-   //    mesh_name << "output/mesh." << setfill('0') << setw(6) << i;
-   //    ofstream mesh_ofs(mesh_name.str().c_str());
-   //    mesh_ofs.precision(8);
-   //    fes[i]->GetMesh()->Print(mesh_ofs);
-   //    GridFunction gf(fes[i]);
-   //    gf = x1[i];
-   //    ostringstream gf_name;
-   //    gf_name << "output/gf." << setfill('0') << setw(6) << i;
-   //    ofstream gf_ofs(gf_name.str().c_str());
-   //    gf_ofs.precision(8);
-   //    gf.Save(gf_ofs);
-   // }
-
-
-
-   int nrsub = nrsubdomains;
-   for (int i = 0 ; i<nrsub; i++)
-   {
-      if (myid == subdomain_rank[i])
-      {
-         socketstream sol_sock1(vishost, visport);
-         sol_sock1.precision(8);
-         sol_sock1 << "parallel " << nrsub << " " << i << "\n";
-         GridFunction gf(fes[i]);
-         GridFunction gf1(fes[i]);
-         gf1.ProjectCoefficient(c1);
-         gf = *x1[i];
-         gf1-=gf;
-         cout << "ip, Diff norm = " <<i<<", " << gf1.Norml2() << endl;
-         sol_sock1 << "solution\n" << *fes[i]->GetMesh() << gf
-                   << keys << flush;
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-   }
-
-   socketstream gf_sock(vishost, visport);
-   gf_sock.precision(8);
-   gf_sock << "parallel " << num_procs << " " << myid << "\n"
-           << "solution\n" << *pfes->GetParMesh() << pgf1 
-           << keys << flush;
-}
-
-
-DofMaps::~DofMaps()
-{
-   for (int i = 0; i<nrsubdomains; i++)
-   {
-      delete fes[i];
-   }
-}

examples/maxwell-solver-dev/ParDST/DofMapsDST.hpp

@@ -1,109 +0,0 @@
-#pragma once
-#include "../common/Utilities.hpp"
-#include "../common/PML.hpp"
-using namespace std;
-using namespace mfem;
-
-double testcoeff(const Vector & x);
-int get_rank(int tdof, std::vector<int> & tdof_offsets);
-
-
-void ComputeTdofOffsets(const MPI_Comm & comm, const ParFiniteElementSpace * pfes, 
-                        std::vector<int> & tdof_offsets);
-
-void GetSubdomainijk(int ip, const Array<int> nxyz, Array<int> & ijk);
-void GetDirectionijk(int id, Array<int> & ijk);
-int GetSubdomainId(const Array<int> nxyz, Array<int> & ijk);
-int GetDirectionId(const Array<int> & ijk);
-
-
-// class handling two types of dof maps
-// 1. Subdomain truedofs ---> Global truedofs
-// 2. Subdomain truedofs ---> Neighbor truedofs 
-class DofMaps
-{
-private:
-   // The FE space of the problem (H1/Hcurl)
-   ParFiniteElementSpace *pfes = nullptr;
-
-   // The given partition of the parmesh
-   ParMeshPartition *part = nullptr;
-   // partition in x-y-z
-   Array<int> nxyz;
-
-   // MPI parameters
-   MPI_Comm comm = MPI_COMM_WORLD;
-   int num_procs, myid;
-
-   // true dof offset and element offset of the processor
-   vector<int> tdof_offsets;
-   int mytoffset;
-   int myelemoffset;
-   
-   int dim;
-   // Total number of subdomains
-   int nrsubdomains;
-   
-   // Array specifying the subdomain rank
-   Array<int> subdomain_rank;
-
-   // Complex flag
-   bool CompFlag;
-
-   // sign factors
-   Array<int> tdof_sign;
-   // Initializing mpi and helper parameters
-   void Init();
-
-   // 1. Setting up the subdomains FE spaces
-   // 2. Setting up the subdomains-to-subdomains maps
-   // 3. Setting up the subdomain-to-global maps
-   void Setup();
-
-   // -----------------------------------------------
-   //           Subdomain to Subdomain maps
-   // -----------------------------------------------
-   std::vector<std::vector<Array<int>>> OvlpElems;
-   void AddElementToOvlpLists(int l, int iel, 
-                            const Array<bool> & neg, 
-                            const Array<bool> & pos);
-   std::vector<std::vector<Array<int>>> OvlpTDofs;
-   void SubdomainToSubdomainMapsSetup();
-   void ComputeOvlpElems();
-   void ComputeOvlpTdofs();
-   void PrintOvlpTdofs();
-
-   // -----------------------------------------------
-   //           Subdomain to Global maps
-   // -----------------------------------------------
-   std::vector<Array<int>> SubdomainGTrueDofs; // Subdomain Tdofs to Global Tdofs
-   std::vector<Array<int>> SubdomainLTrueDofs; // Subdomain Tdofs to Local (on rank) Tdofs
-
-   Array<int> send_count, send_displ;  
-   Array<int> recv_count, recv_displ;  
-   int sbuff_size = 0;
-   int rbuff_size = 0;
-   void SubdomainToGlobalMapsSetup();
-
-   // Testing
-   void TestSubdomainToGlobalMaps();
-   void TestSubdomainToSubdomainMaps();
-
-public:
-   // constructor
-
-   // FiniteElementSpaces of the subdomains
-   Array<FiniteElementSpace *> fes;
-
-   DofMaps(ParFiniteElementSpace *fespace_, ParMeshPartition * part_, bool CompFlag_ = false);
-   ~DofMaps();
-   // Transfering from subdomains SubdomainIds to all their neighbors
-   void TransferToNeighbors(const Array<int> & SubdomainIds, const Array<Vector *> & x, 
-   std::vector<std::vector<Vector * >> & OvlpSol);
-
-   // Prolongation of subdomain solutions to the global solution
-   void SubdomainsToGlobal(const Array<Vector*> & x, Vector & y);
-   // Restriction of global residual to subdomain residuals
-   // bool comp: true for complex valued problems
-   void GlobalToSubdomains(const Vector & y, Array<Vector*> & x);
-};

examples/maxwell-solver-dev/ParDST/ParDST.cpp

@@ -1,849 +0,0 @@
-//Parallel Diagonal Source Transfer Preconditioner
-
-#include "ParDST.hpp"
-
-ParDST::ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
-         double omega_, Coefficient * Q_,  int nrlayers_ , int nx_, int ny_, int nz_)
-   : Solver(2*bf_->ParFESpace()->GetTrueVSize(), 2*bf_->ParFESpace()->GetTrueVSize()), 
-     bf(bf_), Pmllength(Pmllength_), omega(omega_), 
-     Q(Q_), nrlayers(nrlayers_)
-{
-   nx = nx_; ny = ny_; nz = nz_;
-   Init();
-}
-ParDST::ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
-         double omega_, VectorCoefficient * VQ_,  int nrlayers_ , int nx_, int ny_, int nz_)
-   : Solver(2*bf_->ParFESpace()->GetTrueVSize(), 2*bf_->ParFESpace()->GetTrueVSize()), 
-     bf(bf_), Pmllength(Pmllength_), omega(omega_), 
-     VQ(VQ_), nrlayers(nrlayers_)
-{
-   nx = nx_; ny = ny_; nz = nz_;
-   Init();
-}
-ParDST::ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
-         double omega_, MatrixCoefficient * MQ_,  int nrlayers_ , int nx_, int ny_, int nz_)
-   : Solver(2*bf_->ParFESpace()->GetTrueVSize(), 2*bf_->ParFESpace()->GetTrueVSize()), 
-     bf(bf_), Pmllength(Pmllength_), omega(omega_), 
-     MQ(MQ_), nrlayers(nrlayers_)
-{
-   nx = nx_; ny = ny_; nz = nz_;
-   Init();
-}
-
-void ParDST::Init()
-{
-   pfes = bf->ParFESpace();
-   fec = pfes->FEColl();
- 
-   comm = pfes->GetComm();
-   MPI_Comm_size(comm, &num_procs);
-   MPI_Comm_rank(comm, &myid);
-
-   //1. Indentify problem ... Helmholtz or Maxwell
-   prob_kind = fec->GetContType();
-   if (myid == 0)
-   {
-      cout << " 1. Indentify problem to be solved ... " << endl;
-      if (prob_kind == 0) cout << "    Helmholtz" << endl;
-      if (prob_kind == 1) cout << "    Maxwell" << endl; 
-   }
-
-   //2. Create the parallel mesh partition
-   pmesh = pfes->GetParMesh();
-   dim = pmesh->Dimension();
-   if (myid == 0)
-   {
-      cout << "\n 2. Generating ParMesh partitioning ... " << endl;
-   }
-   ovlpnrlayers = nrlayers+1;
-   part = new ParMeshPartition(pmesh,nx,ny,nz,ovlpnrlayers);
-   nxyz.SetSize(3);
-   nxyz[0] = nx = part->nxyz[0]; 
-   nxyz[1] = ny = part->nxyz[1];  
-   nxyz[2] = nz = part->nxyz[2];
-
-   nrsubdomains = part->nrsubdomains;
-   SubdomainRank = part->subdomain_rank;
-
-   for (int ip = 0; ip<nrsubdomains; ip++)
-   {
-      if (myid == SubdomainRank[ip])
-      {
-         RankSubdomains.Append(ip);
-      }
-   }
-
-   cout << " myid: " << myid 
-        << ", nrsubdomains: " << RankSubdomains.Size() << endl;
-
-   MPI_Barrier(comm);
-
-   if (myid == 0)
-   {
-      cout << "    Done ! " << endl;
-   }
-   //3. Setup info for sweeps
-   if (myid == 0)
-   {
-      cout << "\n 3. Computing sweeps info ..." << endl; 
-   }
-   sweeps = new Sweep(dim);
-   if (myid == 0)
-   {
-      cout << "    Done ! " << endl;
-   }
-   //4. Create LocalToGlobal maps 
-   //   (local GridFunctions/Vector to Global ParGridFunction/Vector) 
-   if (myid == 0)
-   {
-      cout << "\n 4. Computing true dofs maps ..." << endl; 
-   }
-
-   // if (myid == SubdomainRank[0])
-   // {
-   //    cout << "myid = " << myid << endl;
-   //    char vishost[] = "localhost";
-   //    int visport = 19916;
-   //    socketstream mesh_sock1(vishost, visport);
-   //    mesh_sock1.precision(8);
-   //    mesh_sock1 << "mesh\n"
-   //               << *part->subdomain_mesh[0] << "window_title 'Subdomain'" << flush;
-   //    part->subdomain_mesh[0]->Print();
-   
-   // }
-   bool comp = true;
-
-   dmaps = new DofMaps(pfes,part, comp);
-   
-   if (myid == 0)
-   {
-      cout << "    Done ! " << endl;
-   }
-   // 4. Setting up the local problems 
-   if (myid == 0)
-   {
-      cout << "\n 5. Setting up the subdomain problems ..." << endl; 
-   }
-
-   SetupSubdomainProblems();
-   
-   if (myid == 0)
-   {
-      cout << "    Done ! " << endl;
-   }
-   if (myid == 0)
-   {
-      cout << "\n 6. Mark subdomain overlap truedofs ..." << endl; 
-   }
-   MarkSubdomainOverlapDofs(comp);
-   if (myid == 0)
-   {
-      cout << "    Done ! " << endl;
-   }
-}
-
-void ParDST::Mult(const Vector &r, Vector &z) const
-{
-   // Initialize transfered residuals to 0.0;
-   for (int ip=0; ip<nrsubdomains; ip++)
-   {
-      if (myid != SubdomainRank[ip]) continue;
-      for (int i=0;i<sweeps->nsweeps; i++)
-      {
-         *f_transf[ip][i] = 0.0;
-      }
-   }
-
-   // restrict given residual to subdomains
-   dmaps->GlobalToSubdomains(r,f_orig);
-
-   for (int ip=0; ip<nrsubdomains; ip++)
-   {
-      if (myid != SubdomainRank[ip]) continue;
-      Array<int> ijk(3);
-      GetSubdomainijk(ip,nxyz,ijk);
-      Array2D<int> direct(dim,2); direct = 0;
-      for (int d=0;d<dim; d++)
-      {
-         if (ijk[d] > 0) direct[d][0] = 1; 
-         if (ijk[d] < part->nxyz[d]-1) direct[d][1] = 1; 
-      }
-      GetChiRes(*f_orig[ip],ip,direct);
-   }
-
-   z = 0.0; 
-   int nsteps;
-   switch(dim)
-   {
-      case 1: nsteps = nx; break;
-      case 2: nsteps = nx+ny-1; break;
-      default: nsteps = nx+ny+nz-2; break;
-   }
-   int nsweeps = sweeps->nsweeps;
-   // 1. Loop through sweeps
-   if (dim == 3 && nz == 1) { nsweeps = 4; } // x-y partition only;
-   for (int l=0; l<nsweeps; l++)
-   {  
-      // 2. loop through diagonals/steps of each sweep   
-      for (int s = 0; s<nsteps; s++)
-      {
-         Array2D<int> subdomains;
-         GetStepSubdomains(l,s,subdomains);
-         int nsubdomains = subdomains.NumRows();
-
-         // 3. Loop through the subdomains on the diagonal
-         Array<int> subdomain_ids;
-         for (int sb=0; sb < nsubdomains; sb++)
-         {
-            Array<int> ijk(dim); ijk = 0;
-            for (int d=0; d<dim; d++) ijk[d] = subdomains[sb][d]; 
-            int ip = GetSubdomainId(nxyz,ijk);
-            subdomain_ids.Append(ip);
-            if (myid != SubdomainRank[ip]) continue;
-
-            int n = dmaps->fes[ip]->GetTrueVSize();
-            Vector res_local(2*n); res_local = 0.0;
-
-            if (l==0)  { res_local += *f_orig[ip]; }
-            res_local += *f_transf[ip][l];
-            if (res_local.Norml2() < 1e-12) 
-            {
-               *subdomain_sol[ip] = 0.0;
-               continue;
-            }
-            PmlMatInv[ip]->Mult(res_local, *subdomain_sol[ip]);
-         }
-         // 4. Transfer solutions to neighbors so that the subdomain
-         // residuals are updated
-         TransferSources(l,subdomain_ids);
-      }
-      // 5. Update the global solution 
-      dmaps->SubdomainsToGlobal(subdomain_sol,z);
-   }
-}
-
-void ParDST::SetupSubdomainProblems()
-{
-   sqf.SetSize(nrsubdomains);
-   Optr.SetSize(nrsubdomains);
-   PmlMat.SetSize(nrsubdomains);
-   PmlMatInv.SetSize(nrsubdomains);
-   f_orig.SetSize(nrsubdomains);
-   f_transf.SetSize(nrsubdomains);
-   subdomain_sol.SetSize(nrsubdomains);
-   for (int ip=0; ip<nrsubdomains; ip++)
-   {
-      sqf[ip] = nullptr;
-      f_orig[ip] = nullptr;
-      subdomain_sol[ip] = nullptr;
-      PmlMat[ip] = nullptr;
-      PmlMatInv[ip] = nullptr;
-      Optr[ip] = nullptr;
-
-      if (myid != SubdomainRank[ip]) continue;
-      subdomain_sol[ip] = new Vector(2*dmaps->fes[ip]->GetTrueVSize());
-      if (prob_kind == 0)
-      {
-         SetHelmholtzPmlSystemMatrix(ip);
-      }
-      else if (prob_kind == 1)
-      {
-         SetMaxwellPmlSystemMatrix(ip);
-      }
-      PmlMat[ip] = Optr[ip]->As<ComplexSparseMatrix>();
-
-      PmlMatInv[ip] = new ComplexUMFPackSolver;
-      PmlMatInv[ip]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
-
-      // HYPRE_Int rowstarts[2]; rowstarts[0] = 0;
-      // rowstarts[1] = dmaps->fes[ip]->GetTrueVSize();
-      // HypreParMatrix * HypreMat_r =
-      //    new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
-      //                       &(PmlMat[ip]->real()));
-      // HypreParMatrix * HypreMat_i =
-      //    new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
-      //                       &(PmlMat[ip]->imag()));
-      // ComplexHypreParMatrix * HypreMat =
-      //    new ComplexHypreParMatrix(HypreMat_r,HypreMat_i,true,true);
-      // PmlMatInv[ip] = new ComplexMUMPSSolver;
-      // PmlMatInv[ip]->SetOperator(*HypreMat);
-      // delete HypreMat;
-      int ndofs = dmaps->fes[ip]->GetTrueVSize();
-      f_transf[ip].SetSize(sweeps->nsweeps);
-      for (int i=0;i<sweeps->nsweeps; i++)
-      {
-         f_transf[ip][i] = new Vector(2*ndofs);
-      }
-   }
-}
-
-
-
-void ParDST::SetHelmholtzPmlSystemMatrix(int ip)
-{
-   MFEM_VERIFY(part->subdomain_mesh[ip], "Null mesh pointer");
-   Mesh * mesh = part->subdomain_mesh[ip];
-   double h = part->MeshSize;
-   Array2D<double> length(dim,2);
-   length = h*(nrlayers);
-
-   Array<int> ijk;
-   GetSubdomainijk(ip,nxyz,ijk);
-   int i = ijk[0];
-   int j = ijk[1];
-   int k = ijk[2];
-
-   if (i == 0 )    length[0][0] = Pmllength[0][0];
-   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
-   if (dim  > 1)
-   {
-      if (j == 0 )    length[1][0] = Pmllength[1][0];
-      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
-   }
-   if (dim  == 3)
-   {
-      if (k == 0 )    length[2][0] = Pmllength[2][0];
-      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
-   }
-   
-   CartesianPML pml(mesh, length);
-   pml.SetOmega(omega);
-
-   Array <int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      dmaps->fes[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient sigma(-pow(omega, 2));
-   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
-   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
-   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
-   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
-   ProductCoefficient c2_re0(sigma, detJ_re);
-   ProductCoefficient c2_im0(sigma, detJ_im);
-   ProductCoefficient c2_re(c2_re0, *Q);
-   ProductCoefficient c2_im(c2_im0, *Q);
-   sqf[ip] = new SesquilinearForm (dmaps->fes[ip],bf->GetConvention());
-
-   sqf[ip]->AddDomainIntegrator(new DiffusionIntegrator(c1_re),
-                         new DiffusionIntegrator(c1_im));
-   sqf[ip]->AddDomainIntegrator(new MassIntegrator(c2_re),
-                         new MassIntegrator(c2_im));
-   sqf[ip]->Assemble(0);
-
-   Optr[ip] = new OperatorPtr;
-   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
-}
-
-void ParDST::SetMaxwellPmlSystemMatrix(int ip)
-{
-   MFEM_VERIFY(part->subdomain_mesh[ip], "Null mesh pointer");
-   Mesh * mesh = part->subdomain_mesh[ip];
-   double h = part->MeshSize;
-   Array2D<double> length(dim,2);
-   length = h*(nrlayers);
-
-   Array<int> ijk;
-   GetSubdomainijk(ip,nxyz,ijk);
-   int i = ijk[0];
-   int j = ijk[1];
-   int k = ijk[2];
-
-   if (i == 0 )    length[0][0] = Pmllength[0][0];
-   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
-   if (dim  > 1)
-   {
-      if (j == 0 )    length[1][0] = Pmllength[1][0];
-      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
-   }
-   if (dim  == 3)
-   {
-      if (k == 0 )    length[2][0] = Pmllength[2][0];
-      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
-   }
-   
-   CartesianPML pml(mesh, length);
-   pml.SetOmega(omega);
-   Array <int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      dmaps->fes[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   ConstantCoefficient omeg(-pow(omega, 2));
-   int cdim = (dim == 2) ? 1 : dim;
-
-   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
-   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
-
-   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
-   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
-   ScalarMatrixProductCoefficient c2_Re0(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im0(omeg,pml_c2_Im);
-
-   MatrixCoefficient * c2_Re=nullptr;
-   MatrixCoefficient * c2_Im=nullptr;
-
-   if (Q)
-   {
-      c2_Re = new ScalarMatrixProductCoefficient(*Q,c2_Re0);
-      c2_Im = new ScalarMatrixProductCoefficient(*Q,c2_Im0);
-   }
-   else if (VQ)
-   {
-      MFEM_ABORT("Vector Coeffiecient not supported ");
-   }
-   else if (MQ)
-   {
-      c2_Re = new MatrixMatrixProductCoefficient(c2_Re0,*MQ);
-      c2_Im = new MatrixMatrixProductCoefficient(c2_Im0,*MQ);
-   }
-   
-   sqf[ip] = new SesquilinearForm(dmaps->fes[ip],bf->GetConvention());
-
-   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(pml_c1_Re),
-                         new CurlCurlIntegrator(pml_c1_Im));
-   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(*c2_Re),
-                         new VectorFEMassIntegrator(*c2_Im));
-   sqf[ip]->Assemble(0);
-
-   Optr[ip] = new OperatorPtr;
-   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
-   delete c2_Re;
-   delete c2_Im;
-}
-
-
-void ParDST::MarkSubdomainOverlapDofs(const bool comp)
-{
-   // First mark the elements
-         // cout<< "Compute Overlap Elements (in each possible direction) " << endl;
-      // Lists of elements
-      // x,y,z = +/- 1 ovlp 
-   NovlpElems.resize(nrsubdomains);
-
-   for (int ip = 0; ip<nrsubdomains; ip++)
-   {
-      if (myid != SubdomainRank[ip]) continue;
-      Array<int> ijk;
-      GetSubdomainijk(ip,nxyz,ijk);
-
-      Mesh * mesh = dmaps->fes[ip]->GetMesh();
-      NovlpElems[ip].resize(2*dim);
-
-      Vector pmin, pmax;
-      mesh->GetBoundingBox(pmin,pmax);
-      double h = part->MeshSize;
-      // Loop through elements
-      for (int iel=0; iel<mesh->GetNE(); iel++)
-      {
-         // Get element center
-         Vector center(dim);
-         int geom = mesh->GetElementBaseGeometry(iel);
-         ElementTransformation * tr = mesh->GetElementTransformation(iel);
-         tr->Transform(Geometries.GetCenter(geom),center);
-
-         // Assign elements to the appropriate lists
-         for (int d=0;d<dim; d++)
-         {
-            if (ijk[d]>0)
-            {
-               if (center[d] >= pmin[d]+h*ovlpnrlayers) 
-               {
-                  NovlpElems[ip][d].Append(iel);
-               }
-            }
-            else
-            {
-               NovlpElems[ip][d].Append(iel);
-            }
-               
-            if (ijk[d]<nxyz[d]-1)
-            {
-               if (center[d] <= pmax[d]-h*ovlpnrlayers) 
-               {
-                  NovlpElems[ip][dim+d].Append(iel);
-               }
-            }
-            else
-            {
-               NovlpElems[ip][dim+d].Append(iel);
-            }
-         }
-      }
-   }
-
-      // mark dofs
-   NovlpDofs.resize(nrsubdomains);
-   int mm = (comp) ? 2 : 1; // complex or real valued
-   for (int ip = 0; ip<nrsubdomains; ip++)
-   {
-      if (myid != SubdomainRank[ip]) continue;
-      FiniteElementSpace * fes = dmaps->fes[ip];
-      // Loop through the marked elements
-      NovlpDofs[ip].resize(2*dim);
-      int n = fes->GetTrueVSize();
-      Array<int> marker(n);
-      for (int d=0;d<2*dim; d++)
-      {
-         marker = 0;
-         int m = 0;
-         int melems = NovlpElems[ip][d].Size(); 
-         for (int iel=0; iel<melems; iel++)
-         {
-            Array<int> ElemDofs;
-            int el =  NovlpElems[ip][d][iel];
-            fes->GetElementDofs(el,ElemDofs);
-            int ndof = ElemDofs.Size();
-            for (int i = 0; i<ndof; ++i)
-            {
-               int eldof = ElemDofs[i];
-               int tdof = (eldof >= 0) ? eldof : abs(eldof) - 1; 
-               if (marker[tdof] == 1) continue;
-               marker[tdof] = 1;
-               m++;
-            }
-         }
-         int k = mm*(n-m);
-         NovlpDofs[ip][d].SetSize(k);
-         int l = 0;
-         for (int i = 0; i<n; i++)
-         {
-            if (marker[i]==0) 
-            {
-               NovlpDofs[ip][d][l] = i;  // real dofs
-               if (comp) 
-               {
-                  NovlpDofs[ip][d][l+k/2] = i+fes->GetTrueVSize();
-               }
-               l++;
-            }
-         }
-      }
-   }
-}
-
-void ParDST::GetChiRes(Vector & res, int ip, Array2D<int> direct) const
-{
-   for (int d=0; d<dim; d++)
-   {
-      // negative direction
-      if (direct[d][0]==1) res.SetSubVector(NovlpDofs[ip][d],0.0);
-      // possitive direction
-      if (direct[d][1]==1) res.SetSubVector(NovlpDofs[ip][d+dim],0.0);
-   }
-}
-
-
-
-void ParDST::PlotLocal(Vector & sol, socketstream & sol_sock, int ip) const
-{
-   FiniteElementSpace * fes = dmaps->fes[ip];
-   Mesh * mesh = fes->GetMesh();
-   GridFunction gf(fes);
-   double * data = sol.GetData();
-   gf.SetData(data);
-   
-   string keys;
-   keys = "keys mrRljc\n";
-   sol_sock << "solution\n" << *mesh << gf << keys << flush;
-}
-
-
-void ParDST::GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const
-{
-   Array<int> aux;
-   switch(dim)
-   {
-      case 2: 
-      for (int i=nx-1;i>=0; i--)
-      {  
-         int j;
-         switch (sweep)
-         {
-            case 0:  j = step-i;         break;
-            case 1:  j = step-nx+i+1;    break;
-            case 2:  j = nx+i-step-1;    break;
-            default: j = nx+ny-i-step-2; break;
-         }
-         if (j<0 || j>=ny) continue;
-         aux.Append(i); aux.Append(j);
-      }
-      break; 
-      default:
-      for (int i=nx-1;i>=0; i--)
-      {  
-         for (int j=ny-1;j>=0; j--)
-         {
-            int k;
-            switch (sweep)
-            {
-               case 0:  k = step-i-j;            break;
-               case 1:  k = step-nx+i+1-j;       break;
-               case 2:  k = step-ny+j+1-i;       break;
-               case 3:  k = step-nx-ny+i+j+2;    break;
-               case 4:  k = i+j+nz-1-step;       break;
-               case 5:  k = nx+nz-i+j-step-2;    break;
-               case 6:  k = ny+nz+i-j-step-2;    break;
-               default: k = nx+ny+nz-i-j-step-3; break;
-            }
-            if (k<0 || k>=nz) continue;   
-            aux.Append(i); aux.Append(j); aux.Append(k); 
-         }
-      }
-      break;
-   }
-
-   int nrows = aux.Size()/dim;
-   int ncols = dim;
-
-   subdomains.SetSize(nrows,ncols);
-   for (int r=0;r<nrows; r++)
-   {
-      for (int c=0; c<ncols; c++)
-      {
-         int k = r*ncols + c;
-         subdomains[r][c] = aux[k];
-      }
-   }
-}
-
-
-void ParDST::TransferSources(int sweep, const Array<int> & subdomain_ids) const
-{
-   OvlpSol.resize(nrsubdomains);
-   int nrneighbors = pow(3,dim);
-   for (int ip = 0; ip<nrsubdomains; ip++)
-   {
-      if (myid == SubdomainRank[ip])
-      {
-         OvlpSol[ip].resize(nrneighbors);
-      }
-   }
-   int m = subdomain_ids.Size();
-   Array<Vector *> x(m);
-   for (int i = 0; i<m; i++)
-   {
-      x[i] = nullptr;
-      int ip = subdomain_ids[i];
-      if (myid != SubdomainRank[ip]) continue;
-      x[i] = new Vector(subdomain_sol[ip]->GetData(),subdomain_sol[ip]->Size());
-   }
-   dmaps->TransferToNeighbors(subdomain_ids,x,OvlpSol);
-   for (int i = 0; i<m; i++)
-   {
-      delete x[i]; x[i] = nullptr;
-   }
-   // Update residuals
-//   Find all neighbors of patch ip0
-   for (int is = 0; is<m; is++)
-   {
-      int ip0 = subdomain_ids[is];
-      Array<int> ijk;
-      Array<int> ijk1(3);
-      GetSubdomainijk(ip0,nxyz,ijk);
-      Array<int> directions(3);   
-      for (int i=-1; i<2; i++)
-      {
-         int i1 = ijk[0] + i;
-         if (i1 <0 || i1>=nx) continue;
-         directions[0] = i;
-         ijk1[0] = i1;
-         for (int j=-1; j<2; j++)
-         {
-            int j1 = ijk[1] + j;
-            if (j1 <0 || j1>=ny) continue;
-            directions[1] = j;
-            ijk1[1] = j1;
-            int kbeg = (dim == 2) ? 0 : -1;
-            int kend = (dim == 2) ? 1 :  2;
-            for (int k=kbeg; k<kend; k++)
-            {
-               int k1 = ijk[2] + k;
-               if (k1 <0 || k1>=nz) continue;
-               directions[2] = (dim == 3) ? k : -1 ;
-               if (i==0 && j==0 && k==0) continue;
-
-               int l = GetSweepToTransfer(sweep,directions);
-               if (l == -1) continue;
-               ijk1[2] = k1;
-               int ip1 = GetSubdomainId(nxyz,ijk1);
-
-               if (myid != SubdomainRank[ip1]) continue;
-               Array<int>directions1(3); directions1 = -1;
-               for (int i = 0; i<dim; i++) directions1[i] = -directions[i];
-               int dir = GetDirectionId(directions1);
-               int n = dmaps->fes[ip1]->GetTrueVSize();
-               Vector res(2*n);
-               PmlMat[ip1]->Mult(*OvlpSol[ip1][dir],res);
-
-               Array2D<int> direct(dim,2); direct = 0;
-               for (int d = 0; d<dim; d++)
-               {
-                  if (directions[d]==1)  direct[d][0] = 1;
-                  if (directions[d]==-1) direct[d][1] = 1;
-               }   
-               GetChiRes(res,ip1,direct);
-               *f_transf[ip1][l] -= res;
-            }
-         }  
-      }
-   }
-
-   for (int ip = 0; ip<nrsubdomains; ip++)
-   {
-      if (myid == SubdomainRank[ip])
-      {
-         for (int i = 0; i<nrneighbors; i++)
-         {
-            if (OvlpSol[ip][i])
-            {
-               delete OvlpSol[ip][i];
-            }
-         }
-         OvlpSol[ip].clear();
-      }
-   }
-}
-
-int ParDST::GetSweepToTransfer(const int s, Array<int> directions) const
-{
-   int l1=-1;
-   int nsweeps = sweeps->nsweeps;
-   Array<int> sweep0;   
-   sweeps->GetSweep(s,sweep0);
-   switch (dim)
-   {
-      case 2:
-      for (int l=s; l<nsweeps; l++)
-      {
-         // Rule 1: the transfer source direction has to be similar with 
-         // the sweep direction
-         Array<int> sweep1;   
-         sweeps->GetSweep(l,sweep1);
-         int ddot = 0;
-         for (int d=0; d<dim; d++) ddot+= sweep1[d] * directions[d];
-         if (ddot <= 0) continue;
-
-         // Rule 2: The horizontal or vertical transfer source cannot be used
-         // Case of horizontal or vertical transfer source 
-         // (it can't be both 0 cause it's skipped)
-         if (directions[0]==0 || directions[1] == 0) 
-         {
-            if (sweep0[0] == -sweep1[0] && sweep0[1] == -sweep1[1]) continue;
-         }
-         l1 = l;
-         break;
-      }
-      break;
-      default:
-      for (int l=s; l<nsweeps; l++)
-      {
-         // Rule 1: (similar directions) the transfer source direction has to be similar with 
-         // the sweep direction
-         Array<int> sweep1;   
-         sweeps->GetSweep(l,sweep1);
-         int ddot = 0;
-         bool similar = true;
-         for (int d=0; d<dim; d++) 
-         {
-            if (sweep1[d] * directions[d] < 0) similar = false;
-            ddot+= sweep1[d] * directions[d];
-         }
-         if (!similar || ddot<=0) continue; // not similar
-
-         // Rule 2: (oposite directions) the transfer source direction has to be similar with 
-         // the sweep direction
-         // 
-         // check any of the projections onto the planes
-         // (xy, xz, yz)
-
-         if ( (directions[0]==0 && directions[1] != 0) || 
-              (directions[0]!=0 && directions[1] == 0) ||
-              (directions[0]==0 && directions[2] != 0) || 
-              (directions[0]!=0 && directions[2] == 0) ||
-              (directions[2]==0 && directions[1] != 0) || 
-              (directions[2]!=0 && directions[1] == 0) ) 
-         {
-            if (sweep0[0] == -sweep1[0] && 
-                sweep0[1] == -sweep1[1] && 
-                sweep0[2] == -sweep1[2]) continue;
-         }
-         l1 = l;
-         break;
-      }
-      break;
-   }
-
-   return l1;   
-}
-
-void ParDST::CorrectOrientation(int ip,Vector &x) const
-{
-   FiniteElementSpace * fespace = dmaps->fes[ip];
-   Mesh * mesh = fespace->GetMesh();
-   int nrelems = mesh->GetNE();
-   // GridFunction test; 
-   // test.SetFromTrueDofs(x)
-   Array<int> signs(fespace->GetTrueVSize()); signs = 0;
-   for (int iel=0; iel<nrelems; iel++)
-   {
-      Array<int> ElemDofs;
-      fespace->GetElementDofs(iel,ElemDofs);
-      int ndofs = ElemDofs.Size();
-      ElemDofs.Print();
-      for (int i = 0; i< ndofs; i++)
-      {
-         int pdof_ = ElemDofs[i];
-         if (pdof_ < 0) 
-         {
-            signs[abs(pdof_)-1] += 1.0 ; 
-         }
-         else
-         {
-            signs[pdof_] -= 1.0 ; 
-         }
-      }
-   }
-
-   cout << "signs = " ; signs.Print();
-   for (int i = 0; i<fespace->GetTrueVSize(); i++)
-   {
-      if (signs[i]<0) 
-      {
-         x(i) *= -1.0;
-         x(i+fespace->GetTrueVSize()) *= -1.0;
-      }
-   }
-}
-
-
-ParDST::~ParDST()
-{
-
-   for (int ip=0; ip<nrsubdomains; ip++)
-   {
-      delete Optr[ip];
-      delete subdomain_sol[ip];
-      delete PmlMatInv[ip];
-      delete sqf[ip];
-      if (myid != SubdomainRank[ip]) continue;
-      for (int i=0;i<sweeps->nsweeps; i++)
-      {
-         delete f_transf[ip][i];
-      }
-      delete f_orig[ip];
-   }
-   f_orig.DeleteAll();
-   delete dmaps;
-   delete sweeps;
-   delete part;
-
-}

examples/maxwell-solver-dev/ParDST/ParDST.hpp

@@ -1,70 +0,0 @@
-#pragma once
-#include "../common/Utilities.hpp"
-#include "../common/PML.hpp"
-#include "DofMapsDST.hpp"
-using namespace std;
-using namespace mfem;
-
-
-class ParDST : public Solver//
-{
-private:
-   MPI_Comm comm = MPI_COMM_WORLD;
-   int num_procs, myid;
-   // Constructor inputs
-   int prob_kind;
-   ParSesquilinearForm *bf=nullptr;
-   ParFiniteElementSpace * pfes = nullptr;
-   ParMesh * pmesh = nullptr;
-   ParMeshPartition * part = nullptr;
-   Array<int> SubdomainRank;
-   Array<int> RankSubdomains;
-   const FiniteElementCollection * fec = nullptr;
-   Array2D<double> Pmllength;
-   int dim = 2;
-   double omega = 0.5;
-   Coefficient * Q=nullptr;
-   VectorCoefficient * VQ=nullptr;
-   MatrixCoefficient * MQ=nullptr;
-   int nrlayers;
-   int ovlpnrlayers;
-   int nrsubdomains = 0;
-   int nx,ny,nz;
-   Array<int> nxyz;
-   Sweep * sweeps = nullptr;
-   DofMaps * dmaps = nullptr;
-   Array< SesquilinearForm * > sqf;
-   Array< OperatorPtr * > Optr;
-   Array<ComplexSparseMatrix *> PmlMat;
-   Array<ComplexUMFPackSolver *> PmlMatInv;
-//    Array<ComplexMUMPSSolver *> PmlMatInv;
-   mutable Array<Vector *> f_orig;
-   mutable Array<Array<Vector * >> f_transf;
-   mutable Array<Vector * > subdomain_sol;
-   mutable std::vector<std::vector<Vector * >> OvlpSol;
-   void SetupSubdomainProblems();
-   std::vector<std::vector<Array<int>>> NovlpElems;
-   std::vector<std::vector<Array<int>>> NovlpDofs;
-   void MarkSubdomainOverlapDofs(const bool comp = false);
-   void SetHelmholtzPmlSystemMatrix(int ip);   
-   void SetMaxwellPmlSystemMatrix(int ip);
-   void GetChiRes(Vector & res, int ip, Array2D<int> direct) const;
-   void PlotLocal(Vector & sol, socketstream & sol_sock, int ip) const;
-   void GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const;
-   void TransferSources(int sweep, const Array<int> & subdomain_ids) const;
-   int GetSweepToTransfer(const int s, Array<int> directions) const;
-   void CorrectOrientation(int ip, Vector & x) const;
-   void Init();
-public:
-   ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
-       double omega_, Coefficient * Q_, int nrlayers_, int nx_=2, int ny_=2, int nz_=2);
-   ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
-       double omega_, VectorCoefficient * VQ_, int nrlayers_, int nx_=2, int ny_=2, int nz_=2);
-   ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
-       double omega_, MatrixCoefficient * MQ_, int nrlayers_, int nx_=2, int ny_=2, int nz_=2);              
-   virtual void SetOperator(const Operator &op) {}
-   virtual void Mult(const Vector &r, Vector &z) const;
-   virtual ~ParDST();
-};
-
-

examples/maxwell-solver-dev/common/MeshPartition.hpp

@@ -1,151 +0,0 @@
-#pragma once
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-using namespace std;
-using namespace mfem;
-
-
-struct UniqueIndexGenerator
-{
-   int counter = 0;
-   std::unordered_map<int,int> idx;
-   int Get(int i)
-   {
-      std::unordered_map<int,int>::iterator f = idx.find(i);
-      if (f == idx.end())
-      {
-         idx[i] = counter;
-         return counter++;
-      }
-      else
-      {
-         return (*f).second;
-      }
-   }
-   void Reset()
-   {
-      counter = 0;
-      idx.clear();
-   }
-};
-
-
-double GetUniformMeshElementSize(Mesh * mesh);
-Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions);
-
-class CartesianMeshPartition 
-{
-private:
-   Mesh *mesh=nullptr;
-public:
-   int nrpatch;
-   int nxyz[3];
-   double MeshSize;
-   std::vector<Array<int>> element_map;
-   Array3D<int>subdomains;
-   // constructor
-   CartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
-   ~CartesianMeshPartition() {};
-};
-
-class OverlappingCartesianMeshPartition 
-{
-private:
-   Mesh *mesh=nullptr;
-public:
-   int nrpatch;
-   double MeshSize;
-   int nxyz[3];
-   std::vector<Array<int>> element_map;
-   Array3D<int> subdomains;
-   // constructor
-   OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
-   OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz, int ovlp_nlayers);
-   ~OverlappingCartesianMeshPartition() {};
-};
-
-class STPOverlappingCartesianMeshPartition // Special layered partition for STP
-{
-private:
-   Mesh *mesh=nullptr;
-public:
-   int nrpatch;
-   int nx, ny, nz;
-   std::vector<Array<int>> element_map;
-   // constructor
-   STPOverlappingCartesianMeshPartition(Mesh * mesh_);
-   ~STPOverlappingCartesianMeshPartition() {};
-};
-
-class MeshPartition
-{
-private:
-   Mesh *mesh=nullptr;
-   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
-   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
-   void PrintElementMap();
-public:
-   int nrpatch;
-   double MeshSize;
-   std::vector<Array<int>> element_map;
-   Array3D<int> subdomains;
-   Array<Mesh *> patch_mesh;
-   int partition_kind;
-   int nxyz[3];
-   // constructor
-   MeshPartition(Mesh * mesh_, int part, int mx=1, int my=1, int mz=1, int ovl_nlayers=0);
-   ~MeshPartition();
-};
-
-void SaveMeshPartition(Array<Mesh * > meshes, 
-                       string mfilename="output/mesh.",
-                       string sfilename="output/sol.");
-
-
-#ifdef MFEM_USE_MPI
-
-class CartesianParMeshPartition 
-{
-private:
-   ParMesh *pmesh=nullptr;
-public:
-   int nrsubdomains;
-   int nxyz[3];
-   double MeshSize;
-   std::vector<Array<int>> local_element_map;
-   Array<int> subdomain_rank;
-   Array3D<int>subdomains;
-   // constructor
-   CartesianParMeshPartition(ParMesh * pmesh_,int & nx, int & ny, int & nz, 
-                             int ovlp_nlayers);
-   ~CartesianParMeshPartition() {};
-};
-
-class ParMeshPartition
-{
-private:
-   MPI_Comm comm;
-   ParMesh *pmesh=nullptr;
-   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
-   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
-   void PrintElementMap();
-public:
-   int nrsubdomains;
-   int OvlpNlayers;
-   int myelem_offset = 0;
-   double MeshSize;
-   std::vector<Array<int>> element_map;
-   std::vector<Array<int>> local_element_map;
-   Array3D<int> subdomains;
-   Array<Mesh *> subdomain_mesh;
-   Array<int> subdomain_rank;
-   int partition_kind;
-   int nxyz[3];
-   // constructor
-   ParMeshPartition(ParMesh * pmesh_, int mx=1, int my=1, int mz=1, int ovl_nlayers=0);
-   void SaveMeshPartition();
-   ~ParMeshPartition();
-};
-
-#endif

examples/maxwell-solver-dev/common/PML.cpp

@@ -1,570 +0,0 @@
-#include "PML.hpp"
-
-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
-   : mesh(mesh_), length(length_)
-{
-   dim = mesh->Dimension();
-   SetBoundaries();
-}
-
-void CartesianPML::SetBoundaries()
-{
-   comp_dom_bdr.SetSize(dim, 2);
-   dom_bdr.SetSize(dim, 2);
-   // initialize
-   for (int i = 0; i < dim; i++)
-   {
-      dom_bdr(i, 0) = infinity();
-      dom_bdr(i, 1) = -infinity();
-   }
-
-   for (int i = 0; i < mesh->GetNBE(); i++)
-   {
-      Array<int> bdr_vertices;
-      mesh->GetBdrElementVertices(i, bdr_vertices);
-      for (int j = 0; j < bdr_vertices.Size(); j++)
-      {
-         for (int k = 0; k < dim; k++)
-         {
-            dom_bdr(k, 0) = min(dom_bdr(k, 0), mesh->GetVertex(bdr_vertices[j])[k]);
-            dom_bdr(k, 1) = max(dom_bdr(k, 1), mesh->GetVertex(bdr_vertices[j])[k]);
-         }
-      }
-   }
-
-#ifdef MFEM_USE_MPI
-   ParMesh * pmesh = dynamic_cast<ParMesh *>(mesh);
-   if (pmesh)
-   {
-      for (int d=0; d<dim; d++)
-      {
-         MPI_Allreduce(MPI_IN_PLACE,&dom_bdr(d,0),1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
-         MPI_Allreduce(MPI_IN_PLACE,&dom_bdr(d,1),1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
-      }
-   }
-#endif
-
-   for (int i = 0; i < dim; i++)
-   {
-      comp_dom_bdr(i, 0) = dom_bdr(i, 0) + length(i, 0);
-      comp_dom_bdr(i, 1) = dom_bdr(i, 1) - length(i, 1);
-   }
-}
-
-void CartesianPML::SetAttributes(Mesh *mesh_)
-{
-   int nrelem = mesh_->GetNE();
-   elems.SetSize(nrelem);
-
-   for (int i = 0; i < nrelem; ++i)
-   {
-      elems[i] = 1;
-      bool in_pml = false;
-      Element *el = mesh_->GetElement(i);
-      Array<int> vertices;
-      // Initialize Attribute
-      el->SetAttribute(1);
-      el->GetVertices(vertices);
-      int nrvert = vertices.Size();
-      // Check if any vertex is in the pml
-      for (int iv = 0; iv < nrvert; ++iv)
-      {
-         int vert_idx = vertices[iv];
-         double *coords = mesh_->GetVertex(vert_idx);
-         for (int comp = 0; comp < dim; ++comp)
-         {
-            if (coords[comp] > comp_dom_bdr(comp, 1) ||
-                coords[comp] < comp_dom_bdr(comp, 0))
-            {
-               in_pml = true;
-               break;
-            }
-         }
-      }
-      if (in_pml)
-      {
-         elems[i] = 0;
-         el->SetAttribute(2);
-      }
-   }
-   mesh_->SetAttributes();
-}
-
-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs, double omega)
-{
-   complex<double> zi = complex<double>(0., 1.);
-
-   double n = 2.0;
-   double c = 10.0;
-   // double c = log(omega);
-   double coeff;
-   // Stretch in each direction independently
-   for (int i = 0; i < dim; ++i)
-   {
-      dxs[i] = 1.0;
-      if (x(i) >= comp_dom_bdr(i, 1))
-      {
-         coeff = n * c / omega / pow(length(i, 1), n);
-         dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 1), n - 1.0));
-      }
-      if (x(i) <= comp_dom_bdr(i, 0))
-      {
-         coeff = n * c / omega / pow(length(i, 0), n);
-         dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 0), n - 1.0));
-      }
-   }
-}
-
-ToroidPML::ToroidPML(Mesh *mesh_)
-   : mesh(mesh_)
-{
-   dim = mesh->Dimension();
-   zlim.SetSize(2);
-   rlim.SetSize(2);
-   alim.SetSize(2);
-   zpml_thickness.SetSize(2);
-   rpml_thickness.SetSize(2);
-   apml_thickness.SetSize(2);
-   SetBoundaries();
-}
-
-void ToroidPML::SetBoundaries()
-{
-   mesh->EnsureNodes();
-   int nrnodes = mesh->GetNodalFESpace()->GetTrueVSize()/dim;
-   double zmin = infinity();
-   double zmax = -infinity();
-   double rmin = infinity();
-   double rmax = -infinity();
-   double amin = infinity(); // in degrees
-   double amax = -infinity(); // in degrees
-   for (int i = 0; i<nrnodes; i++)
-   {
-      Vector coord(dim);
-      mesh->GetNode(i,coord);
-      for (int d = 0; d<dim; d++)
-      {
-         if (abs(coord[d])<1e-13) coord[d] = 0.0;
-      }
-      // Find r and a for this point
-      double x = coord[0];
-      double y = coord[1];
-      double z = 0.0;
-      if (dim == 3) z = coord[2];
-      double a = GetAngle(x,y);
-      double r = sqrt(x*x + y*y);
-      
-      zmin = min(zmin,z);
-      zmax = max(zmax,z);
-      rmin = min(rmin,r);
-      rmax = max(rmax,r);
-      amin = min(amin,a);
-      amax = max(amax,a);
-   }
-
-   zlim[0] = zmin;
-   zlim[1] = zmax;
-   rlim[0] = rmin;
-   rlim[1] = rmax;
-   alim[0] = amin;
-   alim[1] = amax;
-
-
-#ifdef MFEM_USE_MPI
-   ParMesh * pmesh = dynamic_cast<ParMesh *>(mesh);
-   if (pmesh)
-   {
-      MPI_Allreduce(MPI_IN_PLACE,&zlim[0],1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
-      MPI_Allreduce(MPI_IN_PLACE,&zlim[1],1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
-      MPI_Allreduce(MPI_IN_PLACE,&rlim[0],1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
-      MPI_Allreduce(MPI_IN_PLACE,&rlim[1],1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
-      MPI_Allreduce(MPI_IN_PLACE,&alim[0],1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
-      MPI_Allreduce(MPI_IN_PLACE,&alim[1],1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
-   }
-#endif
-}
-
-void ToroidPML::SetAttributes(Mesh *mesh_)
-{
-   int nrelem = mesh_->GetNE();
-   elems.SetSize(nrelem);
-
-   // Loop through the elements and identify which of them are in the PML
-   for (int i = 0; i < nrelem; ++i)
-   {
-      // initialize with 1
-      elems[i] = 1;
-      Element *el = mesh_->GetElement(i);
-      // Initialize attribute
-      el->SetAttribute(1);
-
-      Array<int> vertices;
-      el->GetVertices(vertices);
-      int nrvert = vertices.Size();
-      // Check if any vertex is in the pml
-      bool in_pml = false;
-      for (int iv = 0; iv < nrvert; ++iv)
-      {
-         int vert_idx = vertices[iv];
-         double *coords = mesh_->GetVertex(vert_idx);
-         double x = coords[0];
-         double y = coords[1];
-         double a = GetAngle(x,y);
-         double r = sqrt(x*x + y*y);
-
-         if (astretch)
-         {
-            if ( (a <= alim[0]+apml_thickness[0]) ||
-                 (a >= alim[1]-apml_thickness[1]) )
-            {
-               in_pml = true;
-               break;  
-            }
-         }
-
-         if (rstretch)
-         {
-            if ( (r <= rlim[0]+rpml_thickness[0]) ||
-                 (r >= rlim[1]-rpml_thickness[1]) )
-            {
-               in_pml = true;
-               break;  
-            }
-         }
-      }
-      if (in_pml)
-      {
-         elems[i] = 0;
-         el->SetAttribute(2);
-      }
-
-      // Vector center;
-      // mesh_->GetElementCenter(i,center);
-      // double x = center[0];
-      // double y = center[1];
-      // double a = GetAngle(x,y);
-      // double r = sqrt(x*x + y*y);
-      // // check upper and lower bound
-      // if (astretch)
-      // {
-      //    if ( (a <= alim[0]+apml_thickness[0]) ||
-      //      (a >= alim[1]-apml_thickness[1]) )
-      //    {
-      //       elems[i] = 0;
-      //       el->SetAttribute(2);
-      //    }
-      // }
-      // if (rstretch)
-      // {
-      //    if ( (r <= rlim[0]+rpml_thickness[0]) ||
-      //         (r >= rlim[1]-rpml_thickness[1]) )
-      //    {
-      //       elems[i] = 0;
-      //       el->SetAttribute(2);
-      //    }
-      // }
-   }
-   mesh_->SetAttributes();
-}
-
-
-double ToroidPML::GetAngle(const double x, const double y)
-{
-   // Find r and a for this point
-   double arad;
-   if (x == 0.0)
-   {
-      arad = (y > 0.0)? M_PI/2.0 : 3.0 * M_PI/2.0;
-   }
-   else
-   {
-      arad = atan(y/x);
-      int k = 0;
-      if (x<0)
-      {
-         k = 1;
-      }
-      else if (y<0)
-      {
-         k = 2;
-      }
-      arad += k*M_PI;
-   }
-   return arad * 180.0/M_PI;
-}
-
-// void ToroidPML::StretchFunction(const Vector &X,
-//                                    vector<complex<double>> &dxs, double omega)
-void ToroidPML::StretchFunction(const Vector &X, ComplexDenseMatrix & J, double omega)
-
-{
-   complex<double> zi = complex<double>(0., 1.);
-
-   double n = 2.0;
-   double c = 10.0;
-   // double c = log(omega);
-   // Stretch in the azimuthal direction
-   double x = X[0];
-   double y = X[1];
-   if (abs(x) < 1e-12) x = 0.0;
-   if (abs(y) < 1e-12) y = 0.0;
-   double a = GetAngle(x,y);
-   double r = sqrt(x*x + y*y);
-   // dxs[0] = 1.0;
-   // dxs[1] = 1.0;
-   J = 0.0;
-   J(0,0) = 1.0;
-   J(1,1) = 1.0;
-   if (dim == 3) J(2,2) = 1.0;
-
-   if (astretch)
-   {
-      double th = a * M_PI/180.0;
-      double thl, thL, thH;
-      bool in_pml = false;
-      // negative direction 
-      if (a <= alim[0]+apml_thickness[0])
-      {
-         in_pml = true;
-         thL = alim[1] * M_PI/180.0;
-         thH = apml_thickness[1] * M_PI/180.0;
-         thl = thL + thH;
-      }
-      // positive direction 
-      if (a >= alim[1]-apml_thickness[1])
-      {
-         in_pml = true;
-         thL = alim[1] * M_PI/180.0;
-         thH = apml_thickness[1] * M_PI/180.0;
-         thl = thL - thH;
-      }
-      // double c1 = min(20.0*M_PI/180.0,thH);
-      if (in_pml)
-      {
-         double c1 = thH;
-         double coeff = n * c / omega / pow(c1,n);
-         double f_th = pow(th - thl,n-1);
-         double th_x = - y / (r * r);
-         double th_y =   x / (r * r);
-
-         J(0,0) = 1.0 + zi * coeff * abs(f_th * th_x);
-         J(0,1) = zi * f_th * th_y;
-         J(1,0) = zi * f_th * th_x;
-         J(1,1) = 1.0 + zi * coeff * abs(f_th * th_y);
-      }
-   }
-   // Stretch in the radial direction
-   if (rstretch)
-   {  // negative
-      double rl, rL, rH;
-      bool in_pml = false;
-      if (r <= rlim[0]+rpml_thickness[0])
-      {
-         in_pml = true;
-         rL = rlim[0];     
-         rH = rpml_thickness[0];
-         rl = rL + rH;
-      }
-      // positive direction 
-      if (r >= rlim[1]-rpml_thickness[1])
-      {
-         in_pml = true;
-         rL = rlim[1];     
-         rH = rpml_thickness[1];
-         rl = rL - rH;
-      }
-
-      if (in_pml)
-      {
-         double coeff = n * c / omega / pow (rH,n);
-         double f_r =  pow(r-rl,n-1.0);
-         double r_x = x / r;
-         double r_y = y / r;
-
-         J(0,0) = 1.0 + zi * coeff * abs(f_r*r_x);
-         J(0,1) = zi * f_r * r_y;
-         J(1,0) = zi * f_r * r_x;
-         J(1,1) = 1.0 + zi * coeff * abs(f_r*r_y);
-      }
-   }
-}
-
-double pml_detJ_Re(const Vector & x, CartesianPML * pml)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   std::vector<std::complex<double>> dxs(dim);
-   complex<double> det(1.0,0.0);
-   pml->StretchFunction(x, dxs, omega);
-   for (int i=0; i<dim; ++i) det *= dxs[i];
-   return det.real();
-}
-
-double pml_detJ_Im(const Vector & x, CartesianPML * pml)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   std::vector<std::complex<double>> dxs(dim);
-   complex<double> det(1.0,0.0);
-   pml->StretchFunction(x, dxs, omega);
-   for (int i=0; i<dim; ++i) det *= dxs[i];
-   return det.imag();
-}
-
-void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   std::vector<std::complex<double>> dxs(dim);
-   complex<double> det(1.0,0.0);
-   pml->StretchFunction(x, dxs, omega);
-
-   for (int i = 0; i<dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M=0.0;
-   for (int i = 0; i<dim; ++i)
-   {
-      M(i,i) = (det / pow(dxs[i],2)).real();
-   }
-}
-
-void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-
-   std::vector<std::complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs, omega);
-
-   for (int i = 0; i<dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M=0.0;
-   for (int i = 0; i<dim; ++i)
-   {
-      M(i,i) = (det / pow(dxs[i],2)).imag();
-   }
-}
-
-
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
-   pml->StretchFunction(x, dxs, omega);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M = 0.0;
-   for (int i = 0; i < dim; ++i)
-   {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
-   }
-}
-
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs, omega);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M = 0.0;
-   for (int i = 0; i < dim; ++i)
-   {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
-   }
-}
-
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs, omega);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   M = 0.0;
-   for (int i = 0; i < dim; ++i)
-   {
-      M(i, i) = abs(det / pow(dxs[i], 2));
-   }
-}
-
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   vector<complex<double>> dxs(dim);
-   complex<double> det(1.0, 0.0);
-   pml->StretchFunction(x, dxs, omega);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   // in the 2D case the coefficient is scalar 1/det(J)
-   if (dim == 2)
-   {
-      M = (1.0 / det).real();
-   }
-   else
-   {
-      M = 0.0;
-      for (int i = 0; i < dim; ++i)
-      {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
-      }
-   }
-}
-
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
-{
-   int dim = pml->dim;
-   double omega = pml->omega;
-   vector<complex<double>> dxs(dim);
-   complex<double> det = 1.0;
-   pml->StretchFunction(x, dxs, omega);
-
-   for (int i = 0; i < dim; ++i)
-   {
-      det *= dxs[i];
-   }
-
-   if (dim == 2)
-   {
-      M = (1.0 / det).imag();
-   }
-   else
-   {
-      M = 0.0;
-      for (int i = 0; i < dim; ++i)
-      {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
-      }
-   }
-}

examples/maxwell-solver-dev/common/PML.hpp

@@ -1,179 +0,0 @@
-#pragma once
-#include "mfem.hpp"
-#include "complex_linalg.hpp"
-#include <fstream>
-#include <iostream>
-using namespace std;
-using namespace mfem;
-
-
-// Class for setting up a simple Cartesian PML region
-class CartesianPML
-{
-private:
-   Mesh *mesh;
-
-   // Length of the PML Region in each direction
-   Array2D<double> length;
-
-   // Computational Domain Boundary
-   Array2D<double> comp_dom_bdr;
-
-   // Domain Boundary
-   Array2D<double> dom_bdr;
-
-   // Integer Array identifying elements in the pml
-   // 0: in the pml, 1: not in the pml
-   Array<int> elems;
-
-   // Compute Domain and Computational Domain Boundaries
-   void SetBoundaries();
-
-public:
-   // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
-
-   int dim;
-   double omega;
-   // Return Computational Domain Boundary
-   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
-
-   // Return Domain Boundary
-   Array2D<double> GetDomainBdr() {return dom_bdr;}
-
-   // Return Marker list for elements
-   Array<int> * GetMarkedPMLElements() {return &elems;}
-
-   // Mark element in the PML region
-   void SetAttributes(Mesh *mesh_);
-
-   void SetOmega(double omega_) {omega = omega_;}
-
-   // PML complex stretching function
-   void StretchFunction(const Vector &x, vector<complex<double>> &dxs, double omega);
-};
-
-class ToroidPML
-{
-private:
-   Mesh *mesh;
-
-   Vector zlim, zpml_thickness; // range in axial direction
-   Vector rlim, rpml_thickness; // range in radial direction
-   Vector alim, apml_thickness; // range in azimuthal direction
-
-   // Integer Array identifying elements in the pml
-   // 0: in the pml, 1: not in the pml
-   Array<int> elems;
-
-   double GetAngle(const double x, const double y);
-
-   // Compute Domain and Computational Domain Boundaries
-   void SetBoundaries();
-
-   bool zstretch = false;
-   bool rstretch = false;
-   bool astretch = false;
-
-public:
-   // Constructor
-   ToroidPML(Mesh *mesh_);
-
-   int dim;
-   double omega;
-   // Return Computational Domain Boundary
-
-   // Return Domain Boundary
-   void GetDomainBdrs(Vector & zlim_, Vector & rlim_, Vector & alim_)
-   {
-      zlim_.SetSize(2); zlim_ = zlim;
-      rlim_.SetSize(2); rlim_ = rlim;
-      alim_.SetSize(2); alim_ = alim;
-   }
-
-   void SetPmlWidth(const Vector & zpml, const Vector & rpml, const Vector & apml)
-   {
-      MFEM_VERIFY(zpml.Size() == 2 , "Check zpml size");
-      MFEM_VERIFY(rpml.Size() == 2 , "Check rpml size");
-      MFEM_VERIFY(apml.Size() == 2 , "Check apml size");
-      zpml_thickness = zpml;
-      rpml_thickness = rpml;
-      apml_thickness = apml;
-   }
-
-   void SetPmlAxes(const bool zstretch_, 
-                   const bool rstretch_, 
-                   const bool astretch_ )
-   {
-      zstretch = zstretch_;
-      rstretch = rstretch_;
-      astretch = astretch_;
-   }
-
-   // // Return Marker list for elements
-   Array<int> * GetMarkedPMLElements() {return &elems;}
-
-   // Mark element in the PML region
-   void SetAttributes(Mesh *mesh_);
-
-   void SetOmega(double omega_) {omega = omega_;}
-
-   // PML complex stretching function
-   // void StretchFunction(const Vector &X, vector<complex<double>> &dxs, double omega);
-   void StretchFunction(const Vector &X, ComplexDenseMatrix & J, double omega);
-};
-
-class PmlCoefficient : public Coefficient
-{
-private:
-   CartesianPML * pml = nullptr;
-   double (*Function)(const Vector &, CartesianPML * );
-public:
-   PmlCoefficient(double (*F)(const Vector &, CartesianPML *), CartesianPML * pml_)
-      : pml(pml_), Function(F)
-   {}
-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
-   {
-      double x[3];
-      Vector transip(x, 3);
-      T.Transform(ip, transip);
-      return ((*Function)(transip, pml));
-   }
-};
-
-
-// This includes scalar coefficients
-class PmlMatrixCoefficient : public MatrixCoefficient
-{
-private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
-public:
-   PmlMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
-   {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
-                     const IntegrationPoint &ip)
-   {
-      double x[3];
-      Vector transip(x, 3);
-      T.Transform(ip, transip);
-      K.SetSize(height, width);
-      (*Function)(transip, pml, K);
-   }
-};
-
-// Helmholtz pml Functions
-double pml_detJ_Re(const Vector & x, CartesianPML * pml);
-double pml_detJ_Im(const Vector & x, CartesianPML * pml);
-void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M);
-void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M);
-
-// Maxwell Pml functions
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);

examples/maxwell-solver-dev/common/Utilities.cpp

@@ -1,619 +0,0 @@
-#include "Utilities.hpp"
-
-Sweep::Sweep(int dim_) : dim(dim_)
-{
-   nsweeps = pow(2,dim);
-   sweeps.resize(nsweeps);
-
-   for (int is = 0; is<nsweeps; is++)
-   {
-      sweeps[is].SetSize(dim);
-   }
-
-   switch(dim)
-   {
-      case 1: 
-         sweeps[0][0] =  1; 
-         sweeps[1][0] = -1; 
-         break;
-      case 2: 
-         sweeps[0][0] =  1; sweeps[0][1] =  1; 
-         sweeps[1][0] = -1; sweeps[1][1] =  1; 
-         sweeps[2][0] =  1; sweeps[2][1] = -1; 
-         sweeps[3][0] = -1; sweeps[3][1] = -1;
-         break;
-      default:
-         sweeps[0][0] =  1; sweeps[0][1] =  1; sweeps[0][2] =  1; 
-         sweeps[1][0] = -1; sweeps[1][1] =  1; sweeps[1][2] =  1; 
-         sweeps[2][0] =  1; sweeps[2][1] = -1; sweeps[2][2] =  1; 
-         sweeps[3][0] = -1; sweeps[3][1] = -1; sweeps[3][2] =  1;
-         sweeps[4][0] =  1; sweeps[4][1] =  1; sweeps[4][2] = -1; 
-         sweeps[5][0] = -1; sweeps[5][1] =  1; sweeps[5][2] = -1; 
-         sweeps[6][0] =  1; sweeps[6][1] = -1; sweeps[6][2] = -1; 
-         sweeps[7][0] = -1; sweeps[7][1] = -1; sweeps[7][2] = -1;
-         break;
-   }
-}
-
-Sweep::~Sweep()
-{
-   for (int i = 0; i<nsweeps; i++)
-   {
-      sweeps[i].DeleteAll();
-   }
-}
-
-
-double CutOffFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
-{
-   int dim = pmin.Size();
-   Vector h0(dim);
-   Vector h1(dim);
-   for (int i=0; i<dim; i++)
-   {
-      h0(i) = h_[i][0];
-      h1(i) = h_[i][1];
-   }
-   Vector x0(dim);
-   Vector x1(dim);
-   x0 = pmin; x0+=h0;
-   x1 = pmax; x1-=h1;
-
-   double f = 1.0;
-   for (int i = 0; i<dim; i++)
-   {
-      double val = 1.0;
-      if( x(i) >= pmax(i) || x(i) <= pmin(i))
-      {
-         val = 0.0;
-      }  
-      else if (x(i) < pmax(i) && x(i) >= x1(i))
-      {
-         if(h1(i) != 0.0)
-            // val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
-            val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),1.0); 
-      }
-      else if (x(i) > pmin(i) && x(i) <= x0(i))
-      {
-         if (h0(i) != 0.0)
-            // val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
-            val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),1.0); 
-      }
-
-      if (h0(i) == 0 && x(i) <= x1(i))
-      {
-         val = 1.0;
-      }
-      if (h1(i) == 0 && x(i) >= x0(i))
-      {
-         val = 1.0;
-      }
-      f *= val;
-   }
-   return f;
-}
-
-double ChiFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
-{
-   int dim = pmin.Size();
-   Vector h0(dim);
-   Vector h1(dim);
-   for (int i=0; i<dim; i++)
-   {
-      h0(i) = h_[i][0];
-      h1(i) = h_[i][1];
-   }
-   Vector x0(dim);
-   Vector x1(dim);
-   x0 = pmin; x0+=h0;
-   x1 = pmax; x1-=h1;
-
-   double f = 1.0;
-   for (int i = 0; i<dim; i++)
-   {
-      double val = 1.0;
-      if( x(i) >= pmax(i) || x(i) <= pmin(i))
-      {
-         val = 0.0;
-      }  
-      else if (x(i) < pmax(i) && x(i) >= x1(i))
-      {
-         if(h1(i) != 0.0)
-            val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
-            // This function has to be changed to smth more reasonable
-            // val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),100.0); 
-      }
-      else if (x(i) > pmin(i) && x(i) <= x0(i))
-      {
-         if (h0(i) != 0.0)
-            val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
-            // val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),100.0); 
-      }
-
-      if (h0(i) == 0 && x(i) <= x1(i))
-      {
-         val = 1.0;
-      }
-      if (h1(i) == 0 && x(i) >= x0(i))
-      {
-         val = 1.0;
-      }
-      f *= val;
-   }
-   return f;
-}
-
-
-DofMap::DofMap(FiniteElementSpace * fes , MeshPartition * partition) 
-{
-   const FiniteElementCollection * fec = fes->FEColl();
-   nrpatch = partition->nrpatch;
-
-   fespaces.SetSize(nrpatch);
-
-   Dof2GlobalDof.resize(nrpatch);
-
-   for (int ip=0; ip<nrpatch; ++ip)
-   {
-      // create finite element spaces for each patch 
-      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
-
-      // construct the patch tdof to global tdof map
-      int nrdof = fespaces[ip]->GetTrueVSize();
-      Dof2GlobalDof[ip].SetSize(2*nrdof);
-
-      // loop through the elements in the patch
-      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
-      {
-         // index in the global mesh
-         int iel_idx = partition->element_map[ip][iel];
-         // get the dofs of this element
-         Array<int> ElemDofs;
-         Array<int> GlobalElemDofs;
-         fespaces[ip]->GetElementDofs(iel,ElemDofs);
-         fes->GetElementDofs(iel_idx,GlobalElemDofs);
-         // the sizes have to match
-         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
-                     "Size inconsistency");
-         // loop through the dofs and take into account the signs;
-         int ndof = ElemDofs.Size();
-         for (int i = 0; i<ndof; ++i)
-         {
-            int pdof_ = ElemDofs[i];
-            int gdof_ = GlobalElemDofs[i];
-            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
-            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
-            Dof2GlobalDof[ip][pdof] = gdof;
-            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fes->GetTrueVSize();
-         }
-      }
-   }
-}
-
-DofMap::DofMap(FiniteElementSpace * fes , MeshPartition * partition, int nrlayers) 
-{
-
-   nx = partition->nxyz[0];
-   ny = partition->nxyz[1];
-   nz = partition->nxyz[2];
-
-   int partition_kind = partition->partition_kind;
-   // Mesh * mesh = fespace->GetMesh();
-   const FiniteElementCollection * fec = fes->FEColl();
-   nrpatch = partition->nrpatch;
-
-   fespaces.SetSize(nrpatch);
-   PmlMeshes.SetSize(nrpatch);
-   // Extend patch meshes to include pml
-
-   for  (int ip = 0; ip<nrpatch; ip++)
-   {
-      int k = ip/(nx*ny);
-      int j = (ip-k*nx*ny)/nx;
-      int i = (ip-k*nx*ny)%nx;
-      
-      Array<int> directions;
-      if (i > 0)
-      {
-         for (int i=0; i<nrlayers; i++)
-         {
-            directions.Append(-1);
-         }
-      }
-      if (j > 0)
-      {
-         for (int i=0; i<nrlayers; i++)
-         {
-            directions.Append(-2);
-         }
-      }
-      if (k > 0)
-      {
-         for (int i=0; i<nrlayers; i++)
-         {
-            directions.Append(-3);
-         }
-      }
-      if (i < nx-1)
-      {
-         for (int i=0; i<nrlayers; i++)
-         {
-            if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
-         }
-      }
-      if (j < ny-1)
-      {
-         for (int i=0; i<nrlayers; i++)
-         {
-            if (partition_kind == 3 || partition_kind == 2) directions.Append(2);
-         }
-      }
-      if (k < nz-1)
-      {
-         for (int i=0; i<nrlayers; i++)
-         {
-            if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
-         }
-      }
-      PmlMeshes[ip] = ExtendMesh(partition->patch_mesh[ip],directions);
-   }
-
-   // Save PML_meshes
-   string meshpath;
-   string solpath;
-   if (partition_kind == 3 || partition_kind == 2) 
-   {
-      meshpath = "output/mesh_ovlp_pml.";
-      solpath = "output/sol_ovlp_pml.";
-   }
-   else if (partition_kind == 4)
-   {
-      meshpath = "output/mesh_novlp_pml.";
-      solpath = "output/sol_novlp_pml.";
-   }
-   else
-   {
-      MFEM_ABORT("This partition kind not supported yet");
-   }
-   
-   // SaveMeshPartition(PmlMeshes, meshpath, solpath);
-
-   PmlFespaces.SetSize(nrpatch);
-   Dof2GlobalDof.resize(nrpatch);
-   Dof2PmlDof.resize(nrpatch);
-
-   for (int ip=0; ip<nrpatch; ++ip)
-   {
-      // create finite element spaces for each patch 
-      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
-      PmlFespaces[ip] = new FiniteElementSpace(PmlMeshes[ip],fec);
-
-      // construct the patch tdof to global tdof map
-      int nrdof = fespaces[ip]->GetTrueVSize();
-      Dof2GlobalDof[ip].SetSize(2*nrdof);
-      Dof2PmlDof[ip].SetSize(2*nrdof);
-
-      // build dof maps between patch and extended patch
-      // loop through the patch elements and constract the dof map
-      // The same elements in the extended mesh have the same ordering (but not the dofs)
-
-      // loop through the elements in the patch
-      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
-      {
-         // index in the global mesh
-         int iel_idx = partition->element_map[ip][iel];
-         // get the dofs of this element
-         Array<int> ElemDofs;
-         Array<int> PmlElemDofs;
-         Array<int> GlobalElemDofs;
-         fespaces[ip]->GetElementDofs(iel,ElemDofs);
-         PmlFespaces[ip]->GetElementDofs(iel,PmlElemDofs);
-         fes->GetElementDofs(iel_idx,GlobalElemDofs);
-         // the sizes have to match
-         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
-                     "Size inconsistency");
-         MFEM_VERIFY(ElemDofs.Size() == PmlElemDofs.Size(),
-                     "Size inconsistency");            
-         // loop through the dofs and take into account the signs;
-         int ndof = ElemDofs.Size();
-         for (int i = 0; i<ndof; ++i)
-         {
-            int pdof_ = ElemDofs[i];
-            int gdof_ = GlobalElemDofs[i];
-            int pmldof_ = PmlElemDofs[i];
-            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
-            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
-            int pmldof = (pmldof_ >= 0) ? pmldof_ : abs(pmldof_) - 1;
-
-            Dof2GlobalDof[ip][pdof] = gdof;
-            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fes->GetTrueVSize();
-            Dof2PmlDof[ip][pdof] = pmldof;
-            Dof2PmlDof[ip][pdof+nrdof] = pmldof+PmlFespaces[ip]->GetTrueVSize();
-         }
-      }
-   }
-}
-
-
-LocalDofMap::LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
-               MeshPartition * part2_):fec(fec_), part1(part1_), part2(part2_)
-{
-   // Each overlapping patch has 2 non-overlapping subdomains
-   // Thre are n non-overlapping and and n-1 overlapping subdomains
-   int nrpatch = part2->nrpatch;
-   MFEM_VERIFY(part1->nrpatch-1 == part2->nrpatch, "Check number of subdomains");
-
-   cout << "Constructing local dof maps" << endl; 
-   map1.resize(nrpatch);
-   map2.resize(nrpatch);
-   for (int ip=0; ip<nrpatch; ip++)
-   {
-      // Get the 3 meshes involved
-      Mesh * mesh = part2->patch_mesh[ip];
-      Mesh * mesh1 = part1->patch_mesh[ip];
-      Mesh * mesh2 = part1->patch_mesh[ip+1];
-
-      // Define the fespaces
-      FiniteElementSpace fespace(mesh, fec);
-      FiniteElementSpace fespace1(mesh1, fec);
-      FiniteElementSpace fespace2(mesh2, fec);
-
-      int ndof1 = fespace1.GetTrueVSize();
-      int ndof2 = fespace2.GetTrueVSize();
-
-      map1[ip].SetSize(2*ndof1); // times 2 because it's complex
-      map2[ip].SetSize(2*ndof2); // times 2 because it's complex
-
-      // loop through the elements in the patches
-      // map 1 is constructed by the first half of elements
-      // map 2 is constructed by the second half of elements
-
-      for (int iel = 0; iel<part1->element_map[ip].Size(); ++iel)
-      {
-         // index in the overlapping mesh
-         int iel_idx = iel;
-         Array<int> ElemDofs;
-         Array<int> GlobalElemDofs;
-         fespace1.GetElementDofs(iel,ElemDofs);
-         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
-         // the sizes have to match
-         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
-                     "Size inconsistency");
-         // loop through the dofs and take into account the signs;
-         int ndof = ElemDofs.Size();
-         for (int i = 0; i<ndof; ++i)
-         {
-            int pdof_ = ElemDofs[i];
-            int gdof_ = GlobalElemDofs[i];
-            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
-            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
-            map1[ip][pdof] = gdof;
-            map1[ip][pdof+ndof1] = gdof+fespace.GetTrueVSize();
-         }
-      }
-      for (int iel = 0; iel<part1->element_map[ip+1].Size(); ++iel)
-      {
-         // index in the overlapping mesh
-         int k = part1->element_map[ip].Size();
-         int iel_idx = iel+k;
-         Array<int> ElemDofs;
-         Array<int> GlobalElemDofs;
-         fespace2.GetElementDofs(iel,ElemDofs);
-         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
-         // the sizes have to match
-         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
-                     "Size inconsistency");
-         // loop through the dofs and take into account the signs;
-         int ndof = ElemDofs.Size();
-         for (int i = 0; i<ndof; ++i)
-         {
-            int pdof_ = ElemDofs[i];
-            int gdof_ = GlobalElemDofs[i];
-            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
-            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
-            map2[ip][pdof] = gdof;
-            map2[ip][pdof+ndof2] = gdof+fespace.GetTrueVSize();
-         }
-      }
-   }
-};
-
-
-NeighborDofMaps::NeighborDofMaps(MeshPartition * part_, FiniteElementSpace * fes_,
-                                DofMap * dmap_, 
-                                int ovlp_layers_) : part(part_), fes(fes_), 
-                                dmap(dmap_),
-                                ovlp_layers(ovlp_layers_)
-{
-
-   nrsubdomains = part->nrpatch;
-   nxyz.SetSize(3);
-   mesh = fes->GetMesh();
-   dim = mesh->Dimension();
-   for (int d=0; d<3; d++)  nxyz[d] = part->nxyz[d];
-   MarkOvlpElements();
-   ComputeNeighborDofMaps();
-}
-
-void NeighborDofMaps::MarkOvlpElements()
-{
-   // Lists of elements
-   // x,y,z = +/- 1 ovlp 
-   OvlpElems.resize(nrsubdomains);
-
-   for (int ip = 0; ip<nrsubdomains; ip++)
-   {
-      int i0,j0,k0;
-      Getijk(ip,i0,j0,k0);
-      int ijk[dim]; ijk[0] = i0; ijk[1]=j0;
-      if (dim==3) ijk[2] = k0;
-
-      FiniteElementSpace * sub_fes = dmap->fespaces[ip];
-      Mesh * sub_mesh = sub_fes->GetMesh();
-      // OvlpElems[ip].resize(2*dim);
-      OvlpElems[ip].resize(pow(3,dim)); 
-
-      Vector pmin, pmax;
-      sub_mesh->GetBoundingBox(pmin,pmax);
-      double h = part->MeshSize;
-      // Loop through elements
-      for (int iel=0; iel<sub_mesh->GetNE(); iel++)
-      {
-         // Get element center
-         Vector center(dim);
-         int geom = sub_mesh->GetElementBaseGeometry(iel);
-         ElementTransformation * tr = sub_mesh->GetElementTransformation(iel);
-         tr->Transform(Geometries.GetCenter(geom),center);
-
-         // loop through dimensions   
-         Array<bool> pos(dim); pos = 0;
-         Array<bool> neg(dim); neg = 0;
-
-         for (int d=0;d<dim; d++)
-         {
-            if (ijk[d]>0 && center[d] < pmin[d]+2.0*h*ovlp_layers)
-            {
-               neg[d] = true;
-            }
-
-            if (ijk[d]<nxyz[d]-1 && center[d] > pmax[d]-2.0*h*ovlp_layers) 
-            {
-               pos[d] = true;
-            }
-         }
-         SetElementToOverlap(ip,iel,neg,pos);        
-      }   
-   }
-}
-
-void NeighborDofMaps::ComputeNeighborDofMaps()
-{
-   OvlpDofMaps.resize(nrsubdomains);
-
-   // Array<UniqueIndexGen * > Gen(nrsubdomains);
-   // // construct unique number generator for the elements of a patch
-   // for (int ip = 0; ip<nrsubdomains; ip++)
-   // {
-   //    Gen[ip] = new UniqueIndexGen;
-   //    // register the elements
-   //    int nel = part->element_map[ip].Size();
-   //    for (int iel=0; iel<nel; iel++)
-   //    {
-   //       int iel_idx = part->element_map[ip][iel];
-   //       Gen[ip]->Set(iel_idx);
-   //    }
-   // }
-
-   // construct dof maps
-   int nrneighbors = pow(3,dim); // including its self
-
-   for (int ip0 = 0; ip0<nrsubdomains; ip0++)
-   {
-      OvlpDofMaps[ip0].resize(nrneighbors);
-
-      FiniteElementSpace * fes0 = dmap->fespaces[ip0];
-      int tdofs0 = fes0->GetTrueVSize();
-      Array<int> marker0(tdofs0); marker0 = 0;
-      int i0, j0, k0;
-      Array<int> ijk(dim);
-      Getijk(ip0, i0,j0,k0);
-
-      int kbeg = (dim == 2) ? 0 : -1;
-      int kend = (dim == 2) ? 1 :  2;
-      for (int k=kbeg; k<kend; k++)
-      {
-         int k1 = k0 + k;
-         if (k1 <0 || k1>=nxyz[2]) continue;
-         int kk = (dim == 2) ? -1 : k;
-         for (int j=-1; j<2; j++)
-         {
-            int j1 = j0 + j;
-            if (j1 <0 || j1>=nxyz[1]) continue;
-            for (int i=-1; i<2; i++)
-            {
-               int i1 = i0 + i;
-               if (i1 <0 || i1>=nxyz[0]) continue;
-     
-               Array<int> ip0list;  marker0 = 0;
-               int directionId = GetDirectionId(i,j,kk);
-
-               Array<int> Elems = OvlpElems[ip0][directionId];
-               int nel = Elems.Size();
-
-               for (int iel = 0; iel<nel; ++iel)
-               {
-                  int iel0 = Elems[iel];   
-                  Array<int> ElemDofs0;
-
-                  fes0->GetElementDofs(iel0,ElemDofs0);
-                  int ndof = ElemDofs0.Size();
-                  // since the elements are added to the subdomain meshes 
-                  // in the same ordered fashion (as they come from the
-                  // original mesh) then the ordering of elements in each 
-                  // subdomain is the same. Hence the dof ovlp lists
-                  // can be computed for each subdomain independendly
-                  for (int l = 0; l<ndof; ++l)
-                  {
-                     int dof0_ = ElemDofs0[l];
-                     int dof0 = (dof0_ >= 0) ? dof0_ : abs(dof0_) - 1;
-                     if (!marker0[dof0])
-                     {
-                        ip0list.Append(dof0); // dofs of ip0 in ovlp
-                        marker0[dof0] = 1;
-                     }
-                  }
-               }
-               
-               OvlpDofMaps[ip0][directionId].Append(ip0list);
-               int tsize = fes0->GetTrueVSize();
-               // Imaginary part
-               for (int l=0;l<ip0list.Size(); l++) { ip0list[l] += tsize; }
-               OvlpDofMaps[ip0][directionId].Append(ip0list);
-            }
-         }
-      }
-   }
-}
-
-
-void NeighborDofMaps::GetNeighborDofMap(const int ip,
-                                        const Array<int> & directions, 
-                                        Array<int> & dofmap)
-{
-   int k = (dim == 2) ? -1 : directions[2];
-   int directionid = GetDirectionId(directions[0],directions[1],k);
-   dofmap = OvlpDofMaps[ip][directionid];
-}
-
-
-void NeighborDofMaps::SetElementToOverlap(int ip, int iel,
-                        const Array<bool> & neg, 
-                        const Array<bool> & pos)
-{
-   int kbeg = (dim == 2) ? 0 : -1;
-   int kend = (dim == 2) ? 0 :  1;
-   for (int k = kbeg; k<=kend; k++)
-   {
-      if (dim == 3)
-      {
-         if (k == -1 && !neg[2]) continue;   
-         if (k ==  1 && !pos[2]) continue;   
-      }   
-      for (int j = -1; j<=1; j++)
-      {
-         if (j== -1 && !neg[1]) continue;   
-         if (j==  1 && !pos[1]) continue;   
-         for (int i = -1; i<=1; i++)
-         {
-            // cases to skip 
-            if (i==-1 && !neg[0]) continue;   
-            if (i== 1 && !pos[0]) continue;   
-
-            if (i==0 && j==0 && k == 0) continue;
-            int kk  = (dim==2)?-1 : k;  
-            int DirId = GetDirectionId(i,j,kk);
-            OvlpElems[ip][DirId].Append(iel);
-         }
-      }
-   }
-}