fix path issue

.appveyor.yml

@@ -23,8 +23,9 @@ install:
 - set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
 - set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
 
-# Install METIS, use MFEM's mirror because the original source server is often
-# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
+# Install METIS, use a mirror because the original source server is not always
+# up. Original url:
+#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0

.github/workflows/build-container.yaml

@@ -1,4 +1,4 @@
-name: "Docker"
+name: Build Deploy Container
 
 on:
 
@@ -62,7 +62,7 @@ jobs:
 
       - name: GHCR Login
         if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/builds-and-tests.yml

@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.
 
 # In this CI section, we build different variants of mfem and run test on them.
-name: "Tests"
+name: builds-and-tests
 
 # Github actions can use the default "GITHUB_TOKEN". By default, this token
 # is set to have permissive access. However, this is not a good practice
@@ -47,17 +47,17 @@ jobs:
   builds-and-tests:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-20.04, macos-10.15, windows-2022]
         target: [dbg, opt]
         mpi: [seq, par]
         build-system: [make, cmake]
         hypre-target: [int32]
         exclude:
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
             build-system: cmake
-          - os: macos-latest
+          - os: macos-10.15
             build-system: cmake
-          - os: windows-latest
+          - os: windows-2022
             build-system: make
         # 'include' allows us to:
         # - Add a variable to all jobs without creating a new matrix dimension.
@@ -72,15 +72,15 @@ jobs:
             codecov: NO
           - target: opt
             codecov: YES
-          - os: windows-latest
+          - os: windows-2022
             codecov: NO
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
             target: opt
             codecov: NO
             mpi: par
             build-system: cmake
             hypre-target: int32
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
             target: opt
             codecov: NO
             mpi: par
@@ -94,7 +94,7 @@ jobs:
     # This external action allows to interrupt a workflow already running on
     # the same branch to save resource
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
       with:
         access_token: ${{ github.token }}
 
@@ -102,7 +102,7 @@ jobs:
     # /home/runner/work/mfem/mfem/mfem
     # Note: Done now to access "install-hypre" and "install-metis" actions.
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         path: ${{ env.MFEM_TOP_DIR }}
         # Fetch the complete history for codecov to access commits ID
@@ -112,49 +112,49 @@ jobs:
     # TODO: It would be nice to have only one step, e.g. with a dedicated
     # action, but I (@adrienbernede) don't see how at the moment.
     - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
     - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get install lcov
 
-    # Keep the following section in case we need it again in the future,
-    # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-    # - name: Set up Homebrew
-    #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-    #   uses: Homebrew/actions/setup-homebrew@master
+    - name: Set up Homebrew
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
 
     - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
       run: |
         export HOMEBREW_NO_INSTALL_CLEANUP=1
         brew install openmpi
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
-    - name: get lcov (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
+    - name: get MPI (MacOS)
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
       run: |
         export HOMEBREW_NO_INSTALL_CLEANUP=1
         brew install lcov
 
     - name: get MPI (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-      uses: mpi4py/setup-mpi@v1.1.4
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      uses: mpi4py/setup-mpi@v1.0.3
 
     # Get Hypre through cache, or build it.
     # Install will only run on cache miss.
     - name: cache hypre
       id: hypre-cache
       if: matrix.mpi == 'par'
-      uses: actions/cache@v3
+      uses: actions/cache@v2
       with:
         path: ${{ env.HYPRE_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
 
     - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-2022'
       uses: mfem/github-actions/build-hypre@v2.2
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
@@ -163,7 +163,7 @@ jobs:
         build-system: make
 
     - name: get hypre (Windows)
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-2022'
       uses: mfem/github-actions/build-hypre@v2.2
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
@@ -175,14 +175,14 @@ jobs:
     # Install will only run on cache miss.
     - name: cache metis
       id: metis-cache
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-      uses: actions/cache@v3
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022'
+      uses: actions/cache@v2
       with:
         path: ${{ env.METIS_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
 
     - name: install metis
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022' && steps.metis-cache.outputs.cache-hit != 'true'
       uses: mfem/github-actions/build-metis@v2.2
       with:
         archive: ${{ env.METIS_ARCHIVE }}
@@ -190,23 +190,26 @@ jobs:
 
     - name: cache vcpkg (Windows)
       id: vcpkg-cache
-      if: matrix.os == 'windows-latest'
       uses: actions/cache@v3
       with:
         path: vcpkg_cache
         key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
 
-    - name: prepare vcpkg binary cache location (Windows)
-      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+    - name: prepare binary cache location
+      if: matrix.os == 'windows-2022' && steps.vcpkg-cache.outputs.cache-hit != 'true'
       run: |
         mkdir -p vcpkg_cache
 
     - name: install metis (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
       env:
         VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
       run: |
-        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
+        $PortFile = 'C:\vcpkg\ports\metis\portfile.cmake'
+        $OriginalURL = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-${METIS_VERSION}.tar.gz'
+        $NewURL = 'https://github.com/mfem/tpls/raw/gh-pages/metis-5.1.0.tar.gz'
+        (Get-Content $PortFile).replace($OriginalURL, $NewURL) | Set-Content $PortFile
+        vcpkg install metis --triplet=x64-windows-static
 
     # MFEM build and test
     - name: build
@@ -231,7 +234,11 @@ jobs:
       run: |
         cd ${{ env.MFEM_TOP_DIR }}  && make check
 
-    # Note: 'tests' include the unit tests
+    - name: unit tests
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }} && make unittest
+
     - name: tests
       if: matrix.build-system == 'make' && matrix.target == 'opt'
       run: |
@@ -244,8 +251,8 @@ jobs:
         cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
       shell: bash
 
-    - name: cmake unit tests (Ubuntu)
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
+    - name: cmake unit tests (Ubuntu 20.04)
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-20.04'
       run: |
         CTEST_CONFIG="Release"
         [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
@@ -253,7 +260,7 @@ jobs:
       shell: bash
 
     - name: cmake tests
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-20.04'
       run: |
         CTEST_CONFIG="Release"
         cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
@@ -262,7 +269,7 @@ jobs:
     # Code coverage (process and upload reports)
     - name: codecov
       if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.2
+      uses: mfem/github-actions/upload-coverage@v2.0
       with:
         name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
         project_dir: ${{ env.MFEM_TOP_DIR }}

.github/workflows/codeql-analysis.yml

@@ -1,70 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "Static Analysis"
-
-on:
-  push:
-    branches: [ "master", "next"]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "master" ]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'cpp' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

.github/workflows/mfem-analysis.yml

@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-name: "Build Analysis"
+name: build-analysis
 
 permissions:
   actions: write
@@ -31,26 +31,27 @@ env:
 
 jobs:
   gitignore:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
 
     steps:
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
       with:
         access_token: ${{ github.token }}
 
     - name: checkout MFEM
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         path: mfem
 
     - name: Get MPI (Linux)
       run: |
         sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
     - name: Cache Hypre Install
       id: hypre-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v2
       with:
         path: ${{ env.HYPRE_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
@@ -65,7 +66,7 @@ jobs:
 
     - name: Cache Metis Install
       id: metis-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v2
       with:
         path: ${{ env.METIS_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

.github/workflows/repo-check.yml

@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-name: "Checks"
+name: repo-check
 
 permissions:
   actions: write
@@ -28,18 +28,18 @@ on:
 
 jobs:
   file-headers-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     if: |
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
       with:
         access_token: ${{ github.token }}
 
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
 
     - name: copyright check
       id: copyright
@@ -78,40 +78,35 @@ jobs:
         exit 1
 
   code-style:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     if: |
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
 
     - name: get astyle
       run: |
-        sudo apt-get install astyle
+        sudo apt-get install astyle=3.1-1ubuntu2
 
     - name: style check
       run: |
         ./config/githooks/pre-push --style
 
   documentation:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     if: |
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
 
     - name: get doxygen and graphviz
       run: |
         sudo apt-get install doxygen graphviz
 
-    - name: update doxygen config file
-      run: |
-        cd doc
-        doxygen -u CodeDocumentation.conf.in
-
     - name: build documentation
       run: |
         cd tests/scripts
@@ -123,10 +118,10 @@ jobs:
       github.ref != 'refs/heads/master' &&
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         fetch-depth: 0
 

.gitignore

@@ -131,12 +131,6 @@ examples/hiop/ex9-mesh.*
 examples/hiop/ex9-init.*
 examples/hiop/ex9-final.*
 
-examples/ipopt/exContactBlockTL
-examples/ipopt/exContactBlockTL.mesh
-examples/ipopt/exContactBlockTL-mesh.*
-examples/ipopt/exContactBlockTL-init.*
-examples/ipopt/exContactBlockTL-final.*
-
 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
 examples/petsc/mesh.*
@@ -209,7 +203,6 @@ miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
 miniapps/meshing/trimmer
-miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
@@ -221,12 +214,9 @@ miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
 miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
-miniapps/meshing/mesh-explorer-visit*
-miniapps/meshing/mesh-explorer-paraview/
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
-miniapps/meshing/reflected.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh
@@ -285,8 +275,6 @@ miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
-miniapps/tools/check-tmop-metric
-miniapps/tools/tmop-metric-magnitude
 
 miniapps/toys/automata
 miniapps/toys/life
@@ -319,7 +307,6 @@ miniapps/solvers/sol.*
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh
 
-miniapps/multidomain/multidomain
 miniapps/hooke/hooke
 
 # Unit test binary and outputs

.gitlab/quartz-baseline.yml

@@ -93,7 +93,7 @@ report_baseline:
           git pull && \
           git add ${rundir} && \
           git commit -m "${msg}" && \
-          ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+          git push origin master
         else
           for file in ${rundir}/*; do
             echo "------------------------------"

.gitlab/scripts/git_try_to_push

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Try to push to the remote 5 times. If the push fails, and the local and remote
-# have diverged, then pull from the remote to merge changes, and try pushing
-# again. If some other failure happens
-for i in {1..5}; do
-   git push origin master && exit 0
-   # Wait for 20 seconds in case someone else is pushing to the remote
-   # concurrently
-   sleep 20
-   # Fetch any updates from the remote
-   git remote update
-   # Get the latest commit on the local branch
-   LOCAL=$(git rev-parse @)
-   # Get the latest commit on the remote
-   REMOTE=$(git rev-parse @{u})
-   # Get the common ancestor
-   BASE=$(git merge-base @ @{u})
-   # Have the local and remote diverged?
-   if [[ $LOCAL != $REMOTE && $LOCAL != $BASE && $REMOTE != $BASE ]]; then
-      git pull
-      if [[ $? == 0 ]]; then
-         continue
-      else
-         exit 1 # Something else went wrong trying to pull
-      fi
-   fi
-done
-
-exit 1 # Did not succeed in 5 attempts

.gitlab/scripts/report_build_and_test_failure

@@ -32,7 +32,7 @@ if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
   git pull && \
   git add ${rundir} && \
   git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+  git push origin master
 else
   for file in ${rundir}/*; do
     echo "------------------------------"

.gitlab/scripts/report_build_and_test_success

@@ -29,7 +29,7 @@ if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
   git pull && \
   git add ${rundir} && \
   git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+  git push origin master
 else
   for file in ${rundir}/*; do
     echo "------------------------------"

CHANGELOG

@@ -8,116 +8,31 @@
                                https://mfem.org
 
 
-Version 4.5.1 (development)
+Version 4.4.1 (development)
 ===========================
-- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
-  as the coarsening operator for non-conforming AMR meshes.
 
 Meshing improvements
 --------------------
-- Added support for pyramids in non-conforming meshes. Currently only isotropic
-  refinement is supported in this case.
-
-- Updated logic in FindPointsGSLIB to ignore points found near (but outside) the
-  domain boundary.
-
-- Added support for pyramids in Gmsh meshes.
-
-- Fixed a bug in TMOP metric 301.
-
-- Added an option to auto-balance compound TMOP metrics.
-
-Discretization improvements
----------------------------
-- TBD
-
-Linear and nonlinear solvers
-----------------------------
-- Added a fast normalization-based distance solver, see the Distance miniapp
-  in the miniapps/shifted/ directory.
-
-New and updated examples and miniapps
--------------------------------------
-- Added a new meshing miniapp, reflector, which reflects a high-order or NURBS
-  hexahedral mesh about a plane.
-
-- The mesh-explorer miniapp can now save mesh files in the VisIt or ParaView
-  formats using the corresponding DataCollection objects. See option 'D' in the
-  main menu.
-
-Integrations, testing and documentation
----------------------------------------
-- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
-  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
-
-- The following integrations have updated minimum version requirements:
-  * RAJA >= 2022.10.3
-
-Miscellaneous
--------------
-- VisItDataCollection now correctly handles collection names containing
-  underscores.
-
-- VisItDataCollection::SetPadDigits() no longer alters the number of digits
-  used to represent the MPI rank because VisIt seems to require 6 digits.
-  This parameter can still be explicitly overridden with
-  VisItDataCollection::SetPadDigitsRank().
-
-API changes
------------
-- The implicit cast methods of class Vector to 'double *' and 'const double *'
-  have been deprecated and generate deprecation warnings if used. They will be
-  removed in a future release.
-
-- The methods Mesh::GetFaceBaseGeometry and Mesh::GetFaceGeometryType have been
-  deprecated, and Mesh::GetFaceGeometry (which provides identical functionality)
-  should be used instead.
-
-Version 4.5, released on October 22, 2022
-=========================================
-
-Meshing improvements
---------------------
-- Added new SubMesh and ParSubMesh classes that can be used to extract a subset
-  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
-  and work with all existing MFEM interfaces like finite element spaces etc.
-
-- Added a method, ParMesh::GetSerialMesh(), that reconstructs a partitioned
-  parallel mesh on a given single rank. Also, added ParMesh::PrintAsSerial(),
-  which saves the reconstructed serial mesh to a C++ stream on rank 0.
-
-- Added more 3D TMOP metrics, as well as specialized metrics for mesh
-  untangling and worst-case quality improvement.
-
-- Added a new method, Mesh::NodesUpdated, which should be called after the mesh
-  node coordinates have changed, e.g. after the mesh has moved. This is
-  necessary, for example, with device assembly of linear and bilinear forms.
-
 - Added support for mixed meshes and pyramids in GSLIB-FindPoints.
 
 Discretization improvements
 ---------------------------
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration.
+
+- Added support for partial assembly and fully matrix-free operators on mixed
+  meshes (different element types and p-adaptivity) through libCEED, including
+  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
+  currently limited by MFEM capabilities, i.e. 2D serial meshes. All mixed
+  element topologies are supported in serial and parallel: segment, triangle,
+  square, tetrahedron, cube, prism, and pyramid.
+
 - Added full assembly and device support for several LinearForm integrators:
   * DomainLF: (f, v)
   * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
   * DomainLFGrad: (f, grad(v))
   * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
-  The device assembly of linear forms has to be explicitly enabled by calling
-  LinearForm::UseFastAssembly(true), otherwise the legacy linear form assembly
-  is used by default.
-
-- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration with arbitrary user-supplied coefficients.
-
-- Added a new class FaceQuadratureSpace that allows for the construction of
-  QuadratureFunctions on the interior or boundary faces of a mesh.
-
-- Added a class CoefficientVector for efficient access of variable coefficient
-  values at quadrature points (in particular for GPU/device kernels).
-
-- Added support for GridFunction::GetGradients() and
-  GriFunction::GetVectorGradient() on face-neighbor elements.
 
 - Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
   spatial Gaussian white noise.
@@ -125,25 +40,8 @@ Discretization improvements
 - Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
   See fem/estimators.hpp.
 
-- Various fixes and improvements in LinearFormExtension.
-
 Linear and nonlinear solvers
 ----------------------------
-- Added a new class DGMassInverse that performs a local element-wise CG
-  iteration to solve systems involving the discontinuous Galerkin mass matrix,
-  including support for device/GPU acceleration.
-
-- Added more flexibility to the constrained solver classes:
-  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
-    (necessary for penalty contact)
-  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
-  * All constraint solver classes can take a user-defined preconditioner
-
- - Added functions to toggle additional options for the SuperLU_Dist and Hypre
-   preconditioners (ParaSails, Euclid, ILU).
-
-- Added boundary elimination with device support for `SparseMatrix` and
-  `HypreParMatrix`.
 
 New and updated examples and miniapps
 -------------------------------------
@@ -153,28 +51,12 @@ New and updated examples and miniapps
   automatic differentiation tools like a native dual number implementation or a
   third party library such as Enzyme. See miniapps/elasticity for more details.
 
-- Added example for body-fitted volumetric and shape integration using the
-  Algoim library in miniapps/shifted.
-
 - Add a new example code, Example 33/33p, to demonstrate the solution of
   spectral fractional PDEs with MFEM.
 
 Integrations, testing and documentation
 ---------------------------------------
 - Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
-  More sophisticated developer containers are available in the new repo
-  https://github.com/mfem/containers.
-
-- Added support for the LLVM-based automatic differentiation tool Enzyme, see
-  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
-  header are provided. The functionality and interaction are demonstrated in a
-  new miniapp in miniapps/elasticity.
-
-- Added support for partial assembly and fully matrix-free operators on mixed
-  meshes (different element types and p-adaptivity) through libCEED, including
-  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
-  currently limited to 2D serial meshes. All mixed element topologies are
-  supported in both serial and parallel.
 
 - Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
   which provides parallel non-conforming, non-matching, variational, volumetric
@@ -182,40 +64,29 @@ Integrations, testing and documentation
   between arbitrarily distributed and unrelated finite element meshes in a
   variationally consistent way.
 
-- Fully encapsulated SUNDIALS `N_Vector` object within the `SundialsNVector`
-  class by removing deprecated (e.g. `HypreParVector::ToNVector`) and
-  non-deprecated (e.g. `Vector::ToNVector`) functions in other classes.
+- Added support for the LLVM-based automatic differentiation tool Enzyme, see
+  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
+  header are provided. The functionality and interaction are demonstrated in a
+  new miniapp in miniapps/elasticity.
 
-- New benchmark for the different assembly levels inspired by the CEED
-  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+- Added example for body-fitted volumetric and shape integration using the
+  Algoim library.
 
 - Added Windows 2022 CI testing with GitHub actions.
 
 Miscellaneous
 -------------
-- The method SparseMatrix::EnsureMultTranspose() is now automatically called
-  by the methods AddMultTranspose(), MultTranspose(), and AbsMultTranspose().
-  Added a method with the same name to class HypreParMatrix which is also called
-  automatically by the HypreParMatrix::MultTranspose() methods.
+- Various other simplifications, extensions, and bugfixes in the code.
 
-- Updated various MemoryUsage methods to return 'std::size_t' instead of 'long'
-  since the latter is 32-bit in Win64 builds.
 
+- Added boundary elimination with device support for `SparseMatrix` and
+  `HypreParMatrix`.
+  
 - When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
   outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
   `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).
 
-- In various places in the library, replace the use of 'long' with 'long long'
-  to better support Win64 builds where 'long' is 32-bit and 'long long' is
-  64-bit. On Linux and MacOS, both types are typically 64-bit.
-
-- The behavior of GridFunction::GetTrueVector() has been changed to not return
-  an empty true vector.
-
-- Added support for ordering search points byVDIM in FindPointsGSLIB.
-
-- Various other simplifications, extensions, and bugfixes in the code.
-
+- Added TMOP metrics for mesh untangling and worst-case quality improvement.
 
 Version 4.4, released on March 21, 2022
 =======================================
@@ -248,6 +119,11 @@ Meshing improvements
 - Added a simpler interface to access mesh face information, see FaceInformation
   and GetFaceInformation in the Mesh class.
 
+- Added the method ParMesh::GetSerialMesh() that reconstructs a partitioned
+  parallel mesh on a given single rank. Also, added the method
+  ParMesh::PrintAsSerial() that saves the reconstructed serial mesh to a C++
+  stream on rank 0.
+
 - Gmsh meshes where all elements have zero physical tag (the default Gmsh output
   format if no physical groups are defined) are now successfully loaded, and
   elements are reassigned attribute number 1.
@@ -337,6 +213,9 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
   formatting. See the "make style" target.
 
+- New benchmark for the different assembly levels inspired by the CEED
+  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
@@ -347,6 +226,13 @@ Miscellaneous
 
 - Fixed several MinGW build issues on Windows.
 
+- In various places in the library, replace the use of 'long' with 'long long'
+  to better support Win64 builds where 'long' is 32-bit and 'long long' is
+  64-bit. On Linux and MacOS, both types are typically 64-bit.
+
+- Update various "MemoryUsage" methods to return 'std::size_t' instead of 'long'
+  since the latter is 32-bit in Win64 builds.
+
 - Added 'double' atomicAdd implementation for previous versions of CUDA.
 
 - HypreParVector and Vector now support C++ move semantics, and the copy

CMakeLists.txt

@@ -10,9 +10,7 @@
 # CONTRIBUTING.md for details.
 
 # The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-# Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
-# Version 3.8 or newer is required for direct CUDA support.
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.1)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
@@ -53,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.5.1)
+set(${PROJECT_NAME}_VERSION 4.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -68,7 +66,8 @@ set(XSDK_ENABLE_C OFF)
 set(XSDK_ENABLE_Fortran OFF)
 
 # Check if we need to enable C or Fortran.
-if (MFEM_USE_CONDUIT OR
+if (CMAKE_VERSION VERSION_LESS 3.2 OR
+    MFEM_USE_CONDUIT OR
     MFEM_USE_SIDRE OR
     MFEM_USE_PETSC)
    # This seems to be needed by:
@@ -82,13 +81,7 @@ if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
-if ((MFEM_USE_SUNDIALS OR
-     MFEM_USE_STRUMPACK OR
-     MFEM_USE_GINKGO OR
-     MFEM_USE_RAJA OR
-     MFEM_USE_UMPIRE) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
+if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
   set(CMAKE_CXX_STANDARD 14)
 endif()
 
@@ -106,12 +99,17 @@ if (MFEM_USE_CUDA)
    if (MFEM_USE_HIP)
       message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
    endif()
+   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
+   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
    # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
    if (NOT CMAKE_CUDA_HOST_COMPILER)
       set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
    endif()
    enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
+   set(CMAKE_CUDA_STANDARD 11)
+   if (MFEM_USE_GINKGO)
+     set(CMAKE_CUDA_STANDARD 14)
+   endif()
    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
    set(CMAKE_CUDA_EXTENSIONS OFF)
    set(CUDA_FLAGS "--expt-extended-lambda")
@@ -134,11 +132,12 @@ if (MFEM_USE_CUDA)
       set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
       set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
    endif()
-   set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS})
+   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
+       "CUDA flags set for MFEM" FORCE)
    set(CUSPARSE_FOUND TRUE)
    set(CUSPARSE_LIBRARIES "cusparse")
    set(CUBLAS_FOUND TRUE)
-   set(CUBLAS_LIBRARIES "cublas")
+   set(CUSBLAS_LIBRARIES "cublas")
 endif()
 
 if (XSDK_ENABLE_C)
@@ -196,35 +195,15 @@ if (MFEM_USE_HIP)
   find_package(HIPSPARSE REQUIRED)
 endif()
 
-# OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
-  endif()
-  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
-  if (OPENMP_FOUND)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    if (MFEM_USE_CUDA)
-      set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-    endif()
-  endif()
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
   set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
   if (MFEM_MPIEXEC)
-    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
+    set(MPIEXEC ${MFEM_MPIEXEC})
   endif()
   if (MFEM_MPIEXEC_NP)
-    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
+    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
   endif()
   # Parallel MFEM depends on hypre
   find_package(HYPRE REQUIRED)
@@ -281,6 +260,20 @@ if (MFEM_USE_LAPACK)
   find_package(LAPACK REQUIRED)
 endif()
 
+# OpenMP
+if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
+  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
+    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
+  endif()
+  find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
+  if(APPLE)
+    # On macOS, the compiler needs additional help to find the <omp.h> header.
+    # See issue #2642 for more information.
+    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
+  endif(APPLE)
+endif()
+
 # SuiteSparse (before SUNDIALS which may depend on KLU)
 if (MFEM_USE_SUITESPARSE)
   find_package(SuiteSparse REQUIRED
@@ -299,6 +292,11 @@ if (MFEM_USE_SUNDIALS)
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
+# Mesquite
+if (MFEM_USE_MESQUITE)
+  find_package(Mesquite REQUIRED)
+endif()
+
 # SuperLU_DIST can only be enabled in parallel
 if (MFEM_USE_SUPERLU)
   if (MFEM_USE_MPI)
@@ -404,15 +402,6 @@ if (MFEM_USE_HIOP)
   # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()
 
-# IpOpt optimizer
-if (MFEM_USE_IPOPT)
-  find_package(IPOPT REQUIRED)
-  message(
-      STATUS
-      "IPOPT_INCLUDE_DIRS=${IPOPT_INCLUDE_DIRS}, IPOPT_LIBRARIES=${IPOPT_LIBRARIES}, IPOPT_DIR=${IPOPT_DIR}")
-  # find_package updates IPOPT_FOUND, IPOPT_INCLUDE_DIRS, IPOPT_LIBRARIES
-endif()
-
 # CoDiPack package
 if (MFEM_USE_CODIPACK)
   find_package(CODIPACK REQUIRED)
@@ -488,27 +477,14 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
   endif()
 endif()
 
-# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
-# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
-#   Target "mfem" links to target "Threads::Threads" but the target was not
-#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
-#   an ALIAS target is missing?
-# Call Stack (most recent call first):
-#   CMakeLists.txt:474 (mfem_add_library)
-#
-# NOTE: We need to figure out which TPL library adds the dependency on
-#       "Threads::Threads" and call the next line only when that TPL library is
-#       enabled.  -V. Dobrev
-find_package(Threads REQUIRED)
-
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
-    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
-    NETCDF MPFR PUMI HIOP IPOPT POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
+    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
     ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
     MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
 
@@ -522,12 +498,18 @@ foreach(TPL IN LISTS MFEM_TPLS)
     list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
   endif()
 endforeach(TPL)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
 
+if (OPENMP_FOUND)
+  message(STATUS "MFEM: using package OpenMP")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
+endif()
+
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -706,8 +688,6 @@ add_subdirectory(doc)
 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
 set(INSTALL_INCLUDE_DIR include
   CACHE PATH "Relative path for installing header files.")
-set(INSTALL_BIN_DIR bin
-  CACHE PATH "Relative path for installing the binaries.")
 set(INSTALL_LIB_DIR lib
   CACHE PATH "Relative path for installing the library.")
 # other options: "share/mfem/cmake", "lib/mfem/cmake"
@@ -726,9 +706,7 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Development)
 # Install the library
 install(TARGETS ${PROJECT_NAME}
   EXPORT ${PROJECT_NAME_UC}Targets
-  RUNTIME DESTINATION ${INSTALL_BIN_DIR}
-  LIBRARY DESTINATION ${INSTALL_LIB_DIR}
-  ARCHIVE DESTINATION ${INSTALL_LIB_DIR})
+  DESTINATION ${INSTALL_LIB_DIR})
 
 # Install the master headers
 foreach(Header mfem.hpp mfem-performance.hpp)

CONTRIBUTING.md

@@ -102,9 +102,7 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   ├── cmake
-  │   ├── docker
-  │   ├── githooks
-  │   └── vcpkg
+  │   └── githooks
   ├── data
   ├── doc
   ├── examples
@@ -112,9 +110,7 @@ The MFEM source code has the following structure:
   │   ├── caliper
   │   ├── ginkgo
   │   ├── hiop
-  │   ├── ipopt
   │   ├── jupyter
-  │   ├── moonolith
   │   ├── petsc
   │   ├── pumi
   │   ├── sundials
@@ -122,15 +118,13 @@ The MFEM source code has the following structure:
   ├── fem
   │   ├── ceed
   │   ├── fe
-  │   ├── lor
-  │   ├── moonolith
   │   ├── qinterp
+  │   ├── moonolith
   │   └── tmop
   ├── general
   ├── linalg
   │   └── simd
   ├── mesh
-  │   └── submesh
   ├── miniapps
   │   ├── adjoint
   │   ├── autodiff
@@ -140,7 +134,6 @@ The MFEM source code has the following structure:
   │   ├── hooke
   │   ├── meshing
   │   ├── mtop
-  │   ├── multidomain
   │   ├── navier
   │   ├── nurbs
   │   ├── parelag

INSTALL

@@ -7,10 +7,6 @@
 
                                https://mfem.org
 
-This file provides a detailed description of how to build and install the MFEM
-library. For a simple build, see the step-by-step instructions on the website
-at https://mfem.org/building.
-
 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
 of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
@@ -20,11 +16,7 @@ requires an MPI C++ compiler, as well as the following external libraries:
   https://github.com/hypre-space/hypre
 
 - METIS (a family of multilevel partitioning algorithms)
-  https://github.com/mfem/tpls
-
-  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
-  webpage, http://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
-  and we don't support yet the new repo https://github.com/KarypisLab/METIS.
+  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
 
 The hypre dependency can be downloaded as a tarball from GitHub or from the
 project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
@@ -337,6 +329,10 @@ MFEM_USE_SUNDIALS = YES/NO
    library. When enabled, this option uses the SUNDIALS_* library options,
    see below.
 
+MFEM_USE_MESQUITE = YES/NO
+   Enable MFEM functionality based on the Mesquite library. When enabled, this
+   option uses the MESQUITE_* library options, see below.
+
 MFEM_USE_SUITESPARSE = YES/NO
    Enable MFEM functionality based on the SuiteSparse library. Currently, this
    option adds the classes UMFPackSolver and KLUSolver (both sparse serial
@@ -471,18 +467,15 @@ MFEM_USE_HIOP = YES/NO
    Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
    HPC solver for nonlinear optimization problems.
 
-MFEM_USE_IPOPT = YES/NO
-   Enable the usage of Ipopt in MFEM.
-
 MFEM_USE_CODIPACK = YES/NO
   Enable automatic differentiation using the CoDiPack library.
   www.scicomp.uni-kl.de/codi/
 
 MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical
-  methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM
-  provides interface to Algoim v1. Thus, to check out the specific state use:
+  Enable the usage of Algoim - a collection of high-order accurate numerical 
+  methods and C++ algorithms for working with implicitly-defined geometry and 
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
+  provides interface to Algoim v1. Thus, to check out the specific state use: 
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
   https://algoim.github.io
 
@@ -557,7 +550,7 @@ MFEM_USE_FMS = YES/NO
    Enables support for the FMS library which consists of the DataCollection
    sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
    fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   conversion routines between FMS's FmsDataCollection structure and MFEM's
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
    DataCollection class, see the header file fem/fmsconvert.hpp.
 
 MFEM_USE_PARELAG = YES/NO
@@ -604,7 +597,7 @@ The specific libraries and their options are:
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
+  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
   Options: METIS_OPT, METIS_LIB.
   Versions: METIS 4.0.3 or 5.1.0.
 
@@ -635,6 +628,11 @@ The specific libraries and their options are:
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
   Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.
 
+- Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
+  URL: http://trilinos.org/oldsite/packages/mesquite
+  Options: MESQUITE_OPT, MESQUITE_LIB.
+  The Mesquite support is deprecated and will be removed in the future.
+
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
   Options: SUITESPARSE_OPT, SUITESPARSE_LIB.
@@ -741,11 +739,6 @@ The specific libraries and their options are:
   Options: HIOP_OPT, HIOP_LIB.
   Versions: HIOP >= 0.4.6.
 
-- Ipopt (optional), used when MFEM_USE_IPOPT = YES.
-  URL: https://github.com/coin-or/Ipopt
-  Options: IPOPT_OPT, IPOPT_LIB.
-  Versions: IPOPT >= 3.14
-
 - CoDiPack (optional), used with MFEM_USE_CODIPACK = YES
   URL: https://www.scicomp.uni-kl.de/codi/
   Options: CODIPACK_OPT
@@ -761,12 +754,12 @@ The specific libraries and their options are:
   Options: GSLIB_OPT, GSLIB_LIB.
   Versions: GSLIB >= 1.0.7.
 
-- ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
+- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
   headers so it just needs to be downloaded at the same level as MFEM. Download
   the specific version we use as:
   "git clone https://github.com/algoim/algoim.git;
    git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and the library must be built prior to the MFEM build.
+  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
   Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
   "ln -s blitz-1.0.2 blitz".
   Build Blitz using CMake as:
@@ -801,10 +794,10 @@ The specific libraries and their options are:
   Versions: libCEED >= 0.10.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
+  Beginning with MFEM v4.3, only RAJA v0.14.0+ is supported.
   URL: https://github.com/LLNL/RAJA
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 2022.10.3.
+  Versions: RAJA >= 0.14.0.
 
 - Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
   URL: https://bitbucket.org/zulianp/par_moonolith
@@ -968,6 +961,7 @@ MFEM_USE_LEGACY_OPENMP
 MFEM_USE_OPENMP
 MFEM_USE_MEMALLOC
 MFEM_TIMER_TYPE - Set automatically, can be overwritten.
+MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_MUMPS
@@ -980,7 +974,6 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
-MFEM_USE_IPOPT
 MFEM_USE_CODIPACK
 MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
@@ -1033,6 +1026,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - HYPRE
  - METIS - The option MFEM_USE_METIS_5 is auto-detected.
  - ParMETIS
+ - MESQUITE
  - SuiteSparse
  - SuperLUDist, STRUMPACK
  - Ginkgo
@@ -1044,7 +1038,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - POSIXCLOCKS
  - PUMI
  - HIOP
- - IPOPT
  - CoDiPack
  - OCCA
  - RAJA

config/XSDKDefaults.cmake

@@ -212,6 +212,10 @@ IF (DEFINED TPL_ENABLE_SUNDIALS)
   SET(MFEM_USE_SUNDIALS ${TPL_ENABLE_SUNDIALS} CACHE BOOL "Enable SUNDIALS usage" FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_MESQUITE)
+  SET(MFEM_USE_MESQUITE ${TPL_ENABLE_MESQUITE} CACHE BOOL "Enable MESQUITE usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_SUITESPARSE)
   SET(MFEM_USE_SUITESPARSE ${TPL_ENABLE_SUITESPARSE} CACHE BOOL "Enable SuiteSparse usage" FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -29,6 +29,7 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
@@ -36,7 +37,6 @@ set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
-set(MFEM_USE_IPOPT @MFEM_USE_IPOPT@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)

config/cmake/config.hpp.in

@@ -77,6 +77,9 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC
 
+// Enable MFEM functionality based on the Mesquite library.
+#cmakedefine MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE
 
@@ -131,9 +134,6 @@
 // Enable MFEM functionality based on the HiOp library
 #cmakedefine MFEM_USE_HIOP
 
-// Enable MFEM functionality based on the Ipopt library
-#cmakedefine MFEM_USE_IPOPT
-
 // Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA

config/cmake/modules/FindAlgoim.cmake

@@ -16,7 +16,7 @@
 
 include(MfemCmakeUtilities)
 mfem_find_package(Algoim ALGOIM ALGOIM_DIR
-        "include;src" "algoim_quad.hpp"
+       	"include" "algoim_quad.hpp"
        	"" ""
-        "Paths to headers required by Algoim."
-        "Libraries required by Algoim.")
+	"Paths to headers required by Algoim."
+	"Libraries required by Algoim.")

config/cmake/modules/FindCaliper.cmake

@@ -16,22 +16,7 @@
 
 include(MfemCmakeUtilities)
 mfem_find_package(Caliper CALIPER CALIPER_DIR
-      "include" "caliper/cali.h"
-      "lib" "caliper"
-      "Paths to headers required by Caliper."
-      "Libraries required by Caliper.")
-
-# Append adiak path/lib if the user provided ADIAK_DIR
-if(ADIAK_DIR AND EXISTS ${ADIAK_DIR})
-    find_package(adiak NO_DEFAULT_PATH REQUIRED PATHS ${ADIAK_DIR}/lib/cmake/adiak ${ADIAK_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${adiak_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${adiak_LIBRARIES})
-endif()
-
-# Append gotcha path/lib if the user provided GOTCHA_DIR
-if(GOTCHA_DIR AND EXISTS ${GOTCHA_DIR})
-    find_package(gotcha NO_DEFAULT_PATH REQUIRED PATHS ${GOTCHA_DIR}/lib/cmake/gotcha ${GOTCHA_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${gotcha_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${gotcha_LIBRARIES})
-endif()
-
+       	"include" "caliper/cali.h"
+       	"lib" "caliper"
+  	"Paths to headers required by Caliper."
+       	"Libraries required by Caliper.")

config/cmake/modules/FindENZYME.cmake

@@ -19,7 +19,7 @@ if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
   # Set ENZYME_FOUND
   set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
 
-  # Set CXX flags to accommodate the Enzyme Clang plugin
+  # Set CXX flags to accomodate the Enzyme Clang plugin
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
   set(MFEM_USE_ENZYME YES)
 else()

config/cmake/modules/FindHDF5.cmake

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - HDF5_FOUND - If HDF5 was found
-#   - HDF5_LIBRARIES - The HDF5 libraries
-#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
-
-# NOTE: Using this FindHDF5.cmake instead of the CMake provided version may lead
-#       to issues with some TPL libraries that depend (or may depend) on HDF5.
-#       For this reason, we should consider removing this file, or at least
-#       making it use the CMake provided version by default and apply the logic
-#       below only when specifically requested by a user.  -V. Dobrev
-
-# First Check for HDF5_DIR
-if(NOT HDF5_DIR)
-  message(FATAL_ERROR
-    "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
-endif()
-
-if (NOT HDF5_FIND_QUIETLY)
-  message(STATUS "Looking for HDF5 ...")
-  message(STATUS "   in HDF5_DIR = ${HDF5_DIR}")
-endif()
-
-# Find includes
-find_path( HDF5_INCLUDE_DIRS hdf5.h
-           PATHS  ${HDF5_DIR}/include/
-           NO_DEFAULT_PATH
-           NO_CMAKE_ENVIRONMENT_PATH
-           NO_CMAKE_PATH
-           NO_SYSTEM_ENVIRONMENT_PATH
-           NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
-
-include(FindPackageHandleStandardArgs)
-
-# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
-# listed variables are TRUE
-find_package_handle_standard_args(HDF5
-                                  " *** HDF5 not found. Please set HDF5_DIR."
-                                  HDF5_LIBRARIES
-                                  HDF5_INCLUDE_DIRS
-                                  __HDF5_LIBRARY
-                                  __HDF5_HL_LIBRARY)

config/cmake/modules/FindMesquite.cmake

@@ -9,15 +9,12 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# Sets the following variables:
-#   - IPOPT_FOUND
-#   - IPOPT_INCLUDE_DIRS
-#   - IPOPT_LIBRARIES
+# Defines the following variables:
+#   - MESQUITE_FOUND
+#   - MESQUITE_LIBRARIES
+#   - MESQUITE_INCLUDE_DIRS
 
 include(MfemCmakeUtilities)
-mfem_find_package(IPOPT IPOPT IPOPT_DIR
-  "include" "IpTNLP.hpp"
-  "lib" "ipopt"
-  "Paths to headers required by IPOPT."
-  "Libraries required by IPOPT.")
-
+mfem_find_package(Mesquite MESQUITE MESQUITE_DIR
+  "include" "Mesquite_all_headers.hpp" "lib" "mesquite"
+  "Paths to headers required by Mesquite." "Libraries required by Mesquite.")

config/cmake/modules/FindNetCDF.cmake

@@ -17,24 +17,18 @@
 include(MfemCmakeUtilities)
 
 # FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-# (MFEM's FindHDF5.cmake does not need HDF5_ROOT)
-# set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-
+set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
 # We need to guard against the case where HDF5 was already found but without
 # the HL extensions (in which case mfem_find_package will treat the package
 # as already having been found), so we reset the variable to force FindHDF5.cmake
 # to be called for a second time
 set(HDF5_FOUND OFF)
 enable_language(C) # FindHDF5.cmake uses the C compiler
-
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
   "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-
 # The HL extension libraries are in a separate variable and must precede
 # the "regular" hdf5 library, as hdf5_hl depends on hdf5
 # The netcdf library will always be the first element of NETCDF_LIBRARIES
 # and we need to insert after that library but before the hdf5 library, so
 # position 1 is used
-# (MFEM's FindHDF5.cmake does not set HDF5_C_LIBRARY_hdf5_hl and the HL library
-# is already added to NETCDF_LIBRARIES)
-# list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})

config/cmake/modules/FindRAJA.cmake

@@ -14,27 +14,17 @@
 #   - RAJA_LIBRARIES
 #   - RAJA_INCLUDE_DIRS
 
-if (RAJA_FOUND)
-   return()
+include(MfemCmakeUtilities)
+mfem_find_package(RAJA RAJA RAJA_DIR "include" "RAJA/RAJA.hpp" "lib" "RAJA"
+  "Paths to headers required by RAJA." "Libraries required by RAJA.")
+
+if (NOT RAJA_CONFIG_CMAKE)
+   set(RAJA_CONFIG_CMAKE "${RAJA_DIR}/share/raja/cmake/raja-config.cmake")
 endif()
-message(STATUS "Looking for RAJA ...")
-if (RAJA_DIR)
-   message(STATUS "   in RAJA_DIR = ${RAJA_DIR}")
-   find_package(RAJA CONFIG NO_DEFAULT_PATH PATHS "${RAJA_DIR}")
-endif()
-if (NOT RAJA_FOUND)
-   message(STATUS "   in standard CMake locations")
-   find_package(RAJA CONFIG)
-endif()
-if (RAJA_FOUND)
-   set(RAJA_LIBRARIES "RAJA" CACHE STRING "RAJA imported target." FORCE)
-   set(RAJA_INCLUDE_DIRS "" CACHE STRING "RAJA include dirs (not used)" FORCE)
-   message(STATUS
-      "Found RAJA target: ${RAJA_LIBRARIES} (version: ${RAJA_VERSION})")
-else()
-   set(msg STATUS)
-   if (RAJA_FIND_REQUIRED)
-      set(msg FATAL_ERROR)
+if (EXISTS "${RAJA_CONFIG_CMAKE}")
+   include("${RAJA_CONFIG_CMAKE}")
+   if (ENABLE_CUDA AND NOT MFEM_USE_CUDA)
+      message(FATAL_ERROR
+         "RAJA is built with CUDA: MFEM_USE_CUDA=YES is required")
    endif()
-   message(${msg} "RAJA not found. Please set RAJA_DIR to the RAJA prefix.")
 endif()

config/cmake/modules/FindUMPIRE.cmake

@@ -14,23 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS
 
-if (NOT umpire_DIR AND UMPIRE_DIR)
-  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
-endif()
-message(STATUS "Looking for UMPIRE ...")
-message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
-message(STATUS "      umpire_DIR = ${umpire_DIR}")
-find_package(umpire CONFIG)
-set(UMPIRE_FOUND ${umpire_FOUND})
-set(UMPIRE_LIBRARIES "umpire")
-if (UMPIRE_FOUND)
-  message(STATUS
-    "Found UMPIRE target: ${UMPIRE_LIBRARIES} (version: ${umpire_VERSION})")
-else()
-  set(msg STATUS)
-  if (UMPIRE_FIND_REQUIRED)
-    set(msg FATAL_ERROR)
-  endif()
-  message(${msg}
-    "UMPIRE not found. Please set UMPIRE_DIR to the install prefix.")
-endif()
+include(MfemCmakeUtilities)
+mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
+  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -43,18 +43,22 @@ function(convert_filenames_to_full_paths NAMES)
   set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()
 
-# Wrapper for add_executable
+# Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
-  add_executable(${NAME} ${ARGN})
-  if (MFEM_USE_CUDA)
-    set_target_properties(${NAME} PROPERTIES
-      CUDA_RESOLVE_DEVICE_SYMBOLS ON)
+  if (MFEM_USE_HIP)
+    add_executable(${NAME} ${ARGN})
+  else()
+    add_executable(${NAME} ${ARGN})
   endif()
 endmacro()
 
-# Wrapper for add_library
+# Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
-  add_library(${NAME} ${ARGN})
+  if (MFEM_USE_HIP)
+    add_library(${NAME} ${ARGN})
+  else()
+    add_library(${NAME} ${ARGN})
+  endif()
 endmacro()
 
 # Simple shortcut to add_custom_target() with option to add the target to the
@@ -162,12 +166,27 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
 
   # Append the additional libraries and options
   if (LIBRARIES_LIST)
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${LIBRARIES_LIST})
+    endif()
   endif()
   if (EXTRA_OPTIONS_LIST)
     string(REPLACE ";" " " EXTRA_OPTIONS_STRING "${EXTRA_OPTIONS_LIST}")
     message(STATUS "${MFEM_EXE_NAME}: add flags \"${EXTRA_OPTIONS_STRING}\"")
-    target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    else()
+      get_target_property(THIS_COMPILE_FLAGS ${MFEM_EXE_NAME} COMPILE_FLAGS)
+      if (THIS_COMPILE_FLAGS)
+        set(THIS_COMPILE_FLAGS "${THIS_COMPILE_FLAGS} ${EXTRA_OPTIONS_STRING}")
+      else()
+        set(THIS_COMPILE_FLAGS "${EXTRA_OPTIONS_STRING}")
+      endif()
+      set_target_properties(${MFEM_EXE_NAME}
+        PROPERTIES COMPILE_FLAGS ${THIS_COMPILE_FLAGS})
+    endif()
   endif()
   if (EXTRA_DEFINES_LIST)
     target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
@@ -176,15 +195,17 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
   # Handle the MPI separately
   if (MFEM_USE_MPI)
     # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${MPI_CXX_LIBRARIES})
+    endif()
 
     if (MPI_CXX_INCLUDE_PATH)
       target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
     endif()
     if (MPI_CXX_COMPILE_FLAGS)
-      separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-        "${MPI_CXX_COMPILE_FLAGS}")
-      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_FLAGS})
     endif()
 
     if (MPI_CXX_LINK_FLAGS)
@@ -865,11 +886,11 @@ function(mfem_export_mk_files)
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
       MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
       MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_IPOPT MFEM_USE_GSLIB MFEM_USE_CUDA
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
       MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
       MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
       MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG

config/config.hpp

@@ -31,11 +31,9 @@
 
 // Windows specific options
 #ifdef _WIN32
-#ifndef _USE_MATH_DEFINES
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
-#endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
 // posix_memalign(), strdup(), strerror_r().

config/config.hpp.in

@@ -85,6 +85,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS
 
+// Enable MFEM functionality based on the Mesquite library.
+// #define MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE
 
@@ -141,9 +144,6 @@
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP
 
-// Enable MFEM functionality based on the IPOPT library.
-// #define MFEM_USE_IPOPT
-
 // Enable MFEM functionality based on the GSLIB library
 // #define MFEM_USE_GSLIB
 

config/config.mk.in

@@ -29,6 +29,7 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
@@ -46,7 +47,6 @@ MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
-MFEM_USE_IPOPT          = @MFEM_USE_IPOPT@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
 MFEM_USE_HIP           = @MFEM_USE_HIP@

config/defaults.cmake

@@ -30,6 +30,7 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
@@ -48,7 +49,6 @@ option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
-option(MFEM_USE_IPOPT "Enable Ipopt" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
 option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
@@ -124,6 +124,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")
 
+set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
+    "Path to the Mesquite library.")
+
 set(SuiteSparse_DIR "${MFEM_DIR}/../SuiteSparse" CACHE PATH
     "Path to the SuiteSparse library.")
 set(SuiteSparse_REQUIRED_PACKAGES "BLAS" "METIS"
@@ -185,7 +188,6 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
 
-set(HDF5_DIR "/usr" CACHE PATH "Path to the HDF5 library.")
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
 set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
     "Additional packages required by NetCDF.")
@@ -221,10 +223,6 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
     "Directory where HiOp is installed")
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
     "Packages that HiOp depends on.")
-set(IPOPT_DIR "${MFEM_DIR}/../ipopt/install" CACHE STRING
-    "Directory where IpOpt is installed")
-set(IPOPT_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
-    "Packages that IpOpt depends on.")
 
 set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
@@ -232,12 +230,17 @@ set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
+# If RAJA is built with external CAMP:
+# set(RAJA_REQUIRED_PACKAGES "camp"
+#     CACHE STRING "Packages that RAJA depends on.")
+# set(camp_DIR "${MFEM_DIR}/../camp/lib/cmake/camp"
+#     CACHE PATH "Path to CAMP CMake files.")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
 set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
 set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
-set(Algoim_REQUIRED_PACKAGES "Blitz" CACHE STRING
+set(ALGOIM_REQUIRED_PACKAGES "BLITZ" CACHE STRING
     "Packages that ALGOIM depends on.")
 
 set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH

config/defaults.mk

@@ -131,6 +131,7 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
@@ -148,7 +149,6 @@ MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
-MFEM_USE_IPOPT         = NO
 MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
 MFEM_USE_HIP           = NO
@@ -179,7 +179,7 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
 endif
 
 # ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
 # the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
 ifeq ($(MFEM_USE_HIP),YES)
    HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
@@ -251,16 +251,12 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
-# SUNDIALS >= 6.4.0 requires C++14:
-ifeq ($(MFEM_USE_SUNDIALS),YES)
-   BASE_FLAGS = -std=c++14
-endif
-SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
- $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
- -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
+SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
+SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
  -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+
 ifeq ($(MFEM_USE_MPI),YES)
    SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
 endif
@@ -270,6 +266,11 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
+# MESQUITE library configuration
+MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
+MESQUITE_OPT = -I$(MESQUITE_DIR)/include
+MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
+
 # SuiteSparse library configuration
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
@@ -308,7 +309,7 @@ SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortran library:
+# Additional Fortan library:
 # MPI_FORTRAN_LIB += -lgfortran
 
 # MUMPS library configuration
@@ -319,9 +320,6 @@ MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
 
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
-ifeq ($(MFEM_USE_STRUMPACK),YES)
-   BASE_FLAGS = -std=c++14
-endif
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 # If STRUMPACK was build with OpenMP support, the following may be need:
 # STRUMPACK_OPT += $(OPENMP_OPT)
@@ -448,11 +446,6 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)
 
-# IPOPT
-IPOPT_DIR = @MFEM_DIR@/../ipopt/install
-IPOPT_OPT = -I$(IPOPT_DIR)/include
-IPOPT_LIB = -L$(IPOPT_DIR)/lib -lipopt $(LAPACK_LIB)
-
 # CoDiPack
 CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
 CODIPACK_OPT = -I$(CODIPACK_DIR)
@@ -479,17 +472,7 @@ OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 # CALIPER library configuration
 CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 $(XLINKER)-rpath,$(CALIPER_DIR)/lib -L$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib -lcaliper
-
-ifdef ADIAK_DIR
-   CALIPER_OPT += -I$(ADIAK_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(ADIAK_DIR)/lib64 $(XLINKER)-rpath,$(ADIAK_DIR)/lib -L$(ADIAK_DIR)/lib64 -L$(ADIAK_DIR)/lib -ladiak
-endif
-ifdef GOTCHA_DIR
-   CALIPER_OPT += -I$(GOTCHA_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
-endif
-
+CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
 
 # BLITZ library configuration
 BLITZ_DIR = @MFEM_DIR@/../blitz
@@ -512,29 +495,20 @@ CEED_OPT = -I$(CEED_DIR)/include
 CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed
 
 # RAJA library configuration
-ifeq ($(MFEM_USE_RAJA),YES)
-   BASE_FLAGS = -std=c++14
-endif
 RAJA_DIR = @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
    RAJA_OPT += -I$(CUB_DIR)
 endif
-
-CAMP_LIB = -lcamp
 ifdef CAMP_DIR
    RAJA_OPT += -I$(CAMP_DIR)/include
-   CAMP_LIB = $(XLINKER)-rpath,$(CAMP_DIR)/lib -L$(CAMP_DIR)/lib -lcamp
 endif
-RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA $(CAMP_LIB)
+RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA
 
 # UMPIRE library configuration
-ifeq ($(MFEM_USE_UMPIRE),YES)
-   BASE_FLAGS = -std=c++14
-endif
 UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
-UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire $(CAMP_LIB)
+UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
 # MKL CPardiso library configuration
 MKL_CPARDISO_DIR ?=

config/sample-runs.sh

@@ -58,10 +58,6 @@ groups_serial=(
    "HiOp examples:"
    "examples/hiop"
    "ex9.cpp"'
-'"ipopt"
-   "IpOpt examples:"
-   "examples/ipopt"
-   "ex10.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
@@ -219,10 +215,6 @@ groups_all=(
    "HiOp examples:"
    "examples/hiop"
    "ex9.cpp ex9p.cpp"'
-'"ipopt"
-   "IpOpt examples:"
-   "examples/ipopt"
-   "ex10.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
@@ -562,14 +554,15 @@ function go()
    local cmd_line="${1##+( )}"
    cmd_line="${cmd_line%%+( )}"
    shopt -u extglob
+   eval local cmd=(${cmd_line})
    local res=""
    echo $sep
    echo "<${group}>" "${cmd_line}"
    echo $sep
    if [ "${timing}" == "yes" ]; then
-      timed_run eval "${cmd_line}"
+      timed_run "${cmd[@]}"
    else
-      eval "${cmd_line}"
+      "${cmd[@]}"
    fi
    if [ "$?" -eq 0 ]; then
       res="${green}  OK  ${none}"

config/vcpkg/ports/metis-mfem/disable-programs.patch

@@ -1,8 +0,0 @@
---- a/CMakeLists.txt	Wed Dec 21 18:24:22 2016
-+++ b/CMakeLists.txt	Wed Dec 21 18:24:26 2016
-@@ -20,4 +20,4 @@
- # Recursively look for CMakeLists.txt in subdirs.
- add_subdirectory("include")
- add_subdirectory("libmetis")
--add_subdirectory("programs")
-+# add_subdirectory("programs")

config/vcpkg/ports/metis-mfem/enable-install.patch

@@ -1,15 +0,0 @@
---- a/CMakeLists.txt	Sat Mar 30 17:24:45 2013
-+++ b/CMakeLists.txt	Wed Dec 21 18:23:43 2016
-@@ -4,11 +4,7 @@
- set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
- set(SHARED FALSE CACHE BOOL "build a shared library")
- 
--if(MSVC)
--  set(METIS_INSTALL FALSE)
--else()
--  set(METIS_INSTALL TRUE)
--endif()
-+set(METIS_INSTALL TRUE)
- 
- # Configure libmetis library.
- if(SHARED)

config/vcpkg/ports/metis-mfem/fix-INT_MIN_define.patch

@@ -1,34 +0,0 @@
-diff --git a/include/metis.h b/include/metis.h
-index dc5406a..7732437 100644
---- a/include/metis.h
-+++ b/include/metis.h
-@@ -72,10 +72,14 @@ typedef __int64 int64_t;
- #define PRId64       "I64d"
- #define SCNd32       "ld"
- #define SCNd64       "I64d"
-+#ifdef _WIN32
-+#include <stdint.h>
-+#else
- #define INT32_MIN    ((int32_t)_I32_MIN)
- #define INT32_MAX    _I32_MAX
- #define INT64_MIN    ((int64_t)_I64_MIN)
- #define INT64_MAX    _I64_MAX
-+#endif
- #else
- #include <inttypes.h>
- #endif
-diff --git a/GKlib/gk_arch.h b/GKlib/gk_arch.h
-index 78b1431..7258763 100644
---- a/GKlib/gk_arch.h
-+++ b/GKlib/gk_arch.h
-@@ -32,8 +32,8 @@
- 
- 
- #ifdef __MSC__ 
--  #include "ms_stdint.h"
--  #include "ms_inttypes.h"
-+  #include <stdint.h>
-+  #include <inttypes.h>
-   #include "ms_stat.h"
- #else
- #ifndef SUNOS

config/vcpkg/ports/metis-mfem/fix-gklib-vs14-math.patch

@@ -1,11 +0,0 @@
---- a/GKlib/gk_arch.h	Wed Dec 21 18:34:18 2016
-+++ b/GKlib/gk_arch.h	Wed Dec 21 18:30:49 2016
-@@ -58,7 +58,7 @@
- #define PTRDIFF_MAX  INT64_MAX
- #endif
- 
--#ifdef __MSC__
-+#if defined(__MSC__) && (_MSC_VER < 1900)
- /* MSC does not have rint() function */
- #define rint(x) ((int)((x)+0.5))  
- 

config/vcpkg/ports/metis-mfem/fix-linux-build-error.patch

@@ -1,14 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index e94f050..b9613a7 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -1,7 +1,8 @@
- cmake_minimum_required(VERSION 2.8)
- project(METIS)
- 
--set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
-+set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib")
-+
- set(SHARED FALSE CACHE BOOL "build a shared library")
- 
- set(METIS_INSTALL TRUE)

config/vcpkg/ports/metis-mfem/fix-metis-vs14-math.patch

@@ -1,11 +0,0 @@
---- a/libmetis/metislib.h	Sat Mar 30 17:24:45 2013
-+++ b/libmetis/metislib.h	Wed Dec 21 18:30:59 2016
-@@ -31,7 +31,7 @@
- #include <proto.h>
- 
- 
--#if defined(COMPILER_MSC)
-+#if defined(COMPILER_MSC) && (_MSC_VER < 1900)
- #if defined(rint)
-   #undef rint
- #endif

config/vcpkg/ports/metis-mfem/fix-runtime-install-destination.patch

@@ -1,10 +0,0 @@
---- a/libmetis/CMakeLists.txt	Sat Mar 30 17:24:45 2013
-+++ b/libmetis/CMakeLists.txt	Wed Dec 21 17:41:37 2016
-@@ -11,6 +11,6 @@
- if(METIS_INSTALL)
-   install(TARGETS metis
-     LIBRARY DESTINATION lib
--    RUNTIME DESTINATION lib
-+    RUNTIME DESTINATION bin
-     ARCHIVE DESTINATION lib)
- endif()

config/vcpkg/ports/metis-mfem/install-metisConfig.patch

@@ -1,44 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index b9613a7..e43ffee 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -22,3 +22,23 @@ include_directories(include)
- add_subdirectory("include")
- add_subdirectory("libmetis")
- # add_subdirectory("programs")
-+
-+if(METIS_INSTALL)
-+  set(PRJ_NAME metis)
-+  set(PRJ_VER 5.1.0)
-+  install(EXPORT metisTargets 
-+    FILE ${PRJ_NAME}Targets.cmake
-+    DESTINATION lib/cmake/${PRJ_NAME})
-+  include(CMakePackageConfigHelpers)
-+  write_basic_package_version_file(
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
-+    VERSION ${PRJ_VER}
-+    COMPATIBILITY SameMajorVersion)
-+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
-+    "include(\${CMAKE_CURRENT_LIST_DIR}/${PRJ_NAME}Targets.cmake)")
-+  install(FILES 
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
-+    DESTINATION lib/cmake/${PRJ_NAME})
-+endif()
-+
-diff --git a/libmetis/CMakeLists.txt b/libmetis/CMakeLists.txt
-index 7a5fc74..5a68cf0 100644
---- a/libmetis/CMakeLists.txt
-+++ b/libmetis/CMakeLists.txt
-@@ -9,8 +9,9 @@ if(UNIX)
- endif()
- 
- if(METIS_INSTALL)
--  install(TARGETS metis
-+  install(TARGETS metis EXPORT metisTargets
-     LIBRARY DESTINATION lib
-     RUNTIME DESTINATION bin
--    ARCHIVE DESTINATION lib)
-+    ARCHIVE DESTINATION lib
-+    INCLUDES DESTINATION include)
- endif()

config/vcpkg/ports/metis-mfem/portfile.cmake

@@ -1,41 +0,0 @@
-vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
-set(OPTIONS -DSHARED=OFF)
-
-set(METIS_VERSION 5.1.0)
-
-vcpkg_download_distfile(ARCHIVE
-    URLS "https://github.com/mfem/tpls/raw/gh-pages/metis-${METIS_VERSION}.tar.gz"
-    FILENAME "metis-${METIS_VERSION}.tar.gz"
-    SHA512 deea47749d13bd06fbeaf98a53c6c0b61603ddc17a43dae81d72c8015576f6495fd83c11b0ef68d024879ed5415c14ebdbd87ce49c181bdac680573bea8bdb25
-)
-
-vcpkg_extract_source_archive_ex(
-    OUT_SOURCE_PATH SOURCE_PATH
-    ARCHIVE ${ARCHIVE}
-    REF ${METIS_VERSION}
-    PATCHES
-        enable-install.patch
-        disable-programs.patch
-        fix-runtime-install-destination.patch
-        fix-metis-vs14-math.patch
-        fix-gklib-vs14-math.patch
-        fix-linux-build-error.patch
-        install-metisConfig.patch
-        fix-INT_MIN_define.patch
-)
-
-vcpkg_configure_cmake(
-    SOURCE_PATH ${SOURCE_PATH}
-    PREFER_NINJA
-    OPTIONS ${OPTIONS}
-)
-
-vcpkg_install_cmake()
-vcpkg_copy_pdbs()
-vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/metis)
-
-file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
-
-# Handle copyright
-file(COPY ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/metis)
-file(INSTALL ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)

config/vcpkg/ports/metis-mfem/vcpkg.json

@@ -1,7 +0,0 @@
-{
-  "name": "metis-mfem",
-  "version-string": "5.1.0",
-  "port-version": 0,
-  "description": "Serial Graph Partitioning and Fill-reducing Matrix Ordering",
-  "homepage": "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
-}

data/tinyzoo-3d.mesh

@@ -1,7 +1,7 @@
 MFEM mesh v1.0
 
 #
-# MFEM Geometry Types (see mesh/geom.hpp):
+# MFEM Geomety Types (see mesh/geom.hpp):
 #
 # POINT       = 0
 # SEGMENT     = 1

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.5.1
+PROJECT_NUMBER         = v4.4.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -763,55 +763,36 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \
-                         @MFEM_SOURCE_DIR@/linalg/simd \
                          @MFEM_SOURCE_DIR@/mesh \
-                         @MFEM_SOURCE_DIR@/mesh/submesh \
                          @MFEM_SOURCE_DIR@/fem \
-                         @MFEM_SOURCE_DIR@/fem/ceed \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/convection \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/diffusion \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/mass \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/nlconvection \
-                         @MFEM_SOURCE_DIR@/fem/ceed/interface \
-                         @MFEM_SOURCE_DIR@/fem/ceed/solvers \
+                         @MFEM_SOURCE_DIR@/fem/moonolith \
                          @MFEM_SOURCE_DIR@/fem/fe \
                          @MFEM_SOURCE_DIR@/fem/lor \
-                         @MFEM_SOURCE_DIR@/fem/moonolith \
-                         @MFEM_SOURCE_DIR@/fem/qinterp \
-                         @MFEM_SOURCE_DIR@/fem/tmop \
                          @MFEM_SOURCE_DIR@/examples \
-                         @MFEM_SOURCE_DIR@/examples/amgx \
                          @MFEM_SOURCE_DIR@/examples/caliper \
+                         @MFEM_SOURCE_DIR@/examples/amgx \
                          @MFEM_SOURCE_DIR@/examples/ginkgo \
-                         @MFEM_SOURCE_DIR@/examples/hiop \
-                         @MFEM_SOURCE_DIR@/examples/ipopt \
                          @MFEM_SOURCE_DIR@/examples/moonolith \
+                         @MFEM_SOURCE_DIR@/examples/hiop \
                          @MFEM_SOURCE_DIR@/examples/petsc \
                          @MFEM_SOURCE_DIR@/examples/pumi \
                          @MFEM_SOURCE_DIR@/examples/sundials \
                          @MFEM_SOURCE_DIR@/examples/superlu \
                          @MFEM_SOURCE_DIR@/miniapps/adjoint \
-                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/materials \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/operators \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/preconditioners \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \
                          @MFEM_SOURCE_DIR@/miniapps/mtop \
-                         @MFEM_SOURCE_DIR@/miniapps/multidomain \
+                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                          @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
-                         @MFEM_SOURCE_DIR@/miniapps/parelag \
                          @MFEM_SOURCE_DIR@/miniapps/performance \
                          @MFEM_SOURCE_DIR@/miniapps/shifted \
                          @MFEM_SOURCE_DIR@/miniapps/solvers \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/parelag
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -2368,7 +2349,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_GRAPH_MAX_NODES    = 100
+DOT_GRAPH_MAX_NODES    = 50
 
 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the

examples/CMakeLists.txt

@@ -178,11 +178,6 @@ if (MFEM_USE_HIOP)
   add_subdirectory(hiop)
 endif()
 
-# Include the examples/ipopt directory if IpOpt is enabled
-if (MFEM_USE_IPOPT)
-  add_subdirectory(ipopt)
-endif()
-
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
   add_subdirectory(petsc)

examples/caliper/CMakeLists.txt

@@ -33,13 +33,13 @@ add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})
 if (MFEM_ENABLE_TESTING)
   foreach(SRC_FILE ${CALIPER_EXE_SRCS})
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${PREFIX}${SRC_FILENAME})
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
     set(THIS_TEST_OPTIONS "-no-vis")
 
     if (NOT (${TEST_NAME} MATCHES ".*p$"))
       add_test(NAME ${TEST_NAME}_ser
-        COMMAND $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS})
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
     else()
       add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}

examples/contact.cpp

@@ -1,810 +0,0 @@
-//                                Contact example
-//
-// Compile with: make contact
-//
-// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
-// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include "nodepair.hpp"
-
-using namespace std;
-using namespace mfem;
-
-bool ifequalarray(const Array<int> a1, const Array<int> a2)
-{
-   if (a1.Size()!=a2.Size())
-   {
-      return false;
-   }
-   for (int i=0; i<a1.Size(); i++)
-   {
-      if (a1[i] != a2[i])
-      {
-         return false;
-      }
-   }
-   return true;
-}
-
-void FindSurfaceToProject(Mesh& mesh, const int elem, int& cbdrface)
-{
-   Array<int> attr;
-   attr.Append(2);
-   Array<int> faces;
-   Array<int> ori;
-   std::vector<Array<int> > facesVertices;
-   std::vector<int > faceid;
-   mesh.GetElementFaces(elem, faces, ori);
-   int face = -1;
-   for (int i=0; i<faces.Size(); i++)
-   {
-      face = faces[i];
-      Array<int> faceVert;
-      if (!mesh.FaceIsInterior(face)) // if on the boundary
-      {
-         mesh.GetFaceVertices(face, faceVert);
-         faceVert.Sort();
-         facesVertices.push_back(faceVert);
-         faceid.push_back(face);
-      }
-   }
-   int bdrface = facesVertices.size();
-
-   Array<int> bdryFaces;
-   // This shoulnd't need to be rebuilt
-   std::vector<Array<int> > bdryVerts;
-   for (int b=0; b<mesh.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh.GetBdrAttribute(b)) >= 0)  // found the contact surface
-      {
-         bdryFaces.Append(b);
-         Array<int> vert;
-         mesh.GetBdrElementVertices(b, vert);
-         vert.Sort();
-         bdryVerts.push_back(vert);
-      }
-   }
-
-   int bdrvert = bdryVerts.size();
-   cbdrface = -1;  // the face number of the contact surface element
-   int count_cbdrface = 0;  // the number of matching surfaces, used for checks
-
-   for (int i=0; i<bdrface; i++)
-   {
-      for (int j=0; j<bdrvert; j++)
-      {
-         if (ifequalarray(facesVertices[i], bdryVerts[j]))
-         {
-            cbdrface = faceid[i];
-            count_cbdrface += 1;
-         }
-      }
-   }
-   MFEM_VERIFY(count_cbdrface == 1,"projection surface not found");
-
-};
-
-Vector GetNormalVector(Mesh & mesh, const int elem, const double *ref,
-                       int & refFace, int & refNormal, bool & interior)
-{
-   ElementTransformation *trans = mesh.GetElementTransformation(elem);
-   const int dim = mesh.Dimension();
-   const int spaceDim = trans->GetSpaceDim();
-
-   MFEM_VERIFY(spaceDim == 3, "");
-
-   Vector n(spaceDim);
-
-   IntegrationPoint ip;
-   ip.Set(ref, dim);
-
-   trans->SetIntPoint(&ip);
-   //CalcOrtho(trans->Jacobian(), n);  // Works only for face transformations
-   const DenseMatrix jac = trans->Jacobian();
-
-   int dimNormal = -1;
-   int normalSide = -1;
-
-   const double tol = 1.0e-8;
-   for (int i=0; i<dim; ++i)
-   {
-      const double d0 = std::abs(ref[i]);
-      const double d1 = std::abs(ref[i] - 1.0);
-
-      const double d = std::min(d0, d1);
-      // TODO: this works only for hexahedral meshes!
-
-      if (d < tol)
-      {
-         MFEM_VERIFY(dimNormal == -1, "");
-         dimNormal = i;
-
-         if (d0 < tol)
-         {
-            normalSide = 0;
-         }
-         else
-         {
-            normalSide = 1;
-         }
-      }
-   }
-   // closest point on the boundary
-   if (dimNormal < 0 || normalSide < 0) // node is inside the element
-   {
-      interior = 1;
-      Vector n(3);
-      n = 0.0;
-      return n;
-   }
-
-   MFEM_VERIFY(dimNormal >= 0 && normalSide >= 0, "");
-   refNormal = dimNormal;
-
-   MFEM_VERIFY(dim == 3, "");
-
-   {
-      // Find the reference face
-      if (dimNormal == 0)
-      {
-         refFace = (normalSide == 1) ? 2 : 4;
-      }
-      else if (dimNormal == 1)
-      {
-         refFace = (normalSide == 1) ? 3 : 1;
-      }
-      else
-      {
-         refFace = (normalSide == 1) ? 5 : 0;
-      }
-   }
-
-   std::vector<Vector> tang(2);
-
-   int tangDir[2] = {-1, -1};
-   {
-      int t = 0;
-      for (int i=0; i<dim; ++i)
-      {
-         if (i != dimNormal)
-         {
-            tangDir[t] = i;
-            t++;
-         }
-      }
-
-      MFEM_VERIFY(t == 2, "");
-   }
-
-   for (int i=0; i<2; ++i)
-   {
-      tang[i].SetSize(3);
-
-      Vector tangRef(3);
-      tangRef = 0.0;
-      tangRef[tangDir[i]] = 1.0;
-
-      jac.Mult(tangRef, tang[i]);
-   }
-
-   Vector c(3);  // Cross product
-
-   c[0] = (tang[0][1] * tang[1][2]) - (tang[0][2] * tang[1][1]);
-   c[1] = (tang[0][2] * tang[1][0]) - (tang[0][0] * tang[1][2]);
-   c[2] = (tang[0][0] * tang[1][1]) - (tang[0][1] * tang[1][0]);
-
-   c /= c.Norml2();
-
-   Vector nref(3);
-   nref = 0.0;
-   nref[dimNormal] = 1.0;
-
-   Vector ndir(3);
-   jac.Mult(nref, ndir);
-
-   ndir /= ndir.Norml2();
-
-   const double dp = ndir * c;
-
-   // TODO: eliminate c?
-   n = c;
-   if (dp < 0.0)
-   {
-      n *= -1.0;
-   }
-   interior = 0;
-   return n;
-}
-
-// WARNING: global variable, just for this little example.
-std::array<std::array<int, 3>, 8> HEX_VERT =
-{
-   {  {0,0,0},
-      {1,0,0},
-      {1,1,0},
-      {0,1,0},
-      {0,0,1},
-      {1,0,1},
-      {1,1,1},
-      {0,1,1}
-   }
-};
-
-int GetHexVertex(int cdim, int c, int fa, int fb, Vector & refCrd)
-{
-   int ref[3];
-   ref[cdim] = c;
-   ref[cdim == 0 ? 1 : 0] = fa;
-   ref[cdim == 2 ? 1 : 2] = fb;
-
-   for (int i=0; i<3; ++i) { refCrd[i] = ref[i]; }
-
-   int refv = -1;
-
-   for (int i=0; i<8; ++i)
-   {
-      bool match = true;
-      for (int j=0; j<3; ++j)
-      {
-         if (ref[j] != HEX_VERT[i][j]) { match = false; }
-      }
-
-      if (match) { refv = i; }
-   }
-
-   MFEM_VERIFY(refv >= 0, "");
-
-   return refv;
-}
-
-// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
-// where X is the list of x-coordinates for all points and so on.
-// conn: connectivity of the target surface elements
-// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
-void FindPointsInMesh(Mesh & mesh, Vector const& xyz, Array<int>& conn,
-                      Vector& xi)
-{
-   const int dim = mesh.Dimension();
-   const int np = xyz.Size() / dim;
-
-   MFEM_VERIFY(np * dim == xyz.Size(), "");
-
-   mesh.EnsureNodes();
-
-   //FindPointsGSLIB finder(MPI_COMM_WORLD);
-   FindPointsGSLIB finder;
-
-   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
-
-   const double bb_t = 0.5;
-   finder.Setup(mesh, bb_t);
-
-   finder.FindPoints(xyz);
-
-   /// Return code for each point searched by FindPoints: inside element (0), on
-   /// element boundary (1), or not found (2).
-   Array<unsigned int> codes = finder.GetCode();
-
-   /// Return element number for each point found by FindPoints.
-   Array<unsigned int> elems = finder.GetElem();
-
-   /// Return reference coordinates for each point found by FindPoints.
-   Vector refcrd = finder.GetReferencePosition();
-
-   /// Return distance between the sought and the found point in physical space,
-   /// for each point found by FindPoints.
-   Vector dist = finder.GetDist();
-
-   MFEM_VERIFY(dist.Size() == np, "");
-   MFEM_VERIFY(refcrd.Size() == np * dim, "");
-   MFEM_VERIFY(elems.Size() == np, "");
-   MFEM_VERIFY(codes.Size() == np, "");
-
-   bool allfound = true;
-   for (auto code : codes)
-      if (code == 2) { allfound = false; }
-
-   MFEM_VERIFY(allfound, "A point was not found");
-
-   cout << "Maximum distance of projected points: " << dist.Max() << endl;
-
-   // extract information
-   for (int i=0; i<np; ++i)
-   {
-      /*cout << "Point " << i << ": (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << xyz[i + (j*np)];
-         if (j == dim-1) {cout << ")" << endl;}
-         else{cout << ", ";}
-      }*/
-      //cout << "  element: " << elems[i] << endl;
-      //cout << "  element " << elems[i] << " vertices:" << endl;
-      //Array<int> vert;
-      //mesh.GetElementVertices(elems[i], vert);
-      //for (auto v : vert)
-      //{
-      //   cout << "    " << v << endl;
-      //}
-
-      /*cout << "  reference coordinates: (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << refcrd[(i*dim) + j];
-         if (j == dim-1)
-         {
-            cout << ")" << endl;
-         }
-         else
-         {
-            cout << ", ";
-         }
-      }*/
-
-      int refFace, refNormal, refNormalSide;
-      bool is_interior = -1;
-      Vector normal = GetNormalVector(mesh, elems[i], refcrd.GetData() + (i*dim),
-                                      refFace, refNormal, is_interior);
-      int phyFace;
-      if (is_interior)
-      {
-         phyFace = -1; // the id of the face that has the closest point
-         FindSurfaceToProject(mesh, elems[i], phyFace);
-
-         Array<int> cbdrVert;
-         mesh.GetFaceVertices(phyFace, cbdrVert);
-         Vector xs(dim);
-         xs[0] = xyz[i + 0*np];
-         xs[1] = xyz[i + 1*np];
-         xs[2] = xyz[i + 2*np];
-         Vector xi_tmp(dim-1);
-         // get nodes!
-
-         GridFunction *nodes = mesh.GetNodes();
-         DenseMatrix coords(4,3);
-         for (int i=0; i<4; i++)
-         {
-            for (int j=0; j<3; j++)
-            {
-               coords(i,j) = (*nodes)[cbdrVert[i]*3+j];
-            }
-         }
-         SlaveToMaster(coords, xs, xi_tmp);
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = xi_tmp[j];
-         }
-         // now get get the projection to the surface
-      }
-      else
-      {
-         Vector faceRefCrd(dim-1);
-         {
-            int fd = 0;
-            for (int j=0; j<dim; ++j)
-            {
-               if (j == refNormal)
-               {
-                  refNormalSide = (refcrd[(i*dim) + j] > 0.5);
-               }
-               else
-               {
-                  faceRefCrd[fd] = refcrd[(i*dim) + j];
-                  fd++;
-               }
-            }
-
-            MFEM_VERIFY(fd == dim-1, "");
-         }
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
-         }
-         //cout << "  face reference coordinates: (";
-         /*for (int j=0; j<dim-1; ++j)
-         {
-            cout << faceRefCrd[j];
-            if (j == dim-2){cout << ")" << endl;}
-            else{cout << ", ";}
-         }*/
-      }
-      //cout << "  normal vector: ";
-      //normal.Print();
-
-      // ask, does this do anything?
-      /*
-      IntegrationPoint ip;
-      ip.Set(refcrd.GetData() + (i*dim), dim);
-      ElementTransformation *trans = mesh.GetElementTransformation(elems[i]);
-      Vector phys(trans->GetSpaceDim());
-      trans->Transform(ip, phys);
-      cout << "  physical coordinates: ";
-      phys.Print();
-      */
-
-      // Get the element face
-      Array<int> faces;
-      Array<int> ori;
-      int face;
-
-      if (is_interior)
-      {
-         face = phyFace;
-      }
-      else
-      {
-         mesh.GetElementFaces(elems[i], faces, ori);
-         face = faces[refFace];
-      }
-
-      Array<int> faceVert;
-      mesh.GetFaceVertices(face, faceVert);
-
-      //cout << "  face " << face << " vertices:" << endl;
-      //for (auto v : faceVert){ cout << "    " << v << endl;}
-
-      for (int p=0; p<4; p++)
-      {
-         conn[4*i+p] = faceVert[p];
-      }
-      /*
-      Vector ref(dim);
-
-      for (int p=0; p<2; ++p)
-         for (int q=0; q<2; ++q)
-         {
-            const int refv = GetHexVertex(refNormal, refNormalSide, p, q, ref);
-            cout << "  face reference vertex (" << p << "," << q
-                 << ") is global vertex " << vert[refv] << endl;
-
-            {
-               // Sanity check
-               ip.Set(ref.GetData(), dim);
-               trans->Transform(ip, phys);
-               for (int j=0; j<dim; ++j)
-               {
-                  phys[j] -= mesh.GetVertex(vert[refv])[j];
-               }
-          phys.Print();
-          cout<<vert[refv]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[0]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[1]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[2]<<endl;
-               MFEM_VERIFY(phys.Norml2() < 1.0e-12, "Sanity check failed");
-            }
-         }*/
-   }
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file1 = "block1.mesh";
-   const char *mesh_file2 = "block2.mesh";
-
-   Array<int> attr;
-   Array<int> m_attr;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file1, "-m1", "--mesh1",
-                  "First mesh file to use.");
-   args.AddOption(&mesh_file2, "-m2", "--mesh2",
-                  "Second mesh file to use.");
-   args.AddOption(&attr, "-at", "--attributes-surf",
-                  "Attributes of boundary faces on contact surface for mesh 2.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Mesh mesh1(mesh_file1, 1, 1);
-   Mesh mesh2(mesh_file2, 1, 1);
-
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mesh1a_sock(vishost, visport);
-      mesh1a_sock.precision(8);
-      mesh1a_sock << "mesh\n" << mesh1 << flush;
-      socketstream mesh2a_sock(vishost, visport);
-      mesh2a_sock.precision(8);
-      mesh2a_sock << "mesh\n" << mesh2 << flush;
-   }
-
-   const int dim = mesh1.Dimension();
-   MFEM_VERIFY(dim == mesh2.Dimension(), "");
-
-   // boundary attribute 2 is the potential contact surface of nodes
-   attr.Append(2);
-   // boundary attribute 2 is the potential contact surface for master surface
-   m_attr.Append(2);
-
-   //  Define a finite element space on the mesh. Here we use vector finite
-   //  elements, i.e. dim copies of a scalar finite element space. The vector
-   //  dimension is specified by the last argument of the FiniteElementSpace
-   //  constructor.
-   FiniteElementCollection *fec1;
-   FiniteElementSpace *fespace1;
-   fec1 = new H1_FECollection(1, dim);
-   fespace1 = new FiniteElementSpace(&mesh1, fec1, dim, Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh1: "
-        << fespace1->GetTrueVSize() << endl;
-   mesh1.SetNodalFESpace(fespace1);
-   GridFunction nodes0 = *mesh1.GetNodes(); // undeformed mesh1 nodal grid function
-   GridFunction *nodes1 = mesh1.GetNodes();
-
-   FiniteElementCollection *fec2 = new H1_FECollection(1, dim);
-   FiniteElementSpace *fespace2 = new FiniteElementSpace(&mesh2, fec2, dim,
-                                                         Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh2: "
-        << fespace2->GetTrueVSize() << endl;
-
-   // degrees of freedom of both meshes
-   int ndof_1 = fespace1->GetTrueVSize();
-   int ndof_2 = fespace2->GetTrueVSize();
-   int ndofs = ndof_1 + ndof_2;
-   // number of nodes for each mesh
-   int nnd_1 = mesh1.GetNV();
-   int nnd_2 = mesh2.GetNV();
-   int nnd = nnd_1 + nnd_2;
-   // Determine the list of true (i.e. conforming) essential boundary dofs.
-   // In this example, the boundary conditions are defined by marking only
-   // boundary attribute 1 from the mesh as essential and converting it to a
-   // list of true dofs.
-   Array<int> ess_tdof_list1, ess_bdr1(mesh1.bdr_attributes.Max());
-   ess_bdr1 = 0;
-   //ess_bdr1[0] = 1;
-   // Not ready to be passed on yet
-   // fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   Array<int> ess_tdof_list2, ess_bdr2(mesh2.bdr_attributes.Max());
-   ess_bdr2 = 0;
-   //ess_bdr2[0] = 1;
-
-   // Define the displacement vector x as a finite element grid function
-   // corresponding to fespace. GridFunction is a derived class of Vector.
-   GridFunction x1(fespace1);
-   x1 = 0.0;
-   GridFunction x2(fespace2);
-   x2 = 0.0;
-
-   // Generate force
-   LinearForm *b1 = new LinearForm(fespace1);
-   b1->Assemble();
-
-   LinearForm *b2 = new LinearForm(fespace2);
-   b2->Assemble();
-
-   // Set up the bilinear form a(.,.) on the finite element space
-   //  corresponding to the linear elasticity integrator with piece-wise
-   //  constants coefficient lambda and mu.
-   Vector lambda1(mesh1.attributes.Max());
-   lambda1 = 57.6923076923;
-   PWConstCoefficient lambda1_func(lambda1);
-   Vector mu1(mesh1.attributes.Max());
-   mu1 = 38.4615384615;
-   PWConstCoefficient mu1_func(mu1);
-
-   BilinearForm *a1 = new BilinearForm(fespace1);
-   a1->AddDomainIntegrator(new ElasticityIntegrator(lambda1_func,mu1_func));
-
-   Vector lambda2(mesh2.attributes.Max());
-   lambda2 = 57.6923076923;
-   PWConstCoefficient lambda2_func(lambda2);
-   Vector mu2(mesh2.attributes.Max());
-   mu2 = 38.4615384615;
-   PWConstCoefficient mu2_func(mu2);
-
-   BilinearForm *a2 = new BilinearForm(fespace2);
-   a2->AddDomainIntegrator(new ElasticityIntegrator(lambda2_func,mu2_func));
-
-   a1->Assemble();
-   SparseMatrix A1;
-   Vector B1, X1;
-   a1->FormLinearSystem(ess_tdof_list1, x1, *b1, A1, X1, B1);
-
-   a2->Assemble();
-   SparseMatrix A2;
-   Vector B2, X2;
-   a2->FormLinearSystem(ess_tdof_list2, x2, *b2, A2, X2, B2);
-
-   // Combine elasticity operator for two meshes into one.
-   // Block Matrix
-   SparseMatrix K(ndofs,ndofs);
-   for (int i=0; i<A1.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A1.GetRow(i, col_tmp, v_tmp);
-      K.SetRow(i, col_tmp, v_tmp);
-   }
-   for (int i=0; i<A2.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A2.GetRow(i, col_tmp, v_tmp);
-      for (int j=0; j<col_tmp.Size(); j++)
-      {
-         col_tmp[j] += ndof_1;
-      }
-      K.SetRow(i+ndof_1, col_tmp, v_tmp);  // mesh1 top left corner
-   }
-
-   // Construct node to segment contact constraint.
-
-   attr.Sort();
-   cout << "Boundary attributes for contact surface faces in mesh 2" << endl;
-   for (auto a : attr) { cout << a << endl; }
-
-   Array<int> bdryFaces2;  // TODO: remove this?
-
-   std::set<int> bdryVerts2;
-   for (int b=0; b<mesh2.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh2.GetBdrAttribute(b)) >= 0)
-      {
-         bdryFaces2.Append(b);
-         Array<int> vert;
-         mesh2.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            bdryVerts2.insert(v);
-         }
-      }
-   }
-
-   int npoints = bdryVerts2.size();
-   Array<int> s_conn(npoints); // connectivity of the second/slave mesh
-   Vector xyz(dim * npoints);
-   xyz = 0.0;
-
-   cout << "Boundary vertices for contact surface vertices in mesh 2" << endl;
-
-   // construct the nodal coordinates on mesh2 to be projected, including displacement
-   int count = 0;
-   for (auto v : bdryVerts2)
-   {
-      cout << v << ": " << mesh2.GetVertex(v)[0] << ", "
-           << mesh2.GetVertex(v)[1] << ", "
-           << mesh2.GetVertex(v)[2] << endl;
-
-      for (int i=0; i<dim; ++i)
-      {
-         xyz[count + (i * npoints)] = mesh2.GetVertex(v)[i] + x2[v*dim+i];
-      }
-
-      s_conn[count] = v + nnd_1; // dof1 is the master
-      count++;
-   }
-
-   MFEM_VERIFY(count == npoints, "");
-
-   // gap function
-   Vector g(npoints*dim);
-   g = -1.0;
-   // segment reference coordinates of the closest point
-   Vector m_xi(npoints*(dim-1));
-   m_xi = -1.0;
-   Vector xs(dim*npoints);
-   xs = 0.0;
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<dim; j++)
-      {
-         xs[i*dim+j] = xyz[i + (j*npoints)];
-      }
-   }
-
-   Array<int> m_conn(
-      npoints*4); // only works for linear elements that have 4 vertices!
-   DenseMatrix coordsm(npoints*4, dim);
-
-   // adding displacement to mesh1 using a fixed grid function from mesh1
-   x1 = 1e-4; // x1 order: [xyz xyz... xyz]
-   add(nodes0, x1, *nodes1);
-
-   FindPointsInMesh(mesh1, xyz, m_conn, m_xi);
-
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<4; j++)
-      {
-         for (int k=0; k<dim; k++)
-         {
-            coordsm(i*4+j,k) = mesh1.GetVertex(m_conn[i*4+j])[k]+x1[dim*m_conn[i*4+j]+k];
-         }
-      }
-   }
-   //coordsm.Print();
-   SparseMatrix M(nnd,ndofs);
-   std::vector<SparseMatrix> dM(nnd, SparseMatrix(ndofs,ndofs));
-
-   Assemble_Contact(nnd, npoints, ndofs, xs, m_xi, coordsm,
-                    s_conn, m_conn, g, M, dM);
-
-   std::set<int> dirbdryv2;
-   for (int b=0; b<mesh2.GetNBE(); ++b)
-   {
-      if (mesh2.GetBdrAttribute(b) == 1)
-      {
-         Array<int> vert;
-         mesh2.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            dirbdryv2.insert(v);
-         }
-      }
-   }
-   std::set<int> dirbdryv1;
-   for (int b=0; b<mesh1.GetNBE(); ++b)
-   {
-      if (mesh1.GetBdrAttribute(b) == 1)
-      {
-         Array<int> vert;
-         mesh1.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            dirbdryv1.insert(v);
-         }
-      }
-   }
-
-   Array<int> Dirichlet_dof;
-   Array<double> Dirichlet_val;
-
-   for (auto v : dirbdryv2)
-   {
-      for (int i=0; i<dim; ++i)
-      {
-         Dirichlet_dof.Append(v*dim + i + ndof_1);
-         Dirichlet_val.Append(0.);
-      }
-   }
-   double delta = 0.1;
-   for (auto v : dirbdryv1)
-   {
-      Dirichlet_dof.Append(v*dim + 0);
-      Dirichlet_val.Append(delta);
-      Dirichlet_dof.Append(v*dim + 1);
-      Dirichlet_val.Append(0.);
-      Dirichlet_dof.Append(v*dim + 2);
-      Dirichlet_val.Append(0.);
-   }
-
-
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mesh1_sock(vishost, visport);
-      mesh1_sock.precision(8);
-      mesh1_sock << "mesh\n" << mesh1 << flush;
-      socketstream mesh2_sock(vishost, visport);
-      mesh2_sock.precision(8);
-      mesh2_sock << "mesh\n" << mesh2 << flush;
-   }
-
-   //M.Print();
-   /*Vector eps(ndofs);
-   Vector sol(ndofs); sol = 0.;
-   for(int i=0;i<ndofs;i++) eps[i] = 1e-5 * i ;
-   for(int i=0;i<9;i++)
-   {
-     cout<<i<<endl;
-     dM[s_conn[i]].Mult(eps,sol);
-     sol.Print();
-   }
-   */
-   return 0;
-}

examples/dpg_tests/EM-diffusion/LS_maxwell_2D.cpp

@@ -0,0 +1,310 @@
+// Example run: ./FOSLS2D_maxwell -ref 4  -o 3 -sol 1 -k 3.0
+
+//  ∇ × E - ω H = 0
+// -ω E + ∇ × H = J
+
+// --------------------------------------------------------------------------
+// |   |            E             |             H              |     RHS    | 
+// --------------------------------------------------------------------------
+// | F | (∇ × E,∇ × F)+ ω^2 (E,F) | - ω (∇ × H,F) - ω (H,curF) | - ω (J,F)  |
+// |   |                          |                            |            |
+// | G |-ω (E,∇ × G)-ω (∇ × E,G)  | (∇ × H,∇ × G)+ ω^2(H,G)    | (J,∇ × G)  |
+
+// for E in H1 (scalar) we have ∇ × E = [0 1;-1 0] ∇ E
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Define exact solution
+double E_exact(const Vector &x);
+void H_exact(const Vector &x, Vector &H);
+double frhs(const Vector &x);
+void fvrhs(const Vector &x, Vector &f);
+void get_maxwell_solution(const Vector &x, double & E, Vector & curlE, double & curl2E);
+
+int dim;
+double omega;
+int isol = 0;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/star.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // visualization flag
+   bool visualization = 1;
+   int ref = 1;
+   // number of wavelengths
+   double k = 0.6;
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref, "-ref", "--init-refinements",
+                  "Number of initial mesh refinements");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&isol, "-sol", "--solution",
+                  "Exact Solution: 0) Polynomial, 1) Sinusoidal.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * k;
+
+   // Mesh mesh(1, 1, Element::QUADRILATERAL, true, 1.0, 1.0, false);
+   Mesh mesh(mesh_file, 1, 1);
+
+   dim = mesh.Dimension();
+   if (dim == 3) {MFEM_ABORT("This is 2D Maxwell")};
+
+   for (int i = 0; i < ref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   H1_FECollection H1fec(order,dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   ND_FECollection NDfec(order, dim);
+   FiniteElementSpace NDfes(&mesh, &NDfec);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // Essential BC on E. Nothing on H
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+   Array<int> block_offsets(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = H1fes.GetVSize();
+   block_offsets[2] = NDfes.GetVSize();
+   block_offsets.PartialSum();
+
+   BlockVector x(block_offsets), b(block_offsets);
+   x = 0.0;
+   b = 0.0;
+
+   FunctionCoefficient Eex(E_exact);
+   VectorFunctionCoefficient Hex(dim, H_exact);
+   GridFunction E_gf;
+   GridFunction H_gf;
+   E_gf.MakeRef(&H1fes, x.GetBlock(0));
+   E_gf.ProjectBdrCoefficient(Eex,ess_bdr);
+   H_gf.MakeRef(&NDfes, x.GetBlock(1));
+
+   FunctionCoefficient f(frhs);
+   ProductCoefficient f_E(-omega, f);
+   VectorFunctionCoefficient f_H(1,fvrhs);
+   LinearForm b_E;
+   b_E.Update(&H1fes, b.GetBlock(0), 0);
+   b_E.AddDomainIntegrator(new DomainLFIntegrator(f_E));
+   b_E.Assemble();
+
+   LinearForm b_H;
+   b_H.Update(&NDfes, b.GetBlock(1), 0);
+   b_H.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
+   b_H.Assemble();
+
+   // 7. Bilinear form a(.,.) on the finite element space
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg2(pow(omega, 2));
+   ConstantCoefficient negomega(-(omega));
+   DenseMatrix mat(2);
+   mat(0,0) = 0.; mat(0,1) = 1.;
+   mat(1,0) = -1.; mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(mat);
+
+   BilinearForm a_EE(&H1fes);
+   a_EE.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a_EE.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a_EE.Assemble();
+   a_EE.EliminateEssentialBC(ess_bdr, x.GetBlock(0), b.GetBlock(0));
+   a_EE.Finalize();
+   SparseMatrix &A_EE = a_EE.SpMat();
+
+   ScalarMatrixProductCoefficient c1(-omega, rot);
+   MixedBilinearForm a_EH(&H1fes,&NDfes);
+   // - omega (rot grad E, G) - (omega E, curl G)
+   a_EH.AddDomainIntegrator(new MixedVectorGradientIntegrator(c1));
+   a_EH.AddDomainIntegrator(new MixedScalarWeakCurlIntegrator(negomega));
+   a_EH.Assemble();
+   a_EH.EliminateTrialDofs(ess_bdr, x.GetBlock(0), b.GetBlock(1));
+   a_EH.Finalize();
+   SparseMatrix &A_EH = a_EH.SpMat();
+   SparseMatrix * A_HE = Transpose(A_EH);
+
+   BilinearForm a_HH(&NDfes);
+   a_HH.AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
+   a_HH.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2)); // one is the coeff
+   a_HH.Assemble();
+   a_HH.Finalize();
+   SparseMatrix &A_HH = a_HH.SpMat();
+
+   BlockMatrix LS_Maxwellop(block_offsets);
+   LS_Maxwellop.SetBlock(0, 0, &A_EE);
+   LS_Maxwellop.SetBlock(0, 1, A_HE);
+   LS_Maxwellop.SetBlock(1, 0, &A_EH);
+   LS_Maxwellop.SetBlock(1, 1, &A_HH);
+
+   UMFPackSolver invE;
+   invE.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invE.SetOperator(LS_Maxwellop.GetBlock(0,0));
+
+   UMFPackSolver invH;
+   invH.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invH.SetOperator(LS_Maxwellop.GetBlock(1,1));
+
+   BlockDiagonalPreconditioner prec(block_offsets);
+   prec.SetDiagonalBlock(0, &invE);
+   prec.SetDiagonalBlock(1, &invH);
+
+   int maxit(5000);
+   double rtol(1.e-16);
+   double atol(0.0);
+
+   CGSolver pcg;
+   pcg.SetAbsTol(atol);
+   pcg.SetRelTol(rtol);
+   pcg.SetMaxIter(maxit);
+   pcg.SetOperator(LS_Maxwellop);
+   pcg.SetPreconditioner(prec);
+   pcg.SetPrintLevel(3);
+   pcg.Mult(b, x);
+
+   int order_quad = max(2, 2 * order + 1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+
+   double Error_E = E_gf.ComputeL2Error(Eex, irs);
+   double Error_H = H_gf.ComputeL2Error(Hex, irs);
+   
+   cout << "|| E_h - E || = " << Error_E << "\n";
+   cout << "|| H_h - H || = " << Error_H << "\n";
+   cout << "Total error = " << sqrt(Error_H*Error_H+Error_E*Error_E) << "\n";
+
+   GridFunction E_exgf(&H1fes);
+   E_exgf.ProjectCoefficient(Eex);
+
+   GridFunction H_exgf(&NDfes);
+   H_exgf.ProjectCoefficient(Hex);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      socketstream ex_sock(vishost, visport);
+      ex_sock.precision(8);
+      socketstream sol_sockH(vishost, visport);
+      sol_sockH.precision(8);
+      socketstream ex_sockH(vishost, visport);
+      ex_sockH.precision(8);
+      sol_sock << "solution\n"
+               << mesh << E_gf << "window_title 'Numerical E'" << "keys rRljc\n"
+               << flush;
+      ex_sock << "solution\n"
+               << mesh << E_exgf << "window_title 'Exact E'" << "keys rRljc\n"
+               << flush;
+      sol_sockH << "solution\n"
+               << mesh << H_gf << "window_title 'Numerical H'" << "keys rRljc\n"
+               << flush;
+      ex_sockH << "solution\n"
+               << mesh << H_exgf << "window_title 'Exact H'" << "keys rRljc\n"
+               << flush;        
+   }
+   delete A_HE;
+   return 0;
+}
+
+double E_exact(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   return E;  //Scalar
+}
+
+//define exact solution
+void H_exact(const Vector &x, Vector &H)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   H[0] = curlE[0]/omega;
+   H[1] = curlE[1]/omega;
+}
+
+double frhs(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+
+   // - omega E + curl H = f
+   // - omega E + curl (curl E) / omega = f
+   double f = - omega * E + curl2E / omega;
+   return f;
+}
+
+void fvrhs(const Vector &x, Vector &f)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   f[0] = - omega * E + curl2E / omega;
+}
+
+
+void get_maxwell_solution(const Vector &X, double & E, Vector & curlE, double & curl2E)
+{
+   double x = X[0];
+   double y = X[1];
+   double Ex, Ey, Exx, Eyy;
+   if (isol == 0) // polynomial
+   {
+      E = x * (1.0 - x) * y * (1.0 - y);
+      Ex = (1.0 - 2.0 * x) * y * (1.0 - y);
+      Ey = x * (1.0 - x) * (1.0 - 2.0 * y);
+
+      Exx = -2.0 * y * (1.0 - y);
+      Eyy = -2.0 * x * (1.0 - x);
+   }
+   else 
+   {
+      double s = omega * (y+x);
+      E = cos(s);
+      Ex = -omega * sin(s);  
+      Ey = Ex;
+      Exx = - omega * omega * E;  
+      Eyy = Exx;  
+   }
+   curlE[0] = Ey;
+   curlE[1] = -Ex;
+   curl2E = -Exx - Eyy;
+}

examples/dpg_tests/EM-diffusion/makefile

@@ -10,19 +10,23 @@
 # CONTRIBUTING.md for details.
 
 # Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ipopt/,)
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/EM-diffusion,)
 CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_EXAMPLES = exContactBlockTL
-EXAMPLES = $(SEQ_EXAMPLES)
+SEQ_EXAMPLES = primal_dpg
+	 				
+PAR_EXAMPLES = 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
 
 .SUFFIXES:
 .SUFFIXES: .o .cpp .mk
@@ -35,12 +39,7 @@ EXAMPLES = $(SEQ_EXAMPLES)
 %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
 	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
 
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_IPOPT),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with IPOPT)
-endif
+all: $(EXAMPLES) 
 
 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)
@@ -52,17 +51,11 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
 
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
+clean: clean-build clean-exec 
 
 clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES)
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -rf ParaView
 
 clean-exec:
-	@rm -f exContactBlockTL.mesh exContactBlockTL-mesh.* exContactBlockTL-init.* exContactBlockTL-final.* ExampleContactBlockTL*

examples/dpg_tests/EM-diffusion/primal_dpg.cpp

@@ -0,0 +1,201 @@
+//                     MFEM primal_dpg example
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+int dim;
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   int ref = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");   
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&ref, "-ref", "--refinements",
+                  "Number of refinements.");                         
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   kappa = freq * M_PI;
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+
+   for (int i = 0; i<ref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+   // 3. Define a finite element space on the mesh. Here we use H1 continuous
+   //    high-order Lagrange finite elements of the given order.
+   ND_FECollection fec(order, mesh.Dimension());
+   FiniteElementSpace NDfes(&mesh, &fec);
+
+   FiniteElementCollection * trace_fec = nullptr;
+   if (dim == 3)
+   {
+      trace_fec = new ND_Trace_FECollection(order,mesh.Dimension());
+   }
+   else
+   {
+      trace_fec = new H1_Trace_FECollection(order,mesh.Dimension());
+   }
+   FiniteElementSpace trace_fes(&mesh, trace_fec);
+
+   int test_order = order+1;
+
+   ND_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+
+   trial_fes.Append(&NDfes);
+   trial_fes.Append(&trace_fes);
+   test_fecs.Append(&test_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+
+
+   ConstantCoefficient one(1.0);
+   a->AddTrialIntegrator(new CurlCurlIntegrator(one),0,0);
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(one),0,0);
+   a->AddTrialIntegrator(new TangentTraceIntegrator,1,0);
+   
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),0,0);
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+
+   VectorFunctionCoefficient f(sdim, f_exact);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
+
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      NDfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   Vector X,B;
+   OperatorPtr Ah;
+
+   VectorFunctionCoefficient E(sdim, E_exact);
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = NDfes.GetVSize();
+   offsets[2] = trace_fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets);
+   x = 0.;
+
+
+   GridFunction E_gf(&NDfes);
+   E_gf.MakeRef(&NDfes,x.GetBlock(0));
+   E_gf.ProjectBdrCoefficientTangent(E,ess_bdr);
+   E_gf.ProjectCoefficient(E);
+
+
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+
+
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   GMRESSolver cg;
+   cg.SetRelTol(1e-8);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   E_gf.MakeRef(&NDfes,x.GetData());
+
+   double L2Error = E_gf.ComputeL2Error(E);
+   mfem::out << "L2_error = " << L2Error << endl;
+
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << E_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+   delete trace_fec;
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/dpg_tests/acoustics/complex_uw_dpg.cpp

@@ -0,0 +1,677 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make uw_dpg
+//
+
+//     - Δ p - ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//  ∇ p + i ω u = 0, in Ω
+//  ∇⋅u + i ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/(i ω) 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) + i ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+// -(u , ∇ q) + i ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂  = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | i ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | i ω (p,q) |-(u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void acoustics_solution(const Vector & X, complex<double> & p, 
+                        vector<complex<double>> &dp, complex<double> & d2p);
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);      
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);                                                
+
+double p_exact_r(const Vector &x);
+double p_exact_i(const Vector &x);
+void u_exact_r(const Vector &x, Vector & u);
+void u_exact_i(const Vector &x, Vector & u);
+double rhs_func_r(const Vector &x);
+double rhs_func_i(const Vector &x);
+void gradp_exact_r(const Vector &x, Vector &gradu);
+void gradp_exact_i(const Vector &x, Vector &gradu);
+double divu_exact_r(const Vector &x);
+double divu_exact_i(const Vector &x);
+double d2_exact_r(const Vector &x);
+double d2_exact_i(const Vector &x);
+double hatp_exact_r(const Vector & X);
+double hatp_exact_i(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+void hatu_exact_r(const Vector & X, Vector & hatu);
+void hatu_exact_i(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                     
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   mfem::out << "p_fes space true dofs = " << p_fes->GetTrueVSize() << endl;
+   mfem::out << "u_fes space true dofs = " << u_fes->GetTrueVSize() << endl;
+   mfem::out << "hatp_fes space true dofs = " << hatp_fes->GetTrueVSize() << endl;
+   mfem::out << "hatu_fes space true dofs = " << hatu_fes->GetTrueVSize() << endl;
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ComplexNormalEquations * a = new ComplexNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices();
+
+   // i ω (p,q)
+   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omeg),0,0);
+
+// -(u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(negone)),nullptr,1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),nullptr,0,1);
+
+//  i ω (u,v)
+   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(omeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,nullptr,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,3,0);
+
+   // for impedence condition (only on the boundary)
+   // TODO
+   // a->AddTrialIntegrator(new TraceIntegrator,nullptr,2,0);
+
+
+// test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),nullptr,1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -i ω (∇q,δv)
+      a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negomeg),0,1);
+      // i ω (v,∇ δq)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakDivergenceIntegrator(negomeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),nullptr,1,1);
+
+      // - i ω (∇⋅v,δq)   
+      a->AddTestIntegrator(nullptr,new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // i ω (q,∇⋅v)   
+      a->AddTestIntegrator(nullptr,new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),nullptr,0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs_r(rhs_func_r);
+   FunctionCoefficient f_rhs_i(rhs_func_i);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),new DomainLFIntegrator(f_rhs_i),0);
+   
+   
+   FunctionCoefficient hatpex_r(hatp_exact_r);
+   FunctionCoefficient hatpex_i(hatp_exact_i);
+   VectorFunctionCoefficient hatuex_r(dim,hatu_exact_r);
+   VectorFunctionCoefficient hatuex_i(dim,hatu_exact_i);
+   Array<int> elements_to_refine;
+
+   socketstream p_out_r;
+   socketstream p_out_i;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out_r.open(vishost, visport);
+      p_out_i.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // ess_bdr[1] = 0;
+         // ess_bdr[2] = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         // hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+                        //   + hatp_fes->GetTrueVSize(); 
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ComplexGridFunction hatp_gf(hatp_fes);
+      hatp_gf.real().MakeRef(hatp_fes,&xdata[offsets[2]]);
+      hatp_gf.imag().MakeRef(hatp_fes,&xdata[offsets.Last()+ offsets[2]]);
+      hatp_gf.ProjectBdrCoefficient(hatpex_r,hatpex_i, ess_bdr);
+
+      // ComplexGridFunction hatu_gf(hatu_fes);
+      // hatu_gf.real().MakeRef(hatu_fes,&xdata[offsets[3]]);
+      // hatu_gf.imag().MakeRef(hatu_fes,&xdata[offsets.Last()+ offsets[3]]);
+      // hatu_gf.ProjectBdrCoefficientNormal(hatuex_r,hatuex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      SparseMatrix * Ar = dynamic_cast<BlockMatrix *>(&Ahc->real())->CreateMonolithic();
+      SparseMatrix * Ai = dynamic_cast<BlockMatrix *>(&Ahc->imag())->CreateMonolithic();
+
+      ComplexSparseMatrix Ac(Ar,Ai,true,true);
+      SparseMatrix * A = Ac.GetSystemMatrix();
+
+      mfem::out << "Size of the linear system: " << A->Height() << std::endl;
+
+
+      UMFPackSolver umf(*A);
+      umf.Mult(B,X);
+
+      delete A;
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      double residual = residuals.Norml2();
+
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ComplexGridFunction p(p_fes);
+      p.real().MakeRef(p_fes,x.GetData());
+      p.imag().MakeRef(p_fes,&x.GetData()[offsets.Last()]);
+
+      ComplexGridFunction pgf_ex(p_fes);
+      FunctionCoefficient p_ex_r(p_exact_r);
+      FunctionCoefficient p_ex_i(p_exact_i);
+      pgf_ex.ProjectCoefficient(p_ex_r, p_ex_i);
+
+      int dofs = X.Size()/2;
+
+      double p_err_r = p.real().ComputeL2Error(p_ex_r);
+      double p_err_i = p.imag().ComputeL2Error(p_ex_i);
+
+      double L2Error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  0.0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::setw(10) << std::scientific 
+                << std::endl;
+
+      if (visualization)
+      {
+         p_out_r.precision(8);
+         p_out_r << "solution\n" << mesh << p.real() <<
+                  "window_title 'Real Numerical presure' "
+                  << flush;
+
+         p_out_i.precision(8);
+         p_out_i << "solution\n" << mesh << p.imag() <<
+                  "window_title 'Imag Numerical presure' "
+                  << flush;         
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double p_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return p;
+}
+
+double p_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return p;
+}
+
+double hatp_exact_r(const Vector & X)
+{
+   return p_exact_r(X);
+}
+
+double hatp_exact_i(const Vector & X)
+{
+   return p_exact_i(X);
+}
+
+void gradp_exact_r(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_r(x,p,grad,d2p);
+}
+
+void gradp_exact_i(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_i(x,p,grad,d2p);
+}
+
+double d2_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return d2p;
+}
+
+double d2_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return d2p;
+}
+
+//  u = - ∇ p / (i ω )
+//    = i (∇ p_r + i * ∇ p_i)  / ω 
+//    = - ∇ p_i / ω + i ∇ p_r / ω 
+void u_exact_r(const Vector &x, Vector & u)
+{
+   gradp_exact_i(x,u);
+   u *= -1./omega;
+}
+
+void u_exact_i(const Vector &x, Vector & u)
+{
+   gradp_exact_r(x,u);
+   u *= 1./omega;
+}
+
+void hatu_exact_r(const Vector & X, Vector & hatu)
+{
+   u_exact_r(X,hatu);
+}
+void hatu_exact_i(const Vector & X, Vector & hatu)
+{
+   u_exact_i(X,hatu);
+}
+
+//  ∇⋅u = i Δ p / ω
+//      = i (Δ p_r + i * Δ p_i)  / ω 
+//      = - Δ p_i / ω + i Δ p_r / ω 
+
+double divu_exact_r(const Vector &x)
+{
+   return -d2_exact_i(x)/omega;
+}
+
+double divu_exact_i(const Vector &x)
+{
+   return d2_exact_r(x)/omega;
+}
+
+// f = ∇⋅u + i ω p 
+// f_r = ∇⋅u_r - ω p_i  
+double rhs_func_r(const Vector &x)
+{
+   double p = p_exact_i(x);
+   double divu = divu_exact_r(x);
+   return divu - omega * p;
+}
+
+// f_i = ∇⋅u_i + ω p_r
+double rhs_func_i(const Vector &x)
+{
+   double p = p_exact_r(x);
+   double divu = divu_exact_i(x);
+   return divu + omega * p;
+}
+
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.real();
+   d2p = d2zp.real();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].real();
+   }
+}                           
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.imag();
+   d2p = d2zp.imag();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].imag();
+   }
+}
+
+
+void acoustics_solution(const Vector & X, complex<double> & p, vector<complex<double>> & dp, 
+         complex<double> & d2p)
+{
+   dp.resize(X.Size());
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      double beta = omega/std::sqrt((double)X.Size());
+      complex<double> alpha = beta * zi * X.Sum();
+      p = exp(-alpha);
+      d2p = - dim * beta * beta * p;
+      for (int i = 0; i<X.Size(); i++)
+      {
+         dp[i] = - zi * beta * p;
+      }
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> zi = complex<double>(0., 1.);
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp;
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
+
+      d2p = (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+          + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
+   }
+   break;
+   }
+
+}

examples/dpg_tests/acoustics/fosls.cpp

@@ -0,0 +1,294 @@
+//                 MFEM FOSLS acoustics Example
+//
+// Compile with: make fosls
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// FOSLS:
+//       minimize  1/2(||∇p - ω u||^2 + ||-∇⋅u ± ω p - f||^2)
+
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// -------------------------------------------------------------------
+// |   |            p           |            u             |   RHS    | 
+// -------------------------------------------------------------------
+// | q |  (∇ p,∇ q) + ω^2(p,q)  | ∓ ω (∇⋅u,q) - ω (u, ∇ q) | ± ω(f,q) |
+// |   |                        |                          |          |
+// | v | ∓ ω (p,∇⋅v) - ω (∇ p,v)|  (∇⋅u,∇⋅v) + ω^2 (u,v)   | -(f,∇⋅v) |
+
+// where (q,v) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int sr = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * H1fes = new FiniteElementSpace(&mesh, H1fec);
+   FiniteElementSpace * RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = H1fes;
+   fespaces[1] = RTfes;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+   cout << "H1 fespace = " << H1fes->GetTrueVSize() << endl;
+   cout << "RT fespace = " << RTfes->GetTrueVSize() << endl;
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega*omega);
+
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+
+   // blfi(0,0) = (∇ p,∇ q) + ω^2(p,q)
+   SumIntegrator * integ00 = new SumIntegrator();
+   integ00->AddIntegrator(new DiffusionIntegrator(one));
+   integ00->AddIntegrator(new MassIntegrator(omeg2));
+   blfi(0,0) = integ00;
+
+   // blfi(0,1) = ∓ ω (∇⋅u,q) - ω (u, ∇ q)
+   SumIntegrator * integ01 = new SumIntegrator();
+#ifdef DEFINITE
+   // -ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+#else   
+   // ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+#endif
+   // - ω (u, ∇ q)
+   integ01->AddIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   blfi(0,1) = integ01;
+
+   // blfi(1,0) = ∓ ω (p,∇⋅v) - ω (∇ p,v)
+   SumIntegrator * integ10 = new SumIntegrator();
+#ifdef DEFINITE
+   // - ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+#else
+   // ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+#endif
+   // - ω (∇ p,v)
+   integ10->AddIntegrator(new MixedVectorGradientIntegrator(negomeg));
+   blfi(1,0) = integ10;
+
+   // blfi(1,1) = (∇⋅u,∇⋅v) + ω^2 (u,v)
+   SumIntegrator * integ11 = new SumIntegrator();
+   integ11->AddIntegrator(new DivDivIntegrator(one));
+   integ11->AddIntegrator(new VectorFEMassIntegrator(omeg2));
+   blfi(1,1) = integ11;
+
+
+   BlockLinearForm b(fespaces);
+   Array<LinearFormIntegrator * > lfi(2);
+   // ± ω (f,q)
+   FunctionCoefficient f_rhs(rhs_func);
+#ifdef DEFINITE  
+   ProductCoefficient w_f(omeg,f_rhs);
+#else   
+   ProductCoefficient w_f(negomeg,f_rhs);
+#endif   
+   // lfi[0] = new DomainLFIntegrator(w_f);
+   lfi[0] = new DomainLFIntegrator(w_f);
+
+
+   //  -(f,∇⋅v)
+   ProductCoefficient neg_f(negone,f_rhs);
+   // lfi[1] = new VectorFEDomainLFDivIntegrator(f_rhs);
+   lfi[1] = new VectorFEDomainLFDivIntegrator(neg_f);
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(H1fes);   
+
+   p_gf.MakeRef(H1fes,x,0);
+   // p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+   p_gf.ProjectCoefficient(p_ex);
+   pex_gf.ProjectCoefficient(p_ex);
+
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+   u_gf = 0.;
+
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-10);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   p_gf.MakeRef(H1fes,x,0);
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << p_gf <<
+                "window_title 'Numerical p' "
+                << flush;
+      // socketstream sols_sock(vishost, visport);
+      // sols_sock.precision(8);
+      // sols_sock << "solution\n" << mesh << u_gf <<
+               //  "window_title 'Numerical sigma' "
+               //  << flush;
+
+      socketstream solex_sock(vishost, visport);
+      solex_sock.precision(8);
+      solex_sock << "solution\n" << mesh << pex_gf <<
+                    "window_title 'Exact p' "
+                    << flush;          
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/dpg_tests/acoustics/makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = fosls uw_dpg  strong_dpg complex_uw_dpg
+PAR_EXAMPLES = uw_dpgp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:

examples/dpg_tests/acoustics/pcomplex_uw_dpg.cpp

@@ -0,0 +1,837 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make pcomplex_uw_dpg
+//
+// sample runs
+// ./pcomplex_uw_dpg -o 3 -m ../../../data/inline-quad.mesh -sref 2 -pref 3 -rnum 4.1 -prob 0 -sc -graph-norm
+
+//     - Δ p - ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//  ∇ p + i ω u = 0, in Ω
+//  ∇⋅u + i ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/(i ω) 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) + i ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+// -(u , ∇ q) + i ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂  = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | i ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | i ω (p,q) |-(u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void acoustics_solution(const Vector & X, complex<double> & p, 
+                        vector<complex<double>> &dp, complex<double> & d2p);
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);      
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);                                                
+
+double p_exact_r(const Vector &x);
+double p_exact_i(const Vector &x);
+void u_exact_r(const Vector &x, Vector & u);
+void u_exact_i(const Vector &x, Vector & u);
+double rhs_func_r(const Vector &x);
+double rhs_func_i(const Vector &x);
+void gradp_exact_r(const Vector &x, Vector &gradu);
+void gradp_exact_i(const Vector &x, Vector &gradu);
+double divu_exact_r(const Vector &x);
+double divu_exact_i(const Vector &x);
+double d2_exact_r(const Vector &x);
+double d2_exact_i(const Vector &x);
+double hatp_exact_r(const Vector & X);
+double hatp_exact_i(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+void hatu_exact_r(const Vector & X, Vector & hatu);
+void hatu_exact_i(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+   int sr = 0;
+   int pr = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");     
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                    
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of parallel refinements.");                                              
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");  
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   dim = mesh.Dimension();
+   mesh.EnsureNCMesh();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   // if (myid == 0)
+   // {
+   //    mfem::out << "p_fes space true dofs = " << p_fes->GetTrueVSize() << endl;
+   //    mfem::out << "u_fes space true dofs = " << u_fes->GetTrueVSize() << endl;
+   //    mfem::out << "hatp_fes space true dofs = " << hatp_fes->GetTrueVSize() << endl;
+   //    mfem::out << "hatu_fes space true dofs = " << hatu_fes->GetTrueVSize() << endl;
+   // }
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ComplexParNormalEquations * a = new ComplexParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices();
+   // i ω (p,q)
+   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omeg),0,0);
+
+// -(u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(negone)),nullptr,1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),nullptr,0,1);
+
+//  i ω (u,v)
+   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(omeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,nullptr,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,3,0);
+
+// test integrators 
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),nullptr,1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -i ω (∇q,δv)
+      a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negomeg),0,1);
+      // i ω (v,∇ δq)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakDivergenceIntegrator(negomeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),nullptr,1,1);
+
+      // - i ω (∇⋅v,δq)   
+      a->AddTestIntegrator(nullptr,new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // i ω (q,∇⋅v)   
+      a->AddTestIntegrator(nullptr,new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),nullptr,0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs_r(rhs_func_r);
+   FunctionCoefficient f_rhs_i(rhs_func_i);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),new DomainLFIntegrator(f_rhs_i),0);
+   
+
+   FunctionCoefficient hatpex_r(hatp_exact_r);
+   FunctionCoefficient hatpex_i(hatp_exact_i);
+
+   VectorFunctionCoefficient hatuex_r(dim,hatu_exact_r);
+   VectorFunctionCoefficient hatuex_i(dim,hatu_exact_i);
+   Array<int> elements_to_refine;
+
+   socketstream p_out_r;
+   socketstream p_out_i;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out_r.open(vishost, visport);
+      p_out_i.open(vishost, visport);
+   }
+
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << "\n  Ref |" 
+                << "       Mesh       |"
+                << "    Dofs    |" 
+                << "   ω   |" 
+                << "  L2 Error  |" 
+                << " Relative % |" 
+                << "  Rate  |" 
+                << "  Residual  |" 
+                << "  Rate  |" 
+                << " PCG it |"
+                << " PCG time |"  << endl;
+      mfem::out << " --------------------"      
+                <<  "---------------------"    
+                <<  "---------------------"    
+                <<  "---------------------"    
+                <<  "---------------------"    
+                <<  "-------------------" << endl;      
+   }
+
+
+   for (int it = 0; it<pr; it++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // ess_bdr[1] = 0;
+         // ess_bdr[2] = 0;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         // hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+                           //   + hatp_fes->GetTrueVSize(); 
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ParComplexGridFunction hatp_gf(hatp_fes);
+      hatp_gf.real().MakeRef(hatp_fes,&xdata[offsets[2]]);
+      hatp_gf.imag().MakeRef(hatp_fes,&xdata[offsets.Last()+ offsets[2]]);
+      hatp_gf.ProjectBdrCoefficient(hatpex_r,hatpex_i, ess_bdr);
+      // ParComplexGridFunction hatu_gf(hatu_fes);
+      // hatu_gf.real().MakeRef(hatu_fes,&xdata[offsets[3]]);
+      // hatu_gf.imag().MakeRef(hatu_fes,&xdata[offsets.Last()+ offsets[3]]);
+      // hatu_gf.ProjectCoefficientNormal(hatuex_r,hatuex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      int skip = (static_cond) ? 0 : 2;
+      int k = (static_cond) ? 2 : 0;
+      for (int i=0; i<num_blocks;i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize(); 
+         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize(); 
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+
+      X = 0.;
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(tdof_offsets);
+
+
+      if (!static_cond)
+      {
+         HypreBoomerAMG * solver_p = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(0,0));
+         solver_p->SetPrintLevel(0);
+         solver_p->SetSystemsOptions(dim);
+         HypreBoomerAMG * solver_u = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(1,1));
+         solver_u->SetPrintLevel(0);
+         solver_u->SetSystemsOptions(dim);
+         M->SetDiagonalBlock(0,solver_p);
+         M->SetDiagonalBlock(1,solver_u);
+         M->SetDiagonalBlock(num_blocks,solver_p);
+         M->SetDiagonalBlock(num_blocks+1,solver_u);
+      }
+
+
+      HypreBoomerAMG * solver_hatp = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(skip,skip));
+      // amg->SetCycleNumSweeps(5, 5);
+      solver_hatp->SetPrintLevel(0);
+      
+      HypreSolver * solver_hatu = nullptr;
+      if (dim == 2)
+      {
+         solver_hatu = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),hatu_fes);
+         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
+      }
+      else
+      {
+         solver_hatu = new HypreADS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1), hatu_fes);
+         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
+      }
+      
+
+      M->SetDiagonalBlock(skip,solver_hatp);
+      M->SetDiagonalBlock(skip+1,solver_hatu);
+      M->SetDiagonalBlock(skip+num_blocks,solver_hatp);
+      M->SetDiagonalBlock(skip+num_blocks+1,solver_hatu);
+
+      StopWatch chrono;
+      
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-7);
+      cg.SetAbsTol(1e-7);
+      cg.SetMaxIter(10000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M); 
+      cg.SetOperator(blockA);
+      chrono.Clear();
+      chrono.Start();
+      cg.Mult(B, X);
+      chrono.Stop();
+      delete M;
+
+      int ne = pmesh.GetNE();
+      MPI_Allreduce(MPI_IN_PLACE,&ne,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+      int ne_x = (dim == 2) ? (int)sqrt(ne) : (int)cbrt(ne);
+      ostringstream oss;
+      double pcg_time = chrono.RealTime();
+      if (myid == 0)
+      {
+         if (dim == 2)
+         {
+            oss << ne_x << " x " << ne_x ;
+         }
+         else
+         {
+            oss << ne_x << " x " << ne_x << " x " << ne_x ;
+         }
+      }
+
+      int num_iter = cg.GetNumIterations();
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParComplexGridFunction p(p_fes);
+      p.real().MakeRef(p_fes,x.GetData());
+      p.imag().MakeRef(p_fes,&x.GetData()[offsets.Last()]);
+
+      ParComplexGridFunction u(u_fes);
+      u.real().MakeRef(u_fes,&x.GetData()[offsets[1]]);
+      u.imag().MakeRef(u_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+
+      // Error in pressure 
+      ParComplexGridFunction pgf_ex(p_fes);
+      FunctionCoefficient p_ex_r(p_exact_r);
+      FunctionCoefficient p_ex_i(p_exact_i);
+      pgf_ex.ProjectCoefficient(p_ex_r, p_ex_i);
+
+      double p_err_r = p.real().ComputeL2Error(p_ex_r);
+      double p_err_i = p.imag().ComputeL2Error(p_ex_i);
+      double p_error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
+      double p_norm_r = pgf_ex.real().ComputeL2Error(zero);
+      double p_norm_i = pgf_ex.imag().ComputeL2Error(zero);
+      double p_norm = sqrt(p_norm_r*p_norm_r + p_norm_i*p_norm_i);
+
+      // Error in velocity
+      ParComplexGridFunction ugf_ex(u_fes);
+      VectorFunctionCoefficient u_ex_r(dim,u_exact_r);
+      VectorFunctionCoefficient u_ex_i(dim,u_exact_i);
+
+      double u_err_r = u.real().ComputeL2Error(u_ex_r);
+      double u_err_i = u.imag().ComputeL2Error(u_ex_i);
+      double u_error = sqrt(u_err_r*u_err_r + u_err_i*u_err_i);
+      double u_norm_r = pgf_ex.real().ComputeL2Error(vzero);
+      double u_norm_i = pgf_ex.imag().ComputeL2Error(vzero);
+      double u_norm = sqrt(u_norm_r*u_norm_r + u_norm_i*u_norm_i);
+
+
+      double L2Error = sqrt(p_error*p_error + u_error*u_error);
+      double L2norm = sqrt(p_norm*p_norm + u_norm*u_norm);
+
+      double rel_err = L2Error/L2norm;
+
+      int dofs = p_fes->GlobalTrueVSize()
+               + u_fes->GlobalTrueVSize()
+               + hatp_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize();
+
+
+      double rate_err = (it) ? dim*log(err0/rel_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = rel_err;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(5) << it << " | " 
+                  << std::setw(16) << oss.str() << " | " 
+                  << std::setw(10) <<  dof0 << " | " 
+                  << std::setprecision(0) << std::fixed
+                  << std::setw(2) <<  2*rnum << " π  | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  err0 << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::fixed <<  rel_err * 100. << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_err << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  res0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_res << " | " 
+                  << std::setw(6) << std::fixed << num_iter << " | " 
+                  << std::setprecision(5) 
+                  << std::setw(8) << std::fixed << pcg_time << " | " 
+                  << std::scientific 
+                  << std::endl;
+      }   
+
+      if (visualization)
+      {
+         p_out_r << "parallel " << num_procs << " " << myid << "\n";
+         p_out_r.precision(8);
+         p_out_r << "solution\n" << pmesh << p.real() <<
+                  "window_title 'Real Numerical presure' "
+                  << flush;
+
+         p_out_i << "parallel " << num_procs << " " << myid << "\n";
+         p_out_i.precision(8);
+         p_out_i << "solution\n" << pmesh << p.imag() <<
+                  "window_title 'Imag Numerical presure' "
+                  << flush;         
+      }
+
+      if (it == pr)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();   
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double p_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return p;
+}
+
+double p_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return p;
+}
+
+double hatp_exact_r(const Vector & X)
+{
+   return p_exact_r(X);
+}
+
+double hatp_exact_i(const Vector & X)
+{
+   return p_exact_i(X);
+}
+
+void gradp_exact_r(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_r(x,p,grad,d2p);
+}
+
+void gradp_exact_i(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_i(x,p,grad,d2p);
+}
+
+double d2_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return d2p;
+}
+
+double d2_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return d2p;
+}
+
+//  u = - ∇ p / (i ω )
+//    = i (∇ p_r + i * ∇ p_i)  / ω 
+//    = - ∇ p_i / ω + i ∇ p_r / ω 
+void u_exact_r(const Vector &x, Vector & u)
+{
+   gradp_exact_i(x,u);
+   u *= -1./omega;
+}
+
+void u_exact_i(const Vector &x, Vector & u)
+{
+   gradp_exact_r(x,u);
+   u *= 1./omega;
+}
+
+void hatu_exact_r(const Vector & X, Vector & hatu)
+{
+   u_exact_r(X,hatu);
+}
+void hatu_exact_i(const Vector & X, Vector & hatu)
+{
+   u_exact_i(X,hatu);
+}
+
+//  ∇⋅u = i Δ p / ω
+//      = i (Δ p_r + i * Δ p_i)  / ω 
+//      = - Δ p_i / ω + i Δ p_r / ω 
+
+double divu_exact_r(const Vector &x)
+{
+   return -d2_exact_i(x)/omega;
+}
+
+double divu_exact_i(const Vector &x)
+{
+   return d2_exact_r(x)/omega;
+}
+
+// f = ∇⋅u + i ω p 
+// f_r = ∇⋅u_r - ω p_i  
+double rhs_func_r(const Vector &x)
+{
+   double p = p_exact_i(x);
+   double divu = divu_exact_r(x);
+   return divu - omega * p;
+}
+
+// f_i = ∇⋅u_i + ω p_r
+double rhs_func_i(const Vector &x)
+{
+   double p = p_exact_r(x);
+   double divu = divu_exact_i(x);
+   return divu + omega * p;
+}
+
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.real();
+   d2p = d2zp.real();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].real();
+   }
+}                           
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.imag();
+   d2p = d2zp.imag();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].imag();
+   }
+}
+
+
+void acoustics_solution(const Vector & X, complex<double> & p, vector<complex<double>> & dp, 
+         complex<double> & d2p)
+{
+   dp.resize(X.Size());
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      double beta = omega/std::sqrt((double)X.Size());
+      complex<double> alpha = beta * zi * X.Sum();
+      p = exp(-alpha);
+      d2p = - dim * beta * beta * p;
+      for (int i = 0; i<X.Size(); i++)
+      {
+         dp[i] = - zi * beta * p;
+      }
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp;
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
+
+      d2p = (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+          + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
+   }
+   break;
+   }
+
+}

examples/dpg_tests/acoustics/strong_dpg.cpp

@@ -0,0 +1,271 @@
+//                 MFEM DPG_strong acoustics Example
+//
+// Compile with: make strong_dpg
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// Strong DPG formulation
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// 
+//   (∇ p, v) - ω (u,v) = 0,     in Ω, ∀ v ∈ (L^2)^dim
+//  -(∇⋅u, q) ± ω (p,q) = (f,q), in Ω, ∀ q ∈  L^2 
+//                    p = p_0, in ∂Ω
+// 
+// ------------------------------------
+// |   |      p    |     u    |  RHS  | 
+// ------------------------------------
+// | q | ± ω (p,q) | -(∇⋅u,q) | (f,q) |
+// |   |           |          |       |
+// | v | (∇ p, v)  | -ω (u,v) |       |
+
+// where (q,v) ∈ L^2  × (L^2)^dim
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");                  
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   // Define spaces
+   // H1 space for p
+   FiniteElementCollection *p_fec = new H1_FECollection(order, dim);
+   FiniteElementSpace * p_fes = new FiniteElementSpace(&mesh, p_fec);
+
+   // H(div) for u
+   FiniteElementCollection *u_fec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * u_fes = new FiniteElementSpace(&mesh, u_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new L2_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new L2_FECollection(test_order-1, dim);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->SetTestFECollVdim(1,dim);
+   
+   a->StoreMatrices(true);
+
+//  ± ω (p, q)
+#ifdef DEFINITE
+//  ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(omeg),0,0);
+#else
+// -ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(negomeg),0,0);
+#endif
+
+// -(∇⋅u, q) 
+   a->AddTrialIntegrator(new MixedScalarDivergenceIntegrator(negone),1,0);
+
+// -ω (u,v)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(negomeg),1,1);
+
+// (∇ p, v)
+   a->AddTrialIntegrator(new GradientIntegrator(one),0,1);
+
+// (v,δv)
+   a->AddTestIntegrator(new VectorMassIntegrator(one),1,1);
+    
+// (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(p_fes);   
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = p_fes->GetVSize();
+   offsets[2] = u_fes->GetVSize();
+   offsets.PartialSum();
+   BlockVector x(offsets);
+   x = 0.0;
+
+   p_gf.MakeRef(p_fes,x.GetBlock(0));
+   p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+
+   u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = Ah.As<BlockMatrix>();
+
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream p_out;
+      socketstream u_out;
+      p_out.open(vishost, visport);
+      u_out.open(vishost, visport);
+      p_out.precision(8);
+      p_out << "solution\n" << mesh << p_gf <<
+               "window_title 'Numerical p' "
+               << flush;
+
+      u_out.precision(8);
+      u_out << "solution\n" << mesh << u_gf <<
+            "window_title 'Numerical flux' "
+            << flush;
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/dpg_tests/acoustics/uw_dpg.cpp

@@ -0,0 +1,546 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make uw_dpg
+//
+// ./uw_dpg -m ../../../data/inline-quad.mesh -rnum 40 -theta 0.7 -prob 1 -graph-norm -ref 40 -o 3
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p);
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+double divu_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+   // (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+   // - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+   // < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   // test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   GridFunction hatp_gf;
+
+
+   socketstream p_out;
+   // socketstream u_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+      // u_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-8);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+      GridFunction pex_gf(p_fes);
+      GridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+
+      // Error
+      int dofs = X.Size();
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = pex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+      if (visualization)
+      {
+         p_out.precision(8);
+         p_out << "solution\n" << mesh << p_gf <<
+                  "window_title 'Numerical presure' "
+                  << flush;
+
+         // u_out.precision(8);
+         // u_out << "solution\n" << mesh << u_gf <<
+         //       "window_title 'Numerical velocity' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return p;
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   double p, d2p;
+   acoustics_solution(x,p,u,d2p);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return d2p/omega;
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}
+
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p)
+{
+   dp.SetSize(X.Size());
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      p = sin(omega*X.Sum());
+      dp = omega * cos(omega * X.Sum());
+      d2p = -dim * omega * omega * sin(omega*X.Sum());
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> zi = complex<double>(0., 1.);
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp.real();
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim).real();
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim).real();
+
+      d2p = (  (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+            + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim ).real();
+   }
+   break;
+   }
+
+}

examples/dpg_tests/acoustics/uw_dpgp.cpp

@@ -0,0 +1,525 @@
+//                   MFEM Ultraweak DPG MPI acoustics (Helmholtz) example
+//
+// Compile with: make uw_dpgp
+//
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+//
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+// 
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int sr = 0;
+   int pr = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");      
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of parallel refinements.");                                              
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");  
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                    
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   dim = mesh.Dimension();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   Array<ParFiniteElementSpace * > trial_fes; 
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   Array<FiniteElementCollection * > test_fec; 
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // Integrators
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+// (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+// - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   // test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }   
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatp_gf;
+
+
+
+
+   socketstream p_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+   }
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << "\n Refinement |" 
+               << "    Dofs    |" 
+               << "   ω   |" 
+               << "  L2 Error  |" 
+               << " Relative % |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" 
+               << " PCG it |" << endl;
+      mfem::out << " --------------------"      
+               <<  "---------------------"    
+               <<  "---------------------"    
+               <<  "---------------------"    
+               <<  "----------------" << endl;   
+   }
+
+
+   for (int i = 0; i<pr; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         // amg0->SetRelaxType(16);
+         // amg1->SetRelaxType(16);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip = 2;
+      }
+
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      // amg2->SetRelaxType(16);
+      M->SetDiagonalBlock(skip,amg2);
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatu_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatu_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-7);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      int num_iter = cg.GetNumIterations();
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ParGridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+
+      ParGridFunction pex_gf(p_fes);
+      ParGridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+      int dofs = p_fes->GlobalTrueVSize()
+               + u_fes->GlobalTrueVSize()
+               + hatp_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize();
+
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = pex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      std::ios oldState(nullptr);
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                   << std::setw(10) <<  dof0 << " | " 
+                   << std::setprecision(0) << std::fixed
+                   << std::setw(2) <<  2*rnum << " π  | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  err0 << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_err << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  res0 << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_res << " | " 
+                   << std::setw(6) << std::fixed << num_iter << " | " 
+                   << std::setprecision(5) 
+                   << std::scientific 
+                   << std::endl;
+      }    
+
+      if (visualization)
+      {
+         p_out << "parallel " << num_procs << " " << myid << "\n";
+         p_out.precision(8);
+         p_out << "solution\n" << pmesh << p_gf <<
+                  "window_title 'Numerical pressure' "
+                  << flush;
+      }
+
+      if (i == pr)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}

examples/dpg_tests/convection-diffusion/makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/convection-diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = uw_dpg
+PAR_EXAMPLES = uw_dpgp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:

examples/dpg_tests/convection-diffusion/uw_dpg.cpp

@@ -0,0 +1,652 @@
+//                     MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");        
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                             
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: EJ ,2: General");           
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");       
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");                                              
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatf_fes = new FiniteElementSpace(&mesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   FiniteElementSpace *coeff_fes = new FiniteElementSpace(&mesh,coeff_fec);
+   GridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         // (∇v,∇δv)
+         mfem::out << "\n Test norm: Standard" << endl;
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         mfem::out << "\n Test norm: Robust" << endl;
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+   GridFunction hatu_gf;
+   GridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<=ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(mesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int j = 0; j < n; j++)
+      {
+         ess_tdof_list[j] = ess_tdof_list_uhat[j] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int j = 0; j < m; j++)
+      {
+         ess_tdof_list[j+n] = ess_tdof_list_fhat[j] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      // BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      // M->owns_blocks = 1;
+      // for (int i=0; i<A->NumRowBlocks(); i++)
+      // {
+      //    M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      // }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(0);
+      // cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      // delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }   
+
+      GridFunction uex_gf(u_fes);
+      uex_gf.ProjectCoefficient(uex);
+
+      GridFunction sigmaex_gf(sigma_fes);
+      sigmaex_gf.ProjectCoefficient(sigmaex);
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = X.Size();
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(zero);
+      // mfem::out << "u_err = " << u_err << endl;
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double sigma_norm = sigmaex_gf.ComputeL2Error(vzero);
+      // mfem::out << "sigma_err = " << sigma_err << endl;
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+      double L2norm = sqrt(u_norm * u_norm + sigma_norm * sigma_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "solution\n" << mesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+         u_out.precision(8);
+         u_out << "solution\n" << mesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> dofs;
+         for (int i = 0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+      }
+
+   }
+
+   delete coeff_fes;
+   delete coeff_fec;
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fes;
+   delete sigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}

examples/dpg_tests/convection-diffusion/uw_dpgp.cpp

@@ -0,0 +1,704 @@
+//                                MFEM Ultraweak DPG example
+//
+// Compile with: make  uw_dpgp
+//
+// sample runs 
+// mpirun -np 6 ./uw_dpgp  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, f̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   bool static_cond = false;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                           
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");                         
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");     
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");          
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatf_fes = new ParFiniteElementSpace(&pmesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   ParFiniteElementSpace *coeff_fes = new ParFiniteElementSpace(&pmesh,coeff_fec);
+   ParGridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Standard" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Robust" << endl;
+         }
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+
+   ParGridFunction hatu_gf;
+   ParGridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << " Refinement |" 
+               << "    Dofs    |" 
+               << "  L2 Error  |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" 
+               << "  CG iter  |" << endl;
+      mfem::out << " --------------------"      
+               <<  "-------------------"    
+               <<  "-------------------"    
+               <<  "-------------------" << endl;   
+   }
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(pmesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int j = 0; j < n; j++)
+      {
+         ess_tdof_list[j] = ess_tdof_list_uhat[j] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int j = 0; j < m; j++)
+      {
+         ess_tdof_list[j+n] = ess_tdof_list_fhat[j] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip = 2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(-1);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      int num_iter = cg.GetNumIterations();
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max(); 
+
+      double gresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&gresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      gresidual = sqrt(gresidual);
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = u_fes->GlobalTrueVSize()
+               + sigma_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize()
+               + hatf_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeL2Error(uex);
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/gresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = gresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                  << std::setw(10) <<  dof0 << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  err0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_err << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  res0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_res << " | " 
+                  << std::setw(6) << std::fixed << num_iter << " | " 
+                  << std::resetiosflags(std::ios::showbase)
+                  << std::endl;
+      }
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "parallel " << num_procs << " " << myid << "\n";
+         // uex_out << "solution\n" << pmesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << pmesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref-1)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> edofs;
+         for (int i = 0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,edofs);
+            c1_gf.SetSubVector(edofs,c1);
+            c2_gf.SetSubVector(edofs,c2);
+         }
+      }
+   }
+
+   delete coeff_fes;
+   delete coeff_fec;
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}

examples/dpg_tests/diffusion/blkfosls.cpp

@@ -0,0 +1,203 @@
+//                 MFEM Fosls 1
+//
+// Compile with: make blkfosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec0 = new H1_FECollection(order, dim);
+   FiniteElementCollection *fec1 = new RT_FECollection(order-1, dim);
+   FiniteElementSpace fespace0(&mesh, fec0);
+   FiniteElementSpace fespace1(&mesh, fec1);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = &fespace0;
+   fespaces[1] = &fespace1;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+
+   cout << "H1 fespace = " << fespace0.GetVSize() << endl;
+   cout << "RT fespace = " << fespace1.GetVSize() << endl;
+
+   FiniteElementCollection *fec2 = new RT_Trace_FECollection(order-1, dim);
+   FiniteElementSpace RT_trace_fes(&mesh, fec2);
+   cout << "RT trace = " << RT_trace_fes.GetVSize() << endl;
+
+   // for (int i = 0; i<mesh.GetNE(); i++)
+   // {
+   //    // const FiniteElement * fe = fespace1.GetFE(i);
+   //    // fespace1.GetTraceElement()
+   //    Array<int> faces, ori;
+   //    mesh.GetElementEdges(i, faces, ori);
+   //    for (int f = 0; f<faces.Size(); f++)
+   //    {
+   //       const FiniteElement * fe_trace = RT_trace_fes.GetFaceElement(faces[f]);
+   //       cout << fe_trace->GetDof() << endl;
+   //       Array<int> face_dofs;
+   //       RT_trace_fes.GetFaceDofs(faces[f],face_dofs);
+   //       cout << "face dofs = " << endl;
+   //       face_dofs.Print();
+   //    }
+
+   //    // cout << fe->GetGeomType() << endl;
+   //    Array<int> vdofs;
+   //    RT_trace_fes.GetElementVDofs(i, vdofs);
+   //    cout << "trace dofs = " << endl;
+   //    vdofs.Print();
+   //    fespace1.GetElementVDofs(i, vdofs);
+   //    cout << "elem dofs = " << endl;
+   //    vdofs.Print();
+   //    cin.get();
+   // }
+
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+   blfi(0,0) = new DiffusionIntegrator(one);
+   blfi(0,1) = new MixedVectorWeakDivergenceIntegrator(one);
+   blfi(1,0) = new MixedVectorGradientIntegrator(negone);
+
+   BilinearFormIntegrator * divdiv = new DivDivIntegrator(one);
+   BilinearFormIntegrator * mass = new VectorFEMassIntegrator(one);
+   SumIntegrator * suminteg = new SumIntegrator();
+   suminteg->AddIntegrator(divdiv);
+   suminteg->AddIntegrator(mass);
+   blfi(1,1) = suminteg;
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+
+   BlockLinearForm b(fespaces);
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   Array<LinearFormIntegrator * > lfi(2);
+   lfi[0] = nullptr;
+   lfi[1] = new VectorFEDomainLFDivIntegrator(negone);
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+
+   // need to implement blkgridfunction later but for now Vector would do
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(200);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   GridFunction u_gf, sigma_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(fespaces[0],&data[0]);
+   sigma_gf.MakeRef(fespaces[1],&data[fespaces[0]->GetVSize()]);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   delete fec0;
+
+   return 0;
+}

examples/dpg_tests/diffusion/fosls.cpp

@@ -0,0 +1,223 @@
+//                   MFEM Fosls example
+//
+// Compile with: make fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order,dim);
+   FiniteElementSpace *H1fes = new FiniteElementSpace(&mesh, H1fec);
+
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1,dim);
+   FiniteElementSpace *RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Linear forms
+   LinearForm b_0(H1fes);
+   // (f,∇⋅τ )
+   LinearForm b_1(RTfes);
+   b_1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(negone));
+
+   // Bilinear forms
+   // (∇u,∇v)
+   BilinearForm a_00(H1fes);
+   a_00.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   //   -(σ,∇v)
+   MixedBilinearForm a_01(RTfes, H1fes);
+   a_01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(
+                               one)); // (-1 is included)
+
+   //    // -(∇u,τ)
+   // MixedBilinearForm()
+   MixedBilinearForm a_10(H1fes, RTfes);
+   a_10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negone));
+
+   // (∇⋅σ, ∇⋅τ) + (σ,τ)
+
+   BilinearForm a_11(RTfes);
+   a_11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a_11.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Array<int> block_Toffsets(3);
+   block_Toffsets[0] = 0;
+   block_Toffsets[1] = H1fes->GetTrueVSize();
+   block_Toffsets[2] = RTfes->GetTrueVSize();
+   block_Toffsets.PartialSum();
+
+   Vector rhs_H1(H1fes->GetVSize()); rhs_H1 = 0.;
+   Vector rhs_RT(RTfes->GetVSize()); rhs_RT = 0.;
+
+   Vector x_H1(H1fes->GetVSize()); x_H1 = 0.;
+   Vector x_RT(RTfes->GetVSize()); x_RT = 0.;
+
+
+   Vector RHS_H1(H1fes->GetTrueVSize()); RHS_H1 = 0.0;
+   Vector RHS_RT(RTfes->GetTrueVSize()); RHS_RT = 0.0;
+
+   Vector X_H1(H1fes->GetTrueVSize()); X_H1 = 0.0;
+   Vector X_RT(RTfes->GetTrueVSize()); X_RT = 0.0;
+
+
+   b_0.Update(H1fes,rhs_H1,0);
+   b_0.Assemble();
+
+   b_1.Update(RTfes,rhs_RT,0);
+   b_1.Assemble();
+
+
+   // Assembly and BC
+   a_00.Assemble();
+   SparseMatrix A_00;
+   a_00.FormLinearSystem(ess_tdof_list,x_H1,rhs_H1,
+                         A_00,X_H1,RHS_H1);
+
+   a_01.Assemble();
+   SparseMatrix A_01;
+   Array<int> empty;
+   a_01.FormRectangularSystemMatrix(empty, ess_tdof_list,A_01);
+
+
+   a_10.Assemble();
+   SparseMatrix A_10;
+
+   a_10.FormRectangularLinearSystem(ess_tdof_list,empty,x_H1,rhs_RT,
+                                    A_10,X_H1,RHS_RT);
+
+   a_11.Assemble();
+   SparseMatrix A_11;
+   a_11.FormSystemMatrix(empty,A_11);
+
+
+   BlockMatrix BlockA(block_Toffsets);
+   BlockA.SetBlock(0,0,&A_00);
+   BlockA.SetBlock(0,1,&A_01);
+   BlockA.SetBlock(1,0,&A_10);
+   BlockA.SetBlock(1,1,&A_11);
+
+
+   BlockVector RHS(block_Toffsets);
+   RHS.GetBlock(0) = RHS_H1;
+   RHS.GetBlock(1) = RHS_RT;
+
+   BlockVector X(block_Toffsets);
+   X.GetBlock(0) = X_H1;
+   X.GetBlock(1) = X_RT;
+
+
+   SparseMatrix * A = BlockA.CreateMonolithic();
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(RHS, X);
+
+   GridFunction u_gf(H1fes), sigma_gf(RTfes);
+   u_gf = 0.;
+   sigma_gf = 0.;
+
+   const SparseMatrix * P = H1fes->GetConformingProlongation();
+   if (P)
+   {
+      a_00.RecoverFEMSolution(X.GetBlock(0),rhs_H1,u_gf);
+      a_11.RecoverFEMSolution(X.GetBlock(1),rhs_RT,sigma_gf);
+   }
+   else
+   {
+      u_gf.MakeRef(X.GetBlock(0),0);
+      sigma_gf.MakeRef(X.GetBlock(1),0);
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   return 0;
+}

examples/dpg_tests/diffusion/makefile

@@ -0,0 +1,61 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = blkfosls fosls primal_dpg \
+	 				uw_dpg 
+PAR_EXAMPLES = uw_dpgp 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -rf ParaView
+
+clean-exec:

examples/dpg_tests/diffusion/primal_dpg.cpp

@@ -0,0 +1,179 @@
+//                     MFEM primal_dpg example
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");   
+   args.ParseCheck();
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   // 3. Define a finite element space on the mesh. Here we use H1 continuous
+   //    high-order Lagrange finite elements of the given order.
+   H1_FECollection fec(order, mesh.Dimension());
+   FiniteElementSpace H1fes(&mesh, &fec);
+
+   RT_Trace_FECollection trace_fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTtrace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   H1_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+
+   trial_fes.Append(&H1fes);
+   trial_fes.Append(&RTtrace_fes);
+   test_fecs.Append(&test_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+
+
+   ConstantCoefficient one(1.0);
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
+   a->AddTrialIntegrator(new TraceIntegrator,1,0);
+
+   BilinearFormIntegrator * diffusion = new DiffusionIntegrator(one);
+   BilinearFormIntegrator * mass = new MassIntegrator(one);
+   a->AddTestIntegrator(diffusion,0,0);
+   a->AddTestIntegrator(mass,0,0);
+
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(one),0);
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+   OperatorPtr Ah;
+
+   int size = H1fes.GetVSize() + RTtrace_fes.GetVSize();
+
+   Vector x(size);
+   x = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+
+   a->RecoverFEMSolution(X,x);
+
+   GridFunction u_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(&H1fes,data);
+
+   GridFunction s_gf;
+   s_gf.MakeRef(&RTtrace_fes,&data[H1fes.GetVSize()]);
+
+
+
+   RT_FECollection RTfec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   GridFunction sigma_gf(&RTfes);
+   sigma_gf = 0.0;
+   for (int i = 0; i<mesh.GetNE(); i++)
+   {
+      Array<int> strace_dofs;
+      Array<int> trace_dofs;
+      Vector dofs;
+      RTtrace_fes.GetElementDofs(i,trace_dofs);
+      strace_dofs.SetSize(trace_dofs.Size());
+      // shift dofs;
+      for (int j = 0; j< trace_dofs.Size(); j++)
+      {
+         int offset = trace_dofs[j] < 0 ? -H1fes.GetVSize() : H1fes.GetVSize();
+         strace_dofs[j] = offset + trace_dofs[j];
+      }
+      x.GetSubVector(strace_dofs, dofs);
+      sigma_gf.SetSubVector(trace_dofs,dofs);
+
+   }
+
+
+
+
+
+   ParaViewDataCollection paraview_dc("DPG_example", &mesh);
+   paraview_dc.SetPrefixPath("ParaView");
+   paraview_dc.SetLevelsOfDetail(order);
+   paraview_dc.SetCycle(0);
+   paraview_dc.SetDataFormat(VTKFormat::BINARY);
+   paraview_dc.SetHighOrderOutput(true);
+   paraview_dc.SetTime(0.0); // set the time
+   paraview_dc.RegisterField("field",&u_gf);
+   paraview_dc.RegisterField("flux",&sigma_gf);
+   // paraview_dc.RegisterField("flux",&s_gf);
+   paraview_dc.Save();
+
+
+
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << u_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << sigma_gf <<
+                 "window_title 'Flux sigma_n' "
+                 << flush;
+
+
+
+}

examples/dpg_tests/diffusion/uw_dpg.cpp

@@ -0,0 +1,403 @@
+//                       MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg -m ../lshape2.mesh -o 2 -ref 20 -graph-norm -do 1 -prob 0
+
+//     - Δ u = f,   in Ω
+//         u = u_0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) - (σ , τ) + < û, τ⋅n> = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) + < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//                                û = 0       on ∂Ω 
+
+// Note: 
+// û := u
+// σ̂ := -σ
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           |  <σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u);
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ∇ u
+   sigma = du;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return exact_u(X);
+}
+
+void exact_hatsigma(const Vector & X, Vector & hatsigma)
+{
+   exact_sigma(X,hatsigma);
+   hatsigma *= -1.;
+}
+
+double f_exact(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return -d2u;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");               
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space"); 
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatsigma_fes = new FiniteElementSpace(&mesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negone)),1,0);
+
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   //  <σ̂,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f(f_exact);
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),1);
+   }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   Array<int> elements_to_refine;
+   GridFunction hatu_gf;
+
+
+   socketstream u_out;
+   // socketstream sigma_out;
+   socketstream mesh_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+      mesh_out.open(vishost, visport);
+   }
+
+
+   for (int iref = 0; iref<ref; iref++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new GSSmoother(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      cout << "Residual = " << residual << endl;
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      double theta = 0.7;
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out.precision(8);
+         string keys = (iref == 0) ? "keys em\n" : "keys";
+         u_out << "solution\n" << mesh << u_gf 
+               << "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+
+         mesh_out.precision(8);
+         mesh_out << "mesh\n" << mesh 
+                  << keys
+                  << "window_title 'Mesh' "
+                  << flush;
+
+      }
+
+      mesh.GeneralRefinement(elements_to_refine);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case lshape:
+      {
+         double r = sqrt(x*x + y*y);
+         double alpha = 2./3.;
+         double theta = atan2(y,x);
+         if (theta < 0) theta += 2*M_PI;
+         u = pow(r,alpha) * sin(alpha * theta);
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}

examples/dpg_tests/diffusion/uw_dpgp.cpp

@@ -0,0 +1,404 @@
+//                 MFEM UW DPG parallel example
+//
+// Compile with: make poisson_fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) + < û, τ⋅n> - (σ , τ) = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) - < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//            û = 0        on ∂Ω 
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           | -<σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+double exact(const Vector & X)
+{
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   return pow(r,alpha) * sin(alpha * theta);
+}
+
+void gradexact(const Vector & X, Vector & grad)
+{
+   grad.SetSize(2);
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   double r_x = x/r;
+   double r_y = y/r;
+   double theta_x = - y / (r*r);
+   double theta_y = x / (r*r);
+   double beta = alpha * pow(r,alpha - 1.);
+   grad[0] = beta*(r_x * sin(alpha*theta) + r * theta_x * cos(alpha*theta));
+   grad[1] = beta*(r_y * sin(alpha*theta) + r * theta_y * cos(alpha*theta));
+}
+
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+   double theta = 0.7;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");    
+   args.AddOption(&theta, "-theta", "--theta_factor",
+                  "Refinement factor");                              
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");   
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   TransposeIntegrator * mass = new TransposeIntegrator(new VectorFEMassIntegrator(negone));
+   a->AddTrialIntegrator(mass,1,0);
+
+   // (σ,∇ v)
+   TransposeIntegrator * grad = new TransposeIntegrator(new GradientIntegrator(one));
+   a->AddTrialIntegrator(grad,1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+   // RHS
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(one),1);
+   }
+
+   FunctionCoefficient uex(exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatu_gf;
+
+
+   socketstream u_out;
+   socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      sigma_out.open(vishost, visport);
+   }
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      if (prob == prob_type::lshape)
+      {
+         hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+         hatu_gf.ProjectBdrCoefficient(uex,ess_bdr);
+      }
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip=2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      if (myid == 0)
+      {
+         cout << "Global Residual = " << globalresidual << endl;
+      }
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         sigma_out.precision(8);
+         sigma_out << "solution\n" << pmesh << sigma_gf <<
+                  "window_title 'Numerical flux' "
+                  << flush;
+      }
+
+
+      if (i == ref-1)
+      {
+         break;
+      }
+
+      pmesh.GeneralRefinement(elements_to_refine);
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}

examples/dpg_tests/grad-div/makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/grad-div,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = primal_dpg 
+PAR_EXAMPLES = 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:

examples/dpg_tests/grad-div/primal_dpg.cpp

@@ -0,0 +1,176 @@
+//                                MFEM primal dpg example for grad-dic problem
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+void F_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.ParseCheck();
+
+   kappa = freq * M_PI;
+
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   RT_FECollection fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &fec);
+
+   H1_Trace_FECollection trace_fec(order, mesh.Dimension());
+   FiniteElementSpace H1trace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   RT_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace *> trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+   
+   trial_fes.Append(&RTfes);
+   trial_fes.Append(&H1trace_fes);
+   test_fecs.Append(&test_fec);
+
+
+   GridFunction rt_gf(&RTfes);
+   VectorFunctionCoefficient F(dim, F_exact);
+   rt_gf.ProjectCoefficient(F);
+
+   Vector x(RTfes.GetVSize()+H1trace_fes.GetVSize());
+   x = 0.;
+   x.SetVector(rt_gf,0);
+
+
+   ConstantCoefficient alpha(1.0);
+   ConstantCoefficient beta(1.0);
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+   a->AddTrialIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(beta),0,0);
+   a->AddTrialIntegrator(new NormalTraceIntegrator,1,0);
+   a->AddTestIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTestIntegrator(new VectorFEMassIntegrator(beta),0,0);
+
+
+   VectorFunctionCoefficient f(dim, f_exact);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      RTfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+
+   OperatorPtr Ah;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   // GridFunction u_gf;
+   double *data = x.GetData();
+   rt_gf.MakeRef(&RTfes,data);
+
+   GridFunction exact_gf(&RTfes);
+   exact_gf.ProjectCoefficient(F);
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << rt_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << exact_gf <<
+                 "window_title 'Exact' "
+                 << flush;
+
+}
+
+
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+
+   F(0) = cos(kappa*x)*sin(kappa*y);
+   F(1) = cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      F(2) = 0.0;
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+
+   double temp = 1 + 2*kappa*kappa;
+
+   f(0) = temp*cos(kappa*x)*sin(kappa*y);
+   f(1) = temp*cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      f(2) = 0;
+   }
+}

examples/dpg_tests/lshape2.mesh

@@ -0,0 +1,51 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+3
+1 3 0 1 4 3
+1 3 3 4 7 6
+1 3 1 2 5 4
+
+boundary
+8
+1 1 0 1
+1 1 1 2
+1 1 2 5
+2 1 5 4
+2 1 4 7
+1 1 7 6
+1 1 6 3
+1 1 3 0
+
+vertices
+8
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P1
+VDim: 2
+Ordering: 1
+
+-1 1
+-1 -0
+-1 -1
+0 1
+0 -0
+0 -1
+1 1
+1 -0

examples/dpg_tests/maxwell/complex_uw_dpg.cpp

@@ -0,0 +1,907 @@
+//                   MFEM Ultraweak DPG Maxwell example
+//
+// Compile with: make complex_uw_dpg
+//
+
+//      ∇×(1/μ ∇×E) - ω^2 ϵ E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// First Order System
+
+//  i ω μ H + ∇ × E = 0,   in Ω
+// -i ω ϵ E + ∇ × H = J,   in Ω
+//            E × n = E_0, on ∂Ω
+
+// note: Ĵ = -iωJ
+// in 2D 
+// E is vector valued and H is scalar. 
+//    (∇ × E, F) = (E, ∇ × F) + < n × E , F>
+// or (∇ ⋅ AE , F) = (AE, ∇ F) + < AE ⋅ n, F>
+// where A = A = [0 1; -1 0];
+
+// UW-DPG:
+// 
+// in 3D 
+// E,H ∈ (L^2(Ω))^3 
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γ_h)  
+//  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)      
+// -i ω ϵ (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)      
+//                                   Ê × n = E_0     on ∂Ω 
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) | < n × Ê, F > |              |         |
+// |   |              |             |              |              |         |
+// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < n × Ĥ, G > |  (J,G)  |  
+// where (F,G) ∈  H(curl,Ω) × H(curl,Ω)
+
+// in 2D 
+// E ∈ L^2(Ω)^2, H ∈ L^2(Ω)
+// Ê ∈ H^-1/2(Ω)(Γ_h), Ĥ ∈ H^1/2(Γ_h)  
+//  i ω μ (H,F) + (E, ∇ × F) + < AÊ, F > = 0,      ∀ F ∈ H^1      
+// -i ω ϵ (E,G) + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)      
+//                                    Ê = E_0     on ∂Ω 
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) |   < Ê, F >   |              |         |
+// |   |              |             |              |              |         |
+// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < Ĥ, G × n > |  (J,G)  |  
+
+// where (F,G) ∈  H^1 × H(curl,Ω)
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void E_exact_r(const Vector &x, Vector & E_r);
+void E_exact_i(const Vector &x, Vector & E_i);
+
+void H_exact_r(const Vector &x, Vector & H_r);
+void H_exact_i(const Vector &x, Vector & H_i);
+
+
+void  rhs_func_r(const Vector &x, Vector & J_r);
+void  rhs_func_i(const Vector &x, Vector & J_i);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+void curlE_exact_i(const Vector &x, Vector &curlE_i);
+void curlH_exact_r(const Vector &x,Vector &curlH_r);
+void curlH_exact_i(const Vector &x,Vector &curlH_i);
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r);
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i);
+
+void hatE_exact_r(const Vector & X, Vector & hatE_r);
+void hatE_exact_i(const Vector & X, Vector & hatE_i);
+
+void hatH_exact_r(const Vector & X, Vector & hatH_r);
+void hatH_exact_i(const Vector & X, Vector & hatH_i);
+
+double hatH_exact_scalar_r(const Vector & X);
+double hatH_exact_scalar_i(const Vector & X);
+
+void maxwell_solution(const Vector & X, 
+                      std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE);
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                      Vector &curlE_r, 
+                      Vector &curlcurlE_r);
+
+void maxwell_solution_i(const Vector & X, Vector &E_i, 
+                      Vector &curlE_i, 
+                      Vector &curlcurlE_i);     
+
+int dim;
+int dimc;
+double omega;
+double mu = 1.0;
+double epsilon = 1.0;
+
+enum prob_type
+{
+   polynomial,
+   plane_wave,
+   fichera_oven  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-hex.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");    
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");                  
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: plane wave, 2: Gaussian beam");                     
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+   dimc = (dim == 3) ? 3 : 1;
+   int test_order = order+delta_order;
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *E_fes = new FiniteElementSpace(&mesh,E_fec,dim);
+
+   // Vector L2 space for H 
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *H_fes = new FiniteElementSpace(&mesh,H_fec, dimc); 
+
+   // H^-1/2 (curl) space for Ê   
+   FiniteElementCollection * hatE_fec = nullptr;
+   FiniteElementCollection * hatH_fec = nullptr; 
+   FiniteElementCollection * F_fec = nullptr;
+   if (dim == 3)
+   {
+      hatE_fec = new ND_Trace_FECollection(order,dim);
+      hatH_fec = new ND_Trace_FECollection(order,dim);   
+      F_fec = new ND_FECollection(test_order, dim);
+   }
+   else
+   {
+      hatE_fec = new RT_Trace_FECollection(order-1,dim);
+      hatH_fec = new H1_Trace_FECollection(order,dim);   
+      F_fec = new H1_FECollection(test_order, dim);
+   } 
+   FiniteElementSpace *hatE_fes = new FiniteElementSpace(&mesh,hatE_fec);
+   FiniteElementSpace *hatH_fes = new FiniteElementSpace(&mesh,hatH_fec);
+
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+
+   mfem::out << "E_fes space true dofs = " << E_fes->GetTrueVSize() << endl;
+   mfem::out << "H_fes space true dofs = " << H_fes->GetTrueVSize() << endl;
+   mfem::out << "hatE_fes space true dofs = " << hatE_fes->GetTrueVSize() << endl;
+   mfem::out << "hatH_fes space true dofs = " << hatH_fes->GetTrueVSize() << endl;
+
+
+//    // Coefficients
+   Vector dim_zero(dim); dim_zero = 0.0;
+   Vector dimc_zero(dimc); dimc_zero = 0.0;
+   VectorConstantCoefficient E_zero(dim_zero);
+   VectorConstantCoefficient H_zero(dimc_zero);
+
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient eps2omeg2(epsilon*epsilon*omega*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient negepsomeg(-epsilon*omega);
+   ConstantCoefficient epsomeg(epsilon*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+   ScalarMatrixProductCoefficient epsrot(epsomeg,rot);
+   ScalarMatrixProductCoefficient negepsrot(negepsomeg,rot);
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   ComplexNormalEquations * a = new ComplexNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices();
+
+   // (E,∇ × F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new CurlIntegrator(one)),nullptr,0,0);
+
+   // -i ω ϵ (E , G)
+   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(negepsomeg)),0,1);
+
+   // i ω μ (H, F)
+   if (dim == 3)
+   {
+      a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(muomeg)),1,0);
+   }
+   else
+   {
+      a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(muomeg),1,0);
+   }
+   //  (H,∇ × G) 
+   a->AddTrialIntegrator(new TransposeIntegrator(new CurlIntegrator(one)),nullptr,1,1);
+
+   // < n×Ê,F>
+   if (dim == 3)
+   {
+      a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,2,0);
+   }
+   else
+   {
+      a->AddTrialIntegrator(new TraceIntegrator,nullptr,2,0);
+   }
+
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,3,1);
+
+
+   // test integrators 
+
+   //space-induced norm for H(curl) × H(curl)
+   if (dim == 3)
+   {
+      // (∇×F,∇×δF)
+      a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+      // (F,δF)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+   }
+   else
+   {
+      // (∇F,∇δF)
+      a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
+      // (F,δF)
+      a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
+   }
+
+   // (∇×G ,∇× δG)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,1,1);
+   // (G,δG)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      if(dim == 3)
+      {
+         // μ^2 ω^2 (F,δF)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(mu2omeg2),nullptr,0,0);
+         // -i ω μ (F,∇ × δG) = (F, ω μ ∇ × δ G)
+         a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(negmuomeg),0,1);
+         // -i ω ϵ (∇ × F, δG)
+         a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(negepsomeg),0,1);
+         // i ω μ (∇ × G,δF)
+         a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(epsomeg),1,0);
+         // i ω ϵ (G, ∇ × δF )
+         a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(muomeg),1,0);
+         // ϵ^2 ω^2 (G,δG)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);
+      }
+      else
+      {
+         // μ^2 ω^2 (F,δF)
+         a->AddTestIntegrator(new MassIntegrator(mu2omeg2),nullptr,0,0);
+
+         // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+         a->AddTestIntegrator(nullptr,
+            new TransposeIntegrator(new CurlIntegrator(negmuomeg)),0,1);
+
+         // -i ω ϵ (∇ × F, δG) = i (- ω ϵ A ∇ F,δG), A = [0 1; -1; 0]
+         a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negepsrot),0,1);   
+
+         // i ω μ (∇ × G,δF) = i (ω μ ∇ × G, δF )
+         a->AddTestIntegrator(nullptr,new CurlIntegrator(muomeg),1,0);
+
+         // i ω ϵ (G, ∇ × δF ) =  i (ω ϵ G, A ∇ δF) = i ( G , ω ϵ A ∇ δF) 
+         a->AddTestIntegrator(nullptr,
+                   new TransposeIntegrator(new MixedVectorGradientIntegrator(epsrot)),1,0);
+
+         // or    i ( ω ϵ A^t G, ∇ δF) = i (- ω ϵ A G, ∇ δF)
+         // a->AddTestIntegrator(nullptr,
+                  //  new MixedVectorWeakDivergenceIntegrator(epsrot),1,0);
+         // ϵ^2 ω^2 (G,δG)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);            
+      }
+   }
+
+   // RHS
+   VectorFunctionCoefficient f_rhs_r(dim,rhs_func_r);
+   VectorFunctionCoefficient f_rhs_i(dim,rhs_func_i);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f_rhs_r),
+                            new VectorFEDomainLFIntegrator(f_rhs_i),1);
+   
+   
+   VectorFunctionCoefficient hatEex_r(dim,hatE_exact_r);
+   VectorFunctionCoefficient hatEex_i(dim,hatE_exact_i);
+
+   VectorFunctionCoefficient hatHex_r(dimc,hatH_exact_r);
+   VectorFunctionCoefficient hatHex_i(dimc,hatH_exact_i);
+
+   FunctionCoefficient hatH_2D_ex_r(hatH_exact_scalar_r);
+   FunctionCoefficient hatH_2D_ex_i(hatH_exact_scalar_i);
+
+
+   Array<int> elements_to_refine;
+
+   socketstream E_out_r;
+   socketstream Eex_out_r;
+   // socketstream E_out_i;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      E_out_r.open(vishost, visport);
+      Eex_out_r.open(vishost, visport);
+      // E_out_i.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         // hatH_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+                           // + hatE_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = H_fes->GetVSize();
+      offsets[3] = hatE_fes->GetVSize();
+      offsets[4] = hatH_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ComplexGridFunction hatE_gf(hatE_fes);
+      hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+      hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+      ComplexGridFunction hatH_gf(hatH_fes);
+      hatH_gf.real().MakeRef(hatH_fes,&xdata[offsets[3]]);
+      hatH_gf.imag().MakeRef(hatH_fes,&xdata[offsets.Last()+ offsets[3]]);
+
+      if (dim == 3)
+      {
+         hatE_gf.ProjectBdrCoefficientTangent(hatEex_r,hatEex_i, ess_bdr);
+         // hatH_gf.ProjectBdrCoefficientTangent(hatHex_r,hatHex_i, ess_bdr);
+      }
+      else
+      {
+         hatE_gf.ProjectBdrCoefficientNormal(hatEex_r,hatEex_i, ess_bdr);
+         // hatH_gf.ProjectBdrCoefficient(hatH_2D_ex_r,hatH_2D_ex_i, ess_bdr);
+
+      }
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      SparseMatrix * Ar = dynamic_cast<BlockMatrix *>(&Ahc->real())->CreateMonolithic();
+      SparseMatrix * Ai = dynamic_cast<BlockMatrix *>(&Ahc->imag())->CreateMonolithic();
+
+      ComplexSparseMatrix Ac(Ar,Ai,true,true);
+      SparseMatrix * A = Ac.GetSystemMatrix();
+
+      UMFPackSolver umf(*A);
+      umf.Mult(B,X);
+
+      delete A;
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      double residual = residuals.Norml2();
+
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ComplexGridFunction E(E_fes);
+      E.real().MakeRef(E_fes,x.GetData());
+      E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      VectorFunctionCoefficient E_ex_r(dim,E_exact_r);
+      VectorFunctionCoefficient E_ex_i(dim,E_exact_i);
+
+      ComplexGridFunction H(H_fes);
+      H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+      H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+      VectorFunctionCoefficient H_ex_r(dimc,H_exact_r);
+      VectorFunctionCoefficient H_ex_i(dimc,H_exact_i);
+      
+
+      int dofs = X.Size()/2;
+
+      double E_err_r = E.real().ComputeL2Error(E_ex_r);
+      double E_err_i = E.imag().ComputeL2Error(E_ex_i);
+      double H_err_r = H.real().ComputeL2Error(H_ex_r);
+      double H_err_i = H.imag().ComputeL2Error(H_ex_i);
+
+      double L2Error = sqrt(  E_err_r*E_err_r + E_err_i*E_err_i 
+                            + H_err_r*H_err_r + H_err_i*H_err_i );
+
+      ComplexGridFunction Egf_ex(E_fes);
+      ComplexGridFunction Hgf_ex(H_fes);
+      Egf_ex.ProjectCoefficient(E_ex_r, E_ex_i);
+      Hgf_ex.ProjectCoefficient(H_ex_r, H_ex_i);
+
+   
+
+      double E_norm_r = Egf_ex.real().ComputeL2Error(E_zero);
+      double E_norm_i = Egf_ex.imag().ComputeL2Error(E_zero);
+      double H_norm_r = Hgf_ex.real().ComputeL2Error(H_zero);
+      double H_norm_i = Hgf_ex.imag().ComputeL2Error(H_zero);
+
+      double L2norm = sqrt(  E_norm_r*E_norm_r + E_norm_i*E_norm_i 
+                           + H_norm_r*H_norm_r + H_norm_i*H_norm_i );
+
+      double rel_err = L2Error/L2norm;
+
+
+      double rate_err = (i) ? dim*log(err0/rel_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = rel_err;
+      res0 = residual;
+      dof0 = dofs;
+
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_err*100 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::setw(10) << std::scientific 
+                << std::endl;
+
+      if (visualization)
+      {
+         E_out_r.precision(8);
+         E_out_r << "solution\n" << mesh << E.real() <<
+                  "window_title 'Real Numerical Electric field' "
+                  << flush;
+
+         // E_out_i.precision(8);
+         // E_out_i << "solution\n" << mesh << E.imag() <<
+         //          "window_title 'Imag Numerical Electric field' "
+         //          << flush;         
+
+
+
+
+         Eex_out_r.precision(8);
+         Eex_out_r << "solution\n" << mesh << Egf_ex.real()  
+                  << "window_title 'Real Exact Electric field' " 
+                  << flush;
+         // socketstream E_i_sock(vishost, visport);
+         // E_i_sock.precision(8);
+         // E_i_sock << "solution\n" << mesh << Egf_ex.imag()  
+         //          << "window_title 'Imag Exact Electric field' " 
+         //          << flush;
+
+
+      }
+
+      if (i == ref-1)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+}                                       
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   Vector curlE_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void E_exact_i(const Vector &x, Vector & E_i)
+{
+   Vector curlE_i;
+   Vector curlcurlE_i;
+
+   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   Vector E_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void curlE_exact_i(const Vector &x, Vector &curlE_i)
+{
+   Vector E_i;
+   Vector curlcurlE_i;
+
+   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
+}
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r)
+{
+   Vector E_r;
+   Vector curlE_r;
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i)
+{
+   Vector E_i;
+   Vector curlE_i;
+   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
+}
+
+
+void H_exact_r(const Vector &x, Vector & H_r)
+{
+   // H = i ∇ × E / ω μ  
+   // H_r = - ∇ × E_i / ω μ  
+   Vector curlE_i;
+   curlE_exact_i(x,curlE_i);
+   H_r.SetSize(dimc);
+   for (int i = 0; i<dimc; i++)
+   {
+      H_r(i) = - curlE_i(i) / (omega * mu);
+   }
+}
+
+void H_exact_i(const Vector &x, Vector & H_i)
+{
+   // H = i ∇ × E / ω μ  
+   // H_i =  ∇ × E_r / ω μ  
+   Vector curlE_r;
+   curlE_exact_r(x,curlE_r);
+   H_i.SetSize(dimc);
+   for (int i = 0; i<dimc; i++)
+   {
+      H_i(i) = curlE_r(i) / (omega * mu);
+   }
+}
+
+void curlH_exact_r(const Vector &x,Vector &curlH_r)
+{
+   // ∇ × H_r = - ∇ × ∇ × E_i / ω μ  
+   Vector curlcurlE_i;
+   curlcurlE_exact_i(x,curlcurlE_i);
+   curlH_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      curlH_r(i) = -curlcurlE_i(i) / (omega * mu);
+   }
+}
+
+void curlH_exact_i(const Vector &x,Vector &curlH_i)
+{
+   // ∇ × H_i = ∇ × ∇ × E_r / ω μ  
+   Vector curlcurlE_r;
+   curlcurlE_exact_r(x,curlcurlE_r);
+   curlH_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      curlH_i(i) = curlcurlE_r(i) / (omega * mu);
+   }
+}
+
+
+void hatE_exact_r(const Vector & x, Vector & hatE_r)
+{
+   if (dim == 3)
+   {
+      E_exact_r(x,hatE_r);
+   }
+   else
+   {
+      Vector E_r;
+      E_exact_r(x,E_r);
+      hatE_r.SetSize(hatE_r.Size());
+      // rotate E_hat
+      hatE_r[0] = E_r[1];
+      hatE_r[1] = -E_r[0];
+   }
+}
+
+void hatE_exact_i(const Vector & x, Vector & hatE_i)
+{
+   if (dim == 3)
+   {
+      E_exact_i(x,hatE_i);
+   }
+   else
+   {
+      Vector E_i;
+      E_exact_i(x,E_i);
+      hatE_i.SetSize(hatE_i.Size());
+      // rotate E_hat
+      hatE_i[0] = E_i[1];
+      hatE_i[1] = -E_i[0];
+   }
+}
+
+
+void hatH_exact_r(const Vector & x, Vector & hatH_r)
+{
+   H_exact_r(x,hatH_r);
+}
+
+void hatH_exact_i(const Vector & x, Vector & hatH_i)
+{
+   H_exact_i(x,hatH_i);
+}
+
+double hatH_exact_scalar_r(const Vector & x)
+{
+   Vector hatH_r;
+   H_exact_r(x,hatH_r);
+   return hatH_r[0];
+}
+
+double hatH_exact_scalar_i(const Vector & x)
+{
+   Vector hatH_i;
+   H_exact_i(x,hatH_i);
+   return hatH_i[0];
+}
+
+// J = -i ω ϵ E + ∇ × H 
+// J_r + iJ_i = -i ω ϵ (E_r + i E_i) + ∇ × (H_r + i H_i) 
+void  rhs_func_r(const Vector &x, Vector & J_r)
+{
+   // J_r = ω ϵ E_i + ∇ × H_r
+   Vector E_i, curlH_r;
+   E_exact_i(x,E_i);
+   curlH_exact_r(x,curlH_r);
+   J_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_r(i) = omega * epsilon * E_i(i) + curlH_r(i);
+   }
+}
+
+void  rhs_func_i(const Vector &x, Vector & J_i)
+{
+   // J_i = - ω ϵ E_r + ∇ × H_i
+   Vector E_r, curlH_i;
+   E_exact_r(x,E_r);
+   curlH_exact_i(x,curlH_i);
+   J_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_i(i) = -omega * epsilon * E_r(i) + curlH_i(i);
+   }
+}
+
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE)
+{
+   double x = X(0);
+   double y = X(1);
+   double z;
+   if (dim == 3) z = X(2);
+
+   E.resize(dim);
+   curlE.resize(dimc);
+   curlcurlE.resize(dim);
+   switch (prob)
+   {
+   case prob_type::polynomial:
+   {
+      if (dim == 3)
+      {
+         E[0] = y * z * (1.0 - y) * (1.0 - z);
+         E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+         E[2] = x * y * (1.0 - x) * (1.0 - y);
+         curlE[0] = (1.0 - x) * x * (y*(2.0*z-3.0)+1.0);
+         curlE[1] = 2.0*(1.0 - y)*y*(x-z);
+         curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+         curlcurlE[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+         curlcurlE[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+         curlcurlE[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+      }
+      else
+      {
+         E[0] = y * (1.0 - y);
+         E[1] = x * y * (1.0 - x);
+         curlE[0] = y*(3.0 - 2*x) - 1.0;
+         curlcurlE[0] = 3.0 - 2*x;
+         curlcurlE[1] = 2.0*y;
+      }
+   }
+      break;
+
+   case prob_type::plane_wave:
+   {
+      std::complex<double> zi(0,1);
+      std::complex<double> pw = exp(-zi * omega * (X.Sum()));
+      E[0] = pw;
+      E[1] = 0.0;
+      if (dim == 3)
+      {
+         E[2] = 0.0;
+         curlE[0] = 0.0;
+         curlE[1] = -zi * omega * pw;
+         curlE[2] =  zi * omega * pw;
+
+         curlcurlE[0] = 2.0 * omega * omega * pw;
+         curlcurlE[1] = - omega * omega * pw;
+         curlcurlE[2] = - omega * omega * pw;
+      }
+      else
+      {
+         curlE[0] = zi * omega * pw;
+         curlcurlE[0] =   omega * omega * pw;
+         curlcurlE[1] = - omega * omega * pw ;
+      }
+   }
+      break;
+
+   default:
+      MFEM_ABORT("Fichera 'oven' problem not implemented yet");
+      break;
+   }
+   
+}
+
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                        Vector &curlE_r, 
+                        Vector &curlcurlE_r)
+{
+   E_r.SetSize(dim);
+   curlE_r.SetSize(dimc);
+   curlcurlE_r.SetSize(dim);
+
+   std::vector<complex<double>> E;
+   std::vector<complex<double>> curlE;
+   std::vector<complex<double>> curlcurlE;
+
+   maxwell_solution(X,E,curlE,curlcurlE);
+   for (int i = 0; i<dim ; i++)
+   {
+      E_r(i) = E[i].real();
+      curlcurlE_r(i) = curlcurlE[i].real();
+   }
+   for (int i = 0; i<dimc; i++)
+   {
+      curlE_r(i) = curlE[i].real();
+   }
+}
+
+
+void maxwell_solution_i(const Vector & X, Vector &E_i, 
+                      Vector &curlE_i, 
+                      Vector &curlcurlE_i)
+{
+   E_i.SetSize(dim);
+   curlE_i.SetSize(dimc);
+   curlcurlE_i.SetSize(dim);
+
+   std::vector<complex<double>> E;
+   std::vector<complex<double>> curlE;
+   std::vector<complex<double>> curlcurlE;
+
+   maxwell_solution(X,E,curlE,curlcurlE);
+   for (int i = 0; i<dim; i++)
+   {
+      E_i(i) = E[i].imag();
+      curlcurlE_i(i) = curlcurlE[i].imag();
+   }
+   for (int i = 0; i<dimc; i++)
+   {
+      curlE_i(i) = curlE[i].imag();
+   }
+}  

examples/dpg_tests/maxwell/makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = complex_uw_dpg complex_uw_dpg_2D
+PAR_EXAMPLES = pcomplex_uw_dpg
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:

examples/dpg_tests/maxwell/test_curl_integrator.cpp

@@ -0,0 +1,253 @@
+// Test integrator
+// (∇ × E, F) 
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void E_exact_r(const Vector &x, Vector & E_r);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+
+void maxwell_solution(const Vector & X, 
+                      std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE);
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                      Vector &curlE_r, 
+                      Vector &curlcurlE_r);
+
+int dim;
+int dimc;
+double omega;
+
+enum prob_type
+{
+   polynomial,
+   plane_wave,
+   fichera_oven  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-hex.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: plane wave, 2: Gaussian beam");                     
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   dimc = (dim == 3) ? 3 : 1;
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   FiniteElementSpace *E_fes = new FiniteElementSpace(&mesh,E_fec);
+
+
+   FiniteElementCollection *curlE_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *curlE_fes = new FiniteElementSpace(&mesh,curlE_fec,dimc);
+
+   mfem::out << "E_fes space true dofs = " << E_fes->GetTrueVSize() << endl;
+   mfem::out << "curlE_fes space true dofs = " << curlE_fes->GetTrueVSize() << endl;
+
+
+   GridFunction E_gf(E_fes);
+   VectorFunctionCoefficient E_cf(dim,E_exact_r);
+   E_gf.ProjectCoefficient(E_cf);
+
+   GridFunction curlE_gf(curlE_fes);
+   VectorFunctionCoefficient curlE_cf(dimc,curlE_exact_r);
+   curlE_gf.ProjectCoefficient(curlE_cf);
+
+
+   char vishost[] = "localhost";
+   int visport = 19916;
+   socketstream E_sock(vishost, visport);
+   E_sock.precision(8);
+   E_sock << "solution\n"
+               << mesh << E_gf 
+               << "window_title 'Exact E'" << flush;
+
+   socketstream curlE_sock(vishost, visport);
+   curlE_sock.precision(8);
+   curlE_sock << "solution\n"
+               << mesh << curlE_gf 
+               << "window_title 'Exact curlE'" << flush;
+
+
+
+   MixedBilinearForm a(E_fes,curlE_fes);
+   a.AddDomainIntegrator(new CurlIntegrator());
+   a.Assemble();
+   Array<int> empty;
+   SparseMatrix A;
+   a.FormRectangularSystemMatrix(empty,empty,A);
+
+
+   Vector curl_load(A.Height());
+   A.Mult(E_gf,curl_load);
+   BilinearForm m(curlE_fes);
+   m.AddDomainIntegrator(new VectorMassIntegrator);
+   m.Assemble();
+   SparseMatrix M;
+   m.FormSystemMatrix(empty, M);
+
+
+   GSSmoother prec(M);
+   PCG(M, prec, curl_load, curlE_gf, 1, 200, 1e-12, 0.0);
+
+
+   socketstream curlE2_sock(vishost, visport);
+   curlE2_sock.precision(8);
+   curlE2_sock << "solution\n"
+               << mesh << curlE_gf 
+               << "window_title 'Numerical curlE'" << flush;
+
+
+   delete E_fec;
+   delete E_fes;
+   delete curlE_fec;
+   delete curlE_fes;
+
+   return 0;
+}                                       
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   Vector curlE_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   Vector E_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE)
+{
+   double x = X(0);
+   double y = X(1);
+   double z;
+   if (dim == 3)
+   {
+      z = X(2);
+   } 
+
+   E.resize(dim);
+   curlE.resize(dimc);
+   curlcurlE.resize(dim);
+
+   if (dim == 3)
+   {
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+      E[2] = x * y * (1.0 - x) * (1.0 - y);
+      
+      curlE[0] = (1.0 - x) * x * (y*(2.0*z-3.0)+1.0);
+      curlE[1] = 2.0*(1.0 - y)*y*(x-z);
+      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+      
+      curlcurlE[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+      curlcurlE[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+      curlcurlE[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+   }
+   else if (dim == 2)
+   {
+      double c = 2.0*M_PI;
+      E[0] = sin(c * y);
+      E[1] = sin(c * x);
+      curlE[0] = c * (cos(c*x) - cos(c*y));
+      curlcurlE[0] = c*c * sin(c*y);
+      curlcurlE[1] = c*c * sin(c*x);
+   }
+   else
+   {
+      MFEM_ABORT("Dimension cannot be 1");
+   }
+
+}
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                        Vector &curlE_r, 
+                        Vector &curlcurlE_r)
+{
+   E_r.SetSize(dim);
+   curlE_r.SetSize(dimc);
+   curlcurlE_r.SetSize(dim);
+
+   std::vector<complex<double>> E;
+   std::vector<complex<double>> curlE;
+   std::vector<complex<double>> curlcurlE;
+
+   maxwell_solution(X,E,curlE,curlcurlE);
+   for (int i = 0; i<dim; i++)
+   {
+      E_r(i) = E[i].real();
+      curlcurlE_r(i) = curlcurlE[i].real();
+   }
+   for (int i = 0; i<dimc; i++)
+   {
+      curlE_r(i) = curlE[i].real();
+   }
+}

examples/ex1.cpp

@@ -192,14 +192,7 @@ int main(int argc, char *argv[])
    //    domain integrator.
    BilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,

examples/ex1p.cpp

@@ -219,14 +219,7 @@ int main(int argc, char *argv[])
    //     Diffusion domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 12. Assemble the parallel bilinear form and the corresponding linear

examples/ex27p.cpp

@@ -195,7 +195,7 @@ int main(int argc, char *argv[])
    Array<int> ess_tdof_list(0);
    if (h1 && pmesh.bdr_attributes.Size())
    {
-      // For a continuous basis the linear system must be modified to enforce an
+      // For a continuous basis the linear system must be modifed to enforce an
       // essential (Dirichlet) boundary condition. In the DG case this is not
       // necessary as the boundary condition will only be enforced weakly.
       fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);

examples/ex5.cpp

@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
+      B.EnsureMultTranspose();
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);

examples/ex8.cpp

@@ -72,8 +72,8 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 10,000
    //    elements.
    {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels = 1;
+      // (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -147,6 +147,8 @@ int main(int argc, char *argv[])
    F.AddDomainIntegrator(new DomainLFIntegrator(one));
    F.Assemble();
 
+
+
    // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
    //    the mixed bilinear form for the interfacial unknowns, Bhat,
    //    the inverse stiffness matrix on the discontinuous test space, Sinv,
@@ -187,10 +189,17 @@ int main(int argc, char *argv[])
    // 8. Set up the 1x2 block Least Squares DPG operator, B = [B0  Bhat],
    //    the normal equation operator, A = B^t Sinv B, and
    //    the normal equation right-hand-size, b = B^t Sinv F.
-   BlockOperator B(offsets_test, offsets);
+   // BlockOperator B(offsets_test, offsets);
+
+   BlockMatrix B(offsets_test, offsets);
    B.SetBlock(0,0,&matB0);
    B.SetBlock(0,1,&matBhat);
-   RAPOperator A(B, matSinv, B);
+   SparseMatrix * Bh = B.CreateMonolithic();
+
+   SparseMatrix * A = RAP(*Bh, matSinv, *Bh);
+
+
+   // RAPOperator A(B, matSinv, B);
    {
       Vector SinvF(s_test);
       matSinv.Mult(F,SinvF);
@@ -234,7 +243,20 @@ int main(int argc, char *argv[])
    // 10. Solve the normal equation system using the PCG iterative solver.
    //     Check the weighted norm of residual for the DPG least square problem.
    //     Wrap the primal variable in a GridFunction for visualization purposes.
-   PCG(A, P, b, x, 1, 200, 1e-12, 0.0);
+
+
+   // PCG(*A, P, b, x, 1, 200, 1e-12, 0.0);
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(b, x);
+
 
    {
       Vector LSres(s_test);

examples/ipopt/CMakeLists.txt

@@ -1,60 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(IPOPT_EXAMPLES_SRCS)
-list(APPEND IPOPT_EXAMPLES_SRCS exContactBlockTL.cpp)
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_ipopt" target, see below.
-add_custom_target(test_ipopt
-  ${CMAKE_CTEST_COMMAND} -R ipopt USES_TERMINAL)
-
-# Add one executable per cpp file, adding "ipopt_" as prefix. Sets
-# "test_ipopt" as a target that depends on the given examples.
-set(PFX ipopt_)
-add_mfem_examples(IPOPT_EXAMPLES_SRCS ${PFX} "" test_ipopt)
-
-# Testing.
-# The IPOPT tests can be run separately using the target "test_ipopt"
-# which builds the examples and runs:
-#   ctest -R ipopt
-
-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 9:
-  set(EXCONTACTBTL_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-  set(EXCONTACTBTL_TEST_OPTS ${EXCONTACTBTL_COMMON_OPTS}  -r  2 )
-
-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${IPOPT_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
-
-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()

examples/ipopt/README

@@ -1,19 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM for solving nonlinear constrained optimization problems, including
-features based on the IpOpt, a lightweight HPC solver for nonlinear optimization
-problems.
-
-To use the Ipopt features, make sure that MFEM is configured with the option
-"MFEM_USE_IPOPT = YES", see the top-level INSTALL file for details.
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.

examples/ipopt/block1.mesh

@@ -1,103 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-9
-1   5     0      1     3     2     8    9     11    10
-1   5     2      3     5     4    10    11    13    12
-1   5     4      5     7     6    12    13    15    14
-1   5     8      9    11    10    16    17    19    18
-1   5     10    11    13    12    18    19    21    20
-1   5     12    13    15    14    20    21    23    22
-1   5     16    17    19    18    24    25    27    26
-1   5     18    19    21    20    26    27    29    28
-1   5     20    21    23    22    28    29    31    30
-
-
-
-# 0 nothing
-# 1 dirichlet bc
-# 2 contact
-boundary
-30
-0   3    1     0     2     3
-0   3    3     2     4     5
-0   3    5     4     6     7
-0   3    24    25    27    26
-0   3    26    27    29    28
-0   3    28    29    31    30
-1   3    2     0     8    10
-1   3    4     2    10    12
-1   3    6     4    12    14
-1  3    10    8    16    18
-1  3    12    10    18    20
-1  3    14    12    20    22
-1  3    18    16    24    26
-1  3    20    18    26    28
-1  3    22    20    28    30
-2  3    1     3    11    9
-2  3     3     5    13    11
-2  3     5     7    15    13
-2  3    9    11    19    17
-2  3    11    13    21    19
-2  3    13    15    23    21
-2  3    17    19    27    25
-2  3    19    21    29    27
-2  3    21    23    31    29
-0  3     8     0     1    9
-0  3    16     8    9    17
-0  3    24    16    17    25
-0  3     6    14    15     7
-0  3    14    22    23    15
-0  3    22    30    31    23
-
-
-vertices
-32
-3
-   -1.0000         0         0
-         0         0         0
-   -1.0000    0.3333         0
-         0    0.3333         0
-   -1.0000    0.6667         0
-         0    0.6667         0
-   -1.0000    1.0000         0
-         0    1.0000         0
-   -1.0000         0    0.3333
-         0         0    0.3333
-   -1.0000    0.3333    0.3333
-         0    0.3333    0.3333
-   -1.0000    0.6667    0.3333
-         0    0.6667    0.3333
-   -1.0000    1.0000    0.3333
-         0    1.0000    0.3333
-   -1.0000         0    0.6667
-         0         0    0.6667
-   -1.0000    0.3333    0.6667
-         0    0.3333    0.6667
-   -1.0000    0.6667    0.6667
-         0    0.6667    0.6667
-   -1.0000    1.0000    0.6667
-         0    1.0000    0.6667
-   -1.0000         0    1.0000
-         0         0    1.0000
-   -1.0000    0.3333    1.0000
-         0    0.3333    1.0000
-   -1.0000    0.6667    1.0000
-         0    0.6667    1.0000
-   -1.0000    1.0000    1.0000
-         0    1.0000    1.0000

examples/ipopt/block2.mesh

@@ -1,68 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-# 1 nothing
-elements
-4
-1   5     0     1     3     2     6     7    9     8
-1   5     2     3     5     4     8     9    11    10
-1   5     6     7     9     8    12    13    15    14
-1   5     8     9    11    10    14    15    17    16
-
-# 0 nothing
-# 1 dirichlet bc
-# 2 contact
-boundary
-16
-0   3     1     0     2     3
-0   3     3     2     4     5
-0   3    12    13    15    14
-0   3    14    15    17    16
-2   3     2     0     6     8
-2   3     4     2     8    10
-2   3     8     6    12    14
-2   3    10     8    14    16
-1   3     1     3     9     7
-1   3     3     5    11    9
-1   3     7    9    15    13
-1   3     9    11    17    15
-0   3     6     0     1     7
-0   3    12     6     7    13
-0   3     4    10    11     5
-0   3    10    16    17    11
-
-vertices
-18
-3
-         0    0.2464    0.2464
-    0.5071    0.2464    0.2464
-         0    0.5000    0.2464
-    0.5071    0.5000    0.2464
-         0    0.7536    0.2464
-    0.5071    0.7536    0.2464
-         0    0.2464    0.5000
-    0.5071    0.2464    0.5000
-         0    0.5000    0.5000
-    0.5071    0.5000    0.5000
-         0    0.7536    0.5000
-    0.5071    0.7536    0.5000
-         0    0.2464    0.7536
-    0.5071    0.2464    0.7536
-         0    0.5000    0.7536
-    0.5071    0.5000    0.7536
-         0    0.7536    0.7536
-    0.5071    0.7536    0.7536

examples/ipopt/contact.cpp

@@ -1,742 +0,0 @@
-//                                Contact example
-//
-// Compile with: make contact
-//
-// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
-// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include "nodepair.hpp"
-
-using namespace std;
-using namespace mfem;
-
-bool ifequalarray(const Array<int> a1, const Array<int> a2)
-{
-   if (a1.Size()!=a2.Size())
-   {
-      return false;
-   }
-   for (int i=0; i<a1.Size(); i++)
-   {
-      if (a1[i] != a2[i])
-      {
-         return false;
-      }
-   }
-   return true;
-}
-
-void FindSurfaceToProject(Mesh& mesh, const int elem, int& cbdrface)
-{
-   Array<int> attr;
-   attr.Append(2);
-   Array<int> faces;
-   Array<int> ori;
-   std::vector<Array<int> > facesVertices;
-   std::vector<int > faceid;
-   mesh.GetElementFaces(elem, faces, ori);
-   int face = -1;
-   for (int i=0; i<faces.Size(); i++)
-   {
-      face = faces[i];
-      Array<int> faceVert;
-      if (!mesh.FaceIsInterior(face)) // if on the boundary
-      {
-         mesh.GetFaceVertices(face, faceVert);
-         faceVert.Sort();
-         facesVertices.push_back(faceVert);
-         faceid.push_back(face);
-      }
-   }
-   int bdrface = facesVertices.size();
-
-   Array<int> bdryFaces;
-   // This shoulnd't need to be rebuilt
-   std::vector<Array<int> > bdryVerts;
-   for (int b=0; b<mesh.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh.GetBdrAttribute(b)) >= 0)  // found the contact surface
-      {
-         bdryFaces.Append(b);
-         Array<int> vert;
-         mesh.GetBdrElementVertices(b, vert);
-         vert.Sort();
-         bdryVerts.push_back(vert);
-      }
-   }
-
-   int bdrvert = bdryVerts.size();
-   cbdrface = -1;  // the face number of the contact surface element
-   int count_cbdrface = 0;  // the number of matching surfaces, used for checks
-
-   for (int i=0; i<bdrface; i++)
-   {
-      for (int j=0; j<bdrvert; j++)
-      {
-         if (ifequalarray(facesVertices[i], bdryVerts[j]))
-         {
-            cbdrface = faceid[i];
-            count_cbdrface += 1;
-         }
-      }
-   }
-   MFEM_VERIFY(count_cbdrface == 1,"projection surface not found");
-
-};
-
-Vector GetNormalVector(Mesh & mesh, const int elem, const double *ref,
-                       int & refFace, int & refNormal, bool & interior)
-{
-   ElementTransformation *trans = mesh.GetElementTransformation(elem);
-   const int dim = mesh.Dimension();
-   const int spaceDim = trans->GetSpaceDim();
-
-   MFEM_VERIFY(spaceDim == 3, "");
-
-   Vector n(spaceDim);
-
-   IntegrationPoint ip;
-   ip.Set(ref, dim);
-
-   trans->SetIntPoint(&ip);
-   //CalcOrtho(trans->Jacobian(), n);  // Works only for face transformations
-   const DenseMatrix jac = trans->Jacobian();
-
-   int dimNormal = -1;
-   int normalSide = -1;
-
-   const double tol = 1.0e-8;
-   for (int i=0; i<dim; ++i)
-   {
-      const double d0 = std::abs(ref[i]);
-      const double d1 = std::abs(ref[i] - 1.0);
-
-      const double d = std::min(d0, d1);
-      // TODO: this works only for hexahedral meshes!
-
-      if (d < tol)
-      {
-         MFEM_VERIFY(dimNormal == -1, "");
-         dimNormal = i;
-
-         if (d0 < tol)
-         {
-            normalSide = 0;
-         }
-         else
-         {
-            normalSide = 1;
-         }
-      }
-   }
-   // closest point on the boundary
-   if (dimNormal < 0 || normalSide < 0) // node is inside the element
-   {
-      interior = 1;
-      Vector n(3);
-      n = 0.0;
-      return n;
-   }
-
-   MFEM_VERIFY(dimNormal >= 0 && normalSide >= 0, "");
-   refNormal = dimNormal;
-
-   MFEM_VERIFY(dim == 3, "");
-
-   {
-      // Find the reference face
-      if (dimNormal == 0)
-      {
-         refFace = (normalSide == 1) ? 2 : 4;
-      }
-      else if (dimNormal == 1)
-      {
-         refFace = (normalSide == 1) ? 3 : 1;
-      }
-      else
-      {
-         refFace = (normalSide == 1) ? 5 : 0;
-      }
-   }
-
-   std::vector<Vector> tang(2);
-
-   int tangDir[2] = {-1, -1};
-   {
-      int t = 0;
-      for (int i=0; i<dim; ++i)
-      {
-         if (i != dimNormal)
-         {
-            tangDir[t] = i;
-            t++;
-         }
-      }
-
-      MFEM_VERIFY(t == 2, "");
-   }
-
-   for (int i=0; i<2; ++i)
-   {
-      tang[i].SetSize(3);
-
-      Vector tangRef(3);
-      tangRef = 0.0;
-      tangRef[tangDir[i]] = 1.0;
-
-      jac.Mult(tangRef, tang[i]);
-   }
-
-   Vector c(3);  // Cross product
-
-   c[0] = (tang[0][1] * tang[1][2]) - (tang[0][2] * tang[1][1]);
-   c[1] = (tang[0][2] * tang[1][0]) - (tang[0][0] * tang[1][2]);
-   c[2] = (tang[0][0] * tang[1][1]) - (tang[0][1] * tang[1][0]);
-
-   c /= c.Norml2();
-
-   Vector nref(3);
-   nref = 0.0;
-   nref[dimNormal] = 1.0;
-
-   Vector ndir(3);
-   jac.Mult(nref, ndir);
-
-   ndir /= ndir.Norml2();
-
-   const double dp = ndir * c;
-
-   // TODO: eliminate c?
-   n = c;
-   if (dp < 0.0)
-   {
-      n *= -1.0;
-   }
-   interior = 0;
-   return n;
-}
-
-// WARNING: global variable, just for this little example.
-std::array<std::array<int, 3>, 8> HEX_VERT =
-{
-   {  {0,0,0},
-      {1,0,0},
-      {1,1,0},
-      {0,1,0},
-      {0,0,1},
-      {1,0,1},
-      {1,1,1},
-      {0,1,1}
-   }
-};
-
-int GetHexVertex(int cdim, int c, int fa, int fb, Vector & refCrd)
-{
-   int ref[3];
-   ref[cdim] = c;
-   ref[cdim == 0 ? 1 : 0] = fa;
-   ref[cdim == 2 ? 1 : 2] = fb;
-
-   for (int i=0; i<3; ++i) { refCrd[i] = ref[i]; }
-
-   int refv = -1;
-
-   for (int i=0; i<8; ++i)
-   {
-      bool match = true;
-      for (int j=0; j<3; ++j)
-      {
-         if (ref[j] != HEX_VERT[i][j]) { match = false; }
-      }
-
-      if (match) { refv = i; }
-   }
-
-   MFEM_VERIFY(refv >= 0, "");
-
-   return refv;
-}
-
-// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
-// where X is the list of x-coordinates for all points and so on.
-// conn: connectivity of the target surface elements
-// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
-void FindPointsInMesh(Mesh & mesh, Vector const& xyz, Array<int>& conn,
-                      Vector& xi)
-{
-   const int dim = mesh.Dimension();
-   const int np = xyz.Size() / dim;
-
-   MFEM_VERIFY(np * dim == xyz.Size(), "");
-
-   mesh.EnsureNodes();
-
-   //FindPointsGSLIB finder(MPI_COMM_WORLD);
-   FindPointsGSLIB finder;
-
-   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
-
-   const double bb_t = 0.5;
-   finder.Setup(mesh, bb_t);
-
-   finder.FindPoints(xyz);
-
-   /// Return code for each point searched by FindPoints: inside element (0), on
-   /// element boundary (1), or not found (2).
-   Array<unsigned int> codes = finder.GetCode();
-
-   /// Return element number for each point found by FindPoints.
-   Array<unsigned int> elems = finder.GetElem();
-
-   /// Return reference coordinates for each point found by FindPoints.
-   Vector refcrd = finder.GetReferencePosition();
-
-   /// Return distance between the sought and the found point in physical space,
-   /// for each point found by FindPoints.
-   Vector dist = finder.GetDist();
-
-   MFEM_VERIFY(dist.Size() == np, "");
-   MFEM_VERIFY(refcrd.Size() == np * dim, "");
-   MFEM_VERIFY(elems.Size() == np, "");
-   MFEM_VERIFY(codes.Size() == np, "");
-
-   bool allfound = true;
-   for (auto code : codes)
-      if (code == 2) { allfound = false; }
-
-   MFEM_VERIFY(allfound, "A point was not found");
-
-   cout << "Maximum distance of projected points: " << dist.Max() << endl;
-
-   // extract information
-   for (int i=0; i<np; ++i)
-   {
-      cout << "Point " << i << ": (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << xyz[i + (j*np)];
-         if (j == dim-1) {cout << ")" << endl;}
-         else {cout << ", ";}
-      }
-      //cout << "  element: " << elems[i] << endl;
-      //cout << "  element " << elems[i] << " vertices:" << endl;
-      //Array<int> vert;
-      //mesh.GetElementVertices(elems[i], vert);
-      //for (auto v : vert)
-      //{
-      //   cout << "    " << v << endl;
-      //}
-
-      /*cout << "  reference coordinates: (";
-      for (int j=0; j<dim; ++j)
-      {
-         cout << refcrd[(i*dim) + j];
-         if (j == dim-1)
-         {
-            cout << ")" << endl;
-         }
-         else
-         {
-            cout << ", ";
-         }
-      }*/
-
-      int refFace, refNormal, refNormalSide;
-      bool is_interior = -1;
-      Vector normal = GetNormalVector(mesh, elems[i], refcrd.GetData() + (i*dim),
-                                      refFace, refNormal, is_interior);
-      int phyFace;
-      if (is_interior)
-      {
-         phyFace = -1; // the id of the face that has the closest point
-         FindSurfaceToProject(mesh, elems[i], phyFace);
-
-         Array<int> cbdrVert;
-         mesh.GetFaceVertices(phyFace, cbdrVert);
-         Vector xs(dim);
-         xs[0] = xyz[i + 0*np];
-         xs[1] = xyz[i + 1*np];
-         xs[2] = xyz[i + 2*np];
-         Vector xi_tmp(dim-1);
-         // get nodes!
-
-         GridFunction *nodes = mesh.GetNodes();
-         DenseMatrix coords(4,3);
-         for (int i=0; i<4; i++)
-         {
-            for (int j=0; j<3; j++)
-            {
-               coords(i,j) = (*nodes)[cbdrVert[i]*3+j];
-            }
-         }
-         SlaveToMaster(coords, xs, xi_tmp);
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = xi_tmp[j];
-         }
-         // now get get the projection to the surface
-      }
-      else
-      {
-         Vector faceRefCrd(dim-1);
-         {
-            int fd = 0;
-            for (int j=0; j<dim; ++j)
-            {
-               if (j == refNormal)
-               {
-                  refNormalSide = (refcrd[(i*dim) + j] > 0.5);
-               }
-               else
-               {
-                  faceRefCrd[fd] = refcrd[(i*dim) + j];
-                  fd++;
-               }
-            }
-
-            MFEM_VERIFY(fd == dim-1, "");
-         }
-
-         for (int j=0; j<dim-1; ++j)
-         {
-            xi[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
-         }
-         //cout << "  face reference coordinates: (";
-         for (int j=0; j<dim-1; ++j)
-         {
-            cout << faceRefCrd[j];
-            if (j == dim-2) {cout << ")" << endl;}
-            else {cout << ", ";}
-         }
-      }
-      //cout << "  normal vector: ";
-      //normal.Print();
-
-      // ask, does this do anything?
-      /*
-      IntegrationPoint ip;
-      ip.Set(refcrd.GetData() + (i*dim), dim);
-      ElementTransformation *trans = mesh.GetElementTransformation(elems[i]);
-      Vector phys(trans->GetSpaceDim());
-      trans->Transform(ip, phys);
-      cout << "  physical coordinates: ";
-      phys.Print();
-      */
-
-      // Get the element face
-      Array<int> faces;
-      Array<int> ori;
-      int face;
-
-      if (is_interior)
-      {
-         face = phyFace;
-      }
-      else
-      {
-         mesh.GetElementFaces(elems[i], faces, ori);
-         face = faces[refFace];
-      }
-
-      Array<int> faceVert;
-      mesh.GetFaceVertices(face, faceVert);
-
-      //cout << "  face " << face << " vertices:" << endl;
-      //for (auto v : faceVert){ cout << "    " << v << endl;}
-
-      for (int p=0; p<4; p++)
-      {
-         conn[4*i+p] = faceVert[p];
-      }
-      /*
-      Vector ref(dim);
-
-      for (int p=0; p<2; ++p)
-         for (int q=0; q<2; ++q)
-         {
-            const int refv = GetHexVertex(refNormal, refNormalSide, p, q, ref);
-            cout << "  face reference vertex (" << p << "," << q
-                 << ") is global vertex " << vert[refv] << endl;
-
-            {
-               // Sanity check
-               ip.Set(ref.GetData(), dim);
-               trans->Transform(ip, phys);
-               for (int j=0; j<dim; ++j)
-               {
-                  phys[j] -= mesh.GetVertex(vert[refv])[j];
-               }
-          phys.Print();
-          cout<<vert[refv]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[0]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[1]<<endl;
-               cout<<mesh.GetVertex(vert[refv])[2]<<endl;
-               MFEM_VERIFY(phys.Norml2() < 1.0e-12, "Sanity check failed");
-            }
-         }*/
-   }
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file1 = "block1.mesh";
-   const char *mesh_file2 = "block2.mesh";
-
-   Array<int> attr;
-   Array<int> m_attr;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file1, "-m1", "--mesh1",
-                  "First mesh file to use.");
-   args.AddOption(&mesh_file2, "-m2", "--mesh2",
-                  "Second mesh file to use.");
-   args.AddOption(&attr, "-at", "--attributes-surf",
-                  "Attributes of boundary faces on contact surface for mesh 2.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Mesh mesh1(mesh_file1, 1, 1);
-   Mesh mesh2(mesh_file2, 1, 1);
-
-   const int dim = mesh1.Dimension();
-   MFEM_VERIFY(dim == mesh2.Dimension(), "");
-
-   // boundary attribute 2 is the potential contact surface of nodes
-   attr.Append(2);
-   // boundary attribute 2 is the potential contact surface for master surface
-   m_attr.Append(2);
-
-   //  Define a finite element space on the mesh. Here we use vector finite
-   //  elements, i.e. dim copies of a scalar finite element space. The vector
-   //  dimension is specified by the last argument of the FiniteElementSpace
-   //  constructor.
-   FiniteElementCollection *fec1;
-   FiniteElementSpace *fespace1;
-   fec1 = new H1_FECollection(1, dim);
-   fespace1 = new FiniteElementSpace(&mesh1, fec1, dim, Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh1: "
-        << fespace1->GetTrueVSize() << endl;
-   mesh1.SetNodalFESpace(fespace1);
-   GridFunction nodes0 = *mesh1.GetNodes(); // undeformed mesh1 nodal grid function
-   GridFunction *nodes1 = mesh1.GetNodes();
-
-   FiniteElementCollection *fec2 = new H1_FECollection(1, dim);
-   FiniteElementSpace *fespace2 = new FiniteElementSpace(&mesh2, fec2, dim,
-                                                         Ordering::byVDIM);
-   cout << "Number of finite element unknowns for mesh2: "
-        << fespace2->GetTrueVSize() << endl;
-
-   // degrees of freedom of both meshes
-   int ndof_1 = fespace1->GetTrueVSize();
-   int ndof_2 = fespace2->GetTrueVSize();
-   int ndofs = ndof_1 + ndof_2;
-   // number of nodes for each mesh
-   int nnd_1 = mesh1.GetNV();
-   int nnd_2 = mesh2.GetNV();
-   int nnd = nnd_1 + nnd_2;
-   // Determine the list of true (i.e. conforming) essential boundary dofs.
-   // In this example, the boundary conditions are defined by marking only
-   // boundary attribute 1 from the mesh as essential and converting it to a
-   // list of true dofs.
-   Array<int> ess_tdof_list1, ess_bdr1(mesh1.bdr_attributes.Max());
-   cout<<mesh1.bdr_attributes.Max()<<endl;
-   ess_bdr1 = 0;
-   //ess_bdr1[0] = 1;
-   // Not ready to be passed on yet
-   // fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   Array<int> ess_tdof_list2, ess_bdr2(mesh2.bdr_attributes.Max());
-   ess_bdr2 = 0;
-   //ess_bdr2[0] = 1;
-
-   // Define the displacement vector x as a finite element grid function
-   // corresponding to fespace. GridFunction is a derived class of Vector.
-   GridFunction x1(fespace1);
-   x1 = 0.0;
-   GridFunction x2(fespace2);
-   x2 = 0.0;
-
-   // Generate force
-   LinearForm *b1 = new LinearForm(fespace1);
-   b1->Assemble();
-
-   LinearForm *b2 = new LinearForm(fespace2);
-   b2->Assemble();
-
-   // Set up the bilinear form a(.,.) on the finite element space
-   //  corresponding to the linear elasticity integrator with piece-wise
-   //  constants coefficient lambda and mu.
-   Vector lambda1(mesh1.attributes.Max());
-   lambda1 = 57.6923076923;
-   PWConstCoefficient lambda1_func(lambda1);
-   Vector mu1(mesh1.attributes.Max());
-   mu1 = 38.4615384615;
-   PWConstCoefficient mu1_func(mu1);
-
-   BilinearForm *a1 = new BilinearForm(fespace1);
-   a1->AddDomainIntegrator(new ElasticityIntegrator(lambda1_func,mu1_func));
-
-   Vector lambda2(mesh2.attributes.Max());
-   lambda2 = 57.6923076923;
-   PWConstCoefficient lambda2_func(lambda2);
-   Vector mu2(mesh2.attributes.Max());
-   mu2 = 38.4615384615;
-   PWConstCoefficient mu2_func(mu2);
-
-   BilinearForm *a2 = new BilinearForm(fespace2);
-   a2->AddDomainIntegrator(new ElasticityIntegrator(lambda2_func,mu2_func));
-
-   a1->Assemble();
-   SparseMatrix A1;
-   Vector B1, X1;
-   a1->FormLinearSystem(ess_tdof_list1, x1, *b1, A1, X1, B1);
-
-   a2->Assemble();
-   SparseMatrix A2;
-   Vector B2, X2;
-   a2->FormLinearSystem(ess_tdof_list2, x2, *b2, A2, X2, B2);
-
-   // Combine elasticity operator for two meshes into one.
-   // Block Matrix
-   SparseMatrix K(ndofs,ndofs);
-   for (int i=0; i<A1.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A1.GetRow(i, col_tmp, v_tmp);
-      K.SetRow(i, col_tmp, v_tmp);
-   }
-   for (int i=0; i<A2.Height(); i++)
-   {
-      Array<int> col_tmp;
-      Vector v_tmp;
-      col_tmp = 0;
-      v_tmp = 0.0;
-      A2.GetRow(i, col_tmp, v_tmp);
-      for (int j=0; j<col_tmp.Size(); j++)
-      {
-         col_tmp[j] += ndof_1;
-      }
-      K.SetRow(i+ndof_1, col_tmp, v_tmp);  // mesh1 top left corner
-   }
-
-   // Construct node to segment contact constraint.
-
-   attr.Sort();
-   cout << "Boundary attributes for contact surface faces in mesh 2" << endl;
-   for (auto a : attr)
-   {
-      cout << a << endl;
-   }
-
-   Array<int> bdryFaces2;  // TODO: remove this?
-
-   std::set<int> bdryVerts2;
-   for (int b=0; b<mesh2.GetNBE(); ++b)
-   {
-      if (attr.FindSorted(mesh2.GetBdrAttribute(b)) >= 0)
-      {
-         bdryFaces2.Append(b);
-         Array<int> vert;
-         mesh2.GetBdrElementVertices(b, vert);
-         for (auto v : vert)
-         {
-            bdryVerts2.insert(v);
-         }
-      }
-   }
-
-   int npoints = bdryVerts2.size();
-   Array<int> s_conn(npoints); // connectivity of the second/slave mesh
-   Vector xyz(dim * npoints);
-   xyz = 0.0;
-
-   cout << "Boundary vertices for contact surface vertices in mesh 2" << endl;
-
-   // construct the nodal coordinates on mesh2 to be projected, including displacement
-   int count = 0;
-   for (auto v : bdryVerts2)
-   {
-      cout << v << ": " << mesh2.GetVertex(v)[0] << ", "
-           << mesh2.GetVertex(v)[1] << ", "
-           << mesh2.GetVertex(v)[2] << endl;
-
-      for (int i=0; i<dim; ++i)
-      {
-         xyz[count + (i * npoints)] = mesh2.GetVertex(v)[i] + x2[v*dim+i];
-      }
-
-      s_conn[count] = v + nnd_1; // dof1 is the master
-      count++;
-   }
-
-   MFEM_VERIFY(count == npoints, "");
-
-   // gap function
-   Vector g(npoints*dim);
-   g = -1.0;
-   // segment reference coordinates of the closest point
-   Vector m_xi(npoints*(dim-1));
-   m_xi = -1.0;
-   Vector xs(dim*npoints);
-   xs = 0.0;
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<dim; j++)
-      {
-         xs[i*dim+j] = xyz[i + (j*npoints)];
-      }
-   }
-
-   Array<int> m_conn(
-      npoints*4); // only works for linear elements that have 4 vertices!
-   DenseMatrix coordsm(npoints*4, dim);
-
-   // adding displacement to mesh1 using a fixed grid function from mesh1
-   x1 = 1e-4; // x1 order: [xyz xyz... xyz]
-   add(nodes0, x1, *nodes1);
-
-   FindPointsInMesh(mesh1, xyz, m_conn, m_xi);
-
-   for (int i=0; i<npoints; i++)
-   {
-      for (int j=0; j<4; j++)
-      {
-         for (int k=0; k<dim; k++)
-         {
-            coordsm(i*4+j,k) = mesh1.GetVertex(m_conn[i*4+j])[k]+x1[dim*m_conn[i*4+j]+k];
-         }
-      }
-   }
-   //coordsm.Print();
-   SparseMatrix M(nnd,ndofs);
-   std::vector<SparseMatrix> dM(nnd, SparseMatrix(ndofs,ndofs));
-
-   Assemble_Contact(nnd, npoints, ndofs, xs, m_xi, coordsm,
-                    s_conn, m_conn, g, M, dM);
-
-   //M.Print();
-   /*Vector eps(ndofs);
-   Vector sol(ndofs); sol = 0.;
-   for(int i=0;i<ndofs;i++) eps[i] = 1e-5 * i ;
-   for(int i=0;i<9;i++)
-   {
-     cout<<i<<endl;
-     dM[s_conn[i]].Mult(eps,sol);
-     sol.Print();
-   }
-   */
-   return 0;
-}

examples/ipopt/exContactBlockTL.hpp

@@ -1,230 +0,0 @@
-//                                Contact example
-//
-// Compile with: make exContactBlockTL
-//
-// Sample runs:  ./exContactBlockTL -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
-// Sample runs:  ./exContactBlockTL -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
-
-#ifndef EXCONTACTBLOCKTL_HPP
-#define EXCONTACTBLOCKTL_HPP
-
-#include "mfem.hpp"
-#include "IpTNLP.hpp"
-
-using namespace std;
-using namespace mfem;
-using namespace Ipopt;
-
-
-class ExContactBlockTL: public TNLP
-{
-public:
-   /** default constructor */
-   ExContactBlockTL(int argc, char *argv[]);
-
-   /** default destructor */
-   virtual ~ExContactBlockTL();
-
-   /**@name Overloaded from TNLP */
-   /** Method to return some info about the nlp */
-   virtual bool get_nlp_info(
-      Index&          n,
-      Index&          m,
-      Index&          nnz_jac_g,
-      Index&          nnz_h_lag,
-      IndexStyleEnum& index_style
-   );
-
-   /** Method to return the bounds for my problem */
-   virtual bool get_bounds_info(
-      Index   n,
-      Number* x_l,
-      Number* x_u,
-      Index   m,
-      Number* g_l,
-      Number* g_u
-   );
-
-   /** Method to return the starting point for the algorithm */
-   virtual bool get_starting_point(
-      Index   n,
-      bool    init_x,
-      Number* x,
-      bool    init_z,
-      Number* z_L,
-      Number* z_U,
-      Index   m,
-      bool    init_lambda,
-      Number* lambda
-   );
-
-   /** Method to return the objective value */
-   virtual bool eval_f(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Number&       obj_value
-   );
-
-   /** Method to return the gradient of the objective */
-   virtual bool eval_grad_f(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Number*       grad_f
-   );
-
-   /** Method to return the constraint residuals */
-   virtual bool eval_g(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Index         m,
-      Number*       cons
-   );
-
-   /** Method to return:
-    *   1) The structure of the Jacobian (if "values" is NULL)
-    *   2) The values of the Jacobian (if "values" is not NULL)
-    */
-   virtual bool eval_jac_g(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Index         m,
-      Index         nele_jac,
-      Index*        iRow,
-      Index*        jCol,
-      Number*       values
-   );
-
-   /** Method to return:
-    *   1) The structure of the Hessian of the Lagrangian (if "values" is NULL)
-    *   2) The values of the Hessian of the Lagrangian (if "values" is not NULL)
-    */
-   virtual bool eval_h(
-      Index         n,
-      const Number* x,
-      bool          new_x,
-      Number        obj_factor,
-      Index         m,
-      const Number* lambda,
-      bool          new_lambda,
-      Index         nele_hess,
-      Index*        iRow,
-      Index*        jCol,
-      Number*       values
-   );
-
-   /** This method is called when the algorithm is complete so the TNLP can store/write the solution */
-   virtual void finalize_solution(
-      SolverReturn               status,
-      Index                      n,
-      const Number*              x,
-      const Number*              z_L,
-      const Number*              z_U,
-      Index                      m,
-      const Number*              g,
-      const Number*              lambda,
-      Number                     obj_value,
-      const IpoptData*           ip_data,
-      IpoptCalculatedQuantities* ip_cq
-   );
-
-private:
-   void update_g();
-   void update_jac();
-   void update_hess();
-
-private:
-   /**@name Methods to block default compiler methods.
-    *
-    * The compiler automatically generates the following three methods.
-    *  Since the default compiler implementation is generally not what
-    *  you want (for all but the most simple classes), we usually
-    *  put the declarations of these methods in the private section
-    *  and never implement them. This prevents the compiler from
-    *  implementing an incorrect "default" behavior without us
-    *  knowing. (See Scott Meyers book, "Effective C++")
-    */
-   ExContactBlockTL(
-      const ExContactBlockTL&
-   );
-
-   ExContactBlockTL& operator=(
-      const ExContactBlockTL&
-   );
-
-   Array<int> attr;
-   Array<int> m_attr;
-   Array<int> s_conn; // connectivity of the second/slave mesh
-   std::string mesh_file1;
-   std::string mesh_file2;
-   Mesh* mesh1;
-   Mesh* mesh2;
-   FiniteElementCollection* fec1;
-   FiniteElementCollection* fec2;
-   FiniteElementSpace* fespace1;
-   FiniteElementSpace* fespace2;
-   Array<int> ess_tdof_list1;
-   Array<int> ess_tdof_list2;
-   GridFunction nodes0;
-   GridFunction* nodes1;
-   GridFunction* nodes2;
-   GridFunction* x1;
-   GridFunction* x2;
-   LinearForm* b1;
-   LinearForm* b2;
-   PWConstCoefficient* lambda1_func;
-   PWConstCoefficient* lambda2_func;
-   PWConstCoefficient* mu1_func;
-   PWConstCoefficient* mu2_func;
-   BilinearForm* a1;
-   BilinearForm* a2;
-
-   mfem::Vector lambda1;
-   mfem::Vector lambda2;
-   mfem::Vector mu1;
-   mfem::Vector mu2;
-   mfem::Vector xyz;
-
-   std::set<int> bdryVerts2;
-
-   int dim;
-   // degrees of freedom of both meshes
-   int ndof_1;
-   int ndof_2;
-   int ndofs;
-   // number of nodes for each mesh
-   int nnd_1;
-   int nnd_2;
-   int nnd;
-
-   int npoints;
-
-   SparseMatrix A1;
-   mfem::Vector B1, X1;
-   SparseMatrix A2;
-   mfem::Vector B2, X2;
-
-   SparseMatrix* K;
-   mfem::Vector g;
-   mfem::Vector m_xi;
-   mfem::Vector xs;
-
-   Array<int> m_conn; // only works for linear elements that have 4 vertices!
-   DenseMatrix* coordsm;
-   SparseMatrix* M;
-
-   std::vector<SparseMatrix>* dM;
-
-   Array<int> Dirichlet_dof;
-   Array<double> Dirichlet_val;
-
-public: 
-   Mesh * GetMesh1() {return mesh1;}
-   Mesh * GetMesh2() {return mesh2;}
-
-};
-
-#endif

examples/ipopt/nodepair.hpp

@@ -1,888 +0,0 @@
-
-using namespace std;
-using namespace mfem;
-
-void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]);
-   dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);
-   dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]);
-   dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);
-   dNdxi(1,3) = 0.25*(1-xi[0]);
-}
-
-
-void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
-                     DenseMatrix& dN2dxi)
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(2,4); dNdxi = 0.0;
-   dN2dxi.SetSize(3,4);
-   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
-
-   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
-   dN2dxi(1,3) = -0.25;
-}
-
-// returns the vector and matrix form of the shape functions and its derivative
-void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
-                       DenseMatrix& ddNdxi)
-{
-   N.SetSize(3,12); N = 0.0;
-   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
-   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
-   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
-   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
-   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
-   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
-   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
-
-   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
-   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
-   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
-   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
-   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
-   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
-
-   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
-   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
-   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
-   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
-   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
-   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
-   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
-
-   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
-   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
-   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
-   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
-   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
-   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
-}
-
-
-void cross(const Vector a, const Vector b, Vector& c)
-{
-   assert(a.Size()==3);
-   c.SetSize(3);
-   c[0] = a[1]*b[2] - a[2]*b[1];
-   c[1] = -a[0]*b[2] + b[0]*a[2];
-   c[2] = a[0]*b[1] - a[1]*b[0];
-
-}
-// a outer b
-void outer(const Vector a, const Vector b, DenseMatrix& c)
-{
-   int m = a.Size();
-   int n = b.Size();
-   assert(c.Height()==m);
-   assert(c.Width() ==n);
-   for (int i=0; i<m; i++)
-   {
-      for (int j=0; j<n; j++)
-      {
-         c(i,j) = a[i]*b[j];
-      }
-   }
-}
-// dphidxi 2*4
-// coords 4*3
-void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
-                   Vector& normal, double& nnorm)
-{
-
-   DenseMatrix dxdxi(2,3);
-   Mult(dphidxi, coords, dxdxi);
-   Vector dxdxi1(3);
-   Vector dxdxi2(3);
-
-   dxdxi.GetRow(0,dxdxi1);
-   dxdxi.GetRow(1,dxdxi2);
-
-   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
-   // VectorCrossProductCoefficient::Eval has hard-coded cross product
-   nnorm = normal.Norml2( );
-   normal /=  nnorm;
-}
-
-void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
-{
-   bool converged = false;
-   bool pt_on_elem = false;
-   int dim = 3;
-   xi.SetSize(dim-1);
-   xi = 0.0;
-   double r = 1e10;
-   int max_iter = 15;
-   double off_el_xi = 1e-2;
-   double proj_newton_tol = 1e-13;
-   double proj_max_gap = 0.5;
-   Vector gap_v(dim);
-   // warm start from linear solution
-
-   for (int it=0; it<max_iter; it++)
-   {
-      //cout<<it<<endl;
-      Vector m_N(4);
-      m_N = 0.;
-      DenseMatrix m_dN(2,4);
-      m_dN = 0.;
-      DenseMatrix m_dN2(3,4);
-      m_dN2 = 0.;
-      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-      Vector x_c(dim);
-      m_coords.MultTranspose(m_N, x_c);
-
-      gap_v = s_x;
-      gap_v -= x_c;
-
-      DenseMatrix m_dx(2,3);
-      m_dx = 0.;
-      Mult(m_dN, m_coords, m_dx);
-
-      Vector r(dim-1);
-      r = 0.0;
-      m_dx.Mult(gap_v, r);
-
-      if (r.Normlinf() < proj_newton_tol)
-      {
-         converged = true;
-         break;
-      }
-
-      DenseMatrix drdxi(dim-1,dim-1);
-      drdxi = 0.;
-      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
-      drdxi *= -1.0;
-
-      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-      Mult(m_dN2,m_coords, m_dx2);
-
-      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
-      for (int d=0; d<3; d++)
-      {
-         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-         drdxi.Add(gap_v[d], Mtemp);
-      }
-
-      //cond_num = rcond(drdxi);  condition number?
-      //drdxi.TestInversion();
-      DenseMatrixInverse drdxi_inv(drdxi);
-      Vector xi_tmp(dim-1);
-
-      drdxi_inv.Mult(r,xi_tmp);
-      xi -= xi_tmp;
-   }
-   if (!converged)
-   {
-      xi = 0.0;
-   }
-   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
-
-   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
-   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
-   {
-      pt_on_elem = true;
-   }
-
-   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
-   MFEM_VERIFY(converged == true, "projection didn't converge");
-}
-
-
-
-// m_coords is expected to be 4 * 3
-void  ComputeGapJacobian(const Vector x_s, const Vector xi,
-                         const DenseMatrix m_coords,
-                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N, x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   double nnorm = 0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   gap = gap_v * normal; // gap function value, dot product between vectors
-
-   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
-
-   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
-   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
-
-   Vector m_dxrow1(3);
-   m_dx.GetRow(0, m_dxrow1);
-   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
-   dr_dx_res1 *= -1.0;
-
-   Vector m_dxrow2(3);
-   m_dx.GetRow(1, m_dxrow2);
-   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
-   dr_dx_res2 *= -1.0;
-
-   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
-   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
-
-   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
-   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
-   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
-   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
-
-   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
-   dr_dx_res2 += dr_dx_res2_tmp;
-
-
-   DenseMatrix K_dxidx1(2,2); // 2*2
-   K_dxidx1 = 0.;
-   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   // how to get 2nd order? multidimensional matrix?
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= K_dxidx1;
-   K_dxidx += K_dxidx2;
-
-   // resize the vectors and matrices
-   Vector dxidx(24); dxidx = 0.0;
-   Vector drdx_r(24); drdx_r = 0.0;
-
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         drdx_r[4*j+i] = dr_dx_res1(i,j);
-         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
-
-      }
-   }
-   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
-   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
-   DenseMatrix drdx_K(24,24); drdx_K = 0.;
-   for (int i =0; i<12; i++)
-   {
-      drdx_K(i,i) = K_dxidx(0,0);
-      drdx_K(i,12+i) = K_dxidx(0,1);
-      drdx_K(12+i,i) = K_dxidx(1,0);
-      drdx_K(12+i,12+i) = K_dxidx(1,1);
-   }
-
-   DenseMatrixInverse drdxK_inv(drdx_K);
-   drdxK_inv.Mult(drdx_r,dxidx);
-   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
-   dxidx *= -1.0;
-
-
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-
-   Vector dxidxs(6); dxidxs = 0.0;
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
-
-   dgdxm.SetSize(12); dgdxm = 0.;
-   DenseMatrix dgdxm_tmp(4,3);
-   outer(m_N, normal,dgdxm_tmp);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
-      }
-   }
-   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
-
-   dgdxs.SetSize(3);
-   dgdxs += normal;
-   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
-};
-
-void ComputeGapHessian(const Vector x_s, const Vector xi,
-                       const DenseMatrix m_coords,
-                       DenseMatrix& dg2dx)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   int dim = 3;
-   int num_dofs1 = dim;
-   int num_dofs2 = 4*dim;
-   int num_dofs = num_dofs1 + num_dofs2;
-   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N,x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-   Mult(m_dN2,m_coords, m_dx2);
-   double nnorm = 0.0;
-   Vector normal(3); normal = 0.0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   double gap = gap_v * normal; // gap function value, dot product between vectors
-
-   DenseMatrix M(2,2); M = 0.0;
-   MultABt(m_dx, m_dx, M);
-
-   DenseMatrix f(2, num_dofs2); f = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-      M.Add(-gap_v[d], Mtemp);
-
-      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
-      DenseMatrix ftmp(2,4);
-      outer(m_dxcol, m_N, ftmp);
-      ftmp *= -1;
-      ftmp.Add( gap_v[d], m_dN);   // 2*4
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         f(0,d+j*3) = ftmp(0,j);
-         f(1,d+j*3) = ftmp(1,j);
-      }
-   }
-   //fprintf('hess dxidxm\n');
-   DenseMatrixInverse Minv(M);
-   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
-   Minv.Mult(f, dxidxm);
-   //LinearSolve??
-   //dxidxm = M\f;
-
-   DenseMatrix nde2(2,2); nde2 = 0.0;
-   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
-      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
-      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
-
-      nde2 += ndetmp;
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
-         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
-      }
-
-   }
-
-   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
-   Ndn += Nndx2;
-   AddMult(nde2, dxidxm, Ndn);
-
-
-   DenseMatrix M2(2,2); M2 = 0.0;
-   MultABt(m_dx, m_dx, M2);
-   DenseMatrixInverse M2inv(M2);
-   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
-   DenseMatrix m_con(2,2); m_con = 0.0;
-
-   M2inv.Mult(diag2, m_con);
-
-   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
-
-   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
-   MultAtB(Ndn, m_con, dg2dxm_tmp);
-   Mult(dg2dxm_tmp, Ndn, dg2dxm);
-   dg2dxm *= gap;
-
-   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
-   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   dg2dxm_tmp = 0.0;
-   MultAtB(dxidxm, nde2, dg2dxm_tmp);
-
-   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
-
-   dg2dxm_tmp2 = 0.0;
-   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= M2;
-   K_dxidx += K_dxidx2;
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-   Vector dxidxs(6);
-
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
-   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
-   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
-
-   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
-   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
-
-   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
-   dxidxs_row2 = 0.0;
-   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
-   Vector mdx2_row3(3); mdx2_row3 = 0.0;
-   dxidxs_m.GetRow(0,dxidxs_row1);
-   dxidxs_m.GetRow(1,dxidxs_row2);
-   m_dx2.GetRow(0,mdx2_row1);
-   m_dx2.GetRow(1,mdx2_row2);
-   m_dx2.GetRow(2,mdx2_row3);
-
-   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
-   outer(mdx2_row1, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row2, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-
-   outer(mdx2_row2, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row3, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-
-   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
-
-   for (int d=0; d<3; d++)
-   {
-      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
-      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
-      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
-
-      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
-      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
-
-      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
-      m_dx.GetRow(0, m_dxrow);
-      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
-      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
-      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
-
-      dtaodxs_tmp2 += dtaodxs_tmp;
-      dtaodxs.SetCol(d, dtaodxs_tmp2);
-   }
-
-   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
-   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
-   outer(normal, normal, dndxs_tmp);
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
-
-   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
-   MultAtB(m_dx, dxidxs_m, dgvdxs);
-   dgvdxs *= -1;
-   for (int d=0; d<3; d++)
-   {
-      dgvdxs(d,d) += 1.0;
-   }
-   //dxidxs: 2*3
-
-   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
-   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
-   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
-   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
-   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
-   dg2dxs += dg2dxs_tmp2;
-   dg2dxs_tmp2 = 0.0;
-   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
-   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
-
-   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
-   BasisVectorDerivs(xi, Ne, Be, dBe);
-
-   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
-   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
-
-   Vector m_coords_v(12);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         m_coords_v[i*3+j] = m_coords(i,j);
-      }
-   }
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix dBe_tmp(3,12);
-      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
-
-      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
-
-      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
-      dBe_tmp = 0.0;
-      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
-
-   }
-
-   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
-
-   for (int d=0; d<12; d++)
-   {
-      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
-      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
-      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
-      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
-      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
-      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
-
-      cross(tmp1, m_dxrow2, dtaodxm_tmp);
-      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
-      dtaodxm_tmp += dtaodxm_tmp2;
-
-      dtaodxm.SetCol(d, dtaodxm_tmp);
-   }
-
-   DenseMatrix dndxm(3,12); dndxm = 0.0;
-   dndxm += dtaodxm;
-   dndxm *= 1.0/nnorm;
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
-
-   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
-   dgvdxm -= Ne;
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
-      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
-
-   }
-
-   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
-   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
-   MultAtB(dgvdxs, dndxm, dg2dxsxm);
-
-   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
-   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
-
-   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
-   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
-
-   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
-
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
-      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
-      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
-   }
-
-   dg2dxsxm += dgvdxsxmn;
-
-   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
-   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
-   MultAtB(dgvdxm, dndxs, dg2dxmxs);
-   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
-   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
-
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
-
-   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
-   dgvdxmxsn_tmp *= -1.0;
-   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-      Be_tmp.Transpose(); // Be is now 12*3
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
-      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
-      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
-
-   }
-
-   dg2dxmxs += dgvdxmxsn;
-
-   dg2dx.CopyMN(dg2dxs, 0, 0);
-   dg2dx.CopyMN(dg2dxm, 3, 3);
-   dg2dx.CopyMN(dg2dxsxm, 0, 3);
-   dg2dx.CopyMN(dg2dxmxs, 3, 0);
-
-};
-
-
-
-void NodeSegConPairs(const Vector x1, const Vector xi2,
-                     const DenseMatrix coords2,
-                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
-{
-   double gap = 0.0;
-   Vector normal(3); normal = 0.0;
-   Vector dgdxm(12); dgdxm = 0.0;
-   Vector dgdxs(3); dgdxs = 0.0;
-
-   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
-   node_g = gap;
-
-   node_dg.SetSize(12+3);
-   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
-   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
-
-   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   ComputeGapHessian(x1, xi2, coords2, dg2dx);
-
-   node_dg2.SetSize(15,15);
-   node_dg2 = dg2dx;
-
-   /*
-   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
-
-   v1 = 1:3;
-   v2 = 1:12;
-   %v1 = ones(1,3)
-   %v2 = ones(1,12)
-   v2 = reshape(v2,4,3);
-   x1n1 = x1 + 0.01*v1;
-   coords2n1 = coords2 + 0.001*v2;
-      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
-      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
-   x1n2 = x1 - 0.01*v1;
-   coords2n2 = coords2 - 0.001*v2;
-      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
-      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
-   fprintf('fd\n');
-      %gapv1-gapv2
-   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
-
-   %dgdxsn1-dgdxsn2
-   fprintf('code\n');
-   v2n = v2';
-   %dg2dx(1:3,1:3)*0.04*ones(3,1)
-   temp = zeros(12,3);
-   for i = 1:4
-   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
-   temp((i-1)*3+1:i*3,:) = temp1';
-   end
-   temp2 = zeros(3,12);
-   for i = 1:4
-   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
-   temp2(:,(i-1)*3+1:i*3) = temp3';
-   end
-   %dg2dx
-   %dg2dx(4:end,1:3) = temp;
-   %dg2dx(1:3,4:end) = temp2;
-   %dgvdxm * 0.002*v2n(:)
-   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
-   %dg2dx(4:end,1:3)
-   end*/
-
-};
-
-
-// coordsm : (npoints*4, 3) use what class?
-// m_conn: (npoints*4)
-void Assemble_Contact(const int m, const int npoints, const int ndofs,
-                      const Vector x_s,
-                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
-                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
-                      std::vector<SparseMatrix>& dM)
-{
-   int n = ndofs;
-   int ndim = 3;
-
-   g.SetSize(m);
-   g = 0.0;
-
-   //SparseMatrix M(m, n);  // M needs to be the correct size
-
-   //dM.resize(m); // needs to clear?
-
-   double g_tmp = 0.;
-   Vector dg(4*ndim+ndim);
-   dg = 0.;
-   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
-   dg2 = 0.;
-
-   for (int i=0; i<npoints; i++)
-   {
-      Vector x1(ndim);
-      x1[0] = x_s[i*ndim];
-      x1[1] = x_s[i*ndim+1];
-      x1[2] = x_s[i*ndim+2];
-
-      Vector xi2(ndim-1);
-      xi2[0] = xi[i*(ndim-1)];
-      xi2[1] = xi[i*(ndim-1)+1];
-
-      DenseMatrix coords2(4,3);
-      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
-
-      //how to get coords2?
-      dg = 0.0;
-      dg2 = 0.;
-      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
-      //x1.Print();
-      //xi2.Print();
-      //coords2.Print();
-      g[s_conn[i]] = g_tmp; // should be unique
-      Array<int> m_conn_i(4);
-      m_conn.GetSubArray(4*i, 4, m_conn_i);
-
-      Array<int> node_conn(5);
-      node_conn[0] = s_conn[i];
-      for (int j=0; j<4; j++)
-      {
-         node_conn[j+1] = m_conn_i[j];
-      }
-
-      Array<int>  M_i_tmp(1);
-      M_i_tmp[0] = s_conn[i];
-
-      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
-      Array<int> j_idx(5*ndim); j_idx = 0;
-      for (int j=0; j< 5; j++)
-      {
-         for (int k=0; k<ndim; k++)
-         {
-            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
-         }
-      }
-      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
-      M_v_tmp.SetRow(0, dg);
-
-      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
-
-      Array<int> dM_i(ndim*(4+1));
-      Array<int> dM_j(ndim*(4+1));
-
-      for (int j=0; j< ndim*(4+1); j++)
-      {
-         dM_i[j] = j_idx[j];
-         dM_j[j] = j_idx[j];
-      }
-      //dg2.Print();
-      //dM[s_conn[i]].Print();
-      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
-   }
-};
-

examples/makefile

@@ -46,9 +46,6 @@ endif
 ifeq ($(MFEM_USE_HIOP),YES)
    SUBDIRS += hiop
 endif
-ifeq ($(MFEM_USE_IPOPT),YES)
-   SUBDIRS += ipopt
-endif
 ifeq ($(MFEM_USE_PETSC),YES)
    SUBDIRS += petsc
 endif

examples/nodepair.hpp

@@ -1,888 +0,0 @@
-
-using namespace std;
-using namespace mfem;
-
-void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]);
-   dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);
-   dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]);
-   dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);
-   dNdxi(1,3) = 0.25*(1-xi[0]);
-}
-
-
-void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
-                     DenseMatrix& dN2dxi)
-{
-   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
-   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
-   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
-   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(2,4); dNdxi = 0.0;
-   dN2dxi.SetSize(3,4);
-   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
-
-   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
-   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
-   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
-   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
-
-   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
-   dN2dxi(1,3) = -0.25;
-}
-
-// returns the vector and matrix form of the shape functions and its derivative
-void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
-                       DenseMatrix& ddNdxi)
-{
-   N.SetSize(3,12); N = 0.0;
-   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
-   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
-
-   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
-   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
-   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
-   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
-   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
-   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
-   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
-
-   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
-   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
-   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
-   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
-   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
-   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
-
-   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
-   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
-   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
-   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
-   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
-   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
-   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
-
-   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
-   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
-   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
-   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
-   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
-   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
-}
-
-
-void cross(const Vector a, const Vector b, Vector& c)
-{
-   assert(a.Size()==3);
-   c.SetSize(3);
-   c[0] = a[1]*b[2] - a[2]*b[1];
-   c[1] = -a[0]*b[2] + b[0]*a[2];
-   c[2] = a[0]*b[1] - a[1]*b[0];
-
-}
-// a outer b
-void outer(const Vector a, const Vector b, DenseMatrix& c)
-{
-   int m = a.Size();
-   int n = b.Size();
-   assert(c.Height()==m);
-   assert(c.Width() ==n);
-   for (int i=0; i<m; i++)
-   {
-      for (int j=0; j<n; j++)
-      {
-         c(i,j) = a[i]*b[j];
-      }
-   }
-}
-// dphidxi 2*4
-// coords 4*3
-void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
-                   Vector& normal, double& nnorm)
-{
-
-   DenseMatrix dxdxi(2,3);
-   Mult(dphidxi, coords, dxdxi);
-   Vector dxdxi1(3);
-   Vector dxdxi2(3);
-
-   dxdxi.GetRow(0,dxdxi1);
-   dxdxi.GetRow(1,dxdxi2);
-
-   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
-   // VectorCrossProductCoefficient::Eval has hard-coded cross product
-   nnorm = normal.Norml2( );
-   normal /=  nnorm;
-}
-
-void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
-{
-   bool converged = false;
-   bool pt_on_elem = false;
-   int dim = 3;
-   xi.SetSize(dim-1);
-   xi = 0.0;
-   double r = 1e10;
-   int max_iter = 15;
-   double off_el_xi = 1e-2;
-   double proj_newton_tol = 1e-13;
-   double proj_max_gap = 0.5;
-   Vector gap_v(dim);
-   // warm start from linear solution
-
-   for (int it=0; it<max_iter; it++)
-   {
-      //cout<<it<<endl;
-      Vector m_N(4);
-      m_N = 0.;
-      DenseMatrix m_dN(2,4);
-      m_dN = 0.;
-      DenseMatrix m_dN2(3,4);
-      m_dN2 = 0.;
-      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-      Vector x_c(dim);
-      m_coords.MultTranspose(m_N, x_c);
-
-      gap_v = s_x;
-      gap_v -= x_c;
-
-      DenseMatrix m_dx(2,3);
-      m_dx = 0.;
-      Mult(m_dN, m_coords, m_dx);
-
-      Vector r(dim-1);
-      r = 0.0;
-      m_dx.Mult(gap_v, r);
-
-      if (r.Normlinf() < proj_newton_tol)
-      {
-         converged = true;
-         break;
-      }
-
-      DenseMatrix drdxi(dim-1,dim-1);
-      drdxi = 0.;
-      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
-      drdxi *= -1.0;
-
-      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-      Mult(m_dN2,m_coords, m_dx2);
-
-      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
-      for (int d=0; d<3; d++)
-      {
-         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-         drdxi.Add(gap_v[d], Mtemp);
-      }
-
-      //cond_num = rcond(drdxi);  condition number?
-      //drdxi.TestInversion();
-      DenseMatrixInverse drdxi_inv(drdxi);
-      Vector xi_tmp(dim-1);
-
-      drdxi_inv.Mult(r,xi_tmp);
-      xi -= xi_tmp;
-   }
-   if (!converged)
-   {
-      xi = 0.0;
-   }
-   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
-
-   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
-   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
-   {
-      pt_on_elem = true;
-   }
-
-   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
-   MFEM_VERIFY(converged == true, "projection didn't converge");
-}
-
-
-
-// m_coords is expected to be 4 * 3
-void  ComputeGapJacobian(const Vector x_s, const Vector xi,
-                         const DenseMatrix m_coords,
-                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N, x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   double nnorm = 0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   gap = gap_v * normal; // gap function value, dot product between vectors
-
-   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
-
-   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
-   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
-
-   Vector m_dxrow1(3);
-   m_dx.GetRow(0, m_dxrow1);
-   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
-   dr_dx_res1 *= -1.0;
-
-   Vector m_dxrow2(3);
-   m_dx.GetRow(1, m_dxrow2);
-   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
-   dr_dx_res2 *= -1.0;
-
-   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
-   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
-
-   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
-   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
-   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
-   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
-
-   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
-   dr_dx_res2 += dr_dx_res2_tmp;
-
-
-   DenseMatrix K_dxidx1(2,2); // 2*2
-   K_dxidx1 = 0.;
-   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   // how to get 2nd order? multidimensional matrix?
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= K_dxidx1;
-   K_dxidx += K_dxidx2;
-
-   // resize the vectors and matrices
-   Vector dxidx(24); dxidx = 0.0;
-   Vector drdx_r(24); drdx_r = 0.0;
-
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         drdx_r[4*j+i] = dr_dx_res1(i,j);
-         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
-
-      }
-   }
-   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
-   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
-   DenseMatrix drdx_K(24,24); drdx_K = 0.;
-   for (int i =0; i<12; i++)
-   {
-      drdx_K(i,i) = K_dxidx(0,0);
-      drdx_K(i,12+i) = K_dxidx(0,1);
-      drdx_K(12+i,i) = K_dxidx(1,0);
-      drdx_K(12+i,12+i) = K_dxidx(1,1);
-   }
-
-   DenseMatrixInverse drdxK_inv(drdx_K);
-   drdxK_inv.Mult(drdx_r,dxidx);
-   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
-   dxidx *= -1.0;
-
-
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-
-   Vector dxidxs(6); dxidxs = 0.0;
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
-
-   dgdxm.SetSize(12); dgdxm = 0.;
-   DenseMatrix dgdxm_tmp(4,3);
-   outer(m_N, normal,dgdxm_tmp);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
-      }
-   }
-   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
-
-   dgdxs.SetSize(3);
-   dgdxs += normal;
-   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
-};
-
-void ComputeGapHessian(const Vector x_s, const Vector xi,
-                       const DenseMatrix m_coords,
-                       DenseMatrix& dg2dx)
-{
-   Vector m_N(4);
-   DenseMatrix m_dN(2,4);
-   DenseMatrix m_dN2(3,4);
-   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
-
-   int dim = 3;
-   int num_dofs1 = dim;
-   int num_dofs2 = 4*dim;
-   int num_dofs = num_dofs1 + num_dofs2;
-   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
-
-   Vector x_c(3);
-   m_coords.MultTranspose(m_N,x_c);
-
-   Vector gap_v(3); gap_v = 0.0;
-   gap_v  = x_s;
-   gap_v -= x_c;
-
-   DenseMatrix m_dx(2,3);
-   Mult(m_dN, m_coords, m_dx);
-
-   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
-   Mult(m_dN2,m_coords, m_dx2);
-   double nnorm = 0.0;
-   Vector normal(3); normal = 0.0;
-   ComputeNormal(m_dN, m_coords, normal, nnorm);
-
-   double gap = gap_v * normal; // gap function value, dot product between vectors
-
-   DenseMatrix M(2,2); M = 0.0;
-   MultABt(m_dx, m_dx, M);
-
-   DenseMatrix f(2, num_dofs2); f = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
-      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
-      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
-
-      M.Add(-gap_v[d], Mtemp);
-
-      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
-      DenseMatrix ftmp(2,4);
-      outer(m_dxcol, m_N, ftmp);
-      ftmp *= -1;
-      ftmp.Add( gap_v[d], m_dN);   // 2*4
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         f(0,d+j*3) = ftmp(0,j);
-         f(1,d+j*3) = ftmp(1,j);
-      }
-   }
-   //fprintf('hess dxidxm\n');
-   DenseMatrixInverse Minv(M);
-   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
-   Minv.Mult(f, dxidxm);
-   //LinearSolve??
-   //dxidxm = M\f;
-
-   DenseMatrix nde2(2,2); nde2 = 0.0;
-   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
-
-   for (int d=0; d<3; d++)
-   {
-      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
-      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
-      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
-
-      nde2 += ndetmp;
-
-      for (int j=0; j<4; j++)
-      {
-         assert(d+3*j<num_dofs2);
-         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
-         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
-      }
-
-   }
-
-   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
-   Ndn += Nndx2;
-   AddMult(nde2, dxidxm, Ndn);
-
-
-   DenseMatrix M2(2,2); M2 = 0.0;
-   MultABt(m_dx, m_dx, M2);
-   DenseMatrixInverse M2inv(M2);
-   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
-   DenseMatrix m_con(2,2); m_con = 0.0;
-
-   M2inv.Mult(diag2, m_con);
-
-   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
-
-   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
-   MultAtB(Ndn, m_con, dg2dxm_tmp);
-   Mult(dg2dxm_tmp, Ndn, dg2dxm);
-   dg2dxm *= gap;
-
-   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
-   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   dg2dxm_tmp = 0.0;
-   MultAtB(dxidxm, nde2, dg2dxm_tmp);
-
-   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
-
-   dg2dxm_tmp2 = 0.0;
-   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
-   dg2dxm.Add(-1.0, dg2dxm_tmp2);
-
-   Vector v_dxidx2(4);
-   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
-
-   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
-
-   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
-   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
-   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
-   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
-   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
-   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
-
-   DenseMatrix K_dxidx(2,2);
-   K_dxidx -= M2;
-   K_dxidx += K_dxidx2;
-
-   Vector drdxs_r(6);
-   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
-   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
-
-   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
-   for (int i=0; i<3; i++)
-   {
-      drdxs_K(i,i) = K_dxidx(0,0);
-      drdxs_K(i,3+i) = K_dxidx(0,1);
-      drdxs_K(i+3,i) = K_dxidx(1,0);
-      drdxs_K(i+3,i+3) = K_dxidx(1,1);
-   }
-   Vector dxidxs(6);
-
-   DenseMatrixInverse drdxsK_inv(drdxs_K);
-   drdxsK_inv.Mult(drdxs_r,dxidxs);
-   dxidxs *= -1.0;
-   //dxidxs = -drdxs_K\drdxs_r;
-
-   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
-   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
-   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
-
-   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
-   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
-
-   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
-   dxidxs_row2 = 0.0;
-   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
-   Vector mdx2_row3(3); mdx2_row3 = 0.0;
-   dxidxs_m.GetRow(0,dxidxs_row1);
-   dxidxs_m.GetRow(1,dxidxs_row2);
-   m_dx2.GetRow(0,mdx2_row1);
-   m_dx2.GetRow(1,mdx2_row2);
-   m_dx2.GetRow(2,mdx2_row3);
-
-   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
-   outer(mdx2_row1, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row2, dxidxs_row1,dtaotmp);
-   dtao1dxs += dtaotmp; dtaotmp = 0.0;
-
-   outer(mdx2_row2, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-   outer(mdx2_row3, dxidxs_row2, dtaotmp);
-   dtao2dxs += dtaotmp; dtaotmp = 0.0;
-
-   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
-
-   for (int d=0; d<3; d++)
-   {
-      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
-      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
-      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
-
-      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
-      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
-
-      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
-      m_dx.GetRow(0, m_dxrow);
-      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
-      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
-      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
-
-      dtaodxs_tmp2 += dtaodxs_tmp;
-      dtaodxs.SetCol(d, dtaodxs_tmp2);
-   }
-
-   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
-   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
-   outer(normal, normal, dndxs_tmp);
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
-
-   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
-   MultAtB(m_dx, dxidxs_m, dgvdxs);
-   dgvdxs *= -1;
-   for (int d=0; d<3; d++)
-   {
-      dgvdxs(d,d) += 1.0;
-   }
-   //dxidxs: 2*3
-
-   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
-   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
-   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
-   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
-   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
-   dg2dxs += dg2dxs_tmp2;
-   dg2dxs_tmp2 = 0.0;
-   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
-   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
-
-   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
-   BasisVectorDerivs(xi, Ne, Be, dBe);
-
-   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
-   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
-
-   Vector m_coords_v(12);
-   for (int i=0; i<4; i++)
-   {
-      for (int j=0; j<3; j++)
-      {
-         m_coords_v[i*3+j] = m_coords(i,j);
-      }
-   }
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix dBe_tmp(3,12);
-      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
-
-      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
-
-      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
-      dBe_tmp = 0.0;
-      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
-      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
-
-   }
-
-   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
-
-   for (int d=0; d<12; d++)
-   {
-      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
-      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
-      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
-      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
-      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
-      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
-
-      cross(tmp1, m_dxrow2, dtaodxm_tmp);
-      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
-      dtaodxm_tmp += dtaodxm_tmp2;
-
-      dtaodxm.SetCol(d, dtaodxm_tmp);
-   }
-
-   DenseMatrix dndxm(3,12); dndxm = 0.0;
-   dndxm += dtaodxm;
-   dndxm *= 1.0/nnorm;
-   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
-
-   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
-   dgvdxm -= Ne;
-
-   for (int i=0; i<2; i++)
-   {
-      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
-      dxidxm.GetRow(i,dxidxm_tmp);
-
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
-      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
-      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
-
-   }
-
-   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
-   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
-   MultAtB(dgvdxs, dndxm, dg2dxsxm);
-
-   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
-   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
-
-   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
-   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
-   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
-
-   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
-
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
-      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
-      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
-   }
-
-   dg2dxsxm += dgvdxsxmn;
-
-   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
-   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
-   MultAtB(dgvdxm, dndxs, dg2dxmxs);
-   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
-   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
-
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
-
-   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
-   dgvdxmxsn_tmp *= -1.0;
-   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
-
-   for (int i =0; i<2; i++)
-   {
-      DenseMatrix Be_tmp(3,12);
-      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
-      Be_tmp.Transpose(); // Be is now 12*3
-
-      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
-      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
-      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
-      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
-
-   }
-
-   dg2dxmxs += dgvdxmxsn;
-
-   dg2dx.CopyMN(dg2dxs, 0, 0);
-   dg2dx.CopyMN(dg2dxm, 3, 3);
-   dg2dx.CopyMN(dg2dxsxm, 0, 3);
-   dg2dx.CopyMN(dg2dxmxs, 3, 0);
-
-};
-
-
-
-void NodeSegConPairs(const Vector x1, const Vector xi2,
-                     const DenseMatrix coords2,
-                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
-{
-   double gap = 0.0;
-   Vector normal(3); normal = 0.0;
-   Vector dgdxm(12); dgdxm = 0.0;
-   Vector dgdxs(3); dgdxs = 0.0;
-
-   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
-   node_g = gap;
-
-   node_dg.SetSize(12+3);
-   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
-   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
-
-   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
-   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
-   ComputeGapHessian(x1, xi2, coords2, dg2dx);
-
-   node_dg2.SetSize(15,15);
-   node_dg2 = dg2dx;
-
-   /*
-   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
-
-   v1 = 1:3;
-   v2 = 1:12;
-   %v1 = ones(1,3)
-   %v2 = ones(1,12)
-   v2 = reshape(v2,4,3);
-   x1n1 = x1 + 0.01*v1;
-   coords2n1 = coords2 + 0.001*v2;
-      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
-      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
-   x1n2 = x1 - 0.01*v1;
-   coords2n2 = coords2 - 0.001*v2;
-      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
-      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
-   fprintf('fd\n');
-      %gapv1-gapv2
-   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
-
-   %dgdxsn1-dgdxsn2
-   fprintf('code\n');
-   v2n = v2';
-   %dg2dx(1:3,1:3)*0.04*ones(3,1)
-   temp = zeros(12,3);
-   for i = 1:4
-   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
-   temp((i-1)*3+1:i*3,:) = temp1';
-   end
-   temp2 = zeros(3,12);
-   for i = 1:4
-   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
-   temp2(:,(i-1)*3+1:i*3) = temp3';
-   end
-   %dg2dx
-   %dg2dx(4:end,1:3) = temp;
-   %dg2dx(1:3,4:end) = temp2;
-   %dgvdxm * 0.002*v2n(:)
-   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
-   %dg2dx(4:end,1:3)
-   end*/
-
-};
-
-
-// coordsm : (npoints*4, 3) use what class?
-// m_conn: (npoints*4)
-void Assemble_Contact(const int m, const int npoints, const int ndofs,
-                      const Vector x_s,
-                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
-                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
-                      std::vector<SparseMatrix>& dM)
-{
-   int n = ndofs;
-   int ndim = 3;
-
-   g.SetSize(m);
-   g = 0.0;
-
-   //SparseMatrix M(m, n);  // M needs to be the correct size
-
-   //dM.resize(m); // needs to clear?
-
-   double g_tmp = 0.;
-   Vector dg(4*ndim+ndim);
-   dg = 0.;
-   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
-   dg2 = 0.;
-
-   for (int i=0; i<npoints; i++)
-   {
-      Vector x1(ndim);
-      x1[0] = x_s[i*ndim];
-      x1[1] = x_s[i*ndim+1];
-      x1[2] = x_s[i*ndim+2];
-
-      Vector xi2(ndim-1);
-      xi2[0] = xi[i*(ndim-1)];
-      xi2[1] = xi[i*(ndim-1)+1];
-
-      DenseMatrix coords2(4,3);
-      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
-
-      //how to get coords2?
-      dg = 0.0;
-      dg2 = 0.;
-      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
-      //x1.Print();
-      //xi2.Print();
-      //coords2.Print();
-      g[s_conn[i]] = g_tmp; // should be unique
-      Array<int> m_conn_i(4);
-      m_conn.GetSubArray(4*i, 4, m_conn_i);
-
-      Array<int> node_conn(5);
-      node_conn[0] = s_conn[i];
-      for (int j=0; j<4; j++)
-      {
-         node_conn[j+1] = m_conn_i[j];
-      }
-
-      Array<int>  M_i_tmp(1);
-      M_i_tmp[0] = s_conn[i];
-
-      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
-      Array<int> j_idx(5*ndim); j_idx = 0;
-      for (int j=0; j< 5; j++)
-      {
-         for (int k=0; k<ndim; k++)
-         {
-            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
-         }
-      }
-      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
-      M_v_tmp.SetRow(0, dg);
-
-      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
-
-      Array<int> dM_i(ndim*(4+1));
-      Array<int> dM_j(ndim*(4+1));
-
-      for (int j=0; j< ndim*(4+1); j++)
-      {
-         dM_i[j] = j_idx[j];
-         dM_j[j] = j_idx[j];
-      }
-      //dg2.Print();
-      //dM[s_conn[i]].Print();
-      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
-   }
-};
-

examples/petsc/ex10p.cpp

@@ -187,6 +187,7 @@ int main(int argc, char *argv[])
 {
    // 1. Initialize MPI and HYPRE.
    Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
    int myid = Mpi::WorldRank();
    Hypre::Init();
 

examples/petsc/ex2p.cpp

@@ -248,10 +248,7 @@ int main(int argc, char *argv[])
    //     constraints for non-conforming AMR, static condensation, etc.
    if (myid == 0) { cout << "matrix ... " << flush; }
    if (static_cond) { a->EnableStaticCondensation(); }
-   // Here we want to try out block-size aware AMG solver in PETSc.
-   // For that to work properly, we need a fully-compliant block-size
-   // structure and we do not skip zeros when assembling.
-   a->Assemble(use_petsc ? 0 : 1);
+   a->Assemble();
 
    Vector B, X;
    if (!use_petsc)
@@ -297,14 +294,13 @@ int main(int argc, char *argv[])
          cout << "done." << endl;
          cout << "Size of linear system: " << A.M() << endl;
       }
-      // Tell PETSc the matrix has a block structure
-      A.SetBlockSize(dim);
-
-      // The preconditioner for the PCG solver can be specified in the
-      // PETSc config file
       PetscPCGSolver *pcg = new PetscPCGSolver(A);
+
+      // The preconditioner for the PCG solver defined below is specified in the
+      // PETSc config file, rc_ex2p, since a Krylov solver in PETSc can also
+      // customize its preconditioner.
       PetscPreconditioner *prec = NULL;
-      if (use_nonoverlapping) // Specialized BDDC construction
+      if (use_nonoverlapping)
       {
          // Compute dofs belonging to the natural boundary
          Array<int> nat_tdof_list, nat_bdr(pmesh->bdr_attributes.Max());

examples/petsc/ex9p.cpp

@@ -450,7 +450,7 @@ int main(int argc, char *argv[])
       for (int ti = 0; !done; )
       {
          // We cannot match exactly the time history of the Run method
-         // since we are explicitly telling PETSc to use a time step
+         // since we are explictly telling PETSc to use a time step
          double dt_real = min(dt, t_final - t);
          ode_solver->Step(*U, t, dt_real);
          ti++;

examples/petsc/makefile

@@ -78,7 +78,6 @@ EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly -
 EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
 EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
-EX2_ARGS_ASM          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm
 EX3_ARGS              := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
 EX4_ARGS              := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
 EX4_HYB_ARGS          := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
@@ -110,7 +109,6 @@ endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_BDDC))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_ASM))
 ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p

examples/petsc/rc_ex1p

@@ -14,5 +14,4 @@
 -mg_levels_esteig_ksp_type cg
 -mg_levels_esteig_ksp_max_it 10
 -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05
--pc_gamg_use_sa_esteig 0
 -mg_levels_pc_type sor

examples/petsc/rc_ex2p

@@ -1,7 +1,8 @@
 -ksp_converged_reason
 
 # GAMG is still not used at its best,
-# since we are not exploiting the RBMs
+# since we are not exploiting the
+# block size (Ordering::byVDIM) and the RBMs
 
 -ksp_view
 -pc_type gamg

examples/petsc/rc_ex2p_asm

@@ -1,10 +0,0 @@
-# Additive Schwarz with Overlap
-# This is not a good solver for elasticity
-# These options are here only to describe
-# the setup of the solver
--ksp_converged_reason
--ksp_view
--ksp_max_it 10
--pc_type asm
--pc_asm_overlap 1
--sub_pc_type icc

examples/sundials/ex10.cpp

@@ -210,9 +210,6 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
 
 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
    // 1. Parse command-line options.
    const char *mesh_file = "../../data/beam-quad.mesh";
    int ref_levels = 2;

examples/sundials/ex10p.cpp

@@ -215,11 +215,10 @@ void visualize(ostream &os, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI, HYPRE, and SUNDIALS.
+   // 1. Initialize MPI and HYPRE.
    Mpi::Init(argc, argv);
    int myid = Mpi::WorldRank();
    Hypre::Init();
-   Sundials::Init();
 
    // 2. Parse command-line options.
    const char *mesh_file = "../../data/beam-quad.mesh";

examples/sundials/ex16.cpp

@@ -109,9 +109,6 @@ double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
    // 1. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
    int ref_levels = 2;
@@ -293,10 +290,7 @@ int main(int argc, char *argv[])
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
-         }
+         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
          ode_solver = arkode; break;
       case 12:
          arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);