Fixed some bugs introduced by the previous commit.

ex14 and ex14p.

.gitignore

@@ -272,27 +272,16 @@ miniapps/navier/*_output
 
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex3
-miniapps/nurbs/nurbs_ex5
 miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_ex24
-miniapps/nurbs/nurbs_solenoidal
 miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
-miniapps/nurbs/sol_?.gf
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
-miniapps/nurbs/Example3*
-miniapps/nurbs/Example5*
-miniapps/nurbs/Solenoidal*
-miniapps/nurbs/ParaView
 miniapps/nurbs/sin-fit.mesh
-miniapps/nurbs/ex5.mesh
-miniapps/nurbs/exsol.mesh
 miniapps/nurbs/CurveInt
 miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh

.gitlab-ci.yml

@@ -22,7 +22,7 @@ include:
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where ruby resource are allocated/released once for all.
+# - Allocate/Release is where quartz resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
@@ -53,7 +53,7 @@ variables:
   AUTOTEST_COMMIT: "YES"
 
 # Trigger subpipelines:
-ruby-build-and-test:
+quartz-build-and-test:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -61,10 +61,10 @@ ruby-build-and-test:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/ruby-build-and-test.yml
+    include: .gitlab/quartz-build-and-test.yml
     strategy: depend
 
-ruby-baseline:
+quartz-baseline:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -73,7 +73,7 @@ ruby-baseline:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/ruby-baseline.yml
+    include: .gitlab/quartz-baseline.yml
     strategy: depend
 
 lassen-build-and-test:

.gitlab/README.md

@@ -24,7 +24,7 @@ and `test type`.
 
 Machines typically include:
 
-* Ruby: 2nd Gen Intel Xeon (Cascade Lake)
+* Quartz: Intel bi-socket x86
 * Lassen: Power9 + Nvidia GPU
 * Corona: AMD GPU
 
@@ -76,13 +76,13 @@ with a spack spec of MFEM, within the limits permitted by the MFEM spack
 package.
 
 In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on ruby for example resumes to:
+spack spec to use. Adding a job on quartz for example resumes to:
 
 ```yaml
 <job_name>:
   variables:
     SPEC: "<spack_spec>"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 ```
 
 The remaining and non trivial work is to make sure this spec is working. To

.gitlab/configs/common.yml

@@ -24,7 +24,7 @@ variables:
 # TODO: add a clean-up mechanism
   BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
 
-# On LLNL's ruby, there is only one allocation shared among jobs in order to
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}

.gitlab/configs/quartz-config.yml

@@ -9,17 +9,17 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# GitLab pipelines configurations for the Ruby machine at LLNL
+# GitLab pipelines configurations for the Quartz machine at LLNL
 variables:
-  MACHINE_NAME: ruby
+  MACHINE_NAME: quartz
 
-.on_ruby:
+.on_quartz:
   tags:
     - shell
-    - ruby
+    - quartz
   rules:
-    # Don't run ruby jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"'
+    # Don't run quartz jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
       when: never
     # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
@@ -40,13 +40,13 @@ variables:
     - when: on_success
 
 # Spack helped builds
-# Generic ruby build job, extending build script
-.build_and_test_on_ruby:
-  extends: [.on_ruby]
+# Generic quartz build job, extending build script
+.build_and_test_on_quartz:
+  extends: [.on_quartz]
   stage: build_and_test
   script:
     # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=16
+    - export THREADS=12
     - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
     - echo ${JOBID}

.gitlab/configs/setup-baseline.yml

@@ -18,7 +18,7 @@
 setup_baseline:
   tags:
     - shell
-    - ruby
+    - quartz
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/configs/setup-build-and-test.yml

@@ -16,7 +16,7 @@
 setup:
   tags:
     - shell
-    - ruby
+    - quartz
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/quartz-baseline.yml

@@ -19,8 +19,8 @@ stages:
   - cleanup
   - baseline_publish
 
-baselinecheck_mfem_intel_ruby:
-  extends: [.on_ruby]
+baselinecheck_mfem_intel_quartz:
+  extends: [.on_quartz]
   stage: baseline_check
   variables:
     # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
@@ -32,7 +32,7 @@ baselinecheck_mfem_intel_ruby:
     - echo ${BUILD_ROOT}
     - echo ${TPLS_DIR}
     # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=48
+    - export MFEM_TEST_NP=32
     # The next script uses the following environment variables:
     # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
     # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
@@ -44,16 +44,18 @@ baselinecheck_mfem_intel_ruby:
   allow_failure: true
 
 cleanup:
-  extends: .on_ruby
+  extends: .on_quartz
   stage: cleanup
   variables:
     GIT_STRATEGY: none
   script:
     - echo "BUILD_ROOT=${BUILD_ROOT}"
     - rm -rf "${BUILD_ROOT}" || true
+    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
+    - make -C "${CI_PROJECT_DIR}" distclean
 
 report_baseline:
-  extends: [.on_ruby]
+  extends: [.on_quartz]
   stage: baseline_report
   script:
     - echo ${MACHINE_NAME}
@@ -113,8 +115,8 @@ report_baseline:
         exit $err
       ) 9> autotest.lock
 
-baselinepublish_mfem_ruby:
-  extends: [.on_ruby]
+baselinepublish_mfem_quartz:
+  extends: [.on_quartz]
   stage: baseline_publish
   rules:
     # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
@@ -129,5 +131,5 @@ baselinepublish_mfem_ruby:
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/quartz-config.yml
   - local: .gitlab/configs/setup-baseline.yml

.gitlab/quartz-build-and-test.yml

@@ -19,54 +19,54 @@ stages:
 allocate_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_quartz
   stage: allocate_resource
   script:
     - echo ${ALLOC_NAME}
     - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
   timeout: 6h
 
-# GitLab jobs for the Ruby machine at LLNL
+# GitLab jobs for the Quartz machine at LLNL
 debug_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 debug_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10_sundials:
   variables:
     SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10_petsc:
   variables:
     SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 opt_par_gcc_10_pumi:
   variables:
     SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_quartz
 
 # Release
 release_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_quartz
   stage: release_resource_and_report
   script:
     - echo ${ALLOC_NAME}
@@ -78,17 +78,17 @@ release_resource:
 report_job_success:
   stage: release_resource_and_report
   extends:
-    - .on_ruby
+    - .on_quartz
     - .report_job_success
 
 report_job_failure:
   stage: release_resource_and_report
   extends:
-    - .on_ruby
+    - .on_quartz
     - .report_job_failure
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/quartz-config.yml
   - local: .gitlab/configs/setup-build-and-test.yml
   - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -14,7 +14,7 @@
 # locals
 glob_err=${BASELINE_TEST}.err
 base=${BASELINE_TEST}-${SYS_TYPE}
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
+if [[ "${MACHINE_NAME}" == "quartz" ]]; then
   base="${BASELINE_TEST}-${MACHINE_NAME}"
 fi
 base_diff=${base}.diff
@@ -31,8 +31,8 @@ cd tests
 mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
   salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
@@ -41,11 +41,11 @@ else
   echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
   exit 1
 fi
-status="$?"
 
 # post
 mkdir ${artifacts_path}
 
+status=0
 if [[ -f ${BASELINE_TEST}.out ]]; then
   cp ${BASELINE_TEST}.out ${artifacts_path}
 fi

.gitlab/scripts/rebaseline

@@ -11,7 +11,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and quartz baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
@@ -21,7 +21,7 @@ PATCH_FILE=${ARTIFACT_PATH}.patch
 FULL_FILE=${ARTIFACT_PATH}.out
 DIFF_FILE=${ARTIFACT_PATH}.diff
 
-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and quartz baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 SAVED_NAME=baseline-${SYS_TYPE}.saved

CHANGELOG

@@ -11,46 +11,9 @@
 Version 4.7.1 (development)
 ===========================
 
-Discretization improvements
----------------------------
-- Added NURBS-based H(div) and H(curl) elements in 2D and 3D. Only on single
-  patch meshes. Only implemented for serial computations.
-
-- Added support for boundary constraints to the hybridization class.
-
-Meshing improvements
---------------------
-- The ExodusII reader now handles pyramid and wedge element types. Mixed meshes
-  are also supported.
-
-New and updated examples and miniapps
--------------------------------------
-- Added miniapps to demonstrate the H(div) and H(curl) NURBS elements.
-
 - Added an MFEM example for the eikonal equation. This new solver is based on
   the proximal Galerkin method introduced by Keith and Surowiec.
 
-GPU computing
--------------
-- Added support for GPU-accelerated batched linear algebra (using cuBLAS,
-  hipBLAS, MAGMA, or native MFEM functionality) through the BatchedLinAlg class.
-
-Miscellaneous
--------------
-- Refactored the `ARKStepSolver` class (ARKODE interface) to use
-  `TimeDependentOperator::Mult` only when the associated ODE operator is
-  expressed in explicit form (i.e., `TimeDependentOperator::isExplicit()`),
-  otherwise `TimeDependentOperator::ExplicitMult` is used. A check has been
-  added to `ARKStepSolver` to verify that the associated ODE operator is not in
-  explicit form when a mass matrix solver is enabled via a call to either the
-  `UseMFEMMassLinearSolver` or `UseSundialsMassLinearSolver` methods. This is
-  because enabling a mass matrix solver assumes that F(u,k,t) = M k in the
-  associated ODE operator.
-
-- Added support for custom interpolation procedure in FindPointsGSLIB.
-
-API changes
------------
 - API change: in class GridFunction, 'fec' was renamed to 'fec_owned'.
 
 
@@ -77,9 +40,6 @@ Meshing improvements
 
 - Added support for internal boundary elements in nonconforming meshes.
 
-- Added ExodusII output capability. The writer can handle first-order (Pyramid5,
-  Wedge6, Hex8, Tet4) and second-order FE types (Pyramid14, Wedge18, Hex27, Tet10).
-
 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.
 
 Discretization improvements

CMakeLists.txt

@@ -146,9 +146,7 @@ if (MFEM_USE_CUDA)
    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
    find_package(CUDAToolkit REQUIRED)
    set(CUSPARSE_FOUND TRUE)
-   set(CUBLAS_FOUND TRUE)
    get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-   get_target_property(CUBLAS_LIBRARIES CUDA::cublas LOCATION)
 endif()
 
 if (XSDK_ENABLE_C)
@@ -233,7 +231,6 @@ if (MFEM_USE_HIP)
     list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
   endif()
   find_package(HIP REQUIRED)
-  find_package(HIPBLAS REQUIRED)
   find_package(HIPSPARSE REQUIRED)
 endif()
 
@@ -399,10 +396,6 @@ if (MFEM_USE_AMGX)
   find_package(AMGX REQUIRED)
 endif()
 
-if (MFEM_USE_MAGMA)
-  find_package(MAGMA REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint)
 endif()
@@ -522,10 +515,7 @@ endif()
 
 # Enzyme
 if (MFEM_USE_ENZYME)
-    find_package(Enzyme REQUIRED HINTS ${ENZYME_DIR})
-    message(STATUS "Enzyme found in ${ENZYME_DIR}.")
-    set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
-    set(ENZYME_FOUND 1)
+   find_package(ENZYME REQUIRED)
 endif()
 
 # MFEM_TIMER_TYPE
@@ -567,9 +557,8 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
-    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
-    ALGOIM ENZYME)
+    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -632,11 +621,6 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
-
-if (MFEM_USE_ENZYME)
-  target_link_libraries(mfem PUBLIC ClangEnzymeFlags)
-endif()
-
 if (MINGW)
   target_link_libraries(mfem PRIVATE ws2_32)
 endif()
@@ -689,7 +673,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
 
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
       "${PROJECT_BINARY_DIR}/${Header}.tmp"
       "${PROJECT_BINARY_DIR}/${Header}"
     )
@@ -703,7 +687,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 #include \"mfem/${Header}\"
 ")
 
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
       "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
       "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
     )

INSTALL

@@ -273,13 +273,7 @@ Installation options:
    PREFIX  - Specify the installation directory. The library (libmfem.a) will be
              installed in $(PREFIX)/lib, the headers in $(PREFIX)/include, and
              the configuration makefile (config.mk) in $(PREFIX)/share/mfem.
-   INSTALL - Specify the install program, default = /usr/bin/install
-   INSTALL_DEF_PERM - Specify the default install permissions. This affects
-                      headers and configuration makefiles, default = 644
-   INSTALL_BIN_PERM - Specify the install permissions for binaries. This only
-                      affects the shared version of the library, default = 755
-   INSTALL_DIR_PERM - Specify the install permissions for directories and,
-                      on macOS/BSD, for symlinks as well, default = 755
+   INSTALL - Specify the install program, e.g /usr/bin/install
 
 MFEM library features/options (GNU make)
 ----------------------------------------
@@ -394,11 +388,6 @@ MFEM_USE_AMGX = YES/NO
    Allows the user to use SparseMatrices and HypreParMatrices to solve linear
    systems with the routines from the AmgX library.
 
-MFEM_USE_MAGMA = YES/NO
-   Enable MFEM functionality based on the MAGMA high-performance linear algebra
-   library. The MAGMA library provides a BLAS/LAPACK interface, with
-   implementations that have been optimized for Nvidia and AMD GPUs.
-
 MFEM_USE_GNUTLS = YES/NO
    Enable secure socket support in class socketstream, using the auxiliary
    GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -710,11 +699,6 @@ The specific libraries and their options are:
   Options: AMGX_OPT, AMGX_LIB.
   Versions: AmgX >= 2.1, older versions may work too.
 
-- MAGMA (optional), used with MFEM_USE_MAGMA = YES.
-  URL: https://icl.utk.edu/magma/
-  Options: MAGMA_OPT, MAGMA_LIB
-  Versions: MAGMA >= 2.8.0
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
   GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
   one can get the library through the Homebrew package manager (http://brew.sh).

config/cmake/MFEMConfig.cmake.in

@@ -37,7 +37,6 @@ set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
-set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
 set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)

config/cmake/config.hpp.in

@@ -114,9 +114,6 @@
 // Enable MFEM functionality based on the AmgX library.
 #cmakedefine MFEM_USE_AMGX
 
-// Enable MFEM functionality based on the MAGMA library.
-#cmakedefine MFEM_USE_MAGMA
-
 // Enable secure socket streams based on the GNUTLS library.
 #cmakedefine MFEM_USE_GNUTLS
 

config/cmake/modules/FindENZYME.cmake

@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+message(STATUS "Looking for ENZYME ...")
+message(STATUS "   in ENZYME_DIR = ${ENZYME_DIR}")
+
+# Make sure the directory and version combination works. Do nothing otherwise.
+if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
+  message(STATUS "Found ENZYME: ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
+
+  # Set ENZYME_FOUND
+  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
+
+  # Set CXX flags to accommodate the Enzyme Clang plugin
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
+  set(MFEM_USE_ENZYME YES)
+else()
+
+endif()

config/cmake/modules/FindMAGMA.cmake

@@ -1,37 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MAGMA_FOUND
-#   - MAGMA_LIBRARIES
-#   - MAGMA_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(MAGMA MAGMA MAGMA_DIR "include" "magma.h" "lib" "magma"
-  "Paths to headers required by MAGMA." "Libraries required by MAGMA.")
-
-if (MAGMA_FOUND AND MFEM_USE_CUDA)
-  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-  get_target_property(CUBLAS_LIBRARIES CUDA::cublas LOCATION)
-  list(APPEND MAGMA_LIBRARIES ${CUSPARSE_LIBRARIES} ${CUBLAS_LIBRARIES})
-  set(MAGMA_LIBRARIES ${MAGMA_LIBRARIES} CACHE STRING
-      "MAGMA libraries + dependencies." FORCE)
-  message(STATUS "Updated MAGMA_LIBRARIES: ${MAGMA_LIBRARIES}")
-endif()
-
-if (MAGMA_FOUND AND MFEM_USE_HIP)
-  find_package(HIPBLAS REQUIRED)
-  find_package(HIPSPARSE REQUIRED)
-  list(APPEND MAGMA_LIBRARIES ${HIPBLAS_LIBRARIES} ${HIPSPARSE_LIBRARIES})
-  set(MAGMA_LIBRARIES ${MAGMA_LIBRARIES} CACHE STRING
-      "MAGMA libraries + dependencies." FORCE)
-  message(STATUS "Updated MAGMA_LIBRARIES: ${MAGMA_LIBRARIES}")
-endif()

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -846,14 +846,14 @@ function(mfem_export_mk_files)
       MFEM_USE_ZLIB MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE
       MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS
       MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS
-      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_MAGMA
-      MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC
-      MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA
-      MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_TRIBOL MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS
+      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB
+      MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED
+      MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2
+      MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_ADFORWARD
+      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)

config/config.hpp.in

@@ -114,9 +114,6 @@
 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX
 
-// Enable MFEM functionality based on the MAGMA library.
-// #define MFEM_USE_MAGMA
-
 // Enable secure socket streams based on the GNUTLS library.
 // #define MFEM_USE_GNUTLS
 

config/config.mk.in

@@ -38,7 +38,6 @@ MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_AMGX          = @MFEM_USE_AMGX@
-MFEM_USE_MAGMA         = @MFEM_USE_MAGMA@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@

config/defaults.cmake

@@ -40,7 +40,6 @@ option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
-option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -184,10 +183,6 @@ set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")
 
 set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
 
-set(MAGMA_DIR "${MFEM_DIR}/../magma" CACHE PATH "Path to MAGMA")
-set(MAGMA_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
-    "Additional packages required by MAGMA.")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -264,7 +259,7 @@ set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
     "The ParELAG library.")
 
 set(TRIBOL_DIR "${MFEM_DIR}/../tribol" CACHE PATH "Path to Tribol")
-set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING
+set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING 
     "Additional packages required by Tribol")
 
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")

config/defaults.mk

@@ -95,10 +95,6 @@ else
    # Silence unused command line argument warnings when generating dependencies
    # with mpicxx and clang
    DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
-   # Silence "ignoring duplicate libraries" warnings on new (Xcode 15) linker
-   ifneq (,$(findstring PROJECT:dyld,$(shell ld -v 2>&1)))
-      LDFLAGS_INTERNAL = -Xlinker -no_warn_duplicate_libraries
-   endif
 endif
 
 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -143,7 +139,6 @@ MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
-MFEM_USE_MAGMA         = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -395,11 +390,6 @@ AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
 AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt
 
-# MAGMA library configuration
-MAGMA_DIR = @MFEM_DIR@/../magma
-MAGMA_OPT = -I$(MAGMA_DIR)/include
-MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a -lcublas -lcusparse $(LAPACK_LIB)
-
 # GnuTLS library configuration
 GNUTLS_OPT =
 GNUTLS_LIB = -lgnutls
@@ -507,11 +497,11 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse -lcublas
+CUDA_LIB = -lcusparse
 
 # HIP library configuration
 HIP_OPT =
-HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse -lhipblas
+HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa

config/sample-runs.sh

@@ -32,7 +32,7 @@ groups_serial=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,[1-9]}[0-9].cpp"'
+   "ex{,1,2,3}[0-9].cpp"'
 #   "ex1.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -58,10 +58,6 @@ groups_serial=(
    "HiOp examples:"
    "examples/hiop"
    "ex9.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex1.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
@@ -70,38 +66,25 @@ groups_serial=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
-    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
     mesh-optimizer.cpp minimal-surface.cpp"'
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
    "cvsRoberts_ASAi_dns.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "seq_example.cpp seq_test.cpp"' # 'seq_test.cpp' has no sample runs
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
    "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp "'
-# todo: miniapps/mtop
 '"nurbs"
    "NURBS miniapps:"
    "miniapps/nurbs"
    "nurbs_ex1.cpp"'
-# todo: add other nurbs miniapps
-# todo: miniapps/solvers (serial)
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
    "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
     lor-transfer.cpp"'
-# todo: add other tools miniapps
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
@@ -117,7 +100,7 @@ groups_parallel=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,[1-9]}[0-9]p.cpp"'
+   "ex{,1,2,3}[0-9]p.cpp"'
 #   "ex1p.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -143,10 +126,6 @@ groups_parallel=(
    "HiOp examples:"
    "examples/hiop"
    "ex9p.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex{1,2}p.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
@@ -159,41 +138,24 @@ groups_parallel=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp
-    fit-node-position.cpp"'
+   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
    "joule.cpp"'
-#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
+#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
    "adjoint_advection_diffusion.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "par_example.cpp"'
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "p{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
    "pfindpts.cpp schwarz_ex1p.cpp"'
-'"hdiv-linear-solver"
-   "H(div) linear solver miniapps:"
-   "miniapps/hdiv-linear-solver"
-   "grad_div.cpp darcy.cpp"'
-# 'miniapps/hooke/hooke.cpp' has no sample runs
-# todo: miniapps/mtop
-# todo: miniapps/multidomain
 '"navier"
    "Navier miniapps:"
    "miniapps/navier"
    "navier_cht.cpp"'
-# todo: add other navier miniapps
 '"nurbs"
    "NURBS miniapps:"
    "miniapps/nurbs"
@@ -202,18 +164,14 @@ groups_parallel=(
    "Shifted miniapps:"
    "miniapps/shifted"
    "distance.cpp"'
-# todo: add other shifted miniapps
 '"solvers"
    "Solvers miniapps:"
    "miniapps/solvers"
    "block-solvers.cpp"'
-# todo: add other solvers miniapps
-# todo: miniapps/spde
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
-   "convert-dc.cpp get-values.cpp load-dc.cpp"'
-# todo: add other tools miniapps
+   "convert-cd.cpp get-values.cpp load-dc.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -228,7 +186,7 @@ groups_all=(
 '"examples"
    "Examples:"
    "examples"
-   "ex\"{,[1-9]}[0-9]\"{,p}.cpp"'
+   "ex\"{,1,2,3}[0-9]\"{,p}.cpp"'
 '"sundials"
    "SUNDIALS examples:"
    "examples/sundials"
@@ -257,14 +215,10 @@ groups_all=(
    "HiOp examples:"
    "examples/hiop"
    "ex9.cpp ex9p.cpp"'
-'"moonolith"
-   "Moonolith examples:"
-   "examples/moonolith"
-   "ex1.cpp ex{1,2}p.cpp"'
 '"pumi"
    "PUMI examples:"
    "examples/pumi"
-   "ex1.cpp ex2.cpp ex1p.cpp ex6p.cpp"'
+   "ex1.cpp ex1p.cpp ex2.cpp ex6p.cpp"'
 '"superlu"
    "Superlu examples:"
    "examples/superlu"
@@ -272,67 +226,43 @@ groups_all=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
-    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
-    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp
-    fit-node-position.cpp"'
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
+    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
    "joule.cpp"'
-#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
+#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
-   "cvsRoberts_ASAi_dns.cpp adjoint_advection_diffusion.cpp"'
-'"autodiff"
-   "Autodiff miniapps:"
-   "miniapps/autodiff"
-   "seq_example.cpp seq_test.cpp par_example.cpp"'
-# 'seq_test.cpp' has no sample runs
-'"dpg"
-   "DPG miniapps:"
-   "miniapps/dpg"
-   "{,p}{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
+   "adjoint_advection_diffusion.cpp cvsRoberts_ASAi_dns.cpp"'
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
    "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp
     schwarz_ex1p.cpp"'
-'"hdiv-linear-solver"
-   "H(div) linear solver miniapps:"
-   "miniapps/hdiv-linear-solver"
-   "grad_div.cpp darcy.cpp"'
-# 'miniapps/hooke/hooke.cpp' has no sample runs
-# todo: miniapps/mtop
-# todo: miniapps/multidomain
 '"navier"
    "Navier miniapps:"
    "miniapps/navier"
    "navier_cht.cpp"'
-# todo: add other navier miniapps
 '"nurbs"
    "NURBS miniapps:"
    "miniapps/nurbs"
    "nurbs_ex1.cpp nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-# todo: add other nurbs miniapps
 '"shifted"
    "Shifted miniapps:"
    "miniapps/shifted"
    "distance.cpp"'
-# todo: add other shifted miniapps
 '"solvers"
    "Solvers miniapps:"
    "miniapps/solvers"
    "block-solvers.cpp"'
-# todo: add other solvers miniapps
-# todo: miniapps/spde
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
    "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
     lor-transfer.cpp"'
-# todo: add other tools miniapps
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
@@ -456,7 +386,7 @@ function help_message()
       mfem_config [${mfem_config}]
          Set MFEM configuration options
       make [${make}], mpiexec [${mpiexec}], mpiexec_np [${mpiexec_np}]
-         Their values can also be set using the respective uppercase environment
+         Their values can also set using the respective uppercase environment
          variable
       mfem_build_dir [${mfem_build_dir}]
          Same as '-d': set this variable to something different from <mfem_dir>

data/cube-space-time.mesh

@@ -0,0 +1,102 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+24
+1 7 4 5 8 11 13
+1 7 1 4 5 7 8
+1 7 1 4 5 8 11
+1 7 1 5 8 11 13
+1 7 1 5 7 8 13
+1 7 4 5 7 8 13
+1 7 1 3 4 8 11
+1 7 1 3 4 5 11
+1 7 1 5 10 11 13
+1 7 1 8 10 11 13
+1 7 1 3 5 10 11
+1 7 1 2 3 5 10
+1 7 0 1 3 4 8
+1 7 0 1 4 7 8
+1 7 1 5 6 7 13
+1 7 1 6 7 8 13
+1 7 6 7 8 13 15
+1 7 4 7 8 13 15
+1 7 4 8 12 13 15
+1 7 4 8 11 12 13
+1 7 6 8 13 14 15
+1 7 1 6 8 13 14
+1 7 1 8 9 10 13
+1 7 1 8 9 13 14
+
+boundary
+48
+1 4 0 1 3 4
+1 4 0 1 3 8
+1 4 0 3 4 8
+1 4 0 1 4 7
+1 4 0 1 7 8
+1 4 0 4 7 8
+1 4 1 4 5 7
+1 4 1 3 8 11
+1 4 1 3 4 5
+1 4 1 5 10 13
+1 4 1 8 10 11
+1 4 1 3 10 11
+1 4 1 2 3 5
+1 4 1 2 3 10
+1 4 1 2 5 10
+1 4 1 5 6 7
+1 4 1 5 6 13
+1 4 1 6 7 8
+1 4 1 6 8 14
+1 4 1 6 13 14
+1 4 1 8 9 10
+1 4 1 9 10 13
+1 4 1 8 9 14
+1 4 1 9 13 14
+1 4 2 3 5 10
+1 4 3 4 8 11
+1 4 3 4 5 11
+1 4 3 5 10 11
+1 4 4 5 11 13
+1 4 4 5 7 13
+1 4 4 7 8 15
+1 4 4 7 13 15
+1 4 4 8 12 15
+1 4 4 12 13 15
+1 4 4 8 11 12
+1 4 4 11 12 13
+1 4 5 10 11 13
+1 4 5 6 7 13
+1 4 6 7 8 15
+1 4 6 7 13 15
+1 4 6 8 14 15
+1 4 6 13 14 15
+2 4 8 10 11 13
+2 4 8 12 13 15
+2 4 8 11 12 13
+2 4 8 13 14 15
+2 4 8 9 10 13
+2 4 8 9 13 14
+
+vertices
+16
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000

data/cube4d_24.MFEM

@@ -0,0 +1,102 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+24
+17 8 4 5 8 11 13
+11 8 1 4 5 7 8
+15 8 1 4 5 8 11
+16 8 1 5 8 11 13
+12 8 1 5 7 8 13
+13 8 4 5 7 8 13
+7  8 1 3 4 8 11
+2  8 1 3 4 5 11
+4  8 1 5 10 11 13
+10 8 1 8 10 11 13
+3  8 1 3 5 10 11
+1  8 1 2 3 5 10
+6  8 0 1 3 4 8
+8  8 0 1 4 7 8
+5  8 1 5 6 7 13
+20 8 1 6 7 8 13
+19 8 6 7 8 13 15
+14 8 4 7 8 13 15
+21 8 4 8 12 13 15
+22 8 4 8 11 12 13
+23 8 6 8 13 14 15
+24 8 1 6 8 13 14
+9  8 1 8 9 10 13
+18 8 1 8 9 13 14
+
+boundary
+48
+1 4 0 1 3 4
+3 4 0 1 3 8
+3 4 0 3 4 8
+1 4 0 1 4 7
+3 4 0 1 7 8
+3 4 0 4 7 8
+1 4 1 4 5 7
+3 4 1 3 8 11
+1 4 1 3 4 5
+3 4 1 5 10 13
+3 4 1 8 10 11
+3 4 1 3 10 11
+1 4 1 2 3 5
+3 4 1 2 3 10
+3 4 1 2 5 10
+1 4 1 5 6 7
+3 4 1 5 6 13
+3 4 1 6 7 8
+3 4 1 6 8 14
+3 4 1 6 13 14
+3 4 1 8 9 10
+3 4 1 9 10 13
+3 4 1 8 9 14
+3 4 1 9 13 14
+3 4 2 3 5 10
+3 4 3 4 8 11
+3 4 3 4 5 11
+3 4 3 5 10 11
+3 4 4 5 11 13
+3 4 4 5 7 13
+3 4 4 7 8 15
+3 4 4 7 13 15
+3 4 4 8 12 15
+3 4 4 12 13 15
+3 4 4 8 11 12
+3 4 4 11 12 13
+3 4 5 10 11 13
+3 4 5 6 7 13
+3 4 6 7 8 15
+3 4 6 7 13 15
+3 4 6 8 14 15
+3 4 6 13 14 15
+5 4 8 10 11 13
+5 4 8 12 13 15
+5 4 8 11 12 13
+5 4 8 13 14 15
+5 4 8 9 10 13
+5 4 8 9 13 14
+
+vertices
+16
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000

data/cube4d_96.MFEM

@@ -0,0 +1,231 @@
+MFEM mesh v1.0
+
+dimension
+4
+
+elements
+96
+1 8 0 1 7 8 9
+1 8 1 6 7 8 9
+1 8 4 5 6 8 9
+1 8 4 6 7 8 9
+1 8 0 1 3 8 9
+1 8 1 2 3 8 9
+1 8 0 4 7 8 9
+1 8 0 3 4 8 9
+1 8 1 2 6 8 9
+1 8 2 5 6 8 9
+1 8 3 4 5 8 9
+1 8 2 3 5 8 9
+1 8 9 10 11 17 18
+1 8 9 11 16 17 18
+1 8 9 14 15 16 18
+1 8 9 14 16 17 18
+1 8 9 10 11 13 18
+1 8 9 11 12 13 18
+1 8 9 10 14 17 18
+1 8 9 10 13 14 18
+1 8 9 11 12 16 18
+1 8 9 12 15 16 18
+1 8 9 13 14 15 18
+1 8 9 12 13 15 18
+1 8 9 12 15 16 19
+1 8 9 11 12 16 19
+1 8 1 6 9 11 19
+1 8 6 9 11 16 19
+1 8 1 2 6 9 19
+1 8 2 5 6 9 19
+1 8 2 5 9 15 19
+1 8 2 9 12 15 19
+1 8 5 6 9 16 19
+1 8 5 9 15 16 19
+1 8 1 9 11 12 19
+1 8 1 2 9 12 19
+1 8 9 10 13 14 20
+1 8 9 10 14 17 20
+1 8 0 9 10 17 20
+1 8 0 7 9 17 20
+1 8 0 3 4 9 20
+1 8 0 4 7 9 20
+1 8 3 4 9 13 20
+1 8 4 9 13 14 20
+1 8 4 7 9 14 20
+1 8 7 9 14 17 20
+1 8 0 3 9 10 20
+1 8 3 9 10 13 20
+1 8 2 5 9 15 21
+1 8 2 9 12 15 21
+1 8 2 3 5 9 21
+1 8 3 4 5 9 21
+1 8 9 13 14 15 21
+1 8 9 12 13 15 21
+1 8 4 5 9 14 21
+1 8 5 9 14 15 21
+1 8 2 3 9 12 21
+1 8 3 9 12 13 21
+1 8 3 4 9 13 21
+1 8 4 9 13 14 21
+1 8 1 6 9 11 22
+1 8 6 9 11 16 22
+1 8 6 7 9 16 22
+1 8 7 9 16 17 22
+1 8 0 7 9 17 22
+1 8 0 9 10 17 22
+1 8 0 1 9 10 22
+1 8 1 9 10 11 22
+1 8 0 1 7 9 22
+1 8 1 6 7 9 22
+1 8 9 10 11 17 22
+1 8 9 11 16 17 22
+1 8 5 9 15 16 23
+1 8 5 6 9 16 23
+1 8 6 7 9 16 23
+1 8 7 9 16 17 23
+1 8 7 9 14 17 23
+1 8 4 7 9 14 23
+1 8 4 5 9 14 23
+1 8 5 9 14 15 23
+1 8 4 6 7 9 23
+1 8 4 5 6 9 23
+1 8 9 14 15 16 23
+1 8 9 14 16 17 23
+1 8 1 2 9 12 24
+1 8 1 9 11 12 24
+1 8 0 1 9 10 24
+1 8 1 9 10 11 24
+1 8 0 3 9 10 24
+1 8 3 9 10 13 24
+1 8 2 3 9 12 24
+1 8 3 9 12 13 24
+1 8 1 2 3 9 24
+1 8 0 1 3 9 24
+1 8 9 10 11 13 24
+1 8 9 11 12 13 24
+
+boundary
+96
+1 4 0 1 7 8
+1 4 0 1 3 8
+1 4 0 4 7 8
+1 4 0 3 4 8
+2 4 0 10 17 20
+2 4 0 7 17 20
+2 4 0 3 4 20
+2 4 0 4 7 20
+2 4 0 3 10 20
+2 4 0 7 17 22
+2 4 0 10 17 22
+2 4 0 1 10 22
+2 4 0 1 7 22
+2 4 0 1 10 24
+2 4 0 3 10 24
+2 4 0 1 3 24
+1 4 1 6 7 8
+1 4 1 2 3 8
+1 4 1 2 6 8
+2 4 1 6 11 19
+2 4 1 2 6 19
+2 4 1 11 12 19
+2 4 1 2 12 19
+2 4 1 6 11 22
+2 4 1 10 11 22
+2 4 1 6 7 22
+2 4 1 2 12 24
+2 4 1 11 12 24
+2 4 1 10 11 24
+2 4 1 2 3 24
+1 4 2 5 6 8
+1 4 2 3 5 8
+2 4 2 5 6 19
+2 4 2 5 15 19
+2 4 2 12 15 19
+2 4 2 5 15 21
+2 4 2 12 15 21
+2 4 2 3 5 21
+2 4 2 3 12 21
+2 4 2 3 12 24
+1 4 3 4 5 8
+2 4 3 4 13 20
+2 4 3 10 13 20
+2 4 3 4 5 21
+2 4 3 12 13 21
+2 4 3 4 13 21
+2 4 3 10 13 24
+2 4 3 12 13 24
+1 4 4 5 6 8
+1 4 4 6 7 8
+2 4 4 13 14 20
+2 4 4 7 14 20
+2 4 4 5 14 21
+2 4 4 13 14 21
+2 4 4 7 14 23
+2 4 4 5 14 23
+2 4 4 6 7 23
+2 4 4 5 6 23
+2 4 5 6 16 19
+2 4 5 15 16 19
+2 4 5 14 15 21
+2 4 5 15 16 23
+2 4 5 6 16 23
+2 4 5 14 15 23
+2 4 6 11 16 19
+2 4 6 11 16 22
+2 4 6 7 16 22
+2 4 6 7 16 23
+2 4 7 14 17 20
+2 4 7 16 17 22
+2 4 7 16 17 23
+2 4 7 14 17 23
+3 4 10 11 17 18
+3 4 10 11 13 18
+3 4 10 14 17 18
+3 4 10 13 14 18
+2 4 10 13 14 20
+2 4 10 14 17 20
+2 4 10 11 17 22
+2 4 10 11 13 24
+3 4 11 16 17 18
+3 4 11 12 13 18
+3 4 11 12 16 18
+2 4 11 12 16 19
+2 4 11 16 17 22
+2 4 11 12 13 24
+3 4 12 15 16 18
+3 4 12 13 15 18
+2 4 12 15 16 19
+2 4 12 13 15 21
+3 4 13 14 15 18
+2 4 13 14 15 21
+3 4 14 15 16 18
+3 4 14 16 17 18
+2 4 14 15 16 23
+2 4 14 16 17 23
+
+vertices
+25
+4
+0.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 0.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 0.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 0.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 0.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 0.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 0.0000000000000000 1.0000000000000000
+0.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 1.0000000000000000 1.0000000000000000 1.0000000000000000
+1.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.0000000000000000 0.0000000000000000 1.0000000000000000 1.0000000000000000
+0.5000000000000000 0.5000000000000000 0.5000000000000000 1.0000000000000000
+1.0000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.0000000000000000 0.5000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 1.0000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 0.0000000000000000 0.5000000000000000 0.5000000000000000
+0.5000000000000000 0.5000000000000000 1.0000000000000000 0.5000000000000000
+0.5000000000000000 0.5000000000000000 0.0000000000000000 0.5000000000000000

data/line-space-time.mesh

@@ -16,20 +16,21 @@ dimension
 2
 
 elements
-1
-1 3 0 1 2 3
+2
+1 2 2 0 1
+1 2 0 2 3
 
 boundary
 4
 1 1 0 1
-2 1 1 2
-3 1 2 3
-4 1 3 0
+1 1 1 2
+2 1 2 3
+1 1 3 0
 
 vertices
 4
 2
 0 0
-1 0.3
-1.4 1.2
-0.25 1.34
+1 0
+1 1
+0 1

data/pipe-nurbs-2d.mesh

@@ -18,9 +18,9 @@ elements
 boundary
 4
 1 1 0 1
-2 1 2 3
-3 1 3 0
-4 1 1 2
+1 1 2 3
+1 1 3 0
+1 1 1 2
 
 edges
 4

data/square-space-time.mesh

@@ -0,0 +1,52 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+6
+1 4 3 1 7 5
+1 4 1 6 7 4
+1 4 6 1 0 2
+1 4 1 6 4 2
+1 4 6 1 3 0
+1 4 1 6 3 7
+
+boundary
+12
+1 2 6 0 3
+1 2 0 6 2
+1 2 1 3 0
+1 2 3 1 5
+1 2 3 7 6
+1 2 7 3 5
+1 2 4 6 7
+1 2 6 4 2
+2 2 1 7 5
+2 2 7 1 4
+1 2 1 2 4
+1 2 2 1 0
+
+vertices
+8
+3
+0 0 0
+0 0 1
+1 0 0
+0 1 0
+1 0 1
+0 1 1
+1 1 0
+1 1 1

doc/CodeDocumentation.conf.in

@@ -938,7 +938,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \
-                         @MFEM_SOURCE_DIR@/linalg/batched \
                          @MFEM_SOURCE_DIR@/linalg/simd \
                          @MFEM_SOURCE_DIR@/mesh \
                          @MFEM_SOURCE_DIR@/mesh/submesh \
@@ -1050,8 +1049,7 @@ RECURSIVE              = NO
 EXCLUDE                = @MFEM_SOURCE_DIR@/config/_config.hpp \
                          @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp \
                          @MFEM_SOURCE_DIR@/general/tinyxml2.h \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp \
-                         @MFEM_SOURCE_DIR@/linalg/lapack.hpp
+                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -1210,13 +1208,13 @@ STRIP_CODE_COMMENTS    = NO
 # entity all documented functions referencing it will be listed.
 # The default value is: NO.
 
-REFERENCED_BY_RELATION = NO
+REFERENCED_BY_RELATION = YES
 
 # If the REFERENCES_RELATION tag is set to YES then for each documented function
 # all documented entities called/used by that function will be listed.
 # The default value is: NO.
 
-REFERENCES_RELATION    = NO
+REFERENCES_RELATION    = YES
 
 # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
 # to YES then the hyperlinks from functions in REFERENCES_RELATION and

doc/CodeDocumentation.dox

@@ -182,21 +182,6 @@ namespace mfem {
  *   <a class="el" href="examples_2superlu_2ex1p_8cpp_source.html">1p</a>,
  *   demonstrating the use of MFEM's \link superlu.hpp SuperLU integration\endlink.
  *
- * <H4>NURBS Examples</H4>
- * - Variants of Examples
- *  <a class="el" href="nurbs__ex1_8cpp_source.html">1</a>,
- *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
- *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
- *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
- *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
- *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
- *   demonstrating howto perform NURBS-based Isogeometric Analysis.
- * - Variant of Example <a class="el" href="nurbs__patch__ex1_8cpp_source.html">1</a>: demonstrates the use of patch integration
- * - <a class="el" href="nurbs__solenoidal_8cpp_source.html">NURBS Divergence-free</a>: solve a solenoidal vector projection with NURBS-based H(div) elements
- * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
- * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
- * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
- *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code

examples/CMakeLists.txt

@@ -50,27 +50,6 @@ list(APPEND ALL_EXE_SRCS
 
 if (MFEM_USE_MPI)
   list(APPEND ALL_EXE_SRCS
-    dfem_poisson.cpp
-    dfem_stokes.cpp
-    enzyme_interface_smoketest.cpp
-    test_dfem_dual.cpp
-    test_dfem.cpp
-    dfem_laghos.cpp
-    dfem_minimal_example.cpp
-    dfem_test_diffusion_2d.cpp
-    dfem_test_diffusion_3d.cpp
-    dfem_test_ordering.cpp
-    dfem_test_vector_diffusion.cpp
-    dfem_test_elasticity.cpp
-    dfem_test_nonlinear_elasticity_3d.cpp
-    dfem_test_nonlinear_diffusion_3d.cpp
-    dfem_test_interpolate_linear_scalar.cpp
-    dfem_test_interpolate_linear_scalar_3d.cpp
-    dfem_test_interpolate_gradient_linear_scalar_3d.cpp
-    dfem_test_mass_scalar_3d.cpp
-    dfem_test_mass_scalar_2d.cpp
-    dfem_test_interpolate_linear_vector.cpp
-    dfem_test_interpolate_linear_vector_3d.cpp
     ex0p.cpp
     ex1p.cpp
     ex2p.cpp
@@ -94,6 +73,9 @@ if (MFEM_USE_MPI)
     ex20p.cpp
     ex21p.cpp
     ex22p.cpp
+    ex1p_4d.cpp
+    ex3p_4d.cpp
+    ex4D_DivSkew.cpp
     ex24p.cpp
     ex25p.cpp
     ex26p.cpp
@@ -131,16 +113,6 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 # Add one executable per cpp file
 add_mfem_examples(ALL_EXE_SRCS)
 
-target_link_libraries(dfem_poisson ClangEnzymeFlags)
-target_link_libraries(dfem_stokes ClangEnzymeFlags)
-target_link_libraries(enzyme_interface_smoketest ClangEnzymeFlags)
-target_link_libraries(test_dfem ClangEnzymeFlags)
-target_link_libraries(dfem_laghos ClangEnzymeFlags)
-target_link_libraries(dfem_minimal_example ClangEnzymeFlags)
-target_link_libraries(dfem_test_diffusion_3d ClangEnzymeFlags)
-target_link_libraries(dfem_test_nonlinear_diffusion_3d ClangEnzymeFlags)
-target_link_libraries(dfem_test_nonlinear_elasticity_3d ClangEnzymeFlags)
-
 # Add a test for each example
 if (MFEM_ENABLE_TESTING)
   foreach(SRC_FILE ${ALL_EXE_SRCS})

examples/amgx/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/amgx/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/caliper/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/caliper,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/dfem/dfem.hpp

@@ -1,3 +0,0 @@
-#pragma once
-
-#include "dfem_differentiable_operator.hpp"

examples/dfem/dfem_action_callback.icc

@@ -1,232 +0,0 @@
-template <
-   typename kernels_tuple,
-   size_t num_solutions,
-   size_t num_parameters,
-   size_t num_fields,
-   size_t num_kernels
-   >
-template <
-   typename kernel_t
-   >
-void DifferentiableOperator<kernels_tuple,
-     num_solutions,
-     num_parameters,
-     num_fields,
-     num_kernels>::Action::create_action_callback(
-        kernel_t kernel,
-        mult_func_t &func)
-{
-   using entity_t = typename kernel_t::entity_t;
-
-   auto kinput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                     kernel.inputs, std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto koutput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                      kernel.outputs, std::make_index_sequence<kernel.num_koutputs> {});
-
-   constexpr int hardcoded_output_idx = 0;
-   const int test_space_field_idx = koutput_to_field[hardcoded_output_idx];
-
-   const Operator *R = get_restriction<entity_t>(op.fields[test_space_field_idx],
-                                                 element_dof_ordering);
-
-   auto output_fop = mfem::get<hardcoded_output_idx>(kernel.outputs);
-
-   const int num_elements = GetNumEntities<Entity::Element>(op.mesh);
-   const int num_entities = GetNumEntities<entity_t>(op.mesh);
-   const int num_qp = op.integration_rule.GetNPoints();
-
-   // All solutions T-vector sizes make up the width of the operator, since
-   // they are explicitly provided in Mult() for example.
-
-   op.width = GetTrueVSize(op.fields[test_space_field_idx]);
-   op.residual_lsize = GetVSize(op.fields[test_space_field_idx]);
-
-   if constexpr (std::is_same_v<decltype(output_fop), One>)
-   {
-      op.height = 1;
-   }
-   else
-   {
-      op.height = op.residual_lsize;
-   }
-
-   residual_l.SetSize(op.residual_lsize);
-
-   // assume only a single element type for now
-   std::vector<const DofToQuad*> dtq;
-   for (const auto &field : op.fields)
-   {
-      dtq.emplace_back(GetDofToQuad<entity_t>(field, op.integration_rule,
-                                              doftoquad_mode));
-   }
-   const int q1d = (int)floor(pow(num_qp, 1.0/op.mesh.Dimension()) + 0.5);
-
-   residual_e.SetSize(R->Height());
-
-   const int residual_size_on_qp = GetSizeOnQP<entity_t>(
-                                      mfem::get<hardcoded_output_idx>(kernel.outputs),
-                                      op.fields[test_space_field_idx]);
-
-   auto input_dtq_maps = create_dtq_maps<entity_t>(kernel.inputs, dtq,
-                                                   kinput_to_field);
-   auto output_dtq_maps = create_dtq_maps<entity_t>(kernel.outputs, dtq,
-                                                    koutput_to_field);
-
-   auto input_fops = create_bare_fops(kernel.inputs);
-   auto output_fops = create_bare_fops(kernel.outputs);
-
-   const int test_vdim = mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int test_op_dim =
-      mfem::get<hardcoded_output_idx>(output_fops).size_on_qp /
-      mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int num_test_dof = R->Height() /
-                            mfem::get<hardcoded_output_idx>(output_fops).vdim /
-                            num_entities;
-
-   auto ir_weights = Reshape(this->op.integration_rule.GetWeights().Read(),
-                             num_qp);
-
-   auto input_size_on_qp = get_input_size_on_qp(kernel.inputs,
-                                                std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto shmem_info = get_shmem_info<entity_t>(input_dtq_maps,
-                                              output_dtq_maps,
-                                              op.fields,
-                                              num_entities,
-                                              kernel.inputs,
-                                              num_qp,
-                                              input_size_on_qp,
-                                              residual_size_on_qp);
-
-   Vector shmem_cache(shmem_info.total_size);
-
-   print_shared_memory_info(shmem_info);
-
-   func = [=](Vector &ye_mem) mutable
-   {
-      restriction<entity_t>(op.solutions, solutions_l, this->fields_e,
-                            op.element_dof_ordering);
-      restriction<entity_t>(op.parameters, parameters_l, this->fields_e,
-                            op.element_dof_ordering,
-                            op.solutions.size());
-
-      auto ye = Reshape(ye_mem.ReadWrite(), test_vdim, num_test_dof, num_entities);
-      auto wrapped_fields_e = wrap_fields(this->fields_e, shmem_info.field_sizes, num_entities);
-
-      forall([=] MFEM_HOST_DEVICE (int e, void *shmem)
-      {
-         // printf("\ne: %d\n", e);
-         // tic();
-         auto input_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::INPUT_DTQ],
-            shmem_info.input_dtq_sizes,
-            input_dtq_maps);
-
-         auto output_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::OUTPUT_DTQ],
-            shmem_info.output_dtq_sizes,
-            output_dtq_maps);
-
-         auto fields_shmem = load_field_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::FIELD],
-            shmem_info.field_sizes,
-            kinput_to_field,
-            wrapped_fields_e,
-            e);
-
-         // These methods don't copy, they simply create a `DeviceTensor` object
-         // that points to correct chunks of the shared memory pool.
-         auto input_shmem = load_input_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::INPUT],
-            shmem_info.input_sizes,
-            num_qp);
-
-         auto residual_shmem = load_residual_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::OUTPUT],
-            shmem_info.residual_size,
-            num_qp);
-
-         auto scratch_mem = load_scratch_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::TEMP],
-            shmem_info.temp_sizes);
-
-         MFEM_SYNC_THREAD;
-         // printf("shmem load elapsed: %.1fus\n", toc() * 1e6);
-
-         // tic();
-         map_fields_to_quadrature_data<TensorProduct>(
-            input_shmem, fields_shmem, input_dtq_shmem, input_fops, ir_weights, scratch_mem,
-         std::make_index_sequence<kernel.num_kinputs> {});
-         // printf("interpolate elapsed: %.1fus\n", toc() * 1e6);
-
-         // tic();
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  auto kernel_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-                  auto r = Reshape(&residual_shmem(0, q), residual_size_on_qp);
-                  apply_kernel(r, kernel.func, kernel_args, input_shmem, q);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // printf("qf elapsed: %.1fus\n", toc() * 1e6);
-
-         // tic();
-         auto fhat = Reshape(&residual_shmem(0, 0), test_vdim, test_op_dim, num_qp);
-         auto y = Reshape(&ye(0, 0, e), num_test_dof, test_vdim);
-         map_quadrature_data_to_fields<TensorProduct>(y, fhat,
-                                                      mfem::get<0>(output_fops),
-                                                      output_dtq_shmem[hardcoded_output_idx],
-                                                      scratch_mem);
-         // printf("integrate elapsed: %.1fus\n", toc() * 1e6);
-
-      }, num_entities, q1d, q1d, q1d, shmem_info.total_size, shmem_cache.ReadWrite());
-
-      if constexpr (std::is_same_v<decltype(output_fop), None>)
-      {
-         residual_l = ye_mem;
-      }
-      else
-      {
-         R->MultTranspose(ye_mem, residual_l);
-      }
-   };
-
-   if constexpr (std::is_same_v<decltype(output_fop), None>)
-   {
-      prolongation_transpose = [&](Vector &r_local, Vector &y)
-      {
-         y = r_local;
-      };
-   }
-   else if constexpr (std::is_same_v<decltype(output_fop), One>)
-   {
-      prolongation_transpose = [&](Vector &r_local, Vector &y)
-      {
-         double local_sum = r_local.Sum();
-         MPI_Allreduce(&local_sum, y.GetData(), 1, MPI_DOUBLE, MPI_SUM,
-                       op.mesh.GetComm());
-         MFEM_ASSERT(y.Size() == 1, "output size doesn't match kernel description");
-      };
-   }
-   else
-   {
-      auto P = get_prolongation(op.fields[test_space_field_idx]);
-      prolongation_transpose = [P](const Vector &r_local, Vector &y)
-      {
-         P->MultTranspose(r_local, y);
-      };
-   }
-}

examples/dfem/dfem_assemble_hypreparmatrix.icc

@@ -1,308 +0,0 @@
-template <
-   typename kernels_tuple,
-   size_t num_solutions,
-   size_t num_parameters,
-   size_t num_fields,
-   size_t num_kernels
-   >
-template <
-   size_t derivative_idx
-   >
-template <
-   typename kernel_t
-   >
-void DifferentiableOperator<kernels_tuple,
-     num_solutions,
-     num_parameters,
-     num_fields,
-     num_kernels>::Derivative<derivative_idx>::assemble_hypreparmatrix_impl(
-        kernel_t kernel, HypreParMatrix &A)
-{
-   using entity_t = typename kernel_t::entity_t;
-
-   auto kinput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                     kernel.inputs,
-                                                                     std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto koutput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                      kernel.outputs,
-                                                                      std::make_index_sequence<kernel.num_koutputs> {});
-
-   auto output_fop = std::get<0>(kernel.outputs);
-
-   constexpr int hardcoded_output_idx = 0;
-
-   int num_qp = op.integration_rule.GetNPoints();;
-   int num_el = 0;
-   int dimension = 0;
-   if constexpr (std::is_same_v<entity_t, Entity::Element>)
-   {
-      num_el = op.mesh.GetNE();
-      dimension = op.dim;
-   }
-   else if (std::is_same_v<entity_t, Entity::Face>)
-   {
-      num_el = op.mesh.GetNumFacesWithGhost();
-      dimension = op.dim - 1;
-   }
-   else
-   {
-      static_assert(always_false<entity_t>, "not implemented");
-   }
-
-   std::vector<const DofToQuad*> dtqmaps;
-   for (const auto &field : op.fields)
-   {
-      dtqmaps.emplace_back(GetDofToQuad<entity_t>(field, op.integration_rule,
-                                                  doftoquad_mode));
-   }
-
-   // Allocate memory for fields on quadrature points
-   auto input_qp_mem = create_input_qp_memory(num_qp, kernel.inputs,
-                                              std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto directions_qp_mem = create_input_qp_memory(num_qp, kernel.inputs,
-                                                   std::make_index_sequence<kernel.num_kinputs> {});
-
-   for (auto &d_qp_mem : directions_qp_mem)
-   {
-      d_qp_mem = 0.0;
-   }
-
-   std::array<bool, kernel.num_kinputs> kinput_is_dependent;
-   bool no_kinput_is_dependent = true;
-   for (int i = 0; i < kinput_is_dependent.size(); i++)
-   {
-      if (kinput_to_field[i] == derivative_idx)
-      {
-         no_kinput_is_dependent = false;
-         kinput_is_dependent[i] = true;
-         // out << "function input " << i << " is dependent on "
-         //     << op.fields[kinput_to_field[i]].field_label << "\n";
-      }
-      else
-      {
-         kinput_is_dependent[i] = false;
-      }
-   }
-
-   if (no_kinput_is_dependent)
-   {
-      return;
-   }
-
-   auto kernel_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-   auto kernel_shadow_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-
-   DeviceTensor<1, const double> integration_weights(
-      this->op.integration_rule.GetWeights().Read(), num_qp);
-
-   Vector zero;
-   GeometricFactorMaps geometric_factors
-   {
-      DeviceTensor<3, const double>(zero.Read(), 0, 0, 0)
-   };
-
-   // fields interpolated to the quadrature points in the order of
-   // kernel function arguments
-   auto input_qp = map_inputs_to_memory(input_qp_mem, num_qp,
-                                        kernel.inputs,
-                                        std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto directions_qp = map_inputs_to_memory(directions_qp_mem, num_qp,
-                                             kernel.inputs,
-                                             std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto input_dtq_ops = create_dtq_operators<entity_t>(kernel.inputs, dtqmaps,
-                                                       kinput_to_field);
-   auto dependent_input_dtq_ops = create_dtq_operators_conditional<entity_t>(
-                                     kernel.inputs,
-                                     dtqmaps,
-                                     kinput_to_field,
-                                     kinput_is_dependent, std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto output_dtq_ops = create_dtq_operators<entity_t>(kernel.outputs, dtqmaps,
-                                                        koutput_to_field);
-
-   constexpr int fixed_output_idx = 0;
-   auto Bv = output_dtq_ops[fixed_output_idx];
-   auto [num_test_qp, test_op_dim, num_test_dof] = Bv.GetShape();
-   const int test_vdim = std::get<0>(kernel.outputs).vdim;
-
-   const int num_trial_dof = dependent_input_dtq_ops[0].GetShape()[2];
-   int trial_vdim = 0;
-   for (int i = 0; i < kinput_is_dependent.size(); i++)
-   {
-      if (kinput_is_dependent[i])
-      {
-         trial_vdim = GetVDim(op.fields[kinput_to_field[i]]);
-         break;
-      }
-   }
-
-   // All trial operators dimensions accumulated
-   int total_trial_op_dim = 0;
-   for (int s = 0; s < dependent_input_dtq_ops.size(); s++)
-   {
-      total_trial_op_dim += dependent_input_dtq_ops[s].GetShape()[1];
-   }
-
-   Vector a_qp_mem(test_vdim * test_op_dim * trial_vdim * total_trial_op_dim *
-                   num_qp *
-                   num_el);
-   const auto a_qp = Reshape(a_qp_mem.ReadWrite(), test_vdim, test_op_dim,
-                             trial_vdim, total_trial_op_dim, num_qp,
-                             num_el);
-
-   Vector Ae_mem(num_test_dof * test_vdim * num_trial_dof * trial_vdim * num_el);
-   Ae_mem = 0.0;
-
-   auto A_e = Reshape(Ae_mem.ReadWrite(), num_test_dof, test_vdim, num_trial_dof,
-                      trial_vdim, num_el);
-
-   for (int e = 0; e < num_el; e++)
-   {
-      map_fields_to_quadrature_data(
-         input_qp, e, this->fields_e,
-         kinput_to_field, input_dtq_ops,
-         integration_weights, geometric_factors, kernel.inputs,
-         std::make_index_sequence<kernel.num_kinputs> {});
-
-      for (int q = 0; q < num_qp; q++)
-      {
-         for (int j = 0; j < trial_vdim; j++)
-         {
-            size_t m_offset = 0;
-            for (int s = 0; s < dependent_input_dtq_ops.size(); s++)
-            {
-               auto Bu = dependent_input_dtq_ops[s];
-               auto [unused1, trial_op_dim, unused2] = Bu.GetShape();
-               auto d_qp = Reshape(&(directions_qp[Bu.which_input])[0], trial_vdim,
-                                   trial_op_dim, num_qp);
-               for (int m = 0; m < trial_op_dim; m++)
-               {
-                  d_qp(j, m, q) = 1.0;
-                  Vector f_qp = apply_kernel_fwddiff_enzyme(
-                                   kernel.func,
-                                   kernel_args,
-                                   input_qp,
-                                   kernel_shadow_args,
-                                   directions_qp,
-                                   q);
-                  // Vector f_qp = apply_kernel_fwddiff_dual(
-                  //                  kernel.func,
-                  //                  kernel_args,
-                  //                  input_qp,
-                  //                  directions_qp,
-                  //                  q);
-                  d_qp(j, m, q) = 0.0;
-
-                  auto f = Reshape(f_qp.Read(), test_vdim, test_op_dim);
-
-                  for (int i = 0; i < test_vdim; i++)
-                  {
-                     for (int k = 0; k < test_op_dim; k++)
-                     {
-                        a_qp(i, k, j, m + m_offset, q, e) = f(i, k);
-                     }
-                  }
-               }
-               m_offset += trial_op_dim;
-            }
-         }
-      }
-
-      Vector fhat_mem(test_op_dim * num_qp * dimension);
-      auto fhat = Reshape(fhat_mem.ReadWrite(), test_vdim, test_op_dim, num_qp);
-      for (int J = 0; J < num_trial_dof; J++)
-      {
-         for (int j = 0; j < trial_vdim; j++)
-         {
-            fhat_mem = 0.0;
-            size_t m_offset = 0;
-            for (int s = 0; s < dependent_input_dtq_ops.size(); s++)
-            {
-               auto Bu = dependent_input_dtq_ops[s];
-               int trial_op_dim = dependent_input_dtq_ops[s].GetShape()[1];
-               for (int q = 0; q < num_qp; q++)
-               {
-                  for (int i = 0; i < test_vdim; i++)
-                  {
-                     for (int k = 0; k < test_op_dim; k++)
-                     {
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           fhat(i, k, q) += a_qp(i, k, j, m + m_offset, q, e) * Bu(q, m, J);
-                        }
-                     }
-                  }
-               }
-               m_offset += trial_op_dim;
-            }
-
-            auto bvtfhat = Reshape(&A_e(0, 0, J, j, e), num_test_dof, test_vdim);
-            map_quadrature_data_to_fields(bvtfhat, fhat, output_fop,
-                                          output_dtq_ops[hardcoded_output_idx]);
-         }
-      }
-   }
-
-   bool same_test_and_trial = false;
-   if (koutput_to_field[0] ==
-       kinput_to_field[dependent_input_dtq_ops[0].which_input])
-   {
-      same_test_and_trial = true;
-   }
-
-   auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
-                    (&op.fields[kinput_to_field[dependent_input_dtq_ops[0].which_input]].data);
-
-   auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                   (&op.fields[koutput_to_field[0]].data);
-
-   SparseMatrix mat(test_fes->GlobalVSize(), trial_fes->GlobalVSize());
-
-   if (test_fes == nullptr)
-   {
-      MFEM_ABORT("error");
-   }
-
-   for (int e = 0; e < num_el; e++)
-   {
-      auto tmp = Reshape(Ae_mem.ReadWrite(), num_test_dof * test_vdim,
-                         num_trial_dof * trial_vdim,
-                         num_el);
-      DenseMatrix A_e(&tmp(0, 0, e), num_test_dof * test_vdim,
-                      num_trial_dof * trial_vdim);
-      Array<int> test_vdofs, trial_vdofs;
-      test_fes->GetElementVDofs(e, test_vdofs);
-      GetElementVDofs(
-         op.fields[kinput_to_field[dependent_input_dtq_ops[0].which_input]], e,
-         trial_vdofs);
-      mat.AddSubMatrix(test_vdofs, trial_vdofs, A_e, 1);
-   }
-   mat.Finalize();
-
-   if (same_test_and_trial)
-   {
-      HypreParMatrix tmp(test_fes->GetComm(),
-                         test_fes->GlobalVSize(),
-                         test_fes->GetDofOffsets(),
-                         &mat);
-
-      A = *RAP(&tmp, test_fes->Dof_TrueDof_Matrix());
-      A.EliminateBC(op.ess_tdof_list, DiagonalPolicy::DIAG_ONE);
-   }
-   else
-   {
-      HypreParMatrix tmp(test_fes->GetComm(),
-                         test_fes->GlobalVSize(),
-                         trial_fes->GlobalVSize(),
-                         test_fes->GetDofOffsets(),
-                         trial_fes->GetDofOffsets(),
-                         &mat);
-
-      A = *RAP(test_fes->Dof_TrueDof_Matrix(), &tmp, trial_fes->Dof_TrueDof_Matrix());
-      // A.EliminateBC(op.ess_tdof_list, DiagonalPolicy::DIAG_ONE);
-   }
-}

examples/dfem/dfem_assemble_vector.icc

@@ -1,233 +0,0 @@
-template <
-   typename kernels_tuple,
-   size_t num_solutions,
-   size_t num_parameters,
-   size_t num_fields,
-   size_t num_kernels
-   >
-template <
-   size_t derivative_idx
-   >
-template <
-   typename kernel_t
-   >
-void DifferentiableOperator<kernels_tuple,
-     num_solutions,
-     num_parameters,
-     num_fields,
-     num_kernels>::Derivative<derivative_idx>::assemble_vector_impl(
-        kernel_t kernel, Vector &v)
-{
-   using entity_t = typename kernel_t::entity_t;
-
-   auto kinput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                     kernel.inputs,
-                                                                     std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto koutput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                      kernel.outputs,
-                                                                      std::make_index_sequence<kernel.num_koutputs> {});
-
-   auto output_fop = std::get<0>(kernel.outputs);
-
-   constexpr int hardcoded_output_idx = 0;
-
-   int num_qp = op.integration_rule.GetNPoints();;
-   int num_el = 0;
-   int dimension = 0;
-   if constexpr (std::is_same_v<entity_t, Entity::Element>)
-   {
-      num_el = op.mesh.GetNE();
-      dimension = op.dim;
-   }
-   else if (std::is_same_v<entity_t, Entity::Face>)
-   {
-      num_el = op.mesh.GetNumFacesWithGhost();
-      dimension = op.dim - 1;
-   }
-   else
-   {
-      static_assert(always_false<entity_t>, "not implemented");
-   }
-
-   std::vector<const DofToQuad*> dtqmaps;
-   for (const auto &field : op.fields)
-   {
-      dtqmaps.emplace_back(GetDofToQuad<entity_t>(field, op.integration_rule,
-                                                  doftoquad_mode));
-   }
-
-   // Allocate memory for fields on quadrature points
-   auto input_qp_mem = create_input_qp_memory(num_qp, kernel.inputs,
-                                              std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto directions_qp_mem = create_input_qp_memory(num_qp, kernel.inputs,
-                                                   std::make_index_sequence<kernel.num_kinputs> {});
-
-   for (auto &d_qp_mem : directions_qp_mem)
-   {
-      d_qp_mem = 0.0;
-   }
-
-   std::array<bool, kernel.num_kinputs> kinput_is_dependent;
-   bool no_kinput_is_dependent = true;
-   for (int i = 0; i < kinput_is_dependent.size(); i++)
-   {
-      if (kinput_to_field[i] == derivative_idx)
-      {
-         no_kinput_is_dependent = false;
-         kinput_is_dependent[i] = true;
-         // out << "function input " << i << " is dependent on "
-         //     << op.fields[kinput_to_field[i]].field_label << "\n";
-      }
-      else
-      {
-         kinput_is_dependent[i] = false;
-      }
-   }
-
-   if (no_kinput_is_dependent)
-   {
-      return;
-   }
-
-   auto kernel_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-   auto kernel_shadow_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-
-   DeviceTensor<1, const double> integration_weights(
-      this->op.integration_rule.GetWeights().Read(), num_qp);
-
-   Vector zero;
-   GeometricFactorMaps geometric_factors
-   {
-      DeviceTensor<3, const double>(zero.Read(), 0, 0, 0)
-   };
-
-   // fields interpolated to the quadrature points in the order of
-   // kernel function arguments
-   auto input_qp = map_inputs_to_memory(input_qp_mem, num_qp,
-                                        kernel.inputs,
-                                        std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto directions_qp = map_inputs_to_memory(directions_qp_mem, num_qp,
-                                             kernel.inputs,
-                                             std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto input_dtq_ops = create_dtq_operators<entity_t>(kernel.inputs, dtqmaps,
-                                                       kinput_to_field);
-   auto dependent_input_dtq_ops = create_dtq_operators_conditional<entity_t>(
-                                     kernel.inputs,
-                                     dtqmaps,
-                                     kinput_to_field,
-                                     kinput_is_dependent, std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto output_dtq_ops = create_dtq_operators<entity_t>(kernel.outputs, dtqmaps,
-                                                        koutput_to_field);
-
-   constexpr int fixed_output_idx = 0;
-   auto Bv = output_dtq_ops[fixed_output_idx];
-   auto [num_test_qp, test_op_dim, num_test_dof] = Bv.GetShape();
-   const int test_vdim = std::get<0>(kernel.outputs).vdim;
-
-   const int num_trial_dof = dependent_input_dtq_ops[0].GetShape()[2];
-   int trial_vdim = 0;
-   int dependent_field_idx = -1;
-   for (int i = 0; i < kinput_is_dependent.size(); i++)
-   {
-      if (kinput_is_dependent[i])
-      {
-         dependent_field_idx = kinput_to_field[i];
-         break;
-      }
-   }
-
-   trial_vdim = GetVDim(op.fields[dependent_field_idx]);
-
-   // All trial operators dimensions accumulated
-   int total_trial_op_dim = 0;
-   for (int s = 0; s < dependent_input_dtq_ops.size(); s++)
-   {
-      total_trial_op_dim += dependent_input_dtq_ops[s].GetShape()[1];
-   }
-
-   Vector a_qp_mem(trial_vdim * total_trial_op_dim * num_qp * num_el);
-   const auto a_qp = Reshape(a_qp_mem.ReadWrite(), trial_vdim,
-                             total_trial_op_dim, num_qp, num_el);
-   Vector ve_mem(num_trial_dof * trial_vdim * num_el);
-   ve_mem = 0.0;
-
-   for (int e = 0; e < num_el; e++)
-   {
-      map_fields_to_quadrature_data(
-         input_qp, e, this->fields_e,
-         kinput_to_field, input_dtq_ops,
-         integration_weights, geometric_factors, kernel.inputs,
-         std::make_index_sequence<kernel.num_kinputs> {});
-
-      for (int q = 0; q < num_qp; q++)
-      {
-         for (int j = 0; j < trial_vdim; j++)
-         {
-            size_t m_offset = 0;
-            for (int s = 0; s < dependent_input_dtq_ops.size(); s++)
-            {
-               auto Bu = dependent_input_dtq_ops[s];
-               auto [unused1, trial_op_dim, unused2] = Bu.GetShape();
-               auto d_qp = Reshape(&(directions_qp[Bu.which_input])[0], trial_vdim,
-                                   trial_op_dim, num_qp);
-               for (int m = 0; m < trial_op_dim; m++)
-               {
-                  d_qp(j, m, q) = 1.0;
-                  // Vector f_qp = apply_kernel_fwddiff_dual(
-                  //                  kernel.func,
-                  //                  kernel_args,
-                  //                  input_qp,
-                  //                  directions_qp,
-                  //                  q);
-                  Vector f_qp = apply_kernel_fwddiff_enzyme(
-                                   kernel.func,
-                                   kernel_args,
-                                   input_qp,
-                                   kernel_shadow_args,
-                                   directions_qp,
-                                   q);
-                  d_qp(j, m, q) = 0.0;
-
-                  auto f = Reshape(f_qp.Read(), test_vdim);
-                  a_qp(j, m + m_offset, q, e) = f(0);
-               }
-               m_offset += trial_op_dim;
-            }
-         }
-      }
-
-      auto shat = Reshape(ve_mem.ReadWrite(), num_trial_dof, trial_vdim, num_el);
-      for (int J = 0; J < num_trial_dof; J++)
-      {
-         for (int j = 0; j < trial_vdim; j++)
-         {
-            size_t m_offset = 0;
-            for (int s = 0; s < dependent_input_dtq_ops.size(); s++)
-            {
-               auto Bu = dependent_input_dtq_ops[s];
-               int trial_op_dim = dependent_input_dtq_ops[s].GetShape()[1];
-               for (int q = 0; q < num_qp; q++)
-               {
-                  for (int m = 0; m < trial_op_dim; m++)
-                  {
-                     shat(J, j, e) += a_qp(j, m + m_offset, q, e) * Bu(q, m, J);
-                  }
-               }
-               m_offset += trial_op_dim;
-            }
-         }
-      }
-   }
-
-   auto R = get_element_restriction(op.fields[dependent_field_idx],
-                                    element_dof_ordering);
-   Vector ve(R->Width());
-   R->MultTranspose(ve_mem, ve);
-
-   get_prolongation(op.fields[dependent_field_idx])->MultTranspose(ve, v);
-}

examples/dfem/dfem_derivative_callback.icc

@@ -1,244 +0,0 @@
-template <
-   typename kernels_tuple,
-   size_t num_solutions,
-   size_t num_parameters,
-   size_t num_fields,
-   size_t num_kernels
-   >
-template <
-   size_t derivative_idx
-   >
-template <
-   typename kernel_t
-   >
-void DifferentiableOperator<kernels_tuple,
-     num_solutions,
-     num_parameters,
-     num_fields,
-     num_kernels>::Derivative<derivative_idx>::create_callback(kernel_t kernel,
-                                                               mult_func_t &func)
-{
-   using entity_t = typename kernel_t::entity_t;
-
-   auto kinput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                     kernel.inputs, std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto koutput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                      kernel.outputs, std::make_index_sequence<kernel.num_koutputs> {});
-
-   constexpr int hardcoded_output_idx = 0;
-   const int test_space_field_idx = koutput_to_field[hardcoded_output_idx];
-
-   const Operator *R = get_restriction<entity_t>(op.fields[test_space_field_idx],
-                                                 element_dof_ordering);
-
-   auto output_fop = mfem::get<hardcoded_output_idx>(kernel.outputs);
-
-   const int num_elements = GetNumEntities<Entity::Element>(op.mesh);
-   const int num_entities = GetNumEntities<entity_t>(op.mesh);
-   const int num_qp = op.integration_rule.GetNPoints();
-
-   // assume only a single element type for now
-   std::vector<const DofToQuad*> dtq;
-   for (const auto &field : op.fields)
-   {
-      dtq.emplace_back(GetDofToQuad<entity_t>(field, op.integration_rule,
-                                              doftoquad_mode));
-   }
-   const int q1d = dtq[0]->nqpt;
-
-   derivative_action_e.SetSize(R->Height());
-
-   const int da_size_on_qp = GetSizeOnQP<entity_t>(
-                                mfem::get<hardcoded_output_idx>(kernel.outputs),
-                                op.fields[test_space_field_idx]);
-
-   auto input_dtq_maps = create_dtq_maps<entity_t>(kernel.inputs, dtq,
-                                                   kinput_to_field);
-   auto output_dtq_maps = create_dtq_maps<entity_t>(kernel.outputs, dtq,
-                                                    koutput_to_field);
-
-   auto input_fops = create_bare_fops(kernel.inputs);
-   auto output_fops = create_bare_fops(kernel.outputs);
-
-   const int test_vdim = mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int test_op_dim =
-      mfem::get<hardcoded_output_idx>(output_fops).size_on_qp /
-      mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int num_test_dof = R->Height() /
-                            mfem::get<hardcoded_output_idx>(output_fops).vdim /
-                            num_entities;
-
-   auto ir_weights = Reshape(this->op.integration_rule.GetWeights().Read(),
-                             num_qp);
-
-   auto input_size_on_qp = get_input_size_on_qp(kernel.inputs,
-                                                std::make_index_sequence<kernel.num_kinputs> {});
-
-   // Check which qf inputs are dependent on the dependent variable
-   std::array<bool, kernel.num_kinputs> kinput_is_dependent;
-   bool no_kinput_is_dependent = true;
-   for (int i = 0; i < kinput_is_dependent.size(); i++)
-   {
-      if (kinput_to_field[i] == derivative_idx)
-      {
-         no_kinput_is_dependent = false;
-         kinput_is_dependent[i] = true;
-         // out << "function input " << i << " is dependent on "
-         //     << op.fields[kinput_to_field[i]].field_label << "\n";
-      }
-      else
-      {
-         kinput_is_dependent[i] = false;
-      }
-   }
-
-   bool with_derivatives = true;
-   auto shmem_info = get_shmem_info<entity_t>(input_dtq_maps,
-                                              output_dtq_maps,
-                                              op.fields,
-                                              num_entities,
-                                              kernel.inputs,
-                                              num_qp,
-                                              input_size_on_qp,
-                                              da_size_on_qp,
-                                              derivative_idx);
-
-   Vector shmem_cache(shmem_info.total_size);
-
-   print_shared_memory_info(shmem_info);
-
-   func = [=](Vector &ye_mem) mutable
-   {
-      if (no_kinput_is_dependent)
-      {
-         return;
-      }
-
-      restriction<entity_t>(direction, direction_l, direction_e,
-                            op.element_dof_ordering);
-
-      auto ye = Reshape(ye_mem.ReadWrite(), num_test_dof, test_vdim, num_entities);
-      auto wrapped_fields_e = wrap_fields(this->fields_e, shmem_info.field_sizes, num_entities);
-      auto wrapped_direction_e = Reshape(direction_e.Read(), shmem_info.direction_size, num_entities);
-
-      forall([=] MFEM_HOST_DEVICE (int e, double *shmem)
-      {
-         auto input_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::INPUT_DTQ],
-            shmem_info.input_dtq_sizes,
-            input_dtq_maps);
-
-         auto output_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::OUTPUT_DTQ],
-            shmem_info.output_dtq_sizes,
-            output_dtq_maps);
-
-         auto fields_shmem = load_field_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::FIELD],
-            shmem_info.field_sizes,
-            kinput_to_field,
-            wrapped_fields_e,
-            e);
-
-         auto direction_shmem = load_direction_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::DIRECTION],
-            shmem_info.direction_size,
-            wrapped_direction_e,
-            e);
-
-         // These methods don't copy, they simply create a `DeviceTensor` object
-         // that points to correct chunks of the shared memory pool.
-         auto input_shmem = load_input_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::INPUT],
-            shmem_info.input_sizes,
-            num_qp);
-
-         auto shadow_shmem = load_input_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::SHADOW],
-            shmem_info.input_sizes,
-            num_qp);
-
-         auto residual_shmem = load_residual_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::OUTPUT],
-            shmem_info.residual_size,
-            num_qp);
-
-         auto scratch_mem = load_scratch_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::TEMP],
-            shmem_info.temp_sizes);
-
-         map_fields_to_quadrature_data<TensorProduct>(
-            input_shmem, fields_shmem, input_dtq_shmem, input_fops, ir_weights, scratch_mem,
-         std::make_index_sequence<kernel.num_kinputs> {});
-
-         zero_all(shadow_shmem);
-         map_direction_to_quadrature_data_conditional<TensorProduct>(
-            shadow_shmem, direction_shmem, input_dtq_shmem, input_fops, ir_weights,
-            scratch_mem, kinput_is_dependent,
-            std::make_index_sequence<kernel.num_kinputs> {});
-
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-
-                  auto kernel_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-                  auto kernel_shadow_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-
-                  auto r = Reshape(&residual_shmem(0, q), da_size_on_qp);
-                  apply_kernel_fwddiff_enzyme(
-                     r,
-                     kernel.func,
-                     kernel_args,
-                     input_shmem,
-                     kernel_shadow_args,
-                     shadow_shmem,
-                     q);
-                  // printf(">>>>> WARNING: AD DISABLED\n");
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         auto fhat = Reshape(&residual_shmem(0, 0), test_vdim, test_op_dim, num_qp);
-         auto y = Reshape(&ye(0, 0, e), num_test_dof, test_vdim);
-         map_quadrature_data_to_fields<TensorProduct>(y, fhat,
-                                                      mfem::get<0>(output_fops),
-                                                      output_dtq_shmem[hardcoded_output_idx],
-                                                      scratch_mem);
-      }, num_entities, q1d, q1d, 1, shmem_info.total_size, shmem_cache.ReadWrite());
-
-      R->MultTranspose(ye_mem, derivative_action_l);
-   };
-
-   if constexpr (std::is_same_v<decltype(output_fop), One>)
-   {
-      prolongation_transpose = [&](Vector &r_local, Vector &y)
-      {
-         double local_sum = r_local.Sum();
-         MPI_Allreduce(&local_sum, y.GetData(), 1, MPI_DOUBLE, MPI_SUM,
-                       op.mesh.GetComm());
-         MFEM_ASSERT(y.Size() == 1, "output size doesn't match kernel description");
-      };
-   }
-   else
-   {
-      auto P = get_prolongation(op.fields[test_space_field_idx]);
-      prolongation_transpose = [P](const Vector &r_local, Vector &y)
-      {
-         P->MultTranspose(r_local, y);
-      };
-   }
-}

examples/dfem/dfem_differentiable_operator.hpp

@@ -1,806 +0,0 @@
-#pragma once
-
-#include <algorithm>
-#include <cstdlib>
-#include <functional>
-#include <iostream>
-#include <utility>
-#include <variant>
-#include <vector>
-#include <type_traits>
-#include <mfem.hpp>
-#include <type_traits>
-#include "dfem_fieldoperator.hpp"
-#include "dfem_parametricspace.hpp"
-#include "general/tic_toc.hpp"
-#include "tuple.hpp"
-#include <linalg/tensor.hpp>
-#include <enzyme/utils>
-#include <enzyme/enzyme>
-#include "dfem_util.hpp"
-#include "dfem_interpolate.hpp"
-#include "dfem_qfunction.hpp"
-#include "dfem_integrate.hpp"
-
-namespace mfem
-{
-
-using mult_func_t = std::function<void(Vector &)>;
-
-template <
-   typename kernels_tuple,
-   size_t num_solutions,
-   size_t num_parameters,
-   size_t num_fields = num_solutions + num_parameters,
-   size_t num_kernels = mfem::tuple_size<kernels_tuple>::value
-   >
-class DifferentiableOperator : public Operator
-{
-public:
-   DifferentiableOperator(DifferentiableOperator&) = delete;
-   DifferentiableOperator(DifferentiableOperator&&) = delete;
-
-   class Action : public Operator
-   {
-   public:
-      template <typename kernel_t>
-      void create_action_callback(kernel_t kernel, mult_func_t &func);
-
-      template<std::size_t... idx>
-      void materialize_callbacks(kernels_tuple &ks,
-                                 std::array<mult_func_t, num_kernels>,
-                                 std::index_sequence<idx...> const&)
-      {
-         (create_action_callback(mfem::get<idx>(ks), funcs[idx]), ...);
-      }
-
-      Action(DifferentiableOperator &op, kernels_tuple &ks) : op(op)
-      {
-         materialize_callbacks(ks, funcs,
-                               std::make_index_sequence<mfem::tuple_size<kernels_tuple>::value>());
-      }
-
-      void Mult(const Vector &x, Vector &y) const
-      {
-         prolongation(op.solutions, x, solutions_l);
-
-         residual_e = 0.0;
-         for (const auto &f : funcs)
-         {
-            f(residual_e);
-         }
-
-         prolongation_transpose(residual_l, y);
-
-         y.SetSubVector(op.ess_tdof_list, 0.0);
-      }
-
-      void SetParameters(std::vector<Vector *> p) const
-      {
-         MFEM_ASSERT(num_parameters == p.size(),
-                     "number of parameters doesn't match descriptors");
-         for (int i = 0; i < num_parameters; i++)
-         {
-            p[i]->Read();
-            parameters_l[i] = *p[i];
-            // parameters_l[i].MakeRef(p[i], 0, p[i]->Size());
-         }
-      }
-
-   protected:
-      DifferentiableOperator &op;
-      std::array<mult_func_t, num_kernels> funcs;
-
-      std::function<void(Vector &, Vector &)> prolongation_transpose;
-
-      mutable std::array<Vector, num_solutions> solutions_l;
-      mutable std::array<Vector, num_parameters> parameters_l;
-      mutable Vector residual_l;
-
-      mutable std::array<Vector, num_fields> fields_e;
-      mutable Vector residual_e;
-   };
-
-   template <size_t derivative_idx>
-   class Derivative : public Operator
-   {
-   public:
-      template <typename kernel_t>
-      void create_callback(kernel_t kernel, mult_func_t &func);
-
-      template<std::size_t... idx>
-      void materialize_callbacks(kernels_tuple &ks,
-                                 std::array<mult_func_t, num_kernels>,
-                                 std::index_sequence<idx...> const&)
-      {
-         (create_callback(mfem::get<idx>(ks), funcs[idx]), ...);
-      }
-
-      Derivative(
-         DifferentiableOperator &op,
-         std::array<Vector *, num_solutions> &solutions,
-         std::array<Vector *, num_parameters> &parameters,
-         kernels_tuple &ks) : op(op), ks(ks)
-      {
-         for (int i = 0; i < num_solutions; i++)
-         {
-            solutions_l[i] = *solutions[i];
-         }
-
-         for (int i = 0; i < num_parameters; i++)
-         {
-            parameters_l[i] = *parameters[i];
-         }
-
-         // G
-         // if constexpr (std::is_same_v<OperatesOn, OperatesOnElement>)
-         // {
-         element_restriction(op.solutions, solutions_l, fields_e,
-                             op.element_dof_ordering);
-         element_restriction(op.parameters, parameters_l, fields_e,
-                             op.element_dof_ordering,
-                             op.solutions.size());
-         // }
-         // else
-         // {
-         //    MFEM_ABORT("restriction not implemented for OperatesOn");
-         // }
-         direction = op.fields[derivative_idx];
-
-         size_t derivative_action_l_size = 0;
-         for (auto &s : op.solutions)
-         {
-            derivative_action_l_size += GetVSize(s);
-            this->width += GetTrueVSize(s);
-         }
-         this->height = derivative_action_l_size;
-         derivative_action_l.SetSize(derivative_action_l_size);
-
-         materialize_callbacks(ks, funcs,
-                               std::make_index_sequence<num_kernels>());
-      }
-
-      void Mult(const Vector &x, Vector &y) const override
-      {
-         current_direction_t = x;
-         current_direction_t.SetSubVector(op.ess_tdof_list, 0.0);
-
-         prolongation(direction, current_direction_t, direction_l);
-
-         derivative_action_e = 0.0;
-         for (const auto &f : funcs)
-         {
-            f(derivative_action_e);
-         }
-
-         prolongation_transpose(derivative_action_l, y);
-
-         y.SetSubVector(op.ess_tdof_list, 0.0);
-      }
-
-      template <typename kernel_t>
-      void assemble_vector_impl(kernel_t kernel, Vector &v);
-
-      template<std::size_t... idx>
-      void assemble_vector(
-         kernels_tuple &ks,
-         Vector &v,
-         std::index_sequence<idx...> const&)
-      {
-         (assemble_vector_impl(mfem::get<idx>(ks), v), ...);
-      }
-
-      void Assemble(Vector &v)
-      {
-         assemble_vector(ks, v, std::make_index_sequence<num_kernels>());
-      }
-
-      template <typename kernel_t>
-      void assemble_hypreparmatrix_impl(kernel_t kernel, HypreParMatrix &A);
-
-      template<std::size_t... idx>
-      void assemble_hypreparmatrix(
-         kernels_tuple &ks,
-         HypreParMatrix &A,
-         std::index_sequence<idx...> const&)
-      {
-         (assemble_hypreparmatrix_impl(mfem::get<idx>(ks), A), ...);
-      }
-
-      void Assemble(HypreParMatrix &A)
-      {
-         assemble_hypreparmatrix(ks, A, std::make_index_sequence<num_kernels>());
-      }
-
-      void AssembleDiagonal(Vector &d) const override {}
-
-   protected:
-      DifferentiableOperator &op;
-      kernels_tuple &ks;
-      std::array<mult_func_t, num_kernels> funcs;
-
-      std::function<void(Vector &, Vector &)> prolongation_transpose;
-
-      FieldDescriptor direction;
-
-      std::array<Vector, num_solutions> solutions_l;
-      std::array<Vector, num_parameters> parameters_l;
-      mutable Vector direction_l;
-      mutable Vector derivative_action_l;
-
-      mutable std::array<Vector, num_fields> fields_e;
-      mutable Vector direction_e;
-      mutable Vector derivative_action_e;
-
-      mutable Vector current_direction_t;
-   };
-
-   DifferentiableOperator(std::array<FieldDescriptor, num_solutions> s,
-                          std::array<FieldDescriptor, num_parameters> p,
-                          kernels_tuple ks,
-                          ParMesh &m,
-                          const IntegrationRule &integration_rule) :
-      kernels(ks),
-      mesh(m),
-      dim(mesh.Dimension()),
-      integration_rule(integration_rule),
-      solutions(s),
-      parameters(p)
-   {
-      for (int i = 0; i < num_solutions; i++)
-      {
-         fields[i] = solutions[i];
-      }
-
-      for (int i = 0; i < num_parameters; i++)
-      {
-         fields[i + num_solutions] = parameters[i];
-      }
-
-      residual.reset(new Action(*this, kernels));
-   }
-
-   void SetParameters(std::vector<Vector *> p) const
-   {
-      residual->SetParameters(p);
-   }
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      residual->Mult(x, y);
-   }
-
-   template <int derivative_idx>
-   std::shared_ptr<Derivative<derivative_idx>>
-                                            GetDerivativeWrt(std::array<Vector *, num_solutions> solutions,
-                                                             std::array<Vector *, num_parameters> parameters)
-   {
-      return std::shared_ptr<Derivative<derivative_idx>>(
-                new Derivative<derivative_idx>(*this, solutions, parameters, kernels));
-   }
-
-   void SetEssentialTrueDofs(const Array<int> &l)
-   {
-      l.Copy(ess_tdof_list);
-   }
-
-   kernels_tuple kernels;
-   ParMesh &mesh;
-   const int dim;
-   const IntegrationRule &integration_rule;
-
-   std::array<FieldDescriptor, num_solutions> solutions;
-   std::array<FieldDescriptor, num_parameters> parameters;
-   // solutions and parameters
-   std::array<FieldDescriptor, num_fields> fields;
-
-   int residual_lsize = 0;
-
-   mutable std::array<Vector, num_solutions> current_state_l;
-   mutable Vector direction_l;
-
-   mutable Vector current_direction_t;
-
-   Array<int> ess_tdof_list;
-
-   static constexpr ElementDofOrdering element_dof_ordering =
-      ElementDofOrdering::LEXICOGRAPHIC;
-
-   static constexpr DofToQuad::Mode doftoquad_mode =
-      DofToQuad::Mode::TENSOR;
-
-   // static constexpr ElementDofOrdering element_dof_ordering =
-   //    ElementDofOrdering::NATIVE;
-
-   // static constexpr DofToQuad::Mode doftoquad_mode =
-   //    DofToQuad::Mode::FULL;
-
-   std::shared_ptr<Action> residual;
-};
-
-template <
-   typename kernels_tuple,
-   size_t num_solutions,
-   size_t num_parameters,
-   size_t num_fields,
-   size_t num_kernels
-   >
-template <
-   typename kernel_t
-   >
-void DifferentiableOperator<kernels_tuple,
-     num_solutions,
-     num_parameters,
-     num_fields,
-     num_kernels>::Action::create_action_callback(
-        kernel_t kernel,
-        mult_func_t &func)
-{
-   using entity_t = typename kernel_t::entity_t;
-
-   auto kinput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                     kernel.inputs, std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto koutput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                      kernel.outputs, std::make_index_sequence<kernel.num_koutputs> {});
-
-   constexpr int hardcoded_output_idx = 0;
-   const int test_space_field_idx = koutput_to_field[hardcoded_output_idx];
-
-   const Operator *R = get_restriction<entity_t>(op.fields[test_space_field_idx],
-                                                 element_dof_ordering);
-
-   auto output_fop = mfem::get<hardcoded_output_idx>(kernel.outputs);
-
-   const int num_elements = GetNumEntities<Entity::Element>(op.mesh);
-   const int num_entities = GetNumEntities<entity_t>(op.mesh);
-   const int num_qp = op.integration_rule.GetNPoints();
-
-   // All solutions T-vector sizes make up the width of the operator, since
-   // they are explicitly provided in Mult() for example.
-
-   op.width = GetTrueVSize(op.fields[test_space_field_idx]);
-   op.residual_lsize = GetVSize(op.fields[test_space_field_idx]);
-
-   if constexpr (std::is_same_v<decltype(output_fop), One>)
-   {
-      op.height = 1;
-   }
-   else
-   {
-      op.height = op.residual_lsize;
-   }
-
-   residual_l.SetSize(op.residual_lsize);
-
-   // assume only a single element type for now
-   std::vector<const DofToQuad*> dtq;
-   for (const auto &field : op.fields)
-   {
-      dtq.emplace_back(GetDofToQuad<entity_t>(field, op.integration_rule,
-                                              doftoquad_mode));
-   }
-   const int q1d = (int)floor(pow(num_qp, 1.0/op.mesh.Dimension()) + 0.5);
-
-   residual_e.SetSize(R->Height());
-
-   const int residual_size_on_qp = GetSizeOnQP<entity_t>(
-                                      mfem::get<hardcoded_output_idx>(kernel.outputs),
-                                      op.fields[test_space_field_idx]);
-
-   auto input_dtq_maps = create_dtq_maps<entity_t>(kernel.inputs, dtq,
-                                                   kinput_to_field);
-   auto output_dtq_maps = create_dtq_maps<entity_t>(kernel.outputs, dtq,
-                                                    koutput_to_field);
-
-   auto input_fops = create_bare_fops(kernel.inputs);
-   auto output_fops = create_bare_fops(kernel.outputs);
-
-   const int test_vdim = mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int test_op_dim =
-      mfem::get<hardcoded_output_idx>(output_fops).size_on_qp /
-      mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int num_test_dof = R->Height() /
-                            mfem::get<hardcoded_output_idx>(output_fops).vdim /
-                            num_entities;
-
-   auto ir_weights = Reshape(this->op.integration_rule.GetWeights().Read(),
-                             num_qp);
-
-   auto input_size_on_qp = get_input_size_on_qp(kernel.inputs,
-                                                std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto shmem_info = get_shmem_info<entity_t>(input_dtq_maps,
-                                              output_dtq_maps,
-                                              op.fields,
-                                              num_entities,
-                                              kernel.inputs,
-                                              num_qp,
-                                              input_size_on_qp,
-                                              residual_size_on_qp);
-
-   Vector shmem_cache(shmem_info.total_size);
-
-   // print_shared_memory_info(shmem_info);
-
-   func = [=](Vector &ye_mem) mutable
-   {
-      restriction<entity_t>(op.solutions, solutions_l, this->fields_e,
-                            op.element_dof_ordering);
-      restriction<entity_t>(op.parameters, parameters_l, this->fields_e,
-                            op.element_dof_ordering,
-                            op.solutions.size());
-
-      auto ye = Reshape(ye_mem.ReadWrite(), test_vdim, num_test_dof, num_entities);
-      auto wrapped_fields_e = wrap_fields(this->fields_e, shmem_info.field_sizes, num_entities);
-
-      forall([=] MFEM_HOST_DEVICE (int e, void *shmem)
-      {
-         // printf("\ne: %d\n", e);
-         // tic();
-         auto input_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::INPUT_DTQ],
-            shmem_info.input_dtq_sizes,
-            input_dtq_maps);
-
-         auto output_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::OUTPUT_DTQ],
-            shmem_info.output_dtq_sizes,
-            output_dtq_maps);
-
-         auto fields_shmem = load_field_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::FIELD],
-            shmem_info.field_sizes,
-            kinput_to_field,
-            input_fops,
-            wrapped_fields_e,
-            e,
-         std::make_index_sequence<kernel.num_kinputs> {});
-
-         // These functions don't copy, they simply create a `DeviceTensor` object
-         // that points to correct chunks of the shared memory pool.
-         auto input_shmem = load_input_mem(
-                               shmem,
-                               shmem_info.offsets[SharedMemory::Index::INPUT],
-                               shmem_info.input_sizes,
-                               num_qp);
-
-         auto residual_shmem = load_residual_mem(
-                                  shmem,
-                                  shmem_info.offsets[SharedMemory::Index::OUTPUT],
-                                  shmem_info.residual_size,
-                                  num_qp);
-
-         auto scratch_mem = load_scratch_mem(
-                               shmem,
-                               shmem_info.offsets[SharedMemory::Index::TEMP],
-                               shmem_info.temp_sizes);
-
-         MFEM_SYNC_THREAD;
-         // printf("shmem load elapsed: %.1fus\n", toc() * 1e6);
-
-         // tic();
-         map_fields_to_quadrature_data<TensorProduct>(
-            input_shmem, fields_shmem, input_dtq_shmem, input_fops, ir_weights, scratch_mem,
-            std::make_index_sequence<kernel.num_kinputs> {});
-         // printf("interpolate elapsed: %.1fus\n", toc() * 1e6);
-
-         // tic();
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  auto kernel_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-                  auto r = Reshape(&residual_shmem(0, q), residual_size_on_qp);
-                  apply_kernel(r, kernel.func, kernel_args, input_shmem, q);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // printf("qf elapsed: %.1fus\n", toc() * 1e6);
-
-         // tic();
-         auto fhat = Reshape(&residual_shmem(0, 0), test_vdim, test_op_dim, num_qp);
-         auto y = Reshape(&ye(0, 0, e), num_test_dof, test_vdim);
-         map_quadrature_data_to_fields<TensorProduct>(y, fhat,
-                                                      mfem::get<0>(output_fops),
-                                                      output_dtq_shmem[hardcoded_output_idx],
-                                                      scratch_mem);
-         // printf("integrate elapsed: %.1fus\n", toc() * 1e6);
-
-      }, num_entities, q1d, q1d, q1d, shmem_info.total_size, shmem_cache.ReadWrite());
-
-      if constexpr (std::is_same_v<decltype(output_fop), None>)
-      {
-         residual_l = ye_mem;
-      }
-      else
-      {
-         R->MultTranspose(ye_mem, residual_l);
-      }
-   };
-
-   if constexpr (std::is_same_v<decltype(output_fop), None>)
-   {
-      prolongation_transpose = [&](Vector &r_local, Vector &y)
-      {
-         y = r_local;
-      };
-   }
-   else if constexpr (std::is_same_v<decltype(output_fop), One>)
-   {
-      prolongation_transpose = [&](Vector &r_local, Vector &y)
-      {
-         double local_sum = r_local.Sum();
-         MPI_Allreduce(&local_sum, y.GetData(), 1, MPI_DOUBLE, MPI_SUM,
-                       op.mesh.GetComm());
-         MFEM_ASSERT(y.Size() == 1, "output size doesn't match kernel description");
-      };
-   }
-   else
-   {
-      auto P = get_prolongation(op.fields[test_space_field_idx]);
-      prolongation_transpose = [P](const Vector &r_local, Vector &y)
-      {
-         P->MultTranspose(r_local, y);
-      };
-   }
-}
-
-template <
-   typename kernels_tuple,
-   size_t num_solutions,
-   size_t num_parameters,
-   size_t num_fields,
-   size_t num_kernels
-   >
-template <
-   size_t derivative_idx
-   >
-template <
-   typename kernel_t
-   >
-void DifferentiableOperator<kernels_tuple,
-     num_solutions,
-     num_parameters,
-     num_fields,
-     num_kernels>::Derivative<derivative_idx>::create_callback(kernel_t kernel,
-                                                               mult_func_t &func)
-{
-   using entity_t = typename kernel_t::entity_t;
-
-   auto kinput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                     kernel.inputs, std::make_index_sequence<kernel.num_kinputs> {});
-
-   auto koutput_to_field = create_descriptors_to_fields_map<entity_t>(op.fields,
-                                                                      kernel.outputs, std::make_index_sequence<kernel.num_koutputs> {});
-
-   constexpr int hardcoded_output_idx = 0;
-   const int test_space_field_idx = koutput_to_field[hardcoded_output_idx];
-
-   const Operator *R = get_restriction<entity_t>(op.fields[test_space_field_idx],
-                                                 element_dof_ordering);
-
-   auto output_fop = mfem::get<hardcoded_output_idx>(kernel.outputs);
-
-   const int num_elements = GetNumEntities<Entity::Element>(op.mesh);
-   const int num_entities = GetNumEntities<entity_t>(op.mesh);
-   const int num_qp = op.integration_rule.GetNPoints();
-
-   // assume only a single element type for now
-   std::vector<const DofToQuad*> dtq;
-   for (const auto &field : op.fields)
-   {
-      dtq.emplace_back(GetDofToQuad<entity_t>(field, op.integration_rule,
-                                              doftoquad_mode));
-   }
-   const int q1d = dtq[0]->nqpt;
-
-   derivative_action_e.SetSize(R->Height());
-
-   const int da_size_on_qp = GetSizeOnQP<entity_t>(
-                                mfem::get<hardcoded_output_idx>(kernel.outputs),
-                                op.fields[test_space_field_idx]);
-
-   auto input_dtq_maps = create_dtq_maps<entity_t>(kernel.inputs, dtq,
-                                                   kinput_to_field);
-   auto output_dtq_maps = create_dtq_maps<entity_t>(kernel.outputs, dtq,
-                                                    koutput_to_field);
-
-   auto input_fops = create_bare_fops(kernel.inputs);
-   auto output_fops = create_bare_fops(kernel.outputs);
-
-   const int test_vdim = mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int test_op_dim =
-      mfem::get<hardcoded_output_idx>(output_fops).size_on_qp /
-      mfem::get<hardcoded_output_idx>(output_fops).vdim;
-   const int num_test_dof = R->Height() /
-                            mfem::get<hardcoded_output_idx>(output_fops).vdim /
-                            num_entities;
-
-   auto ir_weights = Reshape(this->op.integration_rule.GetWeights().Read(),
-                             num_qp);
-
-   auto input_size_on_qp = get_input_size_on_qp(kernel.inputs,
-                                                std::make_index_sequence<kernel.num_kinputs> {});
-
-   // Check which qf inputs are dependent on the dependent variable
-   std::array<bool, kernel.num_kinputs> kinput_is_dependent;
-   bool no_kinput_is_dependent = true;
-   for (int i = 0; i < kinput_is_dependent.size(); i++)
-   {
-      if (kinput_to_field[i] == derivative_idx)
-      {
-         no_kinput_is_dependent = false;
-         kinput_is_dependent[i] = true;
-         // out << "function input " << i << " is dependent on "
-         //     << op.fields[kinput_to_field[i]].field_label << "\n";
-      }
-      else
-      {
-         kinput_is_dependent[i] = false;
-      }
-   }
-
-   bool with_derivatives = true;
-   auto shmem_info = get_shmem_info<entity_t>(input_dtq_maps,
-                                              output_dtq_maps,
-                                              op.fields,
-                                              num_entities,
-                                              kernel.inputs,
-                                              num_qp,
-                                              input_size_on_qp,
-                                              da_size_on_qp,
-                                              derivative_idx);
-
-   Vector shmem_cache(shmem_info.total_size);
-
-   // print_shared_memory_info(shmem_info);
-
-   func = [=](Vector &ye_mem) mutable
-   {
-      if (no_kinput_is_dependent)
-      {
-         return;
-      }
-
-      restriction<entity_t>(direction, direction_l, direction_e,
-                            op.element_dof_ordering);
-
-      auto ye = Reshape(ye_mem.ReadWrite(), num_test_dof, test_vdim, num_entities);
-      auto wrapped_fields_e = wrap_fields(this->fields_e, shmem_info.field_sizes, num_entities);
-      auto wrapped_direction_e = Reshape(direction_e.ReadWrite(), shmem_info.direction_size, num_entities);
-
-      forall([=] MFEM_HOST_DEVICE (int e, double *shmem)
-      {
-         auto input_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::INPUT_DTQ],
-            shmem_info.input_dtq_sizes,
-            input_dtq_maps);
-
-         auto output_dtq_shmem = load_dtq_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::OUTPUT_DTQ],
-            shmem_info.output_dtq_sizes,
-            output_dtq_maps);
-
-         auto fields_shmem = load_field_mem(
-            shmem,
-            shmem_info.offsets[SharedMemory::Index::FIELD],
-            shmem_info.field_sizes,
-            kinput_to_field,
-            input_fops,
-            wrapped_fields_e,
-            e,
-         std::make_index_sequence<kernel.num_kinputs> {});
-
-         auto direction_shmem = load_direction_mem(
-                                   shmem,
-                                   shmem_info.offsets[SharedMemory::Index::DIRECTION],
-                                   shmem_info.direction_size,
-                                   wrapped_direction_e,
-                                   e);
-
-         // These methods don't copy, they simply create a `DeviceTensor` object
-         // that points to correct chunks of the shared memory pool.
-         auto input_shmem = load_input_mem(
-                               shmem,
-                               shmem_info.offsets[SharedMemory::Index::INPUT],
-                               shmem_info.input_sizes,
-                               num_qp);
-
-         auto shadow_shmem = load_input_mem(
-                                shmem,
-                                shmem_info.offsets[SharedMemory::Index::SHADOW],
-                                shmem_info.input_sizes,
-                                num_qp);
-
-         auto residual_shmem = load_residual_mem(
-                                  shmem,
-                                  shmem_info.offsets[SharedMemory::Index::OUTPUT],
-                                  shmem_info.residual_size,
-                                  num_qp);
-
-         auto scratch_mem = load_scratch_mem(
-                               shmem,
-                               shmem_info.offsets[SharedMemory::Index::TEMP],
-                               shmem_info.temp_sizes);
-
-         map_fields_to_quadrature_data<TensorProduct>(
-            input_shmem, fields_shmem, input_dtq_shmem, input_fops, ir_weights, scratch_mem,
-            std::make_index_sequence<kernel.num_kinputs> {});
-
-         zero_all(shadow_shmem);
-         map_direction_to_quadrature_data_conditional<TensorProduct>(
-            shadow_shmem, direction_shmem, input_dtq_shmem, input_fops, ir_weights,
-            scratch_mem, kinput_is_dependent,
-            std::make_index_sequence<kernel.num_kinputs> {});
-
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-
-                  auto kernel_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-                  auto kernel_shadow_args = decay_tuple<typename kernel_t::kf_param_ts> {};
-
-                  auto r = Reshape(&residual_shmem(0, q), da_size_on_qp);
-                  apply_kernel_fwddiff_enzyme(
-                     r,
-                     kernel.func,
-                     kernel_args,
-                     input_shmem,
-                     kernel_shadow_args,
-                     shadow_shmem,
-                     q);
-                  // printf(">>>>> WARNING: AD DISABLED\n");
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         auto fhat = Reshape(&residual_shmem(0, 0), test_vdim, test_op_dim, num_qp);
-         auto y = Reshape(&ye(0, 0, e), num_test_dof, test_vdim);
-         map_quadrature_data_to_fields<TensorProduct>(y, fhat,
-                                                      mfem::get<0>(output_fops),
-                                                      output_dtq_shmem[hardcoded_output_idx],
-                                                      scratch_mem);
-      }, num_entities, q1d, q1d, 1, shmem_info.total_size, shmem_cache.ReadWrite());
-
-      R->MultTranspose(ye_mem, derivative_action_l);
-   };
-
-   if constexpr (std::is_same_v<decltype(output_fop), One>)
-   {
-      prolongation_transpose = [&](Vector &r_local, Vector &y)
-      {
-         double local_sum = r_local.Sum();
-         MPI_Allreduce(&local_sum, y.GetData(), 1, MPI_DOUBLE, MPI_SUM,
-                       op.mesh.GetComm());
-         MFEM_ASSERT(y.Size() == 1, "output size doesn't match kernel description");
-      };
-   }
-   else
-   {
-      auto P = get_prolongation(op.fields[test_space_field_idx]);
-      prolongation_transpose = [P](const Vector &r_local, Vector &y)
-      {
-         P->MultTranspose(r_local, y);
-      };
-   }
-}
-
-// #include "dfem_assemble_vector.icc"
-// #include "dfem_assemble_hypreparmatrix.icc"
-
-}

examples/dfem/dfem_fieldoperator.hpp

@@ -1,118 +0,0 @@
-#pragma once
-
-#include <string>
-
-class FieldOperator
-{
-public:
-   FieldOperator(std::string field_label = "", int size_on_qp = 0) :
-      field_label(field_label),
-      size_on_qp(size_on_qp) {};
-
-   std::string field_label;
-
-   int size_on_qp = -1;
-
-   int dim = -1;
-
-   int vdim = -1;
-};
-
-class None : public FieldOperator
-{
-public:
-   None(std::string field_label) :
-      FieldOperator(field_label) {}
-};
-
-class Weight : public FieldOperator
-{
-public:
-   Weight() : FieldOperator("quadrature_weights") {};
-};
-
-class Value : public FieldOperator
-{
-public:
-   Value(std::string field_label) : FieldOperator(field_label) {};
-};
-
-class Gradient : public FieldOperator
-{
-public:
-   Gradient(std::string field_label) : FieldOperator(field_label) {};
-};
-
-class Curl : public FieldOperator
-{
-public:
-   Curl(std::string field_label) : FieldOperator(field_label) {};
-};
-
-class Div : public FieldOperator
-{
-public:
-   Div(std::string field_label) : FieldOperator(field_label) {};
-};
-
-class FaceValueLeft : public FieldOperator
-{
-public:
-   FaceValueLeft(std::string field_label) : FieldOperator(field_label) {};
-};
-
-class FaceValueRight : public FieldOperator
-{
-public:
-   FaceValueRight(std::string field_label) : FieldOperator(field_label) {};
-};
-
-class FaceNormal : public FieldOperator
-{
-public:
-   FaceNormal(std::string field_label) : FieldOperator(field_label) {};
-};
-
-class One : public FieldOperator
-{
-public:
-   One(std::string field_label) : FieldOperator(field_label) {};
-};
-
-namespace BareFieldOperator
-{
-
-struct Base
-{
-   Base(FieldOperator &o)
-   {
-      size_on_qp = o.size_on_qp;
-      dim = o.dim;
-      vdim = o.vdim;
-   };
-   int size_on_qp = -1;
-   int dim = -1;
-   int vdim = -1;
-};
-
-struct None : Base
-{
-   None(FieldOperator &o) : Base(o) {}
-};
-
-struct Weight : Base
-{
-   Weight(FieldOperator &o) : Base(o) {}
-};
-
-struct Value : Base
-{
-   Value(FieldOperator &o) : Base(o) {}
-};
-
-struct Gradient : Base
-{
-   Gradient(FieldOperator &o) : Base(o) {}
-};
-
-}

examples/dfem/dfem_integrate.hpp

@@ -1,292 +0,0 @@
-#pragma once
-
-#include "dfem_util.hpp"
-#include <type_traits>
-
-namespace mfem
-{
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_impl(DeviceTensor<2, double> &y,
-                                        const DeviceTensor<3, double> &f,
-                                        const output_t &output,
-                                        const DofToQuadMap &dtq)
-{
-   auto B = dtq.B;
-   auto G = dtq.G;
-   // assuming the quadrature point residual has to "play nice with
-   // the test function"
-   if constexpr (std::is_same_v<std::decay_t<output_t>, BareFieldOperator::Value>)
-   {
-      const auto [num_qp, cdim, num_dof] = B.GetShape();
-      const int vdim = output.vdim > 0 ? output.vdim : cdim ;
-      for (int dof = 0; dof < num_dof; dof++)
-      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            double acc = 0.0;
-            for (int qp = 0; qp < num_qp; qp++)
-            {
-               acc += B(qp, 0, dof) * f(vd, 0, qp);
-            }
-            y(dof, vd) += acc;
-         }
-      }
-   }
-   else if constexpr (
-      std::is_same_v<std::decay_t<output_t>, BareFieldOperator::Gradient>)
-   {
-      const auto [num_qp, dim, num_dof] = G.GetShape();
-      const int vdim = output.vdim;
-      for (int dof = 0; dof < num_dof; dof++)
-      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            double acc = 0.0;
-            for (int d = 0; d < dim; d++)
-            {
-               for (int qp = 0; qp < num_qp; qp++)
-               {
-                  acc += G(qp, d, dof) * f(vd, d, qp);
-               }
-            }
-            y(dof, vd) += acc;
-         }
-      }
-   }
-   // else if constexpr (std::is_same_v<std::decay_t<output_t>, One>)
-   // {
-   //    // This is the "integral over all quadrature points type" applying
-   //    // B = 1 s.t. B^T * C \in R^1.
-   //    const auto [a, b, num_qp] = B.GetShape();
-   //    auto cc = Reshape(&c(0, 0, 0), num_qp);
-   //    for (int i = 0; i < num_qp; i++)
-   //    {
-   //       y(0, 0) += cc(i);
-   //    }
-   // }
-   else if constexpr (
-      std::is_same_v<std::decay_t<output_t>, BareFieldOperator::None>)
-   {
-      const auto [vdim, dim, num_qp] = G.GetShape();
-      auto cc = Reshape(&f(0, 0, 0), num_qp * vdim);
-      auto yy = Reshape(&y(0, 0), num_qp * vdim);
-      for (int i = 0; i < num_qp * vdim; i++)
-      {
-         yy(i) = cc(i);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("quadrature data mapping to field is not implemented for"
-                 " this field descriptor");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_tensor_impl(DeviceTensor<2, double> &y,
-                                               const DeviceTensor<3, double> &f,
-                                               const output_t &output,
-                                               const DofToQuadMap &dtq,
-                                               std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   auto B = dtq.B;
-   auto G = dtq.G;
-
-   if constexpr (std::is_same_v<std::decay_t<output_t>, BareFieldOperator::Value>)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  double acc = 0.0;
-                  for (int qx = 0; qx < q1d; qx++)
-                  {
-                     acc += fqp(vd, 0, qx, qy, qz) * B(qx, 0, dx);
-                  }
-                  s0(qz, qy, dx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  double acc = 0.0;
-                  for (int qy = 0; qy < q1d; qy++)
-                  {
-                     acc += s0(qz, qy, dx) * B(qy, 0, dy);
-                  }
-                  s1(qz, dy, dx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(dz, z, d1d)
-               {
-                  double acc = 0.0;
-                  for (int qz = 0; qz < q1d; qz++)
-                  {
-                     acc += s1(qz, dy, dx) * B(qz, 0, dz);
-                  }
-                  yd(dx, dy, dz, vd) += acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (
-      std::is_same_v<std::decay_t<output_t>, BareFieldOperator::Gradient>)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, q1d, d1d);
-      auto s2 = Reshape(&scratch_mem[2](0), q1d, q1d, d1d);
-      auto s3 = Reshape(&scratch_mem[3](0), q1d, d1d, d1d);
-      auto s4 = Reshape(&scratch_mem[4](0), q1d, d1d, d1d);
-      auto s5 = Reshape(&scratch_mem[5](0), q1d, d1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qx = 0; qx < q1d; qx++)
-                  {
-                     uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
-                     uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
-                     uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
-                  }
-                  s0(qz, qy, dx) = uvw[0];
-                  s1(qz, qy, dx) = uvw[1];
-                  s2(qz, qy, dx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qy = 0; qy < q1d; qy++)
-                  {
-                     uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
-                     uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
-                     uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
-                  }
-                  s3(qz, dy, dx) = uvw[0];
-                  s4(qz, dy, dx) = uvw[1];
-                  s5(qz, dy, dx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qz = 0; qz < q1d; qz++)
-                  {
-                     uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
-                  }
-                  yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (
-      std::is_same_v<std::decay_t<output_t>, BareFieldOperator::None>)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d, q1d);
-      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
-
-      for (int sq = 0; sq < output.size_on_qp; sq++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else
-   {
-      MFEM_ABORT("quadrature data mapping to field is not implemented for"
-                 " this field descriptor with sum factorization on tensor product elements");
-   }
-}
-
-template <typename T = NonTensorProduct, typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields(DeviceTensor<2, double> &y,
-                                   const DeviceTensor<3, double> &f,
-                                   const output_t &output,
-                                   const DofToQuadMap &dtq,
-                                   std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   if constexpr (std::is_same_v<T, NonTensorProduct>)
-   {
-      map_quadrature_data_to_fields_impl(y, f, output, dtq);
-   }
-   else if constexpr (std::is_same_v<T, TensorProduct>)
-   {
-      map_quadrature_data_to_fields_tensor_impl(y, f, output, dtq, scratch_mem);
-   }
-}
-
-}

examples/dfem/dfem_interpolate.hpp

@@ -1,400 +0,0 @@
-#pragma once
-
-#include "dfem_util.hpp"
-#include <type_traits>
-
-namespace mfem
-{
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE inline
-void map_field_to_quadrature_data_tensor_product(
-   DeviceTensor<2> &field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const double> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   auto B = dtq.B;
-   auto G = dtq.G;
-
-   if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, BareFieldOperator::Value>)
-   {
-      auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d, q1d);
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  double acc = 0.0;
-                  for (int dx = 0; dx < d1d; dx++)
-                  {
-                     acc += B(qx, 0, dx) * field(dx, dy, dz, vd);
-                  }
-                  s0(dz, dy, qx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               MFEM_FOREACH_THREAD(qy, y, q1d)
-               {
-                  double acc = 0.0;
-                  for (int dy = 0; dy < d1d; dy++)
-                  {
-                     acc += s0(dz, dy, qx) * B(qy, 0, dy);
-                  }
-                  s1(dz, qy, qx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  double acc = 0.0;
-                  for (int dz = 0; dz < d1d; dz++)
-                  {
-                     acc += s1(dz, qy, qx) * B(qz, 0, dz);
-                  }
-                  fqp(vd, qx, qy, qz) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, BareFieldOperator::Gradient>)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
-
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, d1d, q1d);
-      auto s2 = Reshape(&scratch_mem[2](0), d1d, q1d, q1d);
-      auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
-      auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uv[2] = {0.0, 0.0};
-                  for (int dx = 0; dx < d1d; dx++)
-                  {
-                     const real_t f = field(dx, dy, dz, vd);
-                     uv[0] += f * B(qx, 0, dx);
-                     uv[1] += f * G(qx, 0, dx);
-                  }
-                  s0(dz, dy, qx) = uv[0];
-                  s1(dz, dy, qx) = uv[1];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int dy = 0; dy < d1d; dy++)
-                  {
-                     const real_t s0i = s0(dz, dy, qx);
-                     uvw[0] += s1(dz, dy, qx) * B(qy, 0, dy);
-                     uvw[1] += s0i * G(qy, 0, dy);
-                     uvw[2] += s0i * B(qy, 0, dy);
-                  }
-                  s2(dz, qy, qx) = uvw[0];
-                  s3(dz, qy, qx) = uvw[1];
-                  s4(dz, qy, qx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int dz = 0; dz < d1d; dz++)
-                  {
-                     uvw[0] += s2(dz, qy, qx) * B(qz, 0, dz);
-                     uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
-                     uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
-                  }
-                  fqp(vd, 0, qx, qy, qz) = uvw[0];
-                  fqp(vd, 1, qx, qy, qz) = uvw[1];
-                  fqp(vd, 2, qx, qy, qz) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   // TODO: Create separate function for clarity
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, BareFieldOperator::Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      // TODO: eeek
-      const int q1d = (int)floor(pow(num_qp, 1.0/input.dim) + 0.5);
-      auto w = Reshape(&integration_weights[0], q1d, q1d, q1d);
-      auto f = Reshape(&field_qp[0], q1d, q1d, q1d);
-      MFEM_FOREACH_THREAD(qx, x, q1d)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(qz, z, q1d)
-            {
-               f(qx, qy, qz) = w(qx, qy, qz);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, BareFieldOperator::None>)
-   {
-      const int q1d = B.GetShape()[0];
-      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d * q1d);
-      field_qp = field;
-   }
-   else
-   {
-      static_assert(always_false<std::decay_t<field_operator_t>>,
-                    "can't map field to quadrature data");
-   }
-}
-
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE
-void map_field_to_quadrature_data(
-   DeviceTensor<2> field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1, const double> &field_e,
-   field_operator_t &input,
-   DeviceTensor<1, const double> integration_weights)
-{
-   auto B = dtq.B;
-   auto G = dtq.G;
-   if constexpr (std::is_same_v<field_operator_t, BareFieldOperator::Value>)
-   {
-      auto [num_qp, dim, num_dof] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e(0), num_dof, vdim);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         for (int qp = 0; qp < num_qp; qp++)
-         {
-            double acc = 0.0;
-            for (int dof = 0; dof < num_dof; dof++)
-            {
-               acc += B(qp, 0, dof) * field(dof, vd);
-            }
-            field_qp(vd, qp) = acc;
-         }
-      }
-   }
-   else if constexpr (
-      std::is_same_v<field_operator_t, BareFieldOperator::Gradient>)
-   {
-      const auto [num_qp, dim, num_dof] = G.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e(0), num_dof, vdim);
-
-      auto f = Reshape(&field_qp[0], vdim, dim, num_qp);
-      for (int qp = 0; qp < num_qp; qp++)
-      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            for (int d = 0; d < dim; d++)
-            {
-               double acc = 0.0;
-               for (int dof = 0; dof < num_dof; dof++)
-               {
-                  acc += G(qp, d, dof) * field(dof, vd);
-               }
-               f(vd, d, qp) = acc;
-            }
-         }
-      }
-   }
-   // else if constexpr (std::is_same_v<field_operator_t, FaceNormal>)
-   // {
-   //    auto normal = geometric_factors.normal;
-   //    auto [num_qp, dim, num_entities] = normal.GetShape();
-   //    auto f = Reshape(&field_qp[0], dim, num_qp);
-   //    for (int qp = 0; qp < num_qp; qp++)
-   //    {
-   //       for (int d = 0; d < dim; d++)
-   //       {
-   //          f(d, qp) = normal(qp, d, entity_idx);
-   //       }
-   //    }
-   // }
-   // TODO: Create separate function for clarity
-   else if constexpr (std::is_same_v<field_operator_t, BareFieldOperator::Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      auto f = Reshape(&field_qp[0], num_qp);
-      for (int qp = 0; qp < num_qp; qp++)
-      {
-         f(qp) = integration_weights(qp);
-      }
-   }
-   else if constexpr (std::is_same_v<field_operator_t, BareFieldOperator::None>)
-   {
-      auto [num_qp, unused, num_dof] = B.GetShape();
-      const int size_on_qp = input.size_on_qp;
-      const auto field = Reshape(&field_e[0], size_on_qp * num_qp);
-      auto f = Reshape(&field_qp[0], size_on_qp * num_qp);
-      for (int i = 0; i < size_on_qp * num_qp; i++)
-      {
-         f(i) = field(i);
-      }
-   }
-   else
-   {
-      static_assert(always_false<field_operator_t>,
-                    "can't map field to quadrature data");
-
-   }
-}
-
-template <typename T = NonTensorProduct, size_t num_kinputs, typename field_operator_ts, std::size_t... i>
-MFEM_HOST_DEVICE inline
-void map_fields_to_quadrature_data(
-   std::array<DeviceTensor<2>, num_kinputs> &fields_qp,
-   const std::array<DeviceTensor<1>, num_kinputs> &fields_e,
-   const std::array<DofToQuadMap, num_kinputs> &dtqmaps,
-   const field_operator_ts &fops,
-   const DeviceTensor<1, const double> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   std::index_sequence<i...>)
-{
-   if constexpr (std::is_same_v<T, TensorProduct>)
-   {
-
-      (map_field_to_quadrature_data_tensor_product(fields_qp[i],
-                                                   dtqmaps[i], fields_e[i],
-                                                   mfem::get<i>(fops), integration_weights,
-                                                   scratch_mem),
-       ...);
-   }
-   else
-   {
-      (map_field_to_quadrature_data(fields_qp[i],
-                                    dtqmaps[i], fields_e[i],
-                                    mfem::get<i>(fops), integration_weights),
-       ...);
-   }
-}
-
-template <typename T, typename field_operator_t>
-MFEM_HOST_DEVICE
-void map_field_to_quadrature_data_conditional(
-   DeviceTensor<2> &field_qp,
-   const DeviceTensor<1> &field_e,
-   const DofToQuadMap &dtqmap,
-   field_operator_t &fop,
-   const DeviceTensor<1, const double> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const bool &condition)
-{
-   if (condition)
-   {
-      if constexpr (std::is_same_v<T, TensorProduct>)
-      {
-         map_field_to_quadrature_data_tensor_product(field_qp, dtqmap,
-                                                     field_e, fop,
-                                                     integration_weights,
-                                                     scratch_mem);
-      }
-      else
-      {
-         map_field_to_quadrature_data(field_qp, dtqmap, field_e, fop,
-                                      integration_weights);
-      }
-   }
-}
-
-template <typename T = NonTensorProduct, size_t num_fields, size_t num_kinputs, typename field_operator_ts, std::size_t... i>
-MFEM_HOST_DEVICE
-void map_fields_to_quadrature_data_conditional(
-   std::array<DeviceTensor<2>, num_kinputs> &fields_qp,
-   const std::array<DeviceTensor<1, const double>, num_fields> &fields_e,
-   const std::array<DofToQuadMap, num_kinputs> &dtqmaps,
-   field_operator_ts fops,
-   const DeviceTensor<1, const double> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const std::array<bool, num_kinputs> &conditions,
-   std::index_sequence<i...>)
-{
-   (map_field_to_quadrature_data_conditional<T>(fields_qp[i],
-                                                fields_e[i],
-                                                dtqmaps[i],
-                                                mfem::get<i>(fops),
-                                                integration_weights,
-                                                scratch_mem,
-                                                conditions[i]),
-    ...);
-}
-
-template <typename T = NonTensorProduct, size_t num_kinputs, typename field_operator_ts, std::size_t... i>
-MFEM_HOST_DEVICE
-void map_direction_to_quadrature_data_conditional(
-   std::array<DeviceTensor<2>, num_kinputs> &directions_qp,
-   const DeviceTensor<1> &direction_e,
-   const std::array<DofToQuadMap, num_kinputs> &dtqmaps,
-   field_operator_ts fops,
-   const DeviceTensor<1, const double> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const std::array<bool, num_kinputs> &conditions,
-   std::index_sequence<i...>)
-{
-   (map_field_to_quadrature_data_conditional<T>(directions_qp[i],
-                                                direction_e,
-                                                dtqmaps[i],
-                                                mfem::get<i>(fops),
-                                                integration_weights,
-                                                scratch_mem,
-                                                conditions[i]),
-    ...);
-}
-
-}

examples/dfem/dfem_parametricspace.hpp

@@ -1,99 +0,0 @@
-#pragma once
-
-#include <mfem.hpp>
-
-namespace mfem
-{
-
-class ParametricSpace
-{
-
-public:
-   ParametricSpace(int spatial_dim, int local_size, int element_size,
-                   int total_size) :
-      spatial_dim(spatial_dim),
-      local_size(local_size),
-      element_size(element_size),
-      total_size(total_size),
-      identity(total_size)
-   {
-      dtq.ndof = (int)floor(pow(element_size, 1.0/spatial_dim) + 0.5);
-      dtq.nqpt = dtq.ndof;
-   }
-
-   ParametricSpace(int local_size) :
-      local_size(local_size),
-      element_size(local_size),
-      total_size(local_size),
-      identity(local_size)
-   {
-      dtq.ndof = (int)floor(pow(element_size, 1.0/spatial_dim) + 0.5);
-      dtq.nqpt = dtq.ndof;
-   }
-
-   int Dimension() const
-   {
-      return spatial_dim;
-   }
-
-   int GetLocalSize() const
-   {
-      return local_size;
-   }
-
-   int GetElementSize() const
-   {
-      return element_size;
-   }
-
-   int GetTotalSize() const
-   {
-      return total_size;
-   }
-
-   const DofToQuad &GetDofToQuad() const
-   {
-      return dtq;
-   }
-
-   const Operator *GetProlongation() const
-   {
-      return &identity;
-   }
-
-   const Operator *GetRestriction() const
-   {
-      return &identity;
-   }
-
-private:
-   int spatial_dim;
-
-   // Hint for the local dimension. E.g. the size on the quadrature point or vdim.
-   int local_size;
-
-   // Size of the data on an element
-   int element_size;
-
-   int total_size;
-
-   IdentityOperator identity;
-
-   DofToQuad dtq;
-};
-
-class ParametricFunction : public Vector
-{
-public:
-   ParametricFunction(ParametricSpace &space) :
-      Vector(space.GetTotalSize()),
-      space(space)
-   {}
-
-   ParametricSpace &space;
-
-   using Vector::operator=;
-
-};
-
-}

examples/dfem/dfem_qfunction.hpp

@@ -1,268 +0,0 @@
-#pragma once
-#include "dfem_util.hpp"
-
-namespace mfem
-{
-
-MFEM_HOST_DEVICE inline
-void process_kf_arg(
-   const DeviceTensor<1> &u,
-   double &arg)
-{
-   arg = u(0);
-}
-
-MFEM_HOST_DEVICE inline
-void process_kf_arg(
-   const DeviceTensor<1> &u,
-   internal::tensor<double> &arg)
-{
-   arg(0) = u(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE
-void process_kf_arg(
-   const DeviceTensor<1> &u,
-   internal::tensor<T, n> &arg)
-{
-   for (int i = 0; i < n; i++)
-   {
-      arg(i) = u(i);
-   }
-}
-
-template <int n, int m>
-MFEM_HOST_DEVICE
-void process_kf_arg(
-   const DeviceTensor<1> &u,
-   internal::tensor<double, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i) = u((i * m) + j);
-      }
-   }
-   // assuming col major layout. translating to row major.
-   // i + N_i*j
-   // arg(0, 0) = u(0);
-   // arg(0, 1) = u(0 + 2 * 1);
-   // arg(1, 0) = u(1 + 2 * 0);
-   // arg(1, 1) = u(1 + 2 * 1);
-}
-
-template <typename arg_type>
-MFEM_HOST_DEVICE
-void process_kf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)
-{
-   // out << "qp: " << qp << "\n";
-   // for (int i = 0; i < u.GetShape()[0] * u.GetShape()[1]; i++)
-   // {
-   //    out << (&u(0, 0))[i] << " ";
-   // }
-   // out << "\n";
-
-   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
-   // for (int i = 0; i < u_qp.GetShape()[0]; i++)
-   // {
-   //    out << (&u_qp(0))[i] << " ";
-   // }
-   // out << "\n";
-
-   process_kf_arg(u_qp, arg);
-}
-
-template <size_t num_fields, typename kf_args, std::size_t... i>
-MFEM_HOST_DEVICE
-void process_kf_args(const std::array<DeviceTensor<2>, num_fields> &u,
-                     kf_args &args, int qp, std::index_sequence<i...>)
-{
-   (process_kf_arg(u[i], mfem::get<i>(args), qp), ...);
-}
-
-template <typename T0, typename T1> inline
-Vector process_kf_result(T0, T1)
-{
-   static_assert(always_false<T0, T1>,
-                 "process_kf_result not implemented for result type");
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_kf_result(
-   DeviceTensor<1, T> &r,
-   const double &x)
-{
-   r(0) = x;
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_kf_result(
-   DeviceTensor<1, T> &r,
-   const internal::tensor<T> &x)
-{
-   r(0) = x(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_kf_result(
-   DeviceTensor<1, T> &r,
-   const internal::tensor<T, n> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      r(i) = x(i);
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_kf_result(
-   DeviceTensor<1, T> &r,
-   const internal::tensor<T, n, m> &x)
-{
-   // out << "x: " << x << "\n";
-   for (size_t i = 0; i < n; i++)
-   {
-      for (size_t j = 0; j < m; j++)
-      {
-         r(i + n * j) = x(i, j);
-      }
-   }
-
-   // out << "r: ";
-   // for (int i = 0; i < r.GetShape()[0]; i++)
-   // {
-   //    out << r(i) << " ";
-   // }
-   // out << "\n\n";
-}
-
-template <typename T> inline
-void process_kf_arg(const DeviceTensor<1> &u, const DeviceTensor<1> &v,
-                    double &arg)
-{
-   arg = u(0);
-}
-
-template <int n, int m> inline
-void process_kf_arg(const DeviceTensor<1> &u, const DeviceTensor<1> &v,
-                    internal::tensor<double, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i) = u((i * m) + j);
-      }
-   }
-}
-
-template <typename arg_type> inline
-void process_kf_arg(const DeviceTensor<2> &u, const DeviceTensor<2> &v,
-                    arg_type &arg, int qp)
-{
-   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
-   const auto v_qp = Reshape(&v(0, qp), v.GetShape()[0]);
-   process_kf_arg(u_qp, v_qp, arg);
-}
-
-template <size_t num_fields, typename kf_args, std::size_t... i> inline
-void process_kf_args(std::array<DeviceTensor<2>, num_fields> &u,
-                     std::array<DeviceTensor<2>, num_fields> &v,
-                     kf_args &args, int qp, std::index_sequence<i...>)
-{
-   (process_kf_arg(u[i], v[i], mfem::get<i>(args), qp), ...);
-}
-
-template <typename kernel_func_t, typename kernel_args_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel(
-   DeviceTensor<1, double> &f_qp,
-   const kernel_func_t &kf,
-   kernel_args_ts &args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   int qp)
-{
-   process_kf_args(u, args, qp,
-                   std::make_index_sequence<mfem::tuple_size<kernel_args_ts>::value> {});
-
-   process_kf_result(f_qp, mfem::get<0>(mfem::apply(kf, args)));
-}
-
-// Version for active function arguments only
-//
-// This is an Enzyme regression and can be removed in later versions.
-template <typename kernel_t, typename arg_ts, std::size_t... Is,
-          typename inactive_arg_ts>
-inline auto fwddiff_apply_enzyme_indexed(kernel_t kernel, arg_ts &&args,
-                                         arg_ts &&shadow_args,
-                                         std::index_sequence<Is...>,
-                                         inactive_arg_ts &&inactive_args,
-                                         std::index_sequence<>)
-{
-   using kf_return_t = typename create_function_signature<
-                       decltype(&kernel_t::operator())>::type::return_t;
-   return __enzyme_fwddiff<kf_return_t>(
-             +kernel, enzyme_dup, &mfem::get<Is>(args)..., enzyme_interleave,
-             &mfem::get<Is>(shadow_args)...);
-}
-
-// Interleave function arguments for enzyme
-template <typename kernel_t, typename arg_ts, std::size_t... Is,
-          typename inactive_arg_ts, std::size_t... Js>
-inline auto fwddiff_apply_enzyme_indexed(kernel_t kernel, arg_ts &&args,
-                                         arg_ts &&shadow_args,
-                                         std::index_sequence<Is...>,
-                                         inactive_arg_ts &&inactive_args,
-                                         std::index_sequence<Js...>)
-{
-   using kf_return_t = typename create_function_signature<
-                       decltype(&kernel_t::operator())>::type::return_t;
-   return __enzyme_fwddiff<kf_return_t>(
-             +kernel, enzyme_dup, &std::get<Is>(args)..., enzyme_const,
-             &mfem::get<Js>(inactive_args)..., enzyme_interleave,
-             &mfem::get<Is>(shadow_args)...);
-}
-
-template <typename kernel_t, typename arg_ts, typename inactive_arg_ts>
-inline auto fwddiff_apply_enzyme(kernel_t kernel, arg_ts &&args,
-                                 arg_ts &&shadow_args,
-                                 inactive_arg_ts &&inactive_args)
-{
-   auto arg_indices = std::make_index_sequence<
-                      mfem::tuple_size<std::remove_reference_t<arg_ts>>::value> {};
-
-   auto inactive_arg_indices = std::make_index_sequence<
-                               mfem::tuple_size<std::remove_reference_t<inactive_arg_ts>>::value> {};
-
-   return fwddiff_apply_enzyme_indexed(kernel, args, shadow_args, arg_indices,
-                                       inactive_args, inactive_arg_indices);
-}
-
-template <typename kf_t, typename kernel_arg_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel_fwddiff_enzyme(
-   DeviceTensor<1, double> &f_qp,
-   const kf_t &kf,
-   kernel_arg_ts &args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   kernel_arg_ts &shadow_args,
-   const std::array<DeviceTensor<2>, num_args> &v,
-   int qp_idx)
-{
-   process_kf_args(u, args, qp_idx,
-                   std::make_index_sequence<mfem::tuple_size<kernel_arg_ts>::value> {});
-
-   process_kf_args(v, shadow_args, qp_idx,
-                   std::make_index_sequence<mfem::tuple_size<kernel_arg_ts>::value> {});
-
-   process_kf_result(f_qp,
-                     mfem::get<0>(fwddiff_apply_enzyme(kf, args, shadow_args, mfem::tuple<> {})));
-}
-
-}

examples/dfem/dfem_smm.hpp

@@ -1,116 +0,0 @@
-#pragma once
-
-#include <mfem.hpp>
-
-class SharedMemoryManager
-{
-private:
-   struct MemoryBlock
-   {
-      char* ptr;
-      int size;
-      bool used;
-   };
-
-   MFEM_HOST_DEVICE static const int MAX_BLOCKS = 16;
-   MFEM_HOST_DEVICE static MemoryBlock blocks[MAX_BLOCKS];
-   MFEM_HOST_DEVICE static int num_blocks;
-   MFEM_HOST_DEVICE static char* base_ptr;
-
-public:
-   MFEM_HOST_DEVICE static void init(void* shmem, int total_size)
-   {
-      base_ptr = static_cast<char*>(shmem);
-      num_blocks = 1;
-      blocks[0] = {base_ptr, total_size, false};
-   }
-
-   template<typename T>
-   MFEM_HOST_DEVICE static T* reserve(int n)
-   {
-      int size_bytes = n * sizeof(T);
-      for (int i = 0; i < num_blocks; ++i)
-      {
-         if (!blocks[i].used && blocks[i].size >= size_bytes)
-         {
-            blocks[i].used = true;
-            if (blocks[i].size > size_bytes)
-            {
-               // Split block
-               if (num_blocks < MAX_BLOCKS)
-               {
-                  blocks[num_blocks] = {blocks[i].ptr + size_bytes, blocks[i].size - size_bytes, false};
-                  ++num_blocks;
-                  blocks[i].size = size_bytes;
-               }
-            }
-            return reinterpret_cast<T*>(blocks[i].ptr);
-         }
-      }
-      return nullptr; // Allocation failed
-   }
-
-   MFEM_HOST_DEVICE static void release(void* ptr)
-   {
-      for (int i = 0; i < num_blocks; ++i)
-      {
-         if (blocks[i].ptr == ptr)
-         {
-            blocks[i].used = false;
-            return;
-         }
-      }
-   }
-
-   MFEM_HOST_DEVICE static void release_and_try_merge(void* ptr)
-   {
-      for (int i = 0; i < num_blocks; ++i)
-      {
-         if (blocks[i].ptr == ptr)
-         {
-            blocks[i].used = false;
-            merge_adjacent_free_blocks();
-            return;
-         }
-      }
-   }
-
-private:
-   MFEM_HOST_DEVICE static void merge_adjacent_free_blocks()
-   {
-      // Simple bubble sort for simplicity (can be optimized)
-      for (int i = 0; i < num_blocks - 1; ++i)
-      {
-         for (int j = 0; j < num_blocks - i - 1; ++j)
-         {
-            if (blocks[j].ptr > blocks[j + 1].ptr)
-            {
-               MemoryBlock temp = blocks[j];
-               blocks[j] = blocks[j + 1];
-               blocks[j + 1] = temp;
-            }
-         }
-      }
-
-      for (int i = 0; i < num_blocks - 1; ++i)
-      {
-         if (!blocks[i].used && !blocks[i + 1].used)
-         {
-            blocks[i].size += blocks[i + 1].size;
-            for (int j = i + 1; j < num_blocks - 1; ++j)
-            {
-               blocks[j] = blocks[j + 1];
-            }
-            --num_blocks;
-            --i;
-         }
-      }
-   }
-};
-
-MFEM_HOST_DEVICE SharedMemoryManager::MemoryBlock
-SharedMemoryManager::blocks[SharedMemoryManager::MAX_BLOCKS];
-
-MFEM_HOST_DEVICE int SharedMemoryManager::num_blocks;
-
-MFEM_HOST_DEVICE char* SharedMemoryManager::base_ptr;

examples/dfem/dfem_test_macro.hpp

@@ -1,39 +0,0 @@
-#pragma once
-#include "dfem.hpp"
-
-#define DFEM_TEST_MAIN(function)                                               \
-   int main(int argc, char* argv[])                                            \
-   {                                                                           \
-      Mpi::Init();                                                             \
-                                                                               \
-      const char* device_config = "cpu";                                       \
-      const char* mesh_file = "../data/ref-square.mesh";                       \
-      int polynomial_order = 1;                                                \
-      int ir_order = 2;                                                        \
-      int refinements = 0;                                                     \
-                                                                               \
-      OptionsParser args(argc, argv);                                          \
-      args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");         \
-      args.AddOption(&polynomial_order, "-o", "--order", "");                  \
-      args.AddOption(&refinements, "-r", "--r", "");                           \
-      args.AddOption(&ir_order, "-iro", "--iro", "");                          \
-      args.AddOption(&device_config, "-d", "--device",                         \
-         "Device configuration string, see Device::Configure().");             \
-      args.ParseCheck();                                                       \
-                                                                               \
-      Device device(device_config);                                            \
-      if (Mpi::Root() == 0)                                                    \
-      {                                                                        \
-         device.Print();                                                       \
-      }                                                                        \
-                                                                               \
-      out << std::setprecision(12);                                            \
-                                                                               \
-      int ret;                                                                 \
-                                                                               \
-      ret = function(mesh_file, refinements, polynomial_order);                \
-      out << #function;                                                        \
-      ret ? out << " FAILURE\n" : out << " OK\n";                              \
-                                                                               \
-      return ret;                                                              \
-   }\

examples/dfem/noisy.hpp

@@ -1,130 +0,0 @@
-// SPDX-ArtifactOfProjectName:     noisy
-// SPDX-ArtifactOfProjectHomePage: https://github.com/VincentZalzal/noisy
-// SPDX-FileCopyrightText:         Copyright 2024 Vincent Zalzal
-// SPDX-License-Identifier:        MIT
-
-#pragma once
-
-#include <iomanip>
-#include <iostream>
-
-namespace vz {
-
-struct Counters {
-    unsigned m_def_ctor    = 0;
-    unsigned m_copy_ctor   = 0;
-    unsigned m_move_ctor   = 0;
-    unsigned m_copy_assign = 0;
-    unsigned m_move_assign = 0;
-    unsigned m_dtor        = 0;
-
-    void reset() {
-        *this = {};
-    }
-
-    bool leaks() const {
-        return m_def_ctor + m_copy_ctor + m_move_ctor != m_dtor;
-    }
-
-    friend std::ostream& operator<<(std::ostream& os, const Counters& c) {
-        stream_counter(os, "Default constructor count: ", c.m_def_ctor   );
-        stream_counter(os, "Copy constructor count:    ", c.m_copy_ctor  );
-        stream_counter(os, "Move constructor count:    ", c.m_move_ctor  );
-        stream_counter(os, "Copy assignment count:     ", c.m_copy_assign);
-        stream_counter(os, "Move assignment count:     ", c.m_move_assign);
-        stream_counter(os, "Destructor count:          ", c.m_dtor       );
-        return os;
-    }
-
-    friend bool operator==(const Counters& lhs, const Counters& rhs) {
-        return
-            lhs.m_def_ctor    == rhs.m_def_ctor    &&
-            lhs.m_copy_ctor   == rhs.m_copy_ctor   &&
-            lhs.m_move_ctor   == rhs.m_move_ctor   &&
-            lhs.m_copy_assign == rhs.m_copy_assign &&
-            lhs.m_move_assign == rhs.m_move_assign &&
-            lhs.m_dtor        == rhs.m_dtor        ;
-    }
-
-    friend bool operator!=(const Counters& lhs, const Counters& rhs) { return !(lhs == rhs); }
-
-private:
-    static void stream_counter(std::ostream& os, const char* msg, unsigned value) {
-        if (value != 0)
-            os << msg << std::setw(2) << value << '\n';
-    }
-};
-
-namespace detail {
-
-struct Globals {
-    ~Globals() {
-        if (m_verbose)
-            std::cout << "\n===== Noisy counters =====\n" << m_counters;
-    }
-
-    Counters m_counters;
-    unsigned m_next_id = 0;
-    bool     m_verbose = true;
-};
-
-}
-
-class Noisy {
-private:
-    static detail::Globals& globals() {
-        static detail::Globals s_globals;
-        return s_globals;
-    }
-
-public:
-    static Counters& counters() { return globals().m_counters; }
-    static void set_verbose(bool verbose) { globals().m_verbose = verbose; }
-
-    Noisy() {
-        if (globals().m_verbose)
-            std::cout << *this << ": default constructor\n";
-        globals().m_counters.m_def_ctor++;
-    }
-
-    Noisy(const Noisy& other) {
-        if (globals().m_verbose)
-            std::cout << *this << ": copy constructor from " << other << '\n';
-        globals().m_counters.m_copy_ctor++;
-    }
-
-    Noisy(Noisy&& other) noexcept {
-        if (globals().m_verbose)
-            std::cout << *this << ": move constructor from " << other << '\n';
-        globals().m_counters.m_move_ctor++;
-    }
-
-    ~Noisy() {
-        if (globals().m_verbose)
-            std::cout << *this << ": destructor\n";
-        globals().m_counters.m_dtor++;
-    }
-
-    Noisy& operator=(const Noisy& other) {
-        if (globals().m_verbose)
-            std::cout << *this << ": copy assignment from " << other << '\n';
-        globals().m_counters.m_copy_assign++;
-        return *this;
-    }
-
-    Noisy& operator=(Noisy&& other) noexcept {
-        if (globals().m_verbose)
-            std::cout << *this << ": move assignment from " << other << '\n';
-        globals().m_counters.m_move_assign++;
-        return *this;
-    }
-
-    unsigned id() const { return m_id; }
-
-    friend std::ostream& operator<<(std::ostream& os, const Noisy& noisy) { return os << "Noisy(" << std::setw(2) << noisy.m_id << ')'; }
-
-private:
-    unsigned m_id = globals().m_next_id++;
-};
-
-}

examples/dfem/shared_mem_calc.md

@@ -1,49 +0,0 @@
-* Calculate shared memory requirements
-* Interpolation and integration
----
-* If grad involved, need B and G
-* Fit largest field, depends on polynomial order (#dofs)
-  -> vdim is irrelevant
-* Temporaries for each sum
-  - DDQ (d1d x d1d x q1d) x 2 -> DDQ0, DDQ1
-  - DQQ (d1d x q1d x q1d) x 3 -> DQQ0, DQQ1, DQQ2
-  - QQQ (q1d x q1d x q1d) x 3 -> QQQ0, QQQ1, QQQ2
-
-We need the following combinations at the same time
-(1) DDQ0 + DDQ1 + DQQ0 + DQQ1 + DQQ2
-(2) DQQ0 + DQQ1 + DQQ2 + QQQ0 + QQQ1 + QQQ2
-(3) QQQ0 + QQQ1 + QQQ2 + QQD0 + QQD1 + QQD2
-(4) QQD0 + QQD1 + QQD2 + QDD0 + QDD1 + QDD2
-
-Allocate largest memory footprint from 2, 3 or 4 and
-add memory footprint of fields and B/G.
-
-Annotations with NR and R mean "not reusable" and
-"reusable", respectively. This means the memory location is
-reused for _all_ e.g. interpolation of a value etc.
-
-----
-For the action of nonlinear diffusion in 2D we have
-(rho * |u|^2 \nabla u, \nabla v)
-
-* Load
-RHO (D x D) | R (after interpolation)
-U (D x D x VDIM) | R (after interpolation)
-B (Q x D) | NR
-G (Q x D) | NR
-
-* Interpolate Value
-Temporary (Q x D) | R
-R (Q x Q) | NR
-U (Q x Q x VDIM) | NR
-
-* Interpolate Grad
-Temporaries (Q x D) + (Q x D) | R
-U (Q x Q x DIM x VDIM) | NR
-
-Quadrature point function
--> purely thread local
-
-* Integrate Grad
-R | temp from Interpolation
-R | U from Load

examples/dfem/tuple.hpp

@@ -1,845 +0,0 @@
-// This is serac's tuple implementation
-
-#pragma once
-
-#include "general/backends.hpp"
-#include <utility>
-#include <mfem.hpp>
-#include <tuple>
-
-namespace mfem
-{
-
-/**
- * @tparam T the types stored in the tuple
- * @brief This is a class that mimics most of std::tuple's interface,
- * except that it is usable in CUDA kernels and admits some arithmetic operator overloads.
- *
- * see https://en.cppreference.com/w/cpp/utility/tuple for more information about std::tuple
- */
-template <typename... T>
-struct tuple
-{
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- */
-template <typename T0>
-struct tuple<T0>
-{
-   T0 v0;  ///< The first member of the tuple
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- * @tparam T1 The second type stored in the tuple
- */
-template <typename T0, typename T1>
-struct tuple<T0, T1>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- * @tparam T1 The second type stored in the tuple
- * @tparam T2 The third type stored in the tuple
- */
-template <typename T0, typename T1, typename T2>
-struct tuple<T0, T1, T2>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-   T2 v2;  ///< The third member of the tuple
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- * @tparam T1 The second type stored in the tuple
- * @tparam T2 The third type stored in the tuple
- * @tparam T3 The fourth type stored in the tuple
- */
-template <typename T0, typename T1, typename T2, typename T3>
-struct tuple<T0, T1, T2, T3>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-   T2 v2;  ///< The third member of the tuple
-   T3 v3;  ///< The fourth member of the tuple
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- * @tparam T1 The second type stored in the tuple
- * @tparam T2 The third type stored in the tuple
- * @tparam T3 The fourth type stored in the tuple
- * @tparam T4 The fifth type stored in the tuple
- */
-template <typename T0, typename T1, typename T2, typename T3, typename T4>
-struct tuple<T0, T1, T2, T3, T4>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-   T2 v2;  ///< The third member of the tuple
-   T3 v3;  ///< The fourth member of the tuple
-   T4 v4;  ///< The fifth member of the tuple
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- * @tparam T1 The second type stored in the tuple
- * @tparam T2 The third type stored in the tuple
- * @tparam T3 The fourth type stored in the tuple
- * @tparam T4 The fifth type stored in the tuple
- * @tparam T5 The sixth type stored in the tuple
- */
-template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5>
-struct tuple<T0, T1, T2, T3, T4, T5>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-   T2 v2;  ///< The third member of the tuple
-   T3 v3;  ///< The fourth member of the tuple
-   T4 v4;  ///< The fifth member of the tuple
-   T5 v5;  ///< The sixth member of the tuple
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- * @tparam T1 The second type stored in the tuple
- * @tparam T2 The third type stored in the tuple
- * @tparam T3 The fourth type stored in the tuple
- * @tparam T4 The fifth type stored in the tuple
- * @tparam T5 The sixth type stored in the tuple
- * @tparam T6 The seventh type stored in the tuple
- */
-template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6>
-struct tuple<T0, T1, T2, T3, T4, T5, T6>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-   T2 v2;  ///< The third member of the tuple
-   T3 v3;  ///< The fourth member of the tuple
-   T4 v4;  ///< The fifth member of the tuple
-   T5 v5;  ///< The sixth member of the tuple
-   T6 v6;  ///< The seventh member of the tuple
-};
-
-/**
- * @brief Type that mimics std::tuple
- *
- * @tparam T0 The first type stored in the tuple
- * @tparam T1 The second type stored in the tuple
- * @tparam T2 The third type stored in the tuple
- * @tparam T3 The fourth type stored in the tuple
- * @tparam T4 The fifth type stored in the tuple
- * @tparam T5 The sixth type stored in the tuple
- * @tparam T6 The seventh type stored in the tuple
- * @tparam T7 The eighth type stored in the tuple
- */
-template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7>
-struct tuple<T0, T1, T2, T3, T4, T5, T6, T7>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-   T2 v2;  ///< The third member of the tuple
-   T3 v3;  ///< The fourth member of the tuple
-   T4 v4;  ///< The fifth member of the tuple
-   T5 v5;  ///< The sixth member of the tuple
-   T6 v6;  ///< The seventh member of the tuple
-   T7 v7;  ///< The eighth member of the tuple
-};
-
-template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8>
-struct tuple<T0, T1, T2, T3, T4, T5, T6, T7, T8>
-{
-   T0 v0;  ///< The first member of the tuple
-   T1 v1;  ///< The second member of the tuple
-   T2 v2;  ///< The third member of the tuple
-   T3 v3;  ///< The fourth member of the tuple
-   T4 v4;  ///< The fifth member of the tuple
-   T5 v5;  ///< The sixth member of the tuple
-   T6 v6;  ///< The seventh member of the tuple
-   T7 v7;  ///< The eighth member of the tuple
-   T8 v8;
-};
-
-/**
- * @brief Class template argument deduction rule for tuples
- * @tparam T The variadic template parameter for tuple types
- */
-template <typename... T>
-MFEM_HOST_DEVICE
-tuple(T...) -> tuple<T...>;
-
-/**
- * @brief helper function for combining a list of values into a tuple
- * @tparam T types of the values to be tuple-d
- * @param args the actual values to be put into a tuple
- */
-template <typename... T>
-MFEM_HOST_DEVICE tuple<T...> make_tuple(const T&... args)
-{
-   return tuple<T...> {args...};
-}
-
-template <class... Types>
-struct tuple_size
-{
-};
-
-template <class... Types>
-struct tuple_size<mfem::tuple<Types...>> :
-                                      std::integral_constant<std::size_t, sizeof...(Types)>
-{
-};
-
-/**
- * @tparam i the tuple index to access
- * @tparam T the types stored in the tuple
- * @brief return a reference to the ith tuple entry
- */
-template <int i, typename... T>
-MFEM_HOST_DEVICE constexpr auto& get(tuple<T...>& values)
-{
-   static_assert(i < sizeof...(T), "");
-   if constexpr (i == 0)
-   {
-      return values.v0;
-   }
-   if constexpr (i == 1)
-   {
-      return values.v1;
-   }
-   if constexpr (i == 2)
-   {
-      return values.v2;
-   }
-   if constexpr (i == 3)
-   {
-      return values.v3;
-   }
-   if constexpr (i == 4)
-   {
-      return values.v4;
-   }
-   if constexpr (i == 5)
-   {
-      return values.v5;
-   }
-   if constexpr (i == 6)
-   {
-      return values.v6;
-   }
-   if constexpr (i == 7)
-   {
-      return values.v7;
-   }
-   if constexpr (i == 8)
-   {
-      return values.v8;
-   }
-}
-
-/**
- * @tparam i the tuple index to access
- * @tparam T the types stored in the tuple
- * @brief return a copy of the ith tuple entry
- */
-template <int i, typename... T>
-MFEM_HOST_DEVICE constexpr const auto& get(const tuple<T...>& values)
-{
-   static_assert(i < sizeof...(T), "");
-   if constexpr (i == 0)
-   {
-      return values.v0;
-   }
-   if constexpr (i == 1)
-   {
-      return values.v1;
-   }
-   if constexpr (i == 2)
-   {
-      return values.v2;
-   }
-   if constexpr (i == 3)
-   {
-      return values.v3;
-   }
-   if constexpr (i == 4)
-   {
-      return values.v4;
-   }
-   if constexpr (i == 5)
-   {
-      return values.v5;
-   }
-   if constexpr (i == 6)
-   {
-      return values.v6;
-   }
-   if constexpr (i == 7)
-   {
-      return values.v7;
-   }
-   if constexpr (i == 8)
-   {
-      return values.v8;
-   }
-}
-
-/**
- * @brief a function intended to be used for extracting the ith type from a tuple.
- *
- * @note type<i>(my_tuple) returns a value, whereas get<i>(my_tuple) returns a reference
- *
- * @tparam i the index of the tuple to query
- * @tparam T the types stored in the tuple
- * @param values the tuple of values
- * @return a copy of the ith entry of the input
- */
-template <int i, typename... T>
-MFEM_HOST_DEVICE constexpr auto type(const tuple<T...>& values)
-{
-   static_assert(i < sizeof...(T), "");
-   if constexpr (i == 0)
-   {
-      return values.v0;
-   }
-   if constexpr (i == 1)
-   {
-      return values.v1;
-   }
-   if constexpr (i == 2)
-   {
-      return values.v2;
-   }
-   if constexpr (i == 3)
-   {
-      return values.v3;
-   }
-   if constexpr (i == 4)
-   {
-      return values.v4;
-   }
-   if constexpr (i == 5)
-   {
-      return values.v5;
-   }
-   if constexpr (i == 6)
-   {
-      return values.v6;
-   }
-   if constexpr (i == 7)
-   {
-      return values.v7;
-   }
-   if constexpr (i == 8)
-   {
-      return values.v8;
-   }
-}
-
-/**
- * @brief A helper function for the + operator of tuples
- *
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param y tuple of values
- * @return the returned tuple sum
- */
-template <typename... S, typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto plus_helper(const tuple<S...>& x,
-                                            const tuple<T...>& y,
-                                            std::integer_sequence<int, i...>)
-{
-   return tuple{get<i>(x) + get<i>(y)...};
-}
-
-/**
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @param x a tuple of values
- * @param y a tuple of values
- * @brief return a tuple of values defined by elementwise sum of x and y
- */
-template <typename... S, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator+(const tuple<S...>& x,
-                                          const tuple<T...>& y)
-{
-   static_assert(sizeof...(S) == sizeof...(T));
-   return plus_helper(x, y,
-                      std::make_integer_sequence<int, static_cast<int>(sizeof...(S))>());
-}
-
-/**
- * @brief A helper function for the += operator of tuples
- *
- * @tparam T the types stored in the tuples x and y
- * @tparam i integer sequence used to index the tuples
- * @param x tuple of values to be incremented
- * @param y tuple of increment values
- */
-template <typename... T, int... i>
-MFEM_HOST_DEVICE constexpr void plus_equals_helper(tuple<T...>& x,
-                                                   const tuple<T...>& y,
-                                                   std::integer_sequence<int, i...>)
-{
-   ((get<i>(x) += get<i>(y)), ...);
-}
-
-/**
- * @tparam T the types stored in the tuples x and y
- * @param x a tuple of values
- * @param y a tuple of values
- * @brief add values contained in y, to the tuple x
- */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator+=(tuple<T...>& x,
-                                           const tuple<T...>& y)
-{
-   return plus_equals_helper(x, y,
-                             std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @brief A helper function for the -= operator of tuples
- *
- * @tparam T the types stored in the tuples x and y
- * @tparam i integer sequence used to index the tuples
- * @param x tuple of values to be subracted from
- * @param y tuple of values to subtract from x
- */
-template <typename... T, int... i>
-MFEM_HOST_DEVICE constexpr void minus_equals_helper(tuple<T...>& x,
-                                                    const tuple<T...>& y,
-                                                    std::integer_sequence<int, i...>)
-{
-   ((get<i>(x) -= get<i>(y)), ...);
-}
-
-/**
- * @tparam T the types stored in the tuples x and y
- * @param x a tuple of values
- * @param y a tuple of values
- * @brief add values contained in y, to the tuple x
- */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator-=(tuple<T...>& x,
-                                           const tuple<T...>& y)
-{
-   return minus_equals_helper(x, y,
-                              std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @brief A helper function for the - operator of tuples
- *
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param y tuple of values
- * @return the returned tuple difference
- */
-template <typename... S, typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto minus_helper(const tuple<S...>& x,
-                                             const tuple<T...>& y,
-                                             std::integer_sequence<int, i...>)
-{
-   return tuple{get<i>(x) - get<i>(y)...};
-}
-
-/**
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @param x a tuple of values
- * @param y a tuple of values
- * @brief return a tuple of values defined by elementwise difference of x and y
- */
-template <typename... S, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator-(const tuple<S...>& x,
-                                          const tuple<T...>& y)
-{
-   static_assert(sizeof...(S) == sizeof...(T));
-   return minus_helper(x, y,
-                       std::make_integer_sequence<int, static_cast<int>(sizeof...(S))>());
-}
-
-/**
- * @brief A helper function for the - operator of tuples
- *
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @return the returned tuple difference
- */
-template <typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto unary_minus_helper(const tuple<T...>& x,
-                                                   std::integer_sequence<int, i...>)
-{
-   return tuple{-get<i>(x)...};
-}
-
-/**
- * @tparam T the types stored in the tuple y
- * @param x a tuple of values
- * @brief return a tuple of values defined by applying the unary minus operator to each element of x
- */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator-(const tuple<T...>& x)
-{
-   return unary_minus_helper(x,
-                             std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @brief A helper function for the / operator of tuples
- *
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param y tuple of values
- * @return the returned tuple ratio
- */
-template <typename... S, typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto div_helper(const tuple<S...>& x,
-                                           const tuple<T...>& y,
-                                           std::integer_sequence<int, i...>)
-{
-   return tuple{get<i>(x) / get<i>(y)...};
-}
-
-/**
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @param x a tuple of values
- * @param y a tuple of values
- * @brief return a tuple of values defined by elementwise division of x by y
- */
-template <typename... S, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator/(const tuple<S...>& x,
-                                          const tuple<T...>& y)
-{
-   static_assert(sizeof...(S) == sizeof...(T));
-   return div_helper(x, y,
-                     std::make_integer_sequence<int, static_cast<int>(sizeof...(S))>());
-}
-
-/**
- * @brief A helper function for the / operator of tuples
- *
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param a the constant numerator
- * @return the returned tuple ratio
- */
-template <typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto div_helper(const double a,
-                                           const tuple<T...>& x, std::integer_sequence<int, i...>)
-{
-   return tuple{a / get<i>(x)...};
-}
-
-/**
- * @brief A helper function for the / operator of tuples
- *
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param a the constant denomenator
- * @return the returned tuple ratio
- */
-template <typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto div_helper(const tuple<T...>& x,
-                                           const double a, std::integer_sequence<int, i...>)
-{
-   return tuple{get<i>(x) / a...};
-}
-
-/**
- * @tparam T the types stored in the tuple x
- * @param a the numerator
- * @param x a tuple of denominator values
- * @brief return a tuple of values defined by division of a by the elements of x
- */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator/(const double a, const tuple<T...>& x)
-{
-   return div_helper(a, x,
-                     std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @tparam T the types stored in the tuple y
- * @param x a tuple of numerator values
- * @param a a denominator
- * @brief return a tuple of values defined by elementwise division of x by a
- */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator/(const tuple<T...>& x, const double a)
-{
-   return div_helper(x, a,
-                     std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @brief A helper function for the * operator of tuples
- *
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param y tuple of values
- * @return the returned tuple product
- */
-template <typename... S, typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto mult_helper(const tuple<S...>& x,
-                                            const tuple<T...>& y,
-                                            std::integer_sequence<int, i...>)
-{
-   return tuple{get<i>(x) * get<i>(y)...};
-}
-
-/**
- * @tparam S the types stored in the tuple x
- * @tparam T the types stored in the tuple y
- * @param x a tuple of values
- * @param y a tuple of values
- * @brief return a tuple of values defined by elementwise multiplication of x and y
- */
-template <typename... S, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator*(const tuple<S...>& x,
-                                          const tuple<T...>& y)
-{
-   static_assert(sizeof...(S) == sizeof...(T));
-   return mult_helper(x, y,
-                      std::make_integer_sequence<int, static_cast<int>(sizeof...(S))>());
-}
-
-/**
- * @brief A helper function for the * operator of tuples
- *
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param a a constant multiplier
- * @return the returned tuple product
- */
-template <typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto mult_helper(const double a,
-                                            const tuple<T...>& x, std::integer_sequence<int, i...>)
-{
-   return tuple{a * get<i>(x)...};
-}
-
-/**
- * @brief A helper function for the * operator of tuples
- *
- * @tparam T the types stored in the tuple y
- * @tparam i The integer sequence to i
- * @param x tuple of values
- * @param a a constant multiplier
- * @return the returned tuple product
- */
-template <typename... T, int... i>
-MFEM_HOST_DEVICE constexpr auto mult_helper(const tuple<T...>& x,
-                                            const double a, std::integer_sequence<int, i...>)
-{
-   return tuple{get<i>(x) * a...};
-}
-
-/**
- * @tparam T the types stored in the tuple
- * @param a a scaling factor
- * @param x the tuple object
- * @brief multiply each component of x by the value a on the left
- */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator*(const double a, const tuple<T...>& x)
-{
-   return mult_helper(a, x,
-                      std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @tparam T the types stored in the tuple
- * @param x the tuple object
- * @param a a scaling factor
- * @brief multiply each component of x by the value a on the right
- */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator*(const tuple<T...>& x, const double a)
-{
-   return mult_helper(x, a,
-                      std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @tparam T the types stored in the tuple
- * @tparam i a list of indices used to acces each element of the tuple
- * @param out the ostream to write the output to
- * @param A the tuple of values
- * @brief helper used to implement printing a tuple of values
- */
-template <typename... T, std::size_t... i>
-auto& print_helper(std::ostream& out, const mfem::tuple<T...>& A,
-                   std::integer_sequence<size_t, i...>)
-{
-   out << "tuple{";
-   (..., (out << (i == 0 ? "" : ", ") << mfem::get<i>(A)));
-   out << "}";
-   return out;
-}
-
-/**
- * @tparam T the types stored in the tuple
- * @param out the ostream to write the output to
- * @param A the tuple of values
- * @brief print a tuple of values
- */
-template <typename... T>
-auto& operator<<(std::ostream& out, const mfem::tuple<T...>& A)
-{
-   return print_helper(out, A, std::make_integer_sequence<size_t, sizeof...(T)>());
-}
-
-/**
- * @brief A helper to apply a lambda to a tuple
- *
- * @tparam lambda The functor type
- * @tparam T The tuple types
- * @tparam i The integer sequence to i
- * @param f The functor to apply to the tuple
- * @param args The input tuple
- * @return The functor output
- */
-template <typename lambda, typename... T, int... i>
-MFEM_HOST_DEVICE auto apply_helper(lambda f, tuple<T...>& args,
-                                   std::integer_sequence<int, i...>)
-{
-   return f(get<i>(args)...);
-}
-
-/**
- * @tparam lambda a callable type
- * @tparam T the types of arguments to be passed in to f
- * @param f the callable object
- * @param args a tuple of arguments
- * @brief a way of passing an n-tuple to a function that expects n separate arguments
- *
- *   e.g. foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz));
- */
-template <typename lambda, typename... T>
-MFEM_HOST_DEVICE auto apply(lambda f, tuple<T...>& args)
-{
-   return apply_helper(f, std::move(args),
-                       std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @overload
- */
-template <typename lambda, typename... T, int... i>
-MFEM_HOST_DEVICE auto apply_helper(lambda f, const tuple<T...>& args,
-                                   std::integer_sequence<int, i...>)
-{
-   return f(get<i>(args)...);
-}
-
-/**
- * @tparam lambda a callable type
- * @tparam T the types of arguments to be passed in to f
- * @param f the callable object
- * @param args a tuple of arguments
- * @brief a way of passing an n-tuple to a function that expects n separate arguments
- *
- *   e.g. foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz));
- */
-template <typename lambda, typename... T>
-MFEM_HOST_DEVICE auto apply(lambda f, const tuple<T...>& args)
-{
-   return apply_helper(f, std::move(args),
-                       std::make_integer_sequence<int, static_cast<int>(sizeof...(T))>());
-}
-
-/**
- * @brief a struct used to determine the type at index I of a tuple
- *
- * @note see: https://en.cppreference.com/w/cpp/utility/tuple/tuple_element
- *
- * @tparam I the index of the desired type
- * @tparam T a tuple of different types
- */
-template <size_t I, class T>
-struct tuple_element;
-
-// recursive case
-/// @overload
-template <size_t I, class Head, class... Tail>
-struct tuple_element<I, tuple<Head, Tail...>> : tuple_element<I - 1,
-                                              tuple<Tail...>>
-{
-};
-
-// base case
-/// @overload
-template <class Head, class... Tail>
-struct tuple_element<0, tuple<Head, Tail...>>
-{
-   using type = Head;  ///< the type at the specified index
-};
-
-/**
- * @brief Trait for checking if a type is a @p mfem::tuple
- */
-template <typename T>
-struct is_tuple : std::false_type
-{
-};
-
-/// @overload
-template <typename... T>
-struct is_tuple<mfem::tuple<T...>> : std::true_type
-{
-};
-
-/**
- * @brief Trait for checking if a type if a @p mfem::tuple containing only @p mfem::tuple
- */
-template <typename T>
-struct is_tuple_of_tuples : std::false_type
-{
-};
-
-/**
- * @brief Trait for checking if a type if a @p mfem::tuple containing only @p mfem::tuple
- */
-template <typename... T>
-struct is_tuple_of_tuples<mfem::tuple<T...>>
-{
-   static constexpr bool value = (is_tuple<T>::value &&
-                                  ...);  ///< true/false result of type check
-};
-
-}  // namespace mfem

examples/dfem_dg.cpp

@@ -1,123 +0,0 @@
-#include "dfem/dfem_refactor.hpp"
-#include "fem/bilininteg.hpp"
-#include "fem/coefficient.hpp"
-#include "linalg/auxiliary.hpp"
-#include "linalg/hypre.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   const char *mesh_file = "../data/ref-square.mesh";
-   int polynomial_order = 1;
-   int ir_order = 2;
-   int refinements = 1;
-
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   L2_FECollection fec(polynomial_order, dim, BasisType::GaussLobatto);
-   ParFiniteElementSpace fes(&mesh, &fec);
-
-   const IntegrationRule &ir = IntRules.Get(fes.GetFE(0)->GetGeomType(),
-                                            ir_order * fec.GetOrder());
-   const IntegrationRule &ir_face = IntRules.Get(
-                                       fes.GetTraceElement(0, fes.GetMesh()->GetFaceGeometry(0))->GetGeomType(),
-                                       ir_order * fec.GetOrder());
-   ParGridFunction u(&fes);
-
-   // // -\nabla \cdot (\nabla u + p * I) -> (\nabla u + p * I, \nabla v)
-   // auto advection_kernel = [](const tensor<double, 2> &dudxi,
-   //                            const tensor<double, 2, 2> &J,
-   //                            const double &w)
-   // {
-   //    constexpr tensor<double, 2> b{1.0, 1.0};
-   //    return std::tuple{dot(b, dudxi * inv(J)) * det(J) * w};
-   // };
-
-   // std::tuple argument_operators_0{Gradient{"quantity"}, Gradient{"coordinates"}, Weight{"integration_weights"}};
-   // std::tuple output_operator_0{Value{"quantity"}};
-   // ElementOperator op_0{advection_kernel, argument_operators_0, output_operator_0};
-
-   // std::array solutions{FieldDescriptor{&fes, "quantity"}};
-   // std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   // DifferentiableOperator advection_op{solutions, parameters, std::tuple{op_0}, mesh, ir};
-
-   // auto adv_du = advection_op.template GetDerivativeWrt<0>({&u}, {mesh_nodes});
-   // HypreParMatrix A;
-   // adv_du->Assemble(A);
-
-   // std::ofstream mmatofs("dfem_mat.dat");
-   // A.PrintMatlab(mmatofs);
-   // mmatofs.close();
-
-   auto trace_kernel = [](const double &uL, const double &uR, const double &J,
-                          const double &w)
-   {
-      return std::tuple{1.0 / J * w};
-   };
-
-   std::tuple argument_operators_0
-   {
-      FaceValueLeft{"quantity"},
-      FaceValueRight{"quantity"},
-      Gradient{"coordinates"},
-      Weight{"integration_weights"}
-   };
-   std::tuple output_operator_0{Value{"quantity"}};
-   FaceElementOperator op_0{trace_kernel, argument_operators_0, output_operator_0};
-
-   std::array solutions{FieldDescriptor{&fes, "quantity"}};
-   std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator trace_op{solutions, parameters, std::tuple{op_0}, mesh, ir_face};
-
-   auto vector_func = [](const Vector &, Vector &u)
-   {
-      u = 1.0;
-   };
-
-   VectorFunctionCoefficient vel_coeff(dim, vector_func);
-
-   ParBilinearForm adv_form(&fes);
-   constexpr double alpha = 1.0;
-   auto integ = new ConvectionIntegrator(vel_coeff, alpha);
-   integ->SetIntRule(&ir);
-   adv_form.AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(vel_coeff, alpha));
-   // adv_form.AddDomainIntegrator(integ);
-   adv_form.Assemble();
-   adv_form.Finalize();
-
-   auto K = adv_form.ParallelAssemble();
-   std::ofstream kmatofs("mfem_mat.dat");
-   K->PrintMatlab(kmatofs);
-   kmatofs.close();
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock(vishost, visport);
-   sol_sock << "parallel " << num_procs << " " << myid << "\n";
-   sol_sock.precision(8);
-   sol_sock << "solution\n" << mesh << u << std::flush;
-
-   return 0;
-}

examples/dfem_elasticity.cpp

@@ -1,150 +0,0 @@
-#include "dfem.hpp"
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-
-   std::cout << std::setprecision(9);
-
-   const char *mesh_file = "../data/star.mesh";
-   int polynomial_order = 1;
-   int refinements = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&polynomial_order, "-o", "--order", "");
-   args.AddOption(&refinements, "-r", "--r", "");
-   args.ParseCheck();
-
-   Mesh mesh_serial(mesh_file, 1, 1);
-   mesh_serial.SetCurvature(1);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   const int dim = mesh_serial.Dimension();
-
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-   mesh_serial.Clear();
-
-   constexpr int vdim = 2;
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   ess_bdr = 1;
-   h1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   std::cout << "nqpts = " << ir.GetNPoints() << std::endl;
-   std::cout << "ndofs = " << h1fes.GlobalTrueVSize() << std::endl;
-
-   ParGridFunction u(&h1fes);
-
-   auto exact_solution = [](const Vector &coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      u(0) = x*x + y;
-      u(1) = x + 0.5*y*y;
-   };
-
-   VectorFunctionCoefficient exact_solution_coeff(dim, exact_solution);
-
-   auto elasticity_kernel = [](tensor<double, 2, 2> &dudxi,
-                               tensor<double, 2, 2> &J,
-                               double &w)
-   {
-      using mfem::internal::tensor;
-      using mfem::internal::IsotropicIdentity;
-
-      double lambda, mu;
-      {
-         lambda = 1.0;
-         mu = 1.0;
-      }
-      static constexpr auto I = IsotropicIdentity<2>();
-      auto eps = sym(dudxi * inv(J));
-      auto JxW = transpose(inv(J)) * det(J) * w;
-      auto r = (lambda * tr(eps) * I + 2.0 * mu * eps) * JxW;
-      return r;
-   };
-
-   tensor<double, 2, 2> dudxi, s_dudxi, J;
-   double w = 1.0;
-
-   enzyme::get<0>
-   (enzyme::autodiff<enzyme::Forward,
-    enzyme::DuplicatedNoNeed<tensor<double, 2, 2>>>
-    (+elasticity_kernel,
-     enzyme::Duplicated<tensor<double, 2, 2> *>(&dudxi, &s_dudxi),
-     enzyme::Const<tensor<double, 2, 2>*>(&J),
-     enzyme::Const<double*>(&w)));
-
-   // std::tuple input_descriptors = {Gradient{"displacement"}, Gradient{"coordinates"}, Weight{"integration_weight"}};
-   // std::tuple output_descriptors = {Gradient{"displacement"}};
-   // ElementOperator qf {elasticity_kernel, input_descriptors, output_descriptors};
-
-   // ElementOperator forcing_qf
-   // {
-   //    [](tensor<double, 2> x, tensor<double, 2, 2> J, double w)
-   //    {
-   //       double lambda, mu;
-   //       {
-   //          lambda = 1.0;
-   //          mu = 1.0;
-   //       }
-   //       auto f = x;
-   //       f(0) = 4.0*mu + 2.0*lambda;
-   //       f(1) = 2.0*mu + lambda;
-   //       return f * det(J) * w;
-   //    },
-   //    // inputs
-   //    std::tuple{
-   //       Value{"coordinates"},
-   //       Gradient{"coordinates"},
-   //       Weight{"integration_weight"}},
-   //    // outputs
-   //    std::tuple{
-   //       Value{"displacement"}}
-   // };
-
-   // std::vector<Field> solutions{{&u, "displacement"}};
-   // std::vector<Field> parameters{{mesh.GetNodes(), "coordinates"}};
-   // std::vector<Field> dependent_fields{{&u, "displacement"}};
-   // DifferentiableForm dop(solutions, parameters, dependent_fields, mesh);
-
-   // dop.AddElementOperator<AD::Enzyme>(qf, ir);
-   // dop.AddElementOperator<AD::None>(forcing_qf, ir);
-   // dop.SetEssentialTrueDofs(ess_tdof_list);
-
-   // GMRESSolver gmres(MPI_COMM_WORLD);
-   // gmres.SetRelTol(1e-12);
-   // gmres.SetMaxIter(5000);
-   // gmres.SetPrintLevel(IterativeSolver::PrintLevel().Summary());
-
-   // NewtonSolver newton(MPI_COMM_WORLD);
-   // newton.SetSolver(gmres);
-   // newton.SetOperator(dop);
-   // newton.SetRelTol(1e-12);
-   // newton.SetMaxIter(100);
-   // newton.SetPrintLevel(1);
-
-   // u = 1e-6;
-   // u.ProjectBdrCoefficient(exact_solution_coeff, ess_bdr);
-   // Vector x;
-   // u.GetTrueDofs(x);
-
-   // Vector zero;
-   // newton.Mult(zero, x);
-
-   // u.Distribute(x);
-
-   // std::cout << "|u-u_ex|_L2 = " << u.ComputeL2Error(exact_solution_coeff) << "\n";
-
-   return 0;
-}

examples/dfem_minimal_example.cpp

@@ -1,115 +0,0 @@
-#include <tuple>
-#include <type_traits>
-#include <iostream>
-#include <enzyme/enzyme>
-
-template <typename T>
-constexpr auto get_type_name() -> std::string_view
-{
-#if defined(__clang__)
-   constexpr auto prefix = std::string_view {"[T = "};
-   constexpr auto suffix = "]";
-   constexpr auto function = std::string_view{__PRETTY_FUNCTION__};
-#elif defined(__GNUC__)
-   constexpr auto prefix = std::string_view {"with T = "};
-   constexpr auto suffix = "; ";
-   constexpr auto function = std::string_view{__PRETTY_FUNCTION__};
-#elif defined(_MSC_VER)
-   constexpr auto prefix = std::string_view {"get_type_name<"};
-   constexpr auto suffix = ">(void)";
-   constexpr auto function = std::string_view{__FUNCSIG__};
-#else
-#error Unsupported compiler
-#endif
-
-   const auto start = function.find(prefix) + prefix.size();
-   const auto end = function.find(suffix);
-   const auto size = end - start;
-
-   return function.substr(start, size);
-}
-
-template <typename ... Ts>
-constexpr auto decay_types(std::tuple<Ts...> const &)
--> std::tuple<std::remove_cv_t<std::remove_reference_t<Ts>>...>;
-
-template <typename T>
-using decay_tuple = decltype(decay_types(std::declval<T>()));
-
-template <class F> struct FunctionSignature;
-
-template <typename output_t, typename... input_ts>
-struct FunctionSignature<output_t(input_ts...)>
-{
-   using return_t = output_t;
-   using parameter_ts = std::tuple<input_ts...>;
-};
-
-template <class T> struct create_function_signature;
-
-template <typename output_t, typename T, typename... input_ts>
-struct create_function_signature<output_t (T::*)(input_ts...) const>
-{
-   using type = FunctionSignature<output_t(input_ts...)>;
-};
-
-template <typename arg_ts, std::size_t... Is>
-auto create_enzyme_args(arg_ts &args,
-                        arg_ts &shadow_args,
-                        std::index_sequence<Is...>)
-{
-   ((std::cout << std::get<Is>(shadow_args) << "\n"), ...);
-   return std::tuple<enzyme::Duplicated<decltype(std::get<Is>(args))>...>
-   {
-      { std::get<Is>(args), std::get<Is>(shadow_args) }...
-   };
-}
-
-template <typename kernel_t, typename arg_ts>
-auto fwddiff_apply_enzyme(kernel_t kernel, arg_ts &&args, arg_ts &&shadow_args)
-{
-   auto arg_indices =
-      std::make_index_sequence<std::tuple_size_v<std::remove_reference_t<arg_ts>>> {};
-
-   auto enzyme_args = create_enzyme_args(args, shadow_args, arg_indices);
-
-   using kf_return_t = typename create_function_signature<
-                       decltype(&kernel_t::operator())>::type::return_t;
-
-   std::cout << "args is " << get_type_name<decltype(args)>() << "\n\n";
-   std::cout << "enzyme_args type is " << get_type_name<decltype(enzyme_args)>() <<
-             "\n\n";
-   std::cout << "return type is " << get_type_name<decltype(kf_return_t{})>() <<
-             "\n\n";
-
-   return std::apply([&](auto &&...args)
-   {
-      return enzyme::get<0>(
-                enzyme::autodiff<enzyme::Forward>
-                (+kernel, args...));
-   },
-   enzyme_args);
-}
-
-int main()
-{
-
-   auto func = [](const double &x)
-   {
-      return x*x;
-   };
-
-   using kf_param_ts = typename create_function_signature<
-                       decltype(&decltype(func)::operator())>::type::parameter_ts;
-   using kf_output_t = typename create_function_signature<
-                       decltype(&decltype(func)::operator())>::type::return_t;
-   auto kernel_args = decay_tuple<kf_param_ts> {};
-   auto kernel_shadow_args = decay_tuple<kf_param_ts> {};
-
-   std::get<0>(kernel_args) = 3;
-   std::get<0>(kernel_shadow_args) = 1;
-   const auto res = fwddiff_apply_enzyme(func, kernel_args, kernel_shadow_args);
-   std::cout << res << " == 6\n";
-
-   return 0;
-}

examples/dfem_objective.cpp

@@ -1,114 +0,0 @@
-#include "dfem.hpp"
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-
-   std::cout << std::setprecision(9);
-
-   const char *mesh_file = "../data/star.mesh";
-   int polynomial_order = 1;
-   int refinements = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&polynomial_order, "-o", "--order", "");
-   args.AddOption(&refinements, "-r", "--r", "");
-   args.ParseCheck();
-
-   Mesh mesh_serial(mesh_file, 1, 1);
-   mesh_serial.SetCurvature(1);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   const int dim = mesh_serial.Dimension();
-
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-   mesh_serial.Clear();
-
-   constexpr int vdim = 2;
-
-   // test_partial_assembly_setup_qf(mesh, 1, polynomial_order);
-   // exit(0);
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   ess_bdr = 1;
-   h1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   std::cout << "nqpts = " << ir.GetNPoints() << std::endl;
-   std::cout << "ndofs = " << h1fes.GlobalTrueVSize() << std::endl;
-
-   ParGridFunction u(&h1fes);
-   ParGridFunction g(&h1fes);
-   ParGridFunction rho(&h1fes);
-
-   auto exact_solution = [](const Vector &coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      u(0) = x*x + y;
-      u(1) = x + 0.5*y*y;
-   };
-
-   VectorFunctionCoefficient exact_solution_coeff(dim, exact_solution);
-
-   auto objective = [](tensor<double, 2> u, double rho,
-                       tensor<double, 2, 2> J,
-                       double w)
-   {
-      return sqnorm(u) * det(J) * w;
-   };
-
-   std::tuple inputs{Value{"displacement"}, Value{"density"}, Gradient{"coordinates"}, Weight{"integration_weight"}};
-   std::tuple outputs{ One{"integral"} };
-   ElementOperator objective_eop { objective, inputs, outputs };
-
-   std::vector<Field> solution_fields{{&u, "displacement"}};
-   std::vector<Field> parameter_fields{{mesh.GetNodes(), "coordinates"}, {&rho, "density"}};
-   std::vector<Field> dependent_variables{{&u, "displacement"}};
-   DifferentiableForm dop(solution_fields, parameter_fields, dependent_variables,
-                          mesh);
-
-   dop.AddElementOperator(objective_eop, ir);
-
-   u.ProjectCoefficient(exact_solution_coeff);
-   Vector zero;
-
-   Vector y(1);
-   Vector utdof;
-   u.GetTrueDofs(utdof);
-   dop.Mult(utdof, y);
-
-   // finite difference test
-   Vector dgdu(u.Size());
-   Vector fx(y);
-   out << "g: ";
-   print_vector(fx);
-   out << "\n";
-
-   for (int i = 0; i < u.Size(); i++)
-   {
-      double h = 1e-6;
-      u(i) += h;
-      dop.Mult(u, y);
-      u(i) -= h;
-      y -= fx;
-      y /= h;
-      dgdu(i) = y(0);
-   }
-
-   out << "dgdu: ";
-   print_vector(dgdu);
-
-   // Vector dgdu = dop.GetGradientWrt({&u, "displacement"});
-
-   return 0;
-}

examples/dfem_plaplacian.cpp

@@ -1,138 +0,0 @@
-#include "dfem.hpp"
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-
-   std::cout << std::setprecision(9);
-
-   const char *mesh_file = "../data/star.mesh";
-   int polynomial_order = 1;
-   int refinements = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&polynomial_order, "-o", "--order", "");
-   args.AddOption(&refinements, "-r", "--r", "");
-   args.ParseCheck();
-
-   Mesh mesh_serial(mesh_file, 1, 1);
-   mesh_serial.SetCurvature(1);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   const int dim = mesh_serial.Dimension();
-
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-   mesh_serial.Clear();
-
-   constexpr int vdim = 1;
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   ess_bdr = 1;
-   h1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   std::cout << "nqpts = " << ir.GetNPoints() << std::endl;
-   std::cout << "ndofs = " << h1fes.GlobalTrueVSize() << std::endl;
-
-   ParGridFunction u(&h1fes);
-
-   auto exact_solution = [](const Vector &coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      // PRESENT
-      return pow(x,2) + 0.5*x*pow(y,2);
-   };
-
-   FunctionCoefficient exact_solution_coeff(exact_solution);
-
-   auto plaplacian = [](double u,
-                        tensor<double, 2> dudxi,
-                        tensor<double, 2, 2> J,
-                        double w)
-   {
-      using mfem::internal::tensor;
-      auto dudx = dudxi * inv(J);
-      auto JxW = transpose(inv(J)) * det(J) * w;
-      // PRESENT: Implement (1+u^2) * ∇u
-      return (1.0 + u*u) * dudx * JxW;
-   };
-
-   // PRESENT: Implement descriptors
-   std::tuple input_descriptors = {Value{"potential"}, Gradient{"potential"}, Gradient{"coordinates"}, Weight{"integration_weights"}};
-   // PRESENT: Implement descriptors
-   std::tuple output_descriptors = {Gradient{"potential"}};
-
-   ElementOperator qf {plaplacian, input_descriptors, output_descriptors};
-
-   ElementOperator forcing_qf
-   {
-      [](tensor<double, 2> coords, tensor<double, 2, 2> J, double w)
-      {
-         int p = 2;
-         double x = coords(0);
-         double y = coords(1);
-         // *INDENT-OFF*
-         double mathematica_please_help_me = 2.*pow(x,2)*pow(y,2)*(pow(x,2) + 0.5*x*pow(y,2)) + 2*pow(2*x + 0.5*pow(y,2),2)*(pow(x,2) + 0.5*x*pow(y,2)) + 2*(1 + pow(pow(x,2) + 0.5*x*pow(y,2),2)) + 1.*x*(1 + pow(pow(x,2) + 0.5*x*pow(y,2),2));
-         return mathematica_please_help_me * det(J) * w;
-        // *INDENT-ON*
-      },
-      // inputs
-      std::tuple{
-         Value{"coordinates"},
-         Gradient{"coordinates"},
-         Weight{"integration_weight"}},
-      // outputs
-      std::tuple{
-         Value{"potential"}}
-   };
-
-   std::tuple list_of_qfs{qf_1, qf_2, qf_n};
-
-   std::vector<Field> solutions{{&u, "potential"}};
-   std::vector<Field> parameters{{mesh.GetNodes(), "coordinates"}};
-   DifferentiableForm dop(solutions, parameters, mesh);
-   dop.SetEssentialTrueDofs(ess_tdof_list);
-
-   auto R = dop.GetResidual(list_of_qfs, ir);
-   auto Jacobian_aka_dRdu = dop.GetDerivative<0>(list_of_qfs, ir);
-
-   // R(u) = (\grad u, \grad v) + (f, v)
-   // dop.AddElementOperator<AD::Enzyme>(qf, ir);
-   // dop.AddElementOperator<AD::None>(forcing_qf, ir);
-
-   GMRESSolver gmres(MPI_COMM_WORLD);
-   gmres.SetRelTol(1e-12);
-   gmres.SetMaxIter(5000);
-   gmres.SetPrintLevel(IterativeSolver::PrintLevel().Summary());
-
-   NewtonSolver newton(MPI_COMM_WORLD);
-   newton.SetSolver(gmres);
-   newton.SetOperator(dop);
-   newton.SetRelTol(1e-12);
-   newton.SetMaxIter(100);
-   newton.SetPrintLevel(1);
-
-   u = 1e-6;
-   u.ProjectBdrCoefficient(exact_solution_coeff, ess_bdr);
-   Vector x;
-   u.GetTrueDofs(x);
-
-   Vector zero;
-   newton.Mult(zero, x);
-
-   u.Distribute(x);
-
-   std::cout << "|u-u_ex|_L2 = " << u.ComputeL2Error(exact_solution_coeff) << "\n";
-
-   return 0;
-}

examples/dfem_poisson.cpp

@@ -1,192 +0,0 @@
-#include "dfem/dfem_refactor.hpp"
-#include "linalg/hypre.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-template <typename diffusion_t, typename force_t>
-class DiffusionOperator : public Operator
-{
-   template <typename diffusion_du_t>
-   class DiffusionJacobianOperator : public Operator
-   {
-   public:
-      DiffusionJacobianOperator(const DiffusionOperator *diffusion,
-                                std::shared_ptr<diffusion_du_t> diff_du) :
-         Operator(diffusion->Height()), s(diffusion)
-      {
-         diff_du->Assemble(A);
-         A.EliminateBC(s->ess_tdofs, Operator::DiagonalPolicy::DIAG_ONE);
-      }
-
-      void Mult(const Vector &x, Vector &y) const override
-      {
-         A.Mult(x, y);
-      }
-
-      const DiffusionOperator *s;
-      HypreParMatrix A;
-   };
-
-public:
-   DiffusionOperator(diffusion_t &diffusion, force_t &force,
-                     Array<int> &ess_tdofs) :
-      Operator(diffusion.Height()), diffusion(diffusion),
-      force(force), ess_tdofs(ess_tdofs), f(force.Height()) {}
-
-   void SetParameters(ParGridFunction &mesh_nodes)
-   {
-      diffusion.SetParameters({&mesh_nodes});
-      force.SetParameters({&mesh_nodes});
-
-      Vector zero;
-
-      this->mesh_nodes.SetSpace(mesh_nodes.ParFESpace());
-      this->mesh_nodes = mesh_nodes;
-   }
-
-   void Mult(const Vector &x, Vector &r) const override
-   {
-      diffusion.Mult(x, r);
-      force.Mult(x, f);
-      r -= f;
-      r.SetSubVector(ess_tdofs, 0.0);
-   }
-
-   Operator &GetGradient(const Vector &x) const override
-   {
-      ParGridFunction u(const_cast<ParFiniteElementSpace *>
-                        (*std::get_if<const ParFiniteElementSpace *>
-                         (&diffusion.solutions[0].data)));
-      u.SetFromTrueDofs(x);
-
-      auto dfdu = diffusion.template GetDerivativeWrt<0>({&u}, {&mesh_nodes});
-      dfdu->Assemble(A);
-      A.EliminateBC(ess_tdofs, DiagonalPolicy::DIAG_ONE);
-      return A;
-      // delete jacobian_operator;
-      // jacobian_operator = new
-      // DiffusionJacobianOperator<typename std::remove_pointer<decltype(dfdu.get())>::type>
-      // (this, dfdu);
-      // return *jacobian_operator;
-   }
-
-   diffusion_t &diffusion;
-   force_t &force;
-
-   const Array<int> ess_tdofs;
-   mutable Vector f;
-
-   mutable ParGridFunction mesh_nodes;
-
-   mutable Operator *jacobian_operator = nullptr;
-   mutable HypreParMatrix A;
-};
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   const char *mesh_file = "../data/ref-square.mesh";
-   int polynomial_order = 2;
-   int ir_order = 2;
-   int refinements = 4;
-
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection potential_fec(polynomial_order, dim);
-   ParFiniteElementSpace potential_fes(&mesh, &potential_fec);
-
-   const IntegrationRule &potential_ir =
-      IntRules.Get(potential_fes.GetFE(0)->GetGeomType(),
-                   ir_order * potential_fec.GetOrder());
-
-   Array<int> bdr_attr_is_ess(mesh.bdr_attributes.Max());
-   bdr_attr_is_ess = 1;
-   Array<int> ess_tdofs;
-   potential_fes.GetEssentialTrueDofs(bdr_attr_is_ess, ess_tdofs);
-
-   ParGridFunction u(&potential_fes);
-   u = 0.0;
-
-   auto diffusion_kernel = [](const internal::dual<double, double> &u,
-                              const tensor<internal::dual<double, double>, 2> &dudxi,
-                              const tensor<double, 2, 2> &J,
-                              const double &w)
-   {
-      auto invJ = inv(J);
-      auto dudx = dudxi * invJ;
-      return std::tuple{(1.0 + u * u) * dudx * det(J) * w * transpose(invJ)};
-   };
-
-   std::tuple argument_operators_0{Value{"potential"}, Gradient{"potential"}, Gradient{"coordinates"}, Weight{"integration_weights"}};
-   std::tuple output_operator_0{Gradient{"potential"}};
-   ElementOperator op_0{diffusion_kernel, argument_operators_0, output_operator_0};
-
-   auto force_kernel = [](const tensor<double, 2, 2> &J,
-                          const double &w)
-   {
-      return std::tuple{1.0 * det(J) * w};
-   };
-   std::tuple argument_operators_1{Gradient{"coordinates"}, Weight{"integration_weights"}};
-   std::tuple output_operator_1{Value{"potential"}};
-   ElementOperator op_1{force_kernel, argument_operators_1, output_operator_1};
-
-   std::array solutions{FieldDescriptor{&potential_fes, "potential"}};
-   std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator diffusion_op{solutions, parameters, std::tuple{op_0}, mesh, potential_ir};
-   DifferentiableOperator force_op{solutions, parameters, std::tuple{op_1}, mesh, potential_ir};
-
-   DiffusionOperator diffusion(diffusion_op, force_op, ess_tdofs);
-
-   diffusion.SetParameters({*mesh_nodes});
-
-   HypreBoomerAMG amg;
-   amg.SetPrintLevel(0);
-
-   CGSolver solver(MPI_COMM_WORLD);
-   solver.SetAbsTol(1e-12);
-   solver.SetRelTol(1e-12);
-   solver.SetMaxIter(500);
-   solver.SetPrintLevel(2);
-   solver.SetPreconditioner(amg);
-
-   NewtonSolver newton(MPI_COMM_WORLD);
-   newton.SetOperator(diffusion);
-   newton.SetSolver(solver);
-   newton.SetRelTol(1e-8);
-   newton.SetMaxIter(10);
-   newton.SetPrintLevel(1);
-
-   Vector zero;
-   Vector x(potential_fes.GetTrueVSize());
-   u.ParallelProject(x);
-   newton.Mult(zero, x);
-
-   u.SetFromTrueDofs(x);
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock(vishost, visport);
-   sol_sock << "parallel " << num_procs << " " << myid << "\n";
-   sol_sock.precision(8);
-   sol_sock << "solution\n" << mesh << u << std::flush;
-
-   return 0;
-}

examples/dfem_refactor.cpp

@@ -1,102 +0,0 @@
-#include "mfem.hpp"
-#include "dfem/dfem_refactor.hpp"
-
-using namespace mfem;
-
-auto main(int argc, char *argv[]) -> int
-{
-   Mpi::Init();
-
-   std::cout << std::setprecision(9);
-
-   const char *mesh_file = "../data/star.mesh";
-   int polynomial_order = 1;
-   int refinements = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&polynomial_order, "-o", "--order", "");
-   args.AddOption(&refinements, "-r", "--r", "");
-   args.ParseCheck();
-
-   Mesh mesh_serial(mesh_file, 1, 1);
-   mesh_serial.SetCurvature(1);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   const int dim = mesh_serial.Dimension();
-
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   constexpr int vdim = 1;
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   ess_bdr = 1;
-   h1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   std::cout << "nqpts = " << ir.GetNPoints() << std::endl;
-   std::cout << "ndofs = " << h1fes.GlobalTrueVSize() << std::endl;
-
-   ParGridFunction u(&h1fes);
-
-   auto exact_solution = [](const Vector &coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      return 2.345 + x + y;
-   };
-
-   FunctionCoefficient exact_solution_coeff(exact_solution);
-
-   u.ProjectCoefficient(exact_solution_coeff);
-
-   auto domain_qf = [](const double &u,
-                       const tensor<double, 2, 2> &J,
-                       const double &w)
-   {
-      out << u << "\n" << J << "\n" << w << "\n\n";
-      return std::tuple{u * det(J) * w};
-   };
-
-   std::tuple input_descriptors = {Value{"potential"}, Gradient{"coordinates"}, Weight{"integration_weights"}};
-   std::tuple output_descriptors = {Value{"potential"}};
-   ElementOperator eop{domain_qf, input_descriptors, output_descriptors};
-
-   auto ops = std::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-   DifferentiableOperator dop{solutions, parameters, ops, mesh, ir};
-
-   Vector x(h1fes.GetTrueVSize()), y(h1fes.GetTrueVSize());
-
-   u.GetTrueDofs(x);
-
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-
-   // Derivative wrt "potential", indicated by the index 0 of the set {solutions} \cup {parameters}
-   auto dFd0 = dop.GetDerivativeWrt<0>({&u}, {mesh_nodes});
-   dFd0->Mult(x, y);
-
-   Vector dFd0_vec;
-   dFd0->Assemble(dFd0_vec);
-
-   // Derivative wrt "coordinates", indicated by the index 1 of the set {solutions} \cup {parameters}
-   auto dFd1 = dop.GetDerivativeWrt<1>({&u}, {mesh_nodes});
-   dFd1->Mult(x, y);
-
-   return 0;
-}

examples/dfem_stokes.cpp

@@ -1,302 +0,0 @@
-#include "dfem/dfem.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-template <typename momentum_t, typename mass_conservation_t>
-class NavierStokesOperator : public Operator
-{
-   template <typename momentum_du_t, typename momentum_dp_t>
-   class NavierStokesJacobianOperator : public Operator
-   {
-   public:
-      NavierStokesJacobianOperator(const NavierStokesOperator *ns,
-                                   std::shared_ptr<momentum_du_t> mom_du,
-                                   std::shared_ptr<momentum_dp_t> mom_dp) :
-         Operator(ns->Height()), ns(ns), block_op(ns->block_offsets)
-      {
-         mom_du->Assemble(A);
-         A.EliminateBC(ns->vel_ess_tdofs, Operator::DiagonalPolicy::DIAG_ONE);
-
-         mom_dp->Assemble(D);
-         D.EliminateRows(ns->vel_ess_tdofs);
-
-         Dt = new TransposeOperator(D);
-
-         block_op.SetBlock(0, 0, &A);
-         block_op.SetBlock(0, 1, &D);
-         block_op.SetBlock(1, 0, Dt);
-         // std::ofstream amatofs("dfem_mat.dat");
-         // block_op.PrintMatlab(amatofs);
-         // amatofs.close();
-      }
-
-      void Mult(const Vector &x, Vector &y) const override
-      {
-         block_op.Mult(x, y);
-      }
-
-      ~NavierStokesJacobianOperator()
-      {
-         delete Dt;
-      }
-
-      const NavierStokesOperator *ns = nullptr;
-      HypreParMatrix A, D;
-      TransposeOperator *Dt = nullptr;
-      BlockOperator block_op;
-   };
-
-public:
-   NavierStokesOperator(momentum_t &momentum,
-                        mass_conservation_t &mass_conservation,
-                        Array<int> &offsets, Array<int> &vel_ess_tdofs) :
-      Operator(offsets.Last()), momentum(momentum),
-      mass_conservation(mass_conservation),
-      block_offsets(offsets), vel_ess_tdofs(vel_ess_tdofs) {}
-
-   void SetParameters(ParGridFunction &mesh_nodes)
-   {
-      momentum.SetParameters({&mesh_nodes});
-      mass_conservation.SetParameters({&mesh_nodes});
-      this->mesh_nodes.SetSpace(mesh_nodes.ParFESpace());
-      this->mesh_nodes = mesh_nodes;
-   }
-
-   void Mult(const Vector &x, Vector &r) const override
-   {
-      Vector ru(r.ReadWrite() + block_offsets[0],
-                block_offsets[1] - block_offsets[0]);
-      Vector rp(r.ReadWrite() + block_offsets[1],
-                block_offsets[2] - block_offsets[1]);
-
-      momentum.Mult(x, ru);
-
-      mass_conservation.Mult(x, rp);
-
-      ru.SetSubVector(vel_ess_tdofs, 0.0);
-   }
-
-   Operator &GetGradient(const Vector &x) const override
-   {
-      xtmp = x;
-      BlockVector xb(xtmp.ReadWrite(), block_offsets);
-
-      ParGridFunction u(const_cast<ParFiniteElementSpace *>
-                        (*std::get_if<const ParFiniteElementSpace *>
-                         (&momentum.solutions[0].data)));
-      ParGridFunction p(const_cast<ParFiniteElementSpace *>
-                        (*std::get_if<const ParFiniteElementSpace *>
-                         (&momentum.solutions[1].data)));
-      u.SetFromTrueDofs(xb.GetBlock(0));
-      p.SetFromTrueDofs(xb.GetBlock(1));
-      auto mom_du = momentum.template GetDerivativeWrt<0>({&u, &p}, {&mesh_nodes});
-      auto mom_dp = momentum.template GetDerivativeWrt<1>({&u, &p}, {&mesh_nodes});
-      delete jacobian_operator;
-      jacobian_operator = new NavierStokesJacobianOperator<
-      typename std::remove_pointer<decltype(mom_du.get())>::type,
-      typename std::remove_pointer<decltype(mom_dp.get())>::type>(this, mom_du,
-                                                                  mom_dp);
-      return *jacobian_operator;
-   }
-
-   momentum_t &momentum;
-   mass_conservation_t &mass_conservation;
-
-   const Array<int> block_offsets;
-   const Array<int> vel_ess_tdofs;
-   mutable Vector xtmp;
-
-   mutable ParGridFunction mesh_nodes;
-
-   mutable Operator *jacobian_operator = nullptr;
-};
-
-double reynolds = 10.0;
-
-int main(int argc, char *argv[])
-{
-   constexpr int dim = 3;
-   constexpr int vdim = dim;
-
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   const char *mesh_file = "../data/ref-cube.mesh";
-   int polynomial_order = 2;
-   int ir_order = 2;
-   int refinements = 2;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&refinements, "-r", "--refinements", "");
-   args.AddOption(&reynolds, "-rey", "--reynolds", "");
-   args.ParseCheck();
-
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection velocity_fec(polynomial_order, dim);
-   ParFiniteElementSpace velocity_fes(&mesh, &velocity_fec, dim);
-
-   H1_FECollection pressure_fec(polynomial_order - 1, dim);
-   ParFiniteElementSpace pressure_fes(&mesh, &pressure_fec);
-
-   const IntegrationRule &velocity_ir =
-      IntRules.Get(velocity_fes.GetFE(0)->GetGeomType(),
-                   ir_order * velocity_fec.GetOrder());
-
-   const IntegrationRule &pressure_ir =
-      IntRules.Get(pressure_fes.GetFE(0)->GetGeomType(),
-                   ir_order * pressure_fec.GetOrder());
-
-   Array<int> bdr_attr_is_ess(mesh.bdr_attributes.Max());
-   bdr_attr_is_ess = 1;
-   Array<int> vel_ess_tdofs;
-   velocity_fes.GetEssentialTrueDofs(bdr_attr_is_ess, vel_ess_tdofs);
-
-   ParGridFunction u(&velocity_fes);
-   ParGridFunction p(&pressure_fes);
-
-   auto u_f = [](const Vector &coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double z = coords(2);
-      if (z >= 1.0)
-      {
-         u(0) = 1.0;
-      }
-      else
-      {
-         u(0) = 0.0;
-      }
-      u(1) = 0.0;
-      u(2) = 0.0;
-   };
-   auto u_coef = VectorFunctionCoefficient(dim, u_f);
-
-   u.ProjectCoefficient(u_coef);
-   p = 0.0;
-
-   // -\nabla \cdot (\nabla u + p * I) -> (\nabla u + p * I, \nabla v)
-   auto momentum_kernel = [](const tensor<double, dim> &u,
-                             const tensor<double, dim, dim> &dudxi,
-                             const double &p,
-                             const tensor<double, dim, dim> &J,
-                             const double &w)
-   {
-      static constexpr auto I = mfem::internal::IsotropicIdentity<dim>();
-      auto invJ = inv(J);
-      auto dudx = dudxi * invJ;
-      double Re = reynolds;
-      return mfem::tuple{(outer(u, u) - 1.0 / Re * dudx + p * I) * det(J) * w * transpose(invJ)};
-   };
-
-   mfem::tuple argument_operators_0{Value{"velocity"}, Gradient{"velocity"}, Value{"pressure"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator_0{Gradient{"velocity"}};
-   ElementOperator op_0{momentum_kernel, argument_operators_0, output_operator_0};
-
-   // (\nabla \cdot u, q)
-   auto mass_conservation_kernel = [](const tensor<double, dim, dim> &dudxi,
-                                      const tensor<double, dim, dim> &J,
-                                      const double &w)
-   {
-      return mfem::tuple{tr(dudxi * inv(J)) * det(J) * w};
-   };
-
-   mfem::tuple argument_operators_1{Gradient{"velocity"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator_1{Value{"pressure"}};
-   ElementOperator op_1{mass_conservation_kernel, argument_operators_1, output_operator_1};
-
-   std::array solutions{FieldDescriptor{&velocity_fes, "velocity"}, FieldDescriptor{&pressure_fes, "pressure"}};
-   std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator momentum_op{solutions, parameters, mfem::tuple{op_0}, mesh, velocity_ir};
-   DifferentiableOperator mass_conservation_op{solutions, parameters, mfem::tuple{op_1}, mesh, pressure_ir};
-
-   // Preconditioner form
-   auto pressure_mass_kernel = [](const double &p,
-                                  const tensor<double, dim, dim> &J,
-                                  const double &w)
-   {
-      return mfem::tuple{p * det(J) * w};
-   };
-
-   mfem::tuple pms_args{Value{"pressure"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple pms_outs{Value{"pressure"}};
-   ElementOperator pressure_mass{pressure_mass_kernel, pms_args, pms_outs};
-   std::array pms_sols{FieldDescriptor{&pressure_fes, "pressure"}};
-   std::array pms_params{FieldDescriptor{&mesh_fes, "coordinates"}};
-   DifferentiableOperator pressure_mass_op{pms_sols, pms_params, mfem::tuple{pressure_mass}, mesh, pressure_ir};
-
-   Array<int> block_offsets(3);
-   block_offsets[0] = 0;
-   block_offsets[1] = velocity_fes.GetTrueVSize();
-   block_offsets[2] = pressure_fes.GetTrueVSize();
-   block_offsets.PartialSum();
-
-   NavierStokesOperator navierstokes(momentum_op, mass_conservation_op,
-                                     block_offsets,
-                                     vel_ess_tdofs);
-
-   BlockVector x(block_offsets), y(block_offsets);
-   u.ParallelProject(x.GetBlock(0));
-   // p.ParallelProject(x.GetBlock(1));
-   navierstokes.SetParameters(*mesh_nodes);
-
-   HypreParMatrix A;
-   momentum_op.template GetDerivativeWrt<0>({&u, &p}, {mesh_nodes})->Assemble(A);
-   A.EliminateBC(vel_ess_tdofs, Operator::DiagonalPolicy::DIAG_ONE);
-   HypreBoomerAMG amg(A);
-   amg.SetMaxLevels(50);
-   amg.SetPrintLevel(0);
-
-   HypreParMatrix Mp;
-   pressure_mass_op.template GetDerivativeWrt<0>({&p}, {mesh_nodes})->Assemble(Mp);
-
-   HypreDiagScale Mp_inv(Mp);
-
-   BlockDiagonalPreconditioner prec(block_offsets);
-   prec.SetDiagonalBlock(0, &amg);
-   prec.SetDiagonalBlock(1, &Mp_inv);
-
-   GMRESSolver solver(MPI_COMM_WORLD);
-   solver.SetAbsTol(0.0);
-   solver.SetRelTol(1e-8);
-   solver.SetKDim(100);
-   solver.SetMaxIter(500);
-   solver.SetPrintLevel(2);
-   solver.SetPreconditioner(prec);
-
-   NewtonSolver newton(MPI_COMM_WORLD);
-   newton.SetOperator(navierstokes);
-   newton.SetSolver(solver);
-   newton.SetRelTol(1e-8);
-   newton.SetMaxIter(50);
-   newton.SetPrintLevel(1);
-
-   Vector zero;
-   newton.Mult(zero, x);
-
-   u.SetFromTrueDofs(x.GetBlock(0));
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock(vishost, visport);
-   sol_sock << "parallel " << num_procs << " " << myid << "\n";
-   sol_sock.precision(8);
-   sol_sock << "solution\n" << mesh << u << std::flush;
-
-   return 0;
-}

examples/dfem_test_diffusion_2d.cpp

@@ -1,174 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_diffusion(
-   std::string mesh_file, int refinements, int polynomial_order)
-{
-   Mesh mesh_serial = Mesh(mesh_file);
-   MFEM_ASSERT(mesh_serial.Dimension() == 2, "incorrect mesh dimension");
-
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   mesh_serial.Clear();
-
-   out << "#el: " << mesh.GetNE() << "\n";
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction*>(mesh.GetNodes());
-   ParFiniteElementSpace& mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-   out << "#dofs " << h1fes.GetTrueVSize() << "\n";
-
-   const IntegrationRule& ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder());
-
-   out << "#qp: " << ir.GetNPoints() << "\n";
-
-   ParGridFunction f1_g(&h1fes);
-   ParGridFunction rho_g(&h1fes);
-
-   auto kernel = [] MFEM_HOST_DEVICE(const tensor<double, 2, 2>& J,
-                                     const double& w, const tensor<double, 2>& dudxi)
-   {
-      auto invJ = inv(J);
-      return mfem::tuple{dudxi * invJ * transpose(invJ) * det(J) * w};
-   };
-
-   mfem::tuple argument_operators =
-   {
-      Gradient{"coordinates"}, Weight{}, Gradient{"potential"}
-   };
-   mfem::tuple output_operator = {Gradient{"potential"}};
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector& coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      return 2.345 + 0.25 * x * x * y + y * y * x;
-   };
-
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(f1_g), y(h1fes.TrueVSize());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-   y.HostRead();
-
-   ParBilinearForm a(&h1fes);
-   a.AddDomainIntegrator(new DiffusionIntegrator);
-   a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   a.Assemble();
-   a.Finalize();
-
-   Vector y2(h1fes.TrueVSize());
-   a.Mult(x, y2);
-   y2.HostRead();
-   Vector diff(y2);
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      print_vector(diff);
-      print_vector(y2);
-      print_vector(y);
-      return 1;
-   }
-
-   // // Test linearization here as well
-   // auto dFdu = dop.GetDerivativeWrt<0>({&f1_g}, {mesh_nodes});
-
-   // if (dFdu->Height() != h1fes.GetTrueVSize())
-   // {
-   //    out << "dFdu unexpected height of " << dFdu->Height() << "\n";
-   //    return 1;
-   // }
-
-   // dFdu->Mult(x, y);
-   // y.HostRead();
-   // a.Mult(x, y2);
-   // y2.HostRead();
-
-   // diff = y2;
-   // diff -= y;
-   // if (diff.Norml2() > 1e-10)
-   // {
-   //    print_vector(diff);
-   //    print_vector(y2);
-   //    print_vector(y);
-   //    return 1;
-   // }
-
-   // // fd jacobian test
-   // {
-   //    double eps = 1.0e-6;
-   //    Vector v(x), xpv(x), xmv(x), fxpv(x.Size()), fxmv(x.Size());
-   //    v *= eps;
-   //    xpv += v;
-   //    xmv -= v;
-   //    dop.Mult(xpv, fxpv);
-   //    dop.Mult(xmv, fxmv);
-   //    fxpv -= fxmv;
-   //    fxpv /= (2.0*eps);
-
-   //    fxpv -= y;
-   //    if (fxpv.Norml2() > eps)
-   //    {
-   //       out << "||dFdu_FD u^* - ex||_l2 = " << fxpv.Norml2() << "\n";
-   //       return 1;
-   //    }
-   // }
-
-   // f1_g.ProjectCoefficient(f1_c);
-   // rho_g.ProjectCoefficient(rho_c);
-   // auto dFdrho = dop.GetDerivativeWrt<1>({&f1_g}, {&rho_g, mesh_nodes});
-   // if (dFdrho->Height() != h1fes.GetTrueVSize())
-   // {
-   //    out << "dFdrho unexpected height of " << dFdrho->Height() << "\n";
-   //    return 1;
-   // }
-
-   // dFdrho->Mult(rho_g, y);
-
-   // // fd test
-   // {
-   //    double eps = 1.0e-6;
-   //    Vector v(rho_g), rhopv(rho_g), rhomv(rho_g), frhopv(x.Size()),
-   //    frhomv(x.Size()); v *= eps; rhopv += v; rhomv -= v;
-   //    dop.SetParameters({&rhopv, mesh_nodes});
-   //    dop.Mult(x, frhopv);
-   //    dop.SetParameters({&rhomv, mesh_nodes});
-   //    dop.Mult(x, frhomv);
-   //    frhopv -= frhomv;
-   //    frhopv /= (2.0*eps);
-
-   //    frhopv -= y;
-   //    if (frhopv.Norml2() > eps)
-   //    {
-   //       out << "||dFdu_FD u^* - ex||_l2 = " << frhopv.Norml2() << "\n";
-   //       return 1;
-   //    }
-   // }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_diffusion);

examples/dfem_test_diffusion_3d.cpp

@@ -1,296 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-#include "examples/dfem/dfem_parametricspace.hpp"
-#include "fem/bilininteg.hpp"
-#include "general/tic_toc.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-
-int test_diffusion_3d(
-   std::string mesh_file, int refinements, int polynomial_order)
-{
-   constexpr int num_samples = 10;
-   constexpr int dim = 3;
-   Mesh mesh_serial = Mesh(mesh_file);
-   MFEM_ASSERT(mesh_serial.Dimension() == dim, "incorrect mesh dimension");
-
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(polynomial_order);
-   mesh_serial.Clear();
-
-   out << "#el: " << mesh.GetNE() << "\n";
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction*>(mesh.GetNodes());
-   ParFiniteElementSpace& mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-
-   out << "#dofs " << h1fes.GetTrueVSize() << "\n";
-
-   const IntegrationRule& ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(),
-                   h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(
-                      0)->GetDim() - 1);
-
-   printf("#ndof per el = %d\n", h1fes.GetFE(0)->GetDof());
-   printf("#nqp = %d\n", ir.GetNPoints());
-   printf("#q1d = %d\n", (int)floor(pow(ir.GetNPoints(), 1.0/dim) + 0.5));
-
-   ParametricSpace qdata_space(dim, dim * dim, ir.GetNPoints(),
-                               dim * dim * ir.GetNPoints() * mesh.GetNE());
-   ParametricFunction qdata(qdata_space);
-
-   ParGridFunction f1_g(&h1fes);
-   ParGridFunction rho_g(&h1fes);
-
-   auto f1 = [](const Vector& coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      const double z = coords(2);
-      return 2.345 + x + x*y + 1.25 * z*x;
-   };
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(f1_g), y(h1fes.GetTrueVSize());
-   {
-      auto diffusion_mf_kernel =
-         [] MFEM_HOST_DEVICE (
-            const tensor<double, dim>& dudxi,
-            const tensor<double, dim, dim>& J,
-            const double& w)
-      {
-         auto invJ = inv(J);
-         return mfem::tuple{dudxi * invJ * transpose(invJ) * det(J) * w};
-      };
-
-      mfem::tuple argument_operators = {Gradient{"potential"}, Gradient{"coordinates"}, Weight{}};
-      mfem::tuple output_operator = {Gradient{"potential"}};
-
-      ElementOperator eop = {diffusion_mf_kernel, argument_operators, output_operator};
-      auto ops = mfem::tuple{eop};
-
-      auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-      auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-      DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-      dop.SetParameters({mesh_nodes});
-      StopWatch sw;
-      sw.Start();
-      for (int i = 0; i < num_samples; i++)
-      {
-         dop.Mult(x, y);
-      }
-      sw.Stop();
-      printf("dfem mf:       %fs\n", sw.RealTime() / num_samples);
-      y.HostRead();
-   }
-
-   {
-      auto diffusion_setup_kernel =
-         [] MFEM_HOST_DEVICE (
-            const tensor<double, dim, dim>& J,
-            const double& w)
-      {
-         auto invJ = inv(J);
-         return mfem::tuple{invJ * transpose(invJ) * det(J) * w};
-      };
-
-      mfem::tuple argument_operators = {Gradient{"coordinates"}, Weight{}};
-      mfem::tuple output_operator = {None{"qdata"}};
-
-      ElementOperator eop = {diffusion_setup_kernel, argument_operators, output_operator};
-      auto ops = mfem::tuple{eop};
-
-      auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-      auto parameters = std::array
-      {
-         FieldDescriptor{&mesh_fes, "coordinates"},
-         FieldDescriptor{&qdata_space, "qdata"}};
-
-      DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-      dop.SetParameters({mesh_nodes, &qdata});
-      StopWatch sw;
-      sw.Start();
-      for (int i = 0; i < num_samples; i++)
-      {
-         dop.Mult(x, qdata);
-      }
-      sw.Stop();
-      printf("dfem pa setup: %fs\n", sw.RealTime() / num_samples);
-      qdata.HostRead();
-   }
-
-   // printf("qdata: ");
-   // print_vector(qdata);
-
-   {
-      auto diffusion_apply_kernel =
-         [] MFEM_HOST_DEVICE (
-            const tensor<double, dim>& dudxi,
-            const tensor<double, dim, dim>& qdata)
-      {
-         return mfem::tuple{dudxi * qdata};
-      };
-
-      mfem::tuple argument_operators = {Gradient{"potential"}, None{"qdata"}};
-      mfem::tuple output_operator = {Gradient{"potential"}};
-
-      ElementOperator eop = {diffusion_apply_kernel, argument_operators, output_operator};
-      auto ops = mfem::tuple{eop};
-
-      auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-      auto parameters = std::array{FieldDescriptor{&qdata_space, "qdata"}};
-
-      DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-      dop.SetParameters({&qdata});
-      StopWatch sw;
-      sw.Start();
-      for (int i = 0; i < num_samples; i++)
-      {
-         dop.Mult(x, y);
-      }
-      sw.Stop();
-      printf("dfem pa apply: %fs\n", sw.RealTime() / num_samples);
-      y.HostRead();
-   }
-
-   // printf("y: ");
-   // print_vector(y);
-
-   Vector y2(h1fes.TrueVSize());
-   {
-      ParBilinearForm a(&h1fes);
-      auto diff_integ = new DiffusionIntegrator;
-      diff_integ->SetIntRule(&ir);
-      a.AddDomainIntegrator(diff_integ);
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-
-      OperatorPtr A;
-      StopWatch sw;
-      sw.Start();
-      a.Assemble();
-      a.Finalize();
-      Array<int> empty;
-      a.FormSystemMatrix(empty, A);
-      sw.Stop();
-      printf("mfem pa setup: %fs\n", sw.RealTime());
-
-      sw.Clear();
-      sw.Start();
-      for (int i = 0; i < num_samples; i++)
-      {
-         A->Mult(x, y2);
-      }
-      sw.Stop();
-      printf("mfem pa apply: %fs\n", sw.RealTime() / num_samples);
-      y2.HostRead();
-   }
-   // printf("y2: ");
-   // print_vector(y2);
-
-   Vector diff(y2);
-   diff -= y;
-   if (diff.Norml2() > 1e-15)
-   {
-      // printf("y ");
-      // print_vector(y);
-      // printf("y2: ");
-      // print_vector(y2);
-      // printf("diff: ");
-      // print_vector(diff);
-      return 1;
-   }
-
-   // Test linearization here as well
-   // auto dFdu = dop.GetDerivativeWrt<0>({&f1_g}, {mesh_nodes});
-
-   // if (dFdu->Height() != h1fes.GetTrueVSize())
-   // {
-   //    out << "dFdu unexpected height of " << dFdu->Height() << "\n";
-   //    return 1;
-   // }
-
-   // dFdu->Mult(x, y);
-   // y.HostRead();
-   // a.Mult(x, y2);
-   // y2.HostRead();
-
-   // diff = y2;
-   // diff -= y;
-   // if (diff.Norml2() > 1e-10)
-   // {
-   //    print_vector(diff);
-   //    print_vector(y2);
-   //    print_vector(y);
-   //    return 1;
-   // }
-
-   // // fd jacobian test
-   // {
-   //    double eps = 1.0e-6;
-   //    Vector v(x), xpv(x), xmv(x), fxpv(x.Size()), fxmv(x.Size());
-   //    v *= eps;
-   //    xpv += v;
-   //    xmv -= v;
-   //    dop.Mult(xpv, fxpv);
-   //    dop.Mult(xmv, fxmv);
-   //    fxpv -= fxmv;
-   //    fxpv /= (2.0*eps);
-
-   //    fxpv -= y;
-   //    if (fxpv.Norml2() > eps)
-   //    {
-   //       out << "||dFdu_FD u^* - ex||_l2 = " << fxpv.Norml2() << "\n";
-   //       return 1;
-   //    }
-   // }
-
-   // f1_g.ProjectCoefficient(f1_c);
-   // rho_g.ProjectCoefficient(rho_c);
-   // auto dFdrho = dop.GetDerivativeWrt<1>({&f1_g}, {&rho_g, mesh_nodes});
-   // if (dFdrho->Height() != h1fes.GetTrueVSize())
-   // {
-   //    out << "dFdrho unexpected height of " << dFdrho->Height() << "\n";
-   //    return 1;
-   // }
-
-   // dFdrho->Mult(rho_g, y);
-
-   // // fd test
-   // {
-   //    double eps = 1.0e-6;
-   //    Vector v(rho_g), rhopv(rho_g), rhomv(rho_g), frhopv(x.Size()),
-   //    frhomv(x.Size()); v *= eps; rhopv += v; rhomv -= v;
-   //    dop.SetParameters({&rhopv, mesh_nodes});
-   //    dop.Mult(x, frhopv);
-   //    dop.SetParameters({&rhomv, mesh_nodes});
-   //    dop.Mult(x, frhomv);
-   //    frhopv -= frhomv;
-   //    frhopv /= (2.0*eps);
-
-   //    frhopv -= y;
-   //    if (frhopv.Norml2() > eps)
-   //    {
-   //       out << "||dFdu_FD u^* - ex||_l2 = " << frhopv.Norml2() << "\n";
-   //       return 1;
-   //    }
-   // }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_diffusion_3d);

examples/dfem_test_elasticity.cpp

@@ -1,109 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_elasticity(std::string mesh_file,
-                    int refinements,
-                    int polynomial_order)
-{
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   const int vdim = dim;
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   Array<int> ess_tdof;
-   ess_bdr = 1;
-   h1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 6 * h1fec.GetOrder());
-
-   out << "#qp: " << ir.GetNPoints() << "\n";
-   out << "#dof_el: " << h1fes.GetRestrictionMatrix()->Height() / mesh.GetNE() <<
-       "\n";
-
-   ParGridFunction u(&h1fes);
-
-   auto f1 = [](const Vector& coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      u(0) = 2.345 + 0.25 * x * x * y + y * y * x;
-      u(1) = 2.345 - 0.25 * x * y * y + y * x * x;
-   };
-
-   VectorFunctionCoefficient u_c(dim, f1);
-   u.ProjectCoefficient(u_c);
-
-   ConstantCoefficient l_coeff(0.5), m_coeff(0.25);
-
-   ParBilinearForm A_form(&h1fes);
-   auto A_integ = new ElasticityIntegrator(l_coeff, m_coeff);
-   A_integ->SetIntegrationRule(ir);
-   A_form.AddDomainIntegrator(A_integ);
-   A_form.Assemble();
-   A_form.Finalize();
-
-   auto elasticity_kernel = [](const tensor<double, 2, 2> &dudxi,
-                               const tensor<double, 2, 2> &J,
-                               const double &w)
-   {
-      constexpr double lambda = 0.5;
-      constexpr double mu = 0.25;
-      static constexpr auto I = mfem::internal::IsotropicIdentity<2>();
-      auto invJ = inv(J);
-      auto eps = sym(dudxi * invJ);
-      return mfem::tuple{transpose(lambda * tr(eps) * I + 2.0 * mu * eps) * det(J) * w * transpose(invJ)};
-   };
-
-   mfem::tuple argument_operators{Gradient{"displacement"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator{Gradient{"displacement"}};
-
-   ElementOperator op{elasticity_kernel, argument_operators, output_operator};
-
-   std::array solutions{FieldDescriptor{&h1fes, "displacement"}};
-   std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop{solutions, parameters, mfem::tuple{op}, mesh, ir};
-
-   Vector x(u), y1(h1fes.GetTrueVSize()),
-          y2(h1fes.GetTrueVSize());
-
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y1);
-   y1.HostRead();
-
-   A_form.Mult(x, y2);
-   y2.HostRead();
-
-   Vector diff(y2);
-   diff -= y1;
-   if (diff.Norml2() > 1e-10)
-   {
-      out << "||F(u) - ex||_l2 = " << diff.Norml2() << "\n";
-      print_vector(diff);
-      print_vector(y1);
-      print_vector(y2);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_elasticity);

examples/dfem_test_interpolate_gradient_linear_scalar_3d.cpp

@@ -1,115 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-#include "examples/dfem/dfem_parametricspace.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_interpolate_gradient_linear_scalar_3d(std::string mesh_file,
-                                               int refinements,
-                                               int polynomial_order)
-{
-   constexpr int dim = 3;
-   Mesh mesh_serial = Mesh(mesh_file);
-   MFEM_ASSERT(mesh_serial.Dimension() == dim, "wrong mesh dimension");
-
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   mesh_serial.Clear();
-
-   ParGridFunction *mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-   // const IntegrationRule &ir =
-   //    IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-   IntegrationRules gll_rules(0, Quadrature1D::GaussLobatto);
-   const IntegrationRule &ir = gll_rules.Get(h1fes.GetFE(0)->GetGeomType(),
-                                             2 * polynomial_order - 1);
-
-   ParGridFunction f1_g(&h1fes);
-
-   ParametricSpace pspace(dim, dim, ir.GetNPoints(),
-                          dim * ir.GetNPoints() * mesh.GetNE());
-   ParametricFunction qdata(pspace);
-
-   auto kernel = [](const tensor<double, dim> &dudxi,
-                    const tensor<double, dim, dim> &J)
-   {
-      return mfem::tuple{dudxi * inv(J)};
-   };
-
-   mfem::tuple argument_operators = {Gradient{"potential"}, Gradient{"coordinates"}};
-   mfem::tuple output_operator = {None{"qdata"}};
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array
-   {
-      FieldDescriptor{&mesh_fes, "coordinates"},
-      FieldDescriptor{&pspace, "qdata"}
-   };
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector &coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      const double z = coords(2);
-      return 2.345 + x * y * z + y * z;
-   };
-
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(*f1_g.GetTrueDofs()), y(h1fes.TrueVSize() * dim);
-   dop.SetParameters({mesh_nodes, &qdata});
-   dop.Mult(x, y);
-
-   Vector f_test(h1fes.GetElementRestriction(
-                    ElementDofOrdering::LEXICOGRAPHIC)->Height() * dim);
-   for (int e = 0; e < mesh.GetNE(); e++)
-   {
-      ElementTransformation *T = mesh.GetElementTransformation(e);
-
-      for (int qp = 0; qp < ir.GetNPoints(); qp++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(qp);
-         T->SetIntPoint(&ip);
-
-         Vector g(dim);
-         f1_g.GetGradient(*T, g);
-         // printf("(%f, %f, %f): (%f, %f, %f)\n", ip.x, ip.y, ip.z,  g(0), g(1), g(2));
-         for (int d = 0; d < dim; d++)
-         {
-            int qpo = qp * dim;
-            int eo = e * (ir.GetNPoints() * dim);
-            f_test(d + qpo + eo) = g(d);
-         }
-      }
-   }
-
-   Vector diff(f_test);
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      print_vector(diff);
-      print_vector(f_test);
-      print_vector(y);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_interpolate_gradient_linear_scalar_3d);

examples/dfem_test_interpolate_linear_scalar.cpp

@@ -1,91 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_interpolate_linear_scalar(std::string mesh_file,
-                                   int refinements,
-                                   int polynomial_order)
-{
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   ParGridFunction f1_g(&h1fes);
-
-   auto kernel = [](const double &u, const tensor<double, 2, 2> &J,
-                    const double &w)
-   {
-      return mfem::tuple{u};
-   };
-
-   mfem::tuple argument_operators = {Value{"potential"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator = {None{"potential"}};
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector &coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      return 2.345 + x + y;
-   };
-
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(*f1_g.GetTrueDofs()), y(h1fes.TrueVSize());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-
-   Vector f_test(h1fes.GetElementRestriction(
-                    ElementDofOrdering::LEXICOGRAPHIC)->Height());
-   for (int e = 0; e < mesh.GetNE(); e++)
-   {
-      ElementTransformation *T = mesh.GetElementTransformation(e);
-      for (int qp = 0; qp < ir.GetNPoints(); qp++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(qp);
-         T->SetIntPoint(&ip);
-
-         f_test((e * ir.GetNPoints()) + qp) = f1_c.Eval(*T, ip);
-      }
-   }
-
-   Vector diff(f_test);
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      print_vector(diff);
-      print_vector(f_test);
-      print_vector(y);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_interpolate_linear_scalar);

examples/dfem_test_interpolate_linear_scalar_3d.cpp

@@ -1,93 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_interpolate_linear_scalar_3d(std::string mesh_file,
-                                      int refinements,
-                                      int polynomial_order)
-{
-   constexpr int dim = 3;
-   Mesh mesh_serial = Mesh(mesh_file);
-   MFEM_ASSERT(mesh_serial.Dimension() == dim, "wrong mesh dimension");
-
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   mesh_serial.Clear();
-
-   ParGridFunction *mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   ParGridFunction f1_g(&h1fes);
-
-   auto kernel = [](const double &u)
-   {
-      return mfem::tuple{u};
-   };
-
-   mfem::tuple argument_operators = {Value{"potential"}};
-   mfem::tuple output_operator = {None{"potential"}};
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector &coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      const double z = coords(2);
-      return 2.345 + x + y + 1.25 * z;
-   };
-
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(*f1_g.GetTrueDofs()), y(h1fes.TrueVSize());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-
-   Vector f_test(h1fes.GetElementRestriction(
-                    ElementDofOrdering::LEXICOGRAPHIC)->Height());
-   for (int e = 0; e < mesh.GetNE(); e++)
-   {
-      ElementTransformation *T = mesh.GetElementTransformation(e);
-      for (int qp = 0; qp < ir.GetNPoints(); qp++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(qp);
-         T->SetIntPoint(&ip);
-
-         f_test((e * ir.GetNPoints()) + qp) = f1_c.Eval(*T, ip);
-      }
-   }
-
-   Vector diff(f_test);
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      print_vector(diff);
-      print_vector(f_test);
-      print_vector(y);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_interpolate_linear_scalar_3d);

examples/dfem_test_interpolate_linear_vector.cpp

@@ -1,100 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_interpolate_linear_vector(std::string mesh_file, int refinements,
-                                   int polynomial_order)
-{
-   constexpr int vdim = 2;
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   QuadratureSpace qspace(mesh, ir);
-   QuadratureFunction qf(&qspace, vdim);
-
-   ParGridFunction f1_g(&h1fes);
-
-   auto kernel = [](const tensor<double, 2> &u)
-   {
-      return mfem::tuple{u};
-   };
-
-   mfem::tuple argument_operators = {Value{"potential"}};
-   mfem::tuple output_operator = {None{"potential"}};
-
-   ElementOperator eop{kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector &coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      u(0) = 2.345 + x + y;
-      u(1) = 12.345 + x + y;
-   };
-
-   VectorFunctionCoefficient f1_c(vdim, f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(f1_g), y(f1_g.Size());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-
-   Vector f_test(qf.Size());
-   for (int e = 0; e < mesh.GetNE(); e++)
-   {
-      ElementTransformation *T = mesh.GetElementTransformation(e);
-      for (int qp = 0; qp < ir.GetNPoints(); qp++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(qp);
-         T->SetIntPoint(&ip);
-
-         Vector f(vdim);
-         f1_g.GetVectorValue(*T, ip, f);
-         for (int d = 0; d < vdim; d++)
-         {
-            int qpo = qp * vdim;
-            int eo = e * (ir.GetNPoints() * vdim);
-            f_test(d + qpo + eo) = f(d);
-         }
-      }
-   }
-
-   Vector diff(f_test);
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      print_vector(diff);
-      print_vector(f_test);
-      print_vector(y);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_interpolate_linear_vector);

examples/dfem_test_interpolate_linear_vector_3d.cpp

@@ -1,105 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_interpolate_linear_vector_3d(std::string mesh_file, int refinements,
-                                      int polynomial_order)
-{
-
-   constexpr int dim = 3;
-   constexpr int vdim = 3;
-   Mesh mesh_serial = Mesh(mesh_file);
-   MFEM_ASSERT(mesh_serial.Dimension() == dim, "wrong mesh dimension");
-
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-   mesh.SetCurvature(1);
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   QuadratureSpace qspace(mesh, ir);
-   QuadratureFunction qf(&qspace, vdim);
-
-   ParGridFunction f1_g(&h1fes);
-
-   auto kernel = [](const tensor<double, vdim> &u)
-   {
-      return mfem::tuple{u};
-   };
-
-   mfem::tuple argument_operators = {Value{"potential"}};
-   mfem::tuple output_operator = {None{"potential"}};
-
-   ElementOperator eop{kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector &coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      const double z = coords(2);
-      u(0) = 2.345 + x + y + 3.0 * z;
-      u(1) = 12.345 + x + y + 2.0 * z;
-      u(2) = 5.345 + x + y + 1.0 * z;
-   };
-
-   VectorFunctionCoefficient f1_c(vdim, f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(f1_g), y(f1_g.Size());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-
-   Vector f_test(qf.Size());
-   for (int e = 0; e < mesh.GetNE(); e++)
-   {
-      ElementTransformation *T = mesh.GetElementTransformation(e);
-      for (int qp = 0; qp < ir.GetNPoints(); qp++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(qp);
-         T->SetIntPoint(&ip);
-
-         Vector f(vdim);
-         f1_g.GetVectorValue(*T, ip, f);
-         for (int d = 0; d < vdim; d++)
-         {
-            int qpo = qp * vdim;
-            int eo = e * (ir.GetNPoints() * vdim);
-            f_test(d + qpo + eo) = f(d);
-         }
-      }
-   }
-
-   Vector diff(f_test);
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      print_vector(diff);
-      print_vector(f_test);
-      print_vector(y);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_interpolate_linear_vector_3d);

examples/dfem_test_mass_scalar_2d.cpp

@@ -1,113 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-#include "fem/bilininteg.hpp"
-#include "fem/normal_deriv_restriction.hpp"
-#include <fstream>
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int dfem_test_mass_scalar_2d(std::string mesh_file,
-                             int refinements,
-                             int polynomial_order)
-{
-   constexpr int dim = 2;
-   Mesh mesh_serial = Mesh(mesh_file);
-   MFEM_ASSERT(mesh_serial.Dimension() == dim, "wrong mesh dimension");
-
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   mesh_serial.Clear();
-
-   ParGridFunction *mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() + 1);
-
-   // IntegrationRules gll_rules(0, Quadrature1D::GaussLobatto);
-   // const IntegrationRule &ir = gll_rules.Get(h1fes.GetFE(0)->GetGeomType(),
-   //                                           2 * polynomial_order - 1);
-
-   printf("#nqp = %d\n", ir.GetNPoints());
-   printf("#q1d = %d\n", (int)floor(pow(ir.GetNPoints(), 1.0/dim) + 0.5));
-
-   ParGridFunction f1_g(&h1fes);
-
-   auto kernel = [](const double& u,
-                    const tensor<double, dim> x,
-                    const tensor<double, dim, dim> J,
-                    const double& w)
-   {
-      out << x << ": " << u << "\n";
-      return mfem::tuple{u * w * det(J)};
-   };
-
-   mfem::tuple argument_operators = {Value{"potential"}, Value{"coordinates"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator = {Value{"potential"}};
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector &coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      return 2.345 + x + x*y + 1.25 * x;
-   };
-
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector f1_g_e(f1_g.Size());
-   auto R = h1fes.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
-   // R->Mult(f1_g, f1_g_e);
-   auto r_out = std::ofstream("r_mat.mtx");
-   R->PrintMatlab(r_out);
-   r_out.close();
-   print_vector(f1_g);
-   // print_vector(f1_g_e);
-
-   Vector x(*f1_g.GetTrueDofs()), y(h1fes.TrueVSize());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-
-   ParBilinearForm a(&h1fes);
-   auto mass_integ = new MassIntegrator;
-   mass_integ->SetIntRule(&ir);
-   a.AddDomainIntegrator(mass_integ);
-   a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   a.Assemble();
-   a.Finalize();
-
-   Vector y2(h1fes.TrueVSize());
-   a.Mult(x, y2);
-   y2.HostRead();
-
-   Vector diff(y2);
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      print_vector(diff);
-      print_vector(y2);
-      print_vector(y);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(dfem_test_mass_scalar_2d);

examples/dfem_test_mass_scalar_3d.cpp

@@ -1,147 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-#include "fem/bilininteg.hpp"
-#include "fem/fe/fe_base.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int dfem_test_mass_scalar_3d(std::string mesh_file,
-                             int refinements,
-                             int polynomial_order)
-{
-   constexpr int dim = 3;
-   Mesh mesh_serial = Mesh(mesh_file);
-   MFEM_ASSERT(mesh_serial.Dimension() == dim, "wrong mesh dimension");
-
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(polynomial_order);
-   mesh_serial.Clear();
-
-   ParGridFunction *mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-   const IntegrationRule& ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(),
-                   h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(
-                      0)->GetDim() - 1);
-
-   // IntegrationRules gll_rules(0, Quadrature1D::GaussLobatto);
-   // const IntegrationRule &ir = gll_rules.Get(h1fes.GetFE(0)->GetGeomType(),
-   //                                           2 * polynomial_order - 1);
-
-   auto dtq = h1fes.GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-   // printf("\n B: ");
-   // dtq.B.Print(out, dtq.B.Size());
-   // printf("\n G: ");
-   // dtq.G.Print(out, dtq.G.Size());
-   // printf("\n w: ");
-   // ir.GetWeights().Print(out, ir.GetWeights().Size());
-
-   // printf("#ndof per el = %d\n", h1fes.GetFE(0)->GetDof());
-   // printf("#nqp = %d\n", ir.GetNPoints());
-   // printf("#q1d = %d\n", (int)floor(pow(ir.GetNPoints(), 1.0/dim) + 0.5));
-
-   // printf("nodes: ");
-   // print_vector(*mesh_nodes);
-
-   ParGridFunction f1_g(&h1fes);
-
-   auto kernel = [](const double &u,
-                    const tensor<double, dim, dim> &J,
-                    const double &w)
-   {
-      return mfem::tuple{u * det(J) * w};
-   };
-
-   mfem::tuple argument_operators = {Value{"potential"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator = {Value{"potential"}};
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "potential"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector &coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      const double z = coords(2);
-      return 2.345 + x + x*y + 1.25 * z*x;
-   };
-
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-   // printf("\nf1_g: ");
-   // print_vector(f1_g);
-
-   auto R = h1fes.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
-   // Vector f1_g_e(R->Height());
-   // R->Mult(f1_g, f1_g_e);
-   // printf("\nf1_g_e: ");
-   // print_vector(f1_g_e);
-   // auto r_out = std::ofstream("r_mat.mtx");
-   // R->PrintMatlab(r_out);
-   // r_out.close();
-
-   Vector x(*f1_g.GetTrueDofs()), y(h1fes.TrueVSize());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-
-   ParBilinearForm a(&h1fes);
-   auto mass_integ = new MassIntegrator;
-   mass_integ->SetIntRule(&ir);
-   a.AddDomainIntegrator(mass_integ);
-   a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   a.Assemble();
-   a.Finalize();
-
-   Vector y2(h1fes.TrueVSize());
-   a.Mult(x, y2);
-   y2.HostRead();
-
-   Vector diff(y2);
-   diff -= y;
-   if (diff.Norml2() > 1e-15)
-   {
-      printf("y ");
-      print_vector(y);
-      printf("y2: ");
-      print_vector(y2);
-      printf("diff: ");
-      print_vector(diff);
-      return 1;
-   }
-
-   Vector y3(h1fes.TrueVSize());
-   auto dFdu = dop.GetDerivativeWrt<0>({&f1_g}, {mesh_nodes});
-
-   dFdu->Mult(x, y3);
-   diff = y2;
-   diff -= y;
-   if (diff.Norml2() > 1e-15)
-   {
-      printf("y2 ");
-      print_vector(y2);
-      printf("y3: ");
-      print_vector(y3);
-      printf("diff: ");
-      print_vector(diff);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(dfem_test_mass_scalar_3d);

examples/dfem_test_neo_hookean_elasticity_2d.cpp

@@ -1,114 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_neo_hookean_elasticity_2d(
-   std::string mesh_file, int refinements, int polynomial_order)
-{
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   MFEM_ASSERT(dim == 2, "This test is for 2D meshes only");
-   mesh_serial.Clear();
-
-   out << "#el: " << mesh.GetNE() << "\n";
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction*>(mesh.GetNodes());
-   ParFiniteElementSpace& mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, dim);
-
-   out << "#dofs " << h1fes.GetTrueVSize() << "\n";
-
-   const IntegrationRule& ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder());
-
-   out << "#qp: " << ir.GetNPoints() << "\n";
-
-   ParGridFunction u_g(&h1fes);
-
-   auto kernel = [] MFEM_HOST_DEVICE(const tensor<double, 2, 2>& J,
-                                     const double& w,
-                                     const tensor<double, 2, 2>& dudxi)
-   {
-      // Neo-Hookean parameters
-      const double lambda = 1.0;
-      const double mu = 0.5;
-
-      static constexpr auto I = mfem::internal::IsotropicIdentity<2>();
-      auto F = I + (dudxi * inv(J));
-      auto E = 0.5 * (transpose(F) * F - I);
-      auto invF = inv(F);
-
-      // 2D plane strain formulation
-      auto P = mu * (F - transpose(invF)) + lambda * log(det(F)) * transpose(invF);
-
-      return mfem::tuple{P * det(J) * w};
-   };
-
-   mfem::tuple argument_operators = {Gradient{"coordinates"}, Weight{},
-                                     Gradient{"displacement"}
-                                    };
-   mfem::tuple output_operator = {Gradient{"displacement"}};
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array{FieldDescriptor{&h1fes, "displacement"}};
-   auto parameters = std::array{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto displacement = [](const Vector& coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      u(0) = 0.1 * x * y;
-      u(1) = 0.1 * y * x;
-   };
-
-   VectorFunctionCoefficient disp_coeff(2, displacement);
-   u_g.ProjectCoefficient(disp_coeff);
-
-   Vector x(u_g), y(h1fes.TrueVSize());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-   y.HostRead();
-
-   // Test linearization
-   auto dFdu = dop.GetDerivativeWrt<0>({&u_g}, {mesh_nodes});
-   dFdu->Mult(x, y);
-
-   // Finite difference Jacobian test
-   {
-      double eps = 1.0e-6;
-      Vector v(x), xpv(x), xmv(x), fxpv(x.Size()), fxmv(x.Size());
-      v *= eps;
-      xpv += v;
-      xmv -= v;
-      dop.Mult(xpv, fxpv);
-      dop.Mult(xmv, fxmv);
-      fxpv -= fxmv;
-      fxpv /= (2.0*eps);
-
-      fxpv -= y;
-      if (fxpv.Norml2() > eps)
-      {
-         out << "||dFdu_FD u^* - ex||_l2 = " << fxpv.Norml2() << "\n";
-         return 1;
-      }
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_neo_hookean_elasticity_2d);

examples/dfem_test_nonlinear_diffusion_3d.cpp

@@ -1,169 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_nonlinear_diffusion(
-   std::string mesh_file, int refinements, int polynomial_order)
-{
-   constexpr int dim = 3;
-
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   mesh_serial.Clear();
-
-   out << "#el: " << mesh.GetNE() << "\n";
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction*>(mesh.GetNodes());
-   ParFiniteElementSpace& mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec);
-
-   out << "#dofs " << h1fes.GetTrueVSize() << "\n";
-
-   const IntegrationRule& ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(),
-                   h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(
-                      0)->GetDim() - 1);
-
-   out << "#qp: " << ir.GetNPoints() << "\n";
-
-   ParGridFunction f1_g(&h1fes);
-
-   bool inactive_derivative = false;
-
-   auto kernel = [] MFEM_HOST_DEVICE(
-                    const tensor<double, dim, dim>& J,
-                    const double& w,
-                    const tensor<double, dim>& dudxi,
-                    const double& u)
-   {
-      auto invJ = inv(J);
-      return mfem::tuple{(u * u) * dudxi * invJ * transpose(invJ) * det(J) * w};
-   };
-
-   mfem::tuple argument_operators =
-   {
-      Gradient{"coordinates"},
-      Weight{},
-      Gradient{"potential"},
-      Value{"potential"}
-   };
-
-   mfem::tuple output_operator =
-   {
-      Gradient{"potential"}
-   };
-
-   ElementOperator eop = {kernel, argument_operators, output_operator};
-   auto ops = mfem::tuple{eop};
-
-   auto solutions = std::array
-   {
-      FieldDescriptor{&h1fes, "potential"}
-   };
-   auto parameters = std::array
-   {
-      FieldDescriptor{&mesh_fes, "coordinates"}
-   };
-
-   DifferentiableOperator dop(solutions, parameters, ops, mesh, ir);
-
-   auto f1 = [](const Vector& coords)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      const double z = coords(2);
-      return 2.345 + 0.25 * x * x * y + y * y * x + z;
-   };
-
-   FunctionCoefficient f1_c(f1);
-   f1_g.ProjectCoefficient(f1_c);
-
-   Vector x(f1_g), y(h1fes.TrueVSize());
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y);
-   y.HostRead();
-
-   ParBilinearForm a(&h1fes);
-   GridFunctionCoefficient f1gc(&f1_g);
-   TransformedCoefficient tf_c(&f1gc, [](double f) { return f * f; });
-   a.AddDomainIntegrator(new DiffusionIntegrator(tf_c));
-   a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   a.Assemble();
-   a.Finalize();
-
-   Vector y2(h1fes.TrueVSize()), diff(h1fes.TrueVSize());
-   a.Mult(x, y2);
-   y2.HostRead();
-   diff = y2;
-   diff -= y;
-   if (diff.Norml2() > 1e-10)
-   {
-      out << "||F(u) - ex||_l2 = " << diff.Norml2() << "\n";
-      print_vector(diff);
-      print_vector(y);
-      print_vector(y2);
-      return 1;
-   }
-
-   // Test linearization here as well
-   auto dFdu = dop.GetDerivativeWrt<0>({&f1_g}, {mesh_nodes});
-   dFdu->Mult(x, y);
-
-   // fd jacobian test
-   {
-      double eps = 1.0e-6;
-      Vector v(x), xpv(x), xmv(x), fxpv(x.Size()), fxmv(x.Size());
-      v *= eps;
-      xpv += v;
-      xmv -= v;
-      dop.Mult(xpv, fxpv);
-      dop.Mult(xmv, fxmv);
-      fxpv -= fxmv;
-      fxpv /= (2.0*eps);
-
-      fxpv -= y;
-      if (fxpv.Norml2() > eps)
-      {
-         out << "||dFdu_FD u^* - ex||_l2 = " << fxpv.Norml2() << "\n";
-         return 1;
-      }
-   }
-
-   // ParBilinearForm da(&h1fes);
-   // TransformedCoefficient dtf_c(&f1gc, [](double f) { return 2.0 * f; });
-   // da.AddDomainIntegrator(new DiffusionIntegrator(dtf_c));
-   // da.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   // da.Assemble();
-   // da.Finalize();
-
-   // if (dFdu->Height() != h1fes.GetTrueVSize())
-   // {
-   //    out << "dFdu unexpected height of " << dFdu->Height() << "\n";
-   //    return 1;
-   // }
-
-   // dFdu->Mult(x, y);
-   // print_vector(y);
-   // da.Mult(x, y2);
-   // print_vector(y2);
-   // y2 -= y;
-   // out << "||dFdu x - A x||_l2 = " << y2.Norml2() << "\n";
-   // if (y2.Norml2() > 1e-10)
-   // {
-   //    out << "||dFdu u^* - ex||_l2 = " << y2.Norml2() << "\n";
-   // }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_nonlinear_diffusion);

examples/dfem_test_nonlinear_elasticity_3d.cpp

@@ -1,268 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-#include "fem/pfespace.hpp"
-#include "linalg/hypre.hpp"
-#include "linalg/operator.hpp"
-#include "linalg/solvers.hpp"
-#include <fstream>
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-class FDJacobian : public Operator
-{
-public:
-   FDJacobian(const Operator &op, const Vector &x) :
-      Operator(op.Height()),
-      op(op),
-      x(x)
-   {
-      f.SetSize(Height());
-      xpev.SetSize(Height());
-      op.Mult(x, f);
-      xnorm = x.Norml2();
-   }
-
-   void Mult(const Vector &v, Vector &y) const override
-   {
-      x.HostRead();
-
-      // See [1] for choice of eps.
-      //
-      // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
-      // finite difference matrix-vector products in Newton-Krylov solvers for
-      // implicit climate dynamics with spectral elements. Procedia Computer
-      // Science, 51, pp.2036-2045.
-      real_t eps = lambda * (lambda + xnorm / v.Norml2());
-
-      for (int i = 0; i < x.Size(); i++)
-      {
-         xpev(i) = x(i) + eps * v(i);
-      }
-
-      // y = f(x + eps * v)
-      op.Mult(xpev, y);
-
-      // y = (f(x + eps * v) - f(x)) / eps
-      for (int i = 0; i < x.Size(); i++)
-      {
-         y(i) = (y(i) - f(i)) / eps;
-      }
-   }
-
-   virtual MemoryClass GetMemoryClass() const override
-   {
-      return Device::GetDeviceMemoryClass();
-   }
-
-private:
-   const Operator &op;
-   Vector x, f;
-   mutable Vector xpev;
-   real_t lambda = 1.0e-6;
-   real_t xnorm;
-};
-
-template <typename elasticity_t>
-class ElasticityOperator : public Operator
-{
-   template <typename elasticity_du_t>
-   class ElasticityJacobianOperator : public Operator
-   {
-   public:
-      ElasticityJacobianOperator(const ElasticityOperator *elasticity,
-                                 std::shared_ptr<elasticity_du_t> dRdu) :
-         Operator(elasticity->Height()),
-         elasticity(elasticity),
-         dRdu(dRdu),
-         x_ess(dRdu->Height())
-      {
-      }
-
-      void Mult(const Vector &x, Vector &y) const override
-      {
-         x_ess = x;
-         x_ess.SetSubVector(elasticity->ess_tdofs, 0.0);
-
-         dRdu->Mult(x_ess, y);
-
-         for (int i = 0; i < elasticity->ess_tdofs.Size(); i++)
-         {
-            y[elasticity->ess_tdofs[i]] = x[elasticity->ess_tdofs[i]];
-         }
-      }
-
-      const ElasticityOperator *elasticity = nullptr;
-      std::shared_ptr<elasticity_du_t> dRdu;
-      mutable Vector x_ess;
-   };
-
-public:
-   ElasticityOperator(ParFiniteElementSpace &fes, elasticity_t &elasticity,
-                      Array<int> &ess_tdofs) :
-      Operator(fes.GetTrueVSize()),
-      fes(fes),
-      elasticity(elasticity),
-      ess_tdofs(ess_tdofs) {}
-
-   void Mult(const Vector &x, Vector &r) const override
-   {
-      elasticity.Mult(x, r);
-      r.SetSubVector(ess_tdofs, 0.0);
-   }
-
-   Operator &GetGradient(const Vector &x) const override
-   {
-      ParGridFunction u(const_cast<ParFiniteElementSpace *>
-                        (*std::get_if<const ParFiniteElementSpace *>
-                         (&elasticity.solutions[0].data)));
-
-      u.SetFromTrueDofs(x);
-      auto dRdu = elasticity.template GetDerivativeWrt<0>({&u}, {mesh_nodes});
-
-      jacobian.reset(
-         new ElasticityJacobianOperator<
-         typename std::remove_pointer<decltype(dRdu.get())>::type> (this, dRdu));
-
-      // jacobian.reset(new FDJacobian(*this, x));
-
-      return *jacobian;
-   }
-
-   void SetParameters(ParGridFunction &mesh_nodes)
-   {
-      elasticity.SetParameters({&mesh_nodes});
-      this->mesh_nodes = &mesh_nodes;
-   }
-
-   ParFiniteElementSpace &fes;
-   elasticity_t &elasticity;
-   Array<int> ess_tdofs;
-   mutable ParGridFunction *mesh_nodes = nullptr;
-   mutable std::shared_ptr<Operator> jacobian;
-};
-
-int test_nonlinear_elasticity_3d(std::string mesh_file,
-                                 int refinements,
-                                 int polynomial_order)
-{
-   constexpr int dim = 3;
-   constexpr int vdim = dim;
-
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(polynomial_order);
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   Array<int> ess_tdof_list, ess_bdr(mesh.bdr_attributes.Max());
-   ess_bdr = 0;
-   ess_bdr[0] = 1;
-   h1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   const IntegrationRule& ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(),
-                   h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(0)->GetOrder() + h1fes.GetFE(
-                      0)->GetDim() - 1);
-
-   out << "#qp: " << ir.GetNPoints() << "\n";
-   out << "#dof: " << h1fes.GetNDofs() << "\n";
-
-   ParGridFunction u(&h1fes);
-
-   auto elasticity_kernel = [] MFEM_HOST_DEVICE
-                            (const tensor<real_t, dim, dim> &dudxi,
-                             const tensor<real_t, dim, dim> &J,
-                             const double &w)
-   {
-      // shear modulus
-      mfem::real_t D1 = 0.1e6;
-      // bulk modulus
-      mfem::real_t C1 = 1.0e6;
-      constexpr auto I = mfem::internal::IsotropicIdentity<dim>();
-      auto invJ = inv(J);
-      auto dudx = dudxi * invJ;
-      real_t F = det(I + dudx);
-      real_t p = -2.0 * D1 * F * (F - 1);
-      auto devB = dev(dudx + transpose(dudx) + dot(dudx, transpose(dudx)));
-      auto sigma = -(p / F) * I + 2.0 * (C1 / pow(F, 5.0 / 3.0)) * devB;
-
-      return mfem::tuple{sigma * det(J) * w * transpose(invJ)};
-   };
-
-   mfem::tuple argument_operators{Gradient{"displacement"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator{Gradient{"displacement"}};
-
-   ElementOperator op{elasticity_kernel, argument_operators, output_operator};
-
-   std::array solutions{FieldDescriptor{&h1fes, "displacement"}};
-   std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop{solutions, parameters, mfem::tuple{op}, mesh, ir};
-
-   ElasticityOperator elasticity(h1fes, dop, ess_tdof_list);
-
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-   {
-      Vector pull_force(mesh.bdr_attributes.Max());
-      pull_force = 0.0;
-      pull_force(1) = -1.0e-2;
-      f.Set(dim-1, new PWConstCoefficient(pull_force));
-   }
-
-   ParLinearForm b(&h1fes);
-   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
-   b.UseFastAssembly(true);
-   b.Assemble();
-   auto B = b.ParallelAssemble();
-
-   Vector X = u.GetTrueVector();
-
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-8);
-   cg.SetMaxIter(1000);
-   cg.SetPrintLevel(IterativeSolver::PrintLevel().Summary());
-
-   NewtonSolver newton(MPI_COMM_WORLD);
-   newton.SetSolver(cg);
-   newton.SetOperator(elasticity);
-   newton.SetRelTol(1e-6);
-   newton.SetMaxIter(100);
-   newton.SetAdaptiveLinRtol();
-   newton.SetPrintLevel(IterativeSolver::PrintLevel().Iterations());
-
-   elasticity.SetParameters(*mesh_nodes);
-
-   // Vector zero;
-   newton.Mult(*B, X);
-
-   u.SetFromTrueDofs(X);
-
-   ParaViewDataCollection paraview_dc("dfem", &mesh);
-   paraview_dc.SetPrefixPath("ParaView");
-   paraview_dc.SetLevelsOfDetail(polynomial_order);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetTime(0.0);
-   paraview_dc.RegisterField("displacement", &u);
-   paraview_dc.Save();
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_nonlinear_elasticity_3d);

examples/dfem_test_ordering.cpp

@@ -1,82 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-#include "fem/coefficient.hpp"
-#include "fem/pgridfunc.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_ordering(std::string mesh_file,
-                  int refinements,
-                  int polynomial_order)
-{
-   constexpr int dim = 2;
-   constexpr int vdim = dim;
-
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(polynomial_order);
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   const IntegrationRule &ir =
-      IntRules.Get(mesh_fes.GetFE(0)->GetGeomType(),
-                   2 * mesh_fes.FEColl()->GetOrder() - 1);
-
-   for (int q = 0; q < ir.GetNPoints(); q++)
-   {
-      out << "(" << ir.IntPoint(q).x << ", " << ir.IntPoint(q).y << ")\n";
-   }
-
-   ParGridFunction u(&mesh_fes);
-   auto f = [](const Vector &coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      u(0) = x*x*y + 1.0;
-      u(1) = y*y*x*x + 2.0;
-   };
-
-   VectorFunctionCoefficient uc(dim, f);
-   u.ProjectCoefficient(uc);
-
-   auto kernel = [](const tensor<double, dim> &xi,
-                    const tensor<double, vdim, dim> &J,
-                    const tensor<double, dim> &u,
-                    const tensor<double, vdim, dim> &dudxi)
-   {
-      out << "xi: " << xi << "\n";
-      out << "J: " << J << "\n";
-      out << "u: " << u << "\n";
-      out << "dudxi: " << dudxi << "\n\n";
-      return mfem::tuple{J};
-   };
-
-   mfem::tuple argument_operators{Value{"coordinates"}, Gradient{"coordinates"}, Value{"potential"}, Gradient{"potential"}};
-   mfem::tuple output_operator{Gradient{"potential"}};
-
-   ElementOperator op{kernel, argument_operators, output_operator};
-
-   std::array solutions{FieldDescriptor{&mesh_fes, "potential"}};
-   std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop{solutions, parameters, mfem::tuple{op}, mesh, ir};
-
-   Vector y(u);
-
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(u, y);
-
-   print_vector(y);
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_ordering);

examples/dfem_test_vector_diffusion.cpp

@@ -1,102 +0,0 @@
-#include "dfem/dfem.hpp"
-#include "dfem/dfem_test_macro.hpp"
-
-using namespace mfem;
-using mfem::internal::tensor;
-
-int test_vector_diffusion(std::string mesh_file,
-                          int refinements,
-                          int polynomial_order)
-{
-   Mesh mesh_serial = Mesh(mesh_file);
-   for (int i = 0; i < refinements; i++)
-   {
-      mesh_serial.UniformRefinement();
-   }
-   ParMesh mesh(MPI_COMM_WORLD, mesh_serial);
-
-   mesh.SetCurvature(1);
-   const int dim = mesh.Dimension();
-   const int vdim = dim;
-   mesh_serial.Clear();
-
-   ParGridFunction* mesh_nodes = static_cast<ParGridFunction *>(mesh.GetNodes());
-   ParFiniteElementSpace &mesh_fes = *mesh_nodes->ParFESpace();
-
-   H1_FECollection h1fec(polynomial_order, dim);
-   ParFiniteElementSpace h1fes(&mesh, &h1fec, vdim);
-
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   Array<int> ess_tdof;
-   ess_bdr = 1;
-   h1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof);
-
-   const IntegrationRule &ir =
-      IntRules.Get(h1fes.GetFE(0)->GetGeomType(), 2 * h1fec.GetOrder() - 1);
-
-   ParGridFunction u(&h1fes);
-
-   auto f1 = [](const Vector& coords, Vector &u)
-   {
-      const double x = coords(0);
-      const double y = coords(1);
-      u(0) = 2.345 + 0.25 * x * x * y + y * y * x;
-      u(1) = 2.345 - 0.25 * x * y * y + y * x * x;
-   };
-
-   VectorFunctionCoefficient u_c(dim, f1);
-   u.ProjectCoefficient(u_c);
-
-   auto vector_diffusion_kernel = [](const tensor<double, 2> &xi,
-                                     const tensor<double, 2, 2> &dudxi,
-                                     const tensor<double, 2, 2> &J,
-                                     const double &w)
-   {
-      out << "xi: " << xi << "\n";
-      out << "dudxi: " << dudxi << "\n";
-      return mfem::tuple{dudxi * inv(J) * det(J) * w * transpose(inv(J))};
-      // return mfem::tuple{dudxi};
-   };
-
-   mfem::tuple argument_operators{Value{"coordinates"}, Gradient{"potential"}, Gradient{"coordinates"}, Weight{}};
-   mfem::tuple output_operator{Gradient{"potential"}};
-
-   ElementOperator op{vector_diffusion_kernel, argument_operators, output_operator};
-
-   std::array solutions{FieldDescriptor{&h1fes, "potential"}};
-   std::array parameters{FieldDescriptor{&mesh_fes, "coordinates"}};
-
-   DifferentiableOperator dop{solutions, parameters, mfem::tuple{op}, mesh, ir};
-
-   Vector x(u), y1(h1fes.GetTrueVSize()),
-          y2(h1fes.GetTrueVSize());
-
-   ParBilinearForm A_form(&h1fes);
-   auto A_integ = new VectorDiffusionIntegrator(vdim);
-   A_integ->SetIntegrationRule(ir);
-   A_form.AddDomainIntegrator(A_integ);
-   A_form.Assemble();
-   A_form.Finalize();
-
-   dop.SetParameters({mesh_nodes});
-   dop.Mult(x, y1);
-   y1.HostRead();
-
-   A_form.Mult(x, y2);
-   y2.HostRead();
-
-   Vector diff(y2);
-   diff -= y1;
-   if (diff.Norml2() > 1e-10)
-   {
-      out << "||F(u) - ex||_l2 = " << diff.Norml2() << "\n";
-      print_vector(diff);
-      print_vector(y1);
-      print_vector(y2);
-      return 1;
-   }
-
-   return 0;
-}
-
-DFEM_TEST_MAIN(test_vector_diffusion);

examples/enzyme_interface_smoketest.cpp

@@ -1,122 +0,0 @@
-#include <tuple>
-#include <type_traits>
-#include <iostream>
-#include <enzyme/enzyme>
-
-template <typename T>
-constexpr auto get_type_name() -> std::string_view
-{
-#if defined(__clang__)
-   constexpr auto prefix = std::string_view {"[T = "};
-   constexpr auto suffix = "]";
-   constexpr auto function = std::string_view{__PRETTY_FUNCTION__};
-#elif defined(__GNUC__)
-   constexpr auto prefix = std::string_view {"with T = "};
-   constexpr auto suffix = "; ";
-   constexpr auto function = std::string_view{__PRETTY_FUNCTION__};
-#elif defined(_MSC_VER)
-   constexpr auto prefix = std::string_view {"get_type_name<"};
-   constexpr auto suffix = ">(void)";
-   constexpr auto function = std::string_view{__FUNCSIG__};
-#else
-#error Unsupported compiler
-#endif
-
-   const auto start = function.find(prefix) + prefix.size();
-   const auto end = function.find(suffix);
-   const auto size = end - start;
-
-   return function.substr(start, size);
-}
-
-template <typename ... Ts>
-constexpr auto decay_types(std::tuple<Ts...> const &)
--> std::tuple<std::remove_cv_t<std::remove_reference_t<Ts>>...>;
-
-template <typename T>
-using decay_tuple = decltype(decay_types(std::declval<T>()));
-
-template <class F> struct FunctionSignature;
-
-template <typename output_t, typename... input_ts>
-struct FunctionSignature<output_t(input_ts...)>
-{
-   using return_t = output_t;
-   using parameter_ts = std::tuple<input_ts...>;
-};
-
-template <class T> struct create_function_signature;
-
-template <typename output_t, typename T, typename... input_ts>
-struct create_function_signature<output_t (T::*)(input_ts...) const>
-{
-   using type = FunctionSignature<output_t(input_ts...)>;
-};
-
-template <typename arg_ts, std::size_t... Is>
-auto create_enzyme_args(arg_ts &args,
-                        arg_ts &shadow_args,
-                        std::index_sequence<Is...>)
-{
-   // (std::cout << ... << std::get<Is>(shadow_args));
-   return std::tuple<enzyme::Duplicated<decltype(std::get<Is>(args))>...>
-   {
-      { std::get<Is>(args), std::get<Is>(shadow_args) }...
-   };
-}
-
-template <typename kernel_t, typename arg_ts>
-auto fwddiff_apply_enzyme(kernel_t kernel, arg_ts &&args, arg_ts &&shadow_args)
-{
-   auto arg_indices =
-      std::make_index_sequence<std::tuple_size_v<std::remove_reference_t<arg_ts>>> {};
-
-   auto enzyme_args = create_enzyme_args(args, shadow_args, arg_indices);
-
-   // using kf_return_t = typename create_function_signature<
-   //                     decltype(&kernel_t::operator())>::type::return_t;
-
-   std::cout << "\n";
-   std::cout << "args is " << get_type_name<decltype(args)>() << "\n\n";
-   std::cout << "enzyme_args type is " << get_type_name<decltype(enzyme_args)>() <<
-             "\n\n";
-   // std::cout << "return type is " << get_type_name<decltype(kf_return_t{})>() <<
-   //           "\n\n";
-
-   std::cout << "args " << std::get<0>(args) << "\n";
-   std::cout << "shadow args " << std::get<0>(shadow_args) << "\n";
-
-   return std::apply([&](auto &&...args)
-   {
-      // std::cout << enzyme::autodiff<enzyme::Forward>(+kernel, args...) << "\n";
-      return enzyme::get<0>
-             (enzyme::autodiff<enzyme::Forward>(+kernel, args...));
-   },
-   enzyme_args);
-}
-
-int main()
-{
-
-   auto func = [](const double &x, double &y)
-   {
-      std::cout << "func( x = " << x << " )\n";
-      return x*x;
-   };
-
-   using kf_param_ts = typename create_function_signature<
-                       decltype(&decltype(func)::operator())>::type::parameter_ts;
-   using kf_output_t = typename create_function_signature<
-                       decltype(&decltype(func)::operator())>::type::return_t;
-   auto kernel_args = decay_tuple<kf_param_ts> {};
-   auto kernel_shadow_args = decay_tuple<kf_param_ts> {};
-
-   std::get<0>(kernel_args) = 3;
-   std::get<0>(kernel_shadow_args) = 1;
-
-   auto dx = fwddiff_apply_enzyme(func, kernel_args, kernel_shadow_args);
-
-   std::cout << "dfdx = " << dx << "\n";
-
-   return 0;
-}

examples/ex14.cpp

@@ -20,6 +20,7 @@
 //               ex14 -m ../data/fichera-amr.mesh
 //               ex14 -pa -r 1 -o 3
 //               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
+//               ex14 -m ../data/inline-tet.mesh -o 0 -nt 4 -s 1
 //
 // Device sample runs:
 //               ex14 -pa -r 2 -d cuda -o 3
@@ -55,10 +56,16 @@ int main(int argc, char *argv[])
    bool pa = false;
    bool visualization = 1;
    const char *device_config = "cpu";
+   int nt = 0;
+   double st = 1.0;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&nt, "-nt", "--number-of-timeslices",
+                  "Number of slices through the hyper-prism in the 4th coordinate.");
+   args.AddOption(&st, "-st", "--size-time",
+                  "Length of hyper-prims in 4th coordinate (e.g. time).");
    args.AddOption(&ref_levels, "-r", "--refine",
                   "Number of times to refine the mesh uniformly, -1 for auto.");
    args.AddOption(&order, "-o", "--order",
@@ -97,8 +104,17 @@ int main(int argc, char *argv[])
    // 3. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
    //    NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   const int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   if (dim == 3 && nt > 0)
+   {
+      Mesh* spat_mesh = mesh;
+      mesh = new Mesh(spat_mesh, nt, Element::PENTATOPE, true, st);
+      delete spat_mesh;
+
+      ref_levels = 0;
+      dim = 4;
+   }
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
@@ -107,23 +123,23 @@ int main(int argc, char *argv[])
    {
       if (ref_levels < 0)
       {
-         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/(dim < 4 ? dim : 1.));
       }
       for (int l = 0; l < ref_levels; l++)
       {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
       }
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
 
    // 5. Define a finite element space on the mesh. Here we use discontinuous
    //    finite elements of the specified order >= 0.
    const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
    DG_FECollection fec(order, dim, bt);
-   FiniteElementSpace fespace(&mesh, &fec);
+   FiniteElementSpace fespace(mesh, &fec);
    cout << "Number of unknowns: " << fespace.GetVSize() << endl;
 
    // 6. Set up the linear form b(.) which corresponds to the right-hand side of
@@ -198,7 +214,7 @@ int main(int argc, char *argv[])
    //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
    ofstream mesh_ofs("refined.mesh");
    mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
    ofstream sol_ofs("sol.gf");
    sol_ofs.precision(8);
    x.Save(sol_ofs);
@@ -210,8 +226,10 @@ int main(int argc, char *argv[])
       int  visport   = 19916;
       socketstream sol_sock(vishost, visport);
       sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
    }
 
+   delete mesh;
+
    return 0;
 }

examples/ex14p.cpp

@@ -19,6 +19,7 @@
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
 //               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
 //               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
+//               mpirun -np 4 ex14p -m ../data/inline-tet.mesh -o 0 -nt 4 -s 1
 //
 // Device sample runs:
 //               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
@@ -90,10 +91,16 @@ int main(int argc, char *argv[])
    bool pa = false;
    bool visualization = 1;
    const char *device_config = "cpu";
+   int nt = 0;
+   double st = 1.0;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&nt, "-nt", "--number-of-timeslices",
+                  "Number of slices through the hyper-prism in the 4th coordinate.");
+   args.AddOption(&st, "-st", "--size-time",
+                  "Length of hyper-prims in 4th coordinate (e.g. time).");
    args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
                   "Number of times to refine the mesh uniformly in serial,"
                   " -1 for auto.");
@@ -139,8 +146,17 @@ int main(int argc, char *argv[])
    // 3. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   if (dim == 3 && nt > 0)
+   {
+      Mesh* spat_mesh = mesh;
+      mesh = new Mesh(spat_mesh, nt, Element::PENTATOPE, true, st);
+      delete spat_mesh;
+      dim = 4;
+   }
+   if (dim == 4)
+      ser_ref_levels = 0;
 
    // 4. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -149,23 +165,23 @@ int main(int argc, char *argv[])
    {
       if (ser_ref_levels < 0)
       {
-         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
       }
       for (int l = 0; l < ser_ref_levels; l++)
       {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
       }
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
 
    // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
    {
       for (int l = 0; l < par_ref_levels; l++)
       {

examples/ex1p_4d.cpp

@@ -0,0 +1,412 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "./spe10_coeff.cpp"
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_grad.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_grad.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+using namespace std;
+using namespace mfem;
+
+double kappa = 1.0;
+
+double u_exact(const Vector &x)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      return cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+   }
+   else { return 0.0; }
+}
+
+double f_exact(const Vector &x)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      return (kappa + 4.0 * M_PI*M_PI) * cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(
+                                                                              2))*cos(M_PI*x(3));
+   }
+   else { return 0.0; }
+}
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   bool set_bc = true;
+   bool standardCG = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   //   if(dim !=4 || sdim != 4)
+   //   {
+   //    MPI_Finalize();
+   //    return 0;
+   //   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      if (dim==4)
+      {
+         if (order==1) { fec = new LinearFECollection; }
+         else { fec = new QuadraticFECollection; }
+      }
+      else { fec = new H1_FECollection(order, dim); }
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = set_bc ? 1 : 0;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   FunctionCoefficient uExact(u_exact);
+   ParGridFunction x(fespace);
+
+   int NExpo =8;
+   for (int expo=-NExpo; expo<=NExpo; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      x.ProjectCoefficient(uExact);
+
+      ParLinearForm *b = new ParLinearForm(fespace);
+      FunctionCoefficient ffunc(f_exact);
+      b->AddDomainIntegrator(new DomainLFIntegrator(ffunc));
+      b->Assemble();
+
+      x = 0.0;
+
+      // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+      //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+      //     domain integrator.
+
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+      //    FunctionCoefficient *cspe10 = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      Coefficient *beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DiffusionIntegrator);
+      a->AddDomainIntegrator(new MassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreSolver *amg = new HypreBoomerAMG(A);
+
+      int iter = -1;
+      if (standardCG)
+      {
+         IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+         pcg->SetOperator(A);
+         pcg->SetRelTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetPrintLevel(1);
+         pcg->SetPreconditioner(*amg);
+         pcg->Mult(B, X);
+
+         iter = pcg->GetNumIterations();
+
+         delete pcg;
+      }
+      else
+      {
+         HyprePCG *pcg = new HyprePCG(A);
+         pcg->SetTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetResidualConvergenceOptions(1,tol);
+         pcg->SetPrintLevel(2);
+         pcg->SetPreconditioner(*amg);
+         pcg->Mult(B, X);
+
+         pcg->GetNumIterations(iter);
+
+         delete pcg;
+      }
+
+
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      {
+         double err = x.ComputeL2Error(uExact);
+         if (myid == 0)
+         {
+            cout << "\n|| u - u_h ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 14. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //   {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //   }
+
+      // 15. Send the solution by socket to a GLVis server.
+      //   if (visualization)
+      //   {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //   }
+
+      delete amg;
+      delete a;
+      delete beta;
+      delete b;
+   }
+
+   // 16. Free the used memory.
+
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/ex23.cpp

@@ -44,7 +44,7 @@ protected:
    BilinearForm *M;
    BilinearForm *K;
 
-   SparseMatrix Mmat, Kmat;
+   SparseMatrix Mmat, Kmat, Kmat0;
    SparseMatrix *T; // T = M + dt K
    real_t current_dt;
 
@@ -83,24 +83,25 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
    : SecondOrderTimeDependentOperator(f.GetTrueVSize(), (real_t) 0.0),
      fespace(f), M(NULL), K(NULL), T(NULL), current_dt(0.0), z(height)
 {
-   // Assemble Laplace matrix
+   const real_t rel_tol = 1e-8;
+
+   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
    c2 = new ConstantCoefficient(speed*speed);
+
    K = new BilinearForm(&fespace);
    K->AddDomainIntegrator(new DiffusionIntegrator(*c2));
    K->Assemble();
 
-   // Assemble Mass matrix
+   Array<int> dummy;
+   K->FormSystemMatrix(dummy, Kmat0);
+   K->FormSystemMatrix(ess_tdof_list, Kmat);
+
    M = new BilinearForm(&fespace);
    M->AddDomainIntegrator(new MassIntegrator());
    M->Assemble();
-
-   // Apply Bcs
-   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   K->FormSystemMatrix(ess_tdof_list, Kmat);
    M->FormSystemMatrix(ess_tdof_list, Mmat);
 
-   // Configure preconditioner
-   const real_t rel_tol = 1e-8;
    M_solver.iterative_mode = false;
    M_solver.SetRelTol(rel_tol);
    M_solver.SetAbsTol(0.0);
@@ -109,13 +110,14 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(Mmat);
 
-   // Configure solver
    T_solver.iterative_mode = false;
    T_solver.SetRelTol(rel_tol);
    T_solver.SetAbsTol(0.0);
    T_solver.SetMaxIter(100);
    T_solver.SetPrintLevel(0);
    T_solver.SetPreconditioner(T_prec);
+
+   T = NULL;
 }
 
 void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
@@ -124,11 +126,9 @@ void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
    // Compute:
    //    d2udt2 = M^{-1}*-K(u)
    // for d2udt2
-   K->FullMult(u, z);
+   Kmat.Mult(u, z);
    z.Neg(); // z = -z
-   z.SetSubVector(ess_tdof_list, 0.0);
    M_solver.Mult(z, d2udt2);
-   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }
 
 void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
@@ -142,11 +142,14 @@ void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
       T = Add(1.0, Mmat, fac0, Kmat);
       T_solver.SetOperator(*T);
    }
-   K->FullMult(u, z);
+   Kmat0.Mult(u, z);
    z.Neg();
-   z.SetSubVector(ess_tdof_list, 0.0);
+
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      z[ess_tdof_list[i]] = 0.0;
+   }
    T_solver.Mult(z, d2udt2);
-   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }
 
 void WaveOperator::SetParameters(const Vector &u)
@@ -311,6 +314,7 @@ int main(int argc, char *argv[])
          ess_bdr = 0;
       }
    }
+
    WaveOperator oper(fespace, ess_bdr, speed);
 
    u_gf.SetFromTrueDofs(u);

examples/ex3p.cpp

@@ -58,6 +58,180 @@ void f_exact(const Vector &, Vector &);
 real_t freq = 1.0, kappa;
 int dim;
 
+class Curl4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+
+   HypreParMatrix *idMat;
+   HypreParMatrix *H1VecLaplaceMat;
+   HypreBoomerAMG *amgVecH1;
+
+
+   HypreParMatrix *gradMat;
+   HypreParMatrix *H1LaplaceMat;
+   HypreBoomerAMG *amgH1;
+
+   HypreSmoother * smoother;
+   CGSolver *pcgGrad;
+   CGSolver *pcgH1Vec;
+
+   Vector *f;
+   Vector *fGrad, *uGrad;
+   Vector *fH1Vec, *uH1Vec;
+
+   bool exactSolves;
+
+public:
+   Curl4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+              const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //vecH1  --> H(curl)
+      int orderKer=orderKernel; //grad V --> H(curl)
+
+      smoother = new HypreSmoother(*A, 16, 3);
+
+      //    //for the pure dirichlet case
+      //    Array<int> essBnd(pmesh->bdr_attributes.Max()); essBnd = 1;
+
+      Array<int> HCurl_essDof(fespace->GetVSize()); HCurl_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HCurl_essDof);
+
+      //setup the H1 FESpace
+      FiniteElementCollection* fecH1;
+      if (orderKer==1) { fecH1 = new LinearFECollection; }
+      else { fecH1 = new QuadraticFECollection; }
+
+      ParFiniteElementSpace *H1FESpace = new ParFiniteElementSpace(pmesh, fecH1);
+      Array<int> H1_essDof(H1FESpace->GetVSize()); H1_essDof = 0;
+      H1FESpace->GetEssentialVDofs(essBnd, H1_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disGrad = new ParDiscreteLinearOperator(H1FESpace,
+                                                                         fespace);
+      disGrad->AddDomainInterpolator(new GradientInterpolator);
+      disGrad->Assemble();
+      disGrad->Finalize();
+      SparseMatrix* smat = &(disGrad->SpMat());
+      smat->EliminateCols(H1_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smat->EliminateRow(dof); }
+      gradMat = disGrad->ParallelAssemble();
+      delete disGrad;
+
+      //setup the H1 preconditioner
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1FESpace);
+      H1Varf->AddDomainIntegrator(new DiffusionIntegrator);
+      H1Varf->AddDomainIntegrator(new MassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1_essDof.Size(); dof++) if (H1_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1LaplaceMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1 = new HypreBoomerAMG(*H1LaplaceMat);
+
+
+      //setup the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1VecFESpace = new ParFiniteElementSpace(pmesh, fecH1Vec,
+                                                                      dim, Ordering::byVDIM);
+      Array<int> H1Vec_essDof(H1VecFESpace->GetVSize()); H1Vec_essDof = 0;
+      H1VecFESpace->GetEssentialVDofs(essBnd, H1Vec_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1VecFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Vec_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      idMat = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the H1-vec preconditioner
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1VecFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator);
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator);
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Vec_essDof.Size(); dof++) if (H1Vec_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1VecLaplaceMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgVecH1 = new HypreBoomerAMG(*H1VecLaplaceMat);
+      amgVecH1->SetSystemsOptions(dim);
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fGrad = new Vector(H1FESpace->GetTrueVSize());
+      uGrad = new Vector(H1FESpace->GetTrueVSize());
+
+      fH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+      uH1Vec = new Vector(H1VecFESpace->GetTrueVSize());;
+
+
+      amgH1->Mult(*fGrad, *uGrad);
+      amgVecH1->Mult(*fH1Vec, *uH1Vec);
+
+      pcgGrad = new CGSolver(MPI_COMM_WORLD);
+      pcgGrad->SetOperator(*H1LaplaceMat);
+      pcgGrad->SetPreconditioner(*amgH1);
+      pcgGrad->SetRelTol(1e-16);
+      pcgGrad->SetMaxIter(100000000);
+      pcgGrad->SetPrintLevel(-2);
+
+      pcgH1Vec = new CGSolver(MPI_COMM_WORLD);
+      pcgH1Vec->SetOperator(*H1VecLaplaceMat);
+      pcgH1Vec->SetPreconditioner(*amgVecH1);
+      pcgH1Vec->SetRelTol(1e-16);
+      pcgH1Vec->SetMaxIter(100000000);
+      pcgH1Vec->SetPrintLevel(-2);
+
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smoother->Mult(x,y);
+
+      idMat->MultTranspose(x,*fH1Vec);
+      *uH1Vec = 0.0;
+      if (exactSolves) { pcgH1Vec->Mult(*fH1Vec, *uH1Vec); }
+      else { amgVecH1->Mult(*fH1Vec, *uH1Vec); }
+      idMat->Mult(1.0, *uH1Vec, 1.0, y);
+
+      gradMat->MultTranspose(x,*fGrad);
+      *uGrad = 0.0;
+      if (exactSolves) { pcgGrad->Mult(*fGrad, *uGrad); }
+      else { amgH1->Mult(*fGrad, *uGrad); }
+      gradMat->Mult(1.0, *uGrad, 1.0, y);
+
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
 int main(int argc, char *argv[])
 {
    // 1. Initialize MPI and HYPRE.
@@ -162,7 +336,13 @@ int main(int argc, char *argv[])
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementCollection *fec;
+   if (dim==4)
+   {
+      if (order==1) { fec = new ND1_4DFECollection; }
+      else { fec = new ND2_4DFECollection; }
+   }
+   else { fec = new ND_FECollection(order, dim); }
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
    HYPRE_BigInt size = fespace->GlobalTrueVSize();
    if (myid == 0)
@@ -248,15 +428,29 @@ int main(int argc, char *argv[])
 
       ParFiniteElementSpace *prec_fespace =
          (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      Solver *prec;
       HypreAMS ams(*A.As<HypreParMatrix>(), prec_fespace);
-      HyprePCG pcg(*A.As<HypreParMatrix>());
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(ams);
-      pcg.Mult(B, X);
+      if (dim <= 3)
+      {
+         prec = new HypreAMS(*A.As<HypreParMatrix>(), prec_fespace);
+      }
+      else if (dim == 4)
+      {
+         prec = new Curl4dPrec(A.As<HypreParMatrix>(), fespace, ess_bdr, order, false);
+      }
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(*A.As<HypreParMatrix>());
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete prec;
    }
 
+   // 13. Recover the parallel grid function corresponding to X. This is the
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
@@ -312,7 +506,14 @@ int main(int argc, char *argv[])
 
 void E_exact(const Vector &x, Vector &E)
 {
-   if (dim == 3)
+   if (dim==4)
+   {
+      E(0) =  sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(1) = -cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(2) =  cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(M_PI*x(3));
+      E(3) = -cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(M_PI*x(3));
+   }
+   else if (dim == 3)
    {
       E(0) = sin(kappa * x(1));
       E(1) = sin(kappa * x(2));
@@ -328,7 +529,19 @@ void E_exact(const Vector &x, Vector &E)
 
 void f_exact(const Vector &x, Vector &f)
 {
-   if (dim == 3)
+   //f_exact = E +  DivSkew P( curl E ), where P is the 4d permutation operator
+   if (dim==4)
+   {
+      f(0) =  (1.0+4.0*M_PI*M_PI)*sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(1) = -(1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                M_PI*x(3));
+      f(2) =  (1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(3) = -(1.0+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(
+                M_PI*x(3));
+   }
+   else if (dim == 3)
    {
       f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
       f(1) = (1. + kappa * kappa) * sin(kappa * x(2));

examples/ex3p_4d.cpp

@@ -0,0 +1,650 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_curl.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_curl.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa = 1.0;
+int dim;
+
+double osziCoeff(const Vector &x)
+{
+   return 1.0001 + sin(100*x(0))*sin(200*x(1))*sin(300*x(2))*sin(400*x(3));
+}
+
+class Curl4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+   Coefficient *alpha_, *beta_, *neg_beta_;
+
+   HypreParMatrix *idMat;
+   HypreParMatrix *H1VecLaplaceMat;
+   HypreBoomerAMG *amgVecH1;
+
+
+   HypreParMatrix *gradMat;
+   HypreParMatrix *H1LaplaceMat;
+   HypreBoomerAMG *amgH1;
+
+   HypreSmoother * smoother;
+   CGSolver *pcgGrad;
+   CGSolver *pcgH1Vec;
+
+   Vector *f;
+   Vector *fGrad, *uGrad;
+   Vector *fH1Vec, *uH1Vec;
+
+   bool exactSolves;
+
+public:
+   ~Curl4dPrec()
+   {
+      delete pcgH1Vec;
+      delete pcgGrad;
+
+      delete   f, fGrad, uGrad, fH1Vec, uH1Vec;
+
+      delete smoother;
+
+      delete amgVecH1, H1VecLaplaceMat;
+      delete idMat;
+      delete amgH1, H1LaplaceMat;
+      delete gradMat;
+   }
+
+   Curl4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+              Coefficient *alpha, Coefficient *beta, Coefficient *neg_beta,
+              const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+      neg_beta_=neg_beta;
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //vecH1  --> H(curl)
+      int orderKer=orderKernel; //grad V --> H(curl)
+
+      smoother = new HypreSmoother(*A, 16, 3);
+
+      //    //for the pure dirichlet case
+      //    Array<int> essBnd(pmesh->bdr_attributes.Max()); essBnd = 1;
+
+      Array<int> HCurl_essDof(fespace->GetVSize()); HCurl_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HCurl_essDof);
+
+      //setup the H1 FESpace
+      FiniteElementCollection* fecH1;
+      if (orderKer==1) { fecH1 = new LinearFECollection; }
+      else { fecH1 = new QuadraticFECollection; }
+
+      ParFiniteElementSpace *H1FESpace = new ParFiniteElementSpace(pmesh, fecH1);
+      Array<int> H1_essDof(H1FESpace->GetVSize()); H1_essDof = 0;
+      H1FESpace->GetEssentialVDofs(essBnd, H1_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disGrad = new ParDiscreteLinearOperator(H1FESpace,
+                                                                         fespace);
+      disGrad->AddDomainInterpolator(new GradientInterpolator);
+      disGrad->Assemble();
+      disGrad->Finalize();
+      SparseMatrix* smat = &(disGrad->SpMat());
+      smat->EliminateCols(H1_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smat->EliminateRow(dof); }
+      gradMat = disGrad->ParallelAssemble();
+      delete disGrad;
+
+      //setup the H1 preconditioner
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1FESpace);
+      H1Varf->AddDomainIntegrator(new DiffusionIntegrator(*beta_));
+      //    H1Varf->AddDomainIntegrator(new MassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1_essDof.Size(); dof++) if (H1_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1LaplaceMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1 = new HypreBoomerAMG(*H1LaplaceMat);
+
+
+      //setup the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1VecFESpace = new ParFiniteElementSpace(pmesh, fecH1Vec,
+                                                                      dim, Ordering::byVDIM);
+      Array<int> H1Vec_essDof(H1VecFESpace->GetVSize()); H1Vec_essDof = 0;
+      H1VecFESpace->GetEssentialVDofs(essBnd, H1Vec_essDof);
+
+      //setup the discrete gradient
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1VecFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Vec_essDof);
+      for (int dof=0; dof<HCurl_essDof.Size(); dof++) if (HCurl_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      idMat = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the H1-vec preconditioner
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1VecFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_));
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator(*neg_beta_));
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Vec_essDof.Size(); dof++) if (H1Vec_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1VecLaplaceMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgVecH1 = new HypreBoomerAMG(*H1VecLaplaceMat);
+      amgVecH1->SetSystemsOptions(dim);
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fGrad = new Vector(H1FESpace->GetTrueVSize());
+      uGrad = new Vector(H1FESpace->GetTrueVSize());
+
+      fH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+      uH1Vec = new Vector(H1VecFESpace->GetTrueVSize());
+
+
+      amgH1->Mult(*fGrad, *uGrad);
+      amgVecH1->Mult(*fH1Vec, *uH1Vec);
+
+      pcgGrad = new CGSolver(MPI_COMM_WORLD);
+      pcgGrad->SetOperator(*H1LaplaceMat);
+      pcgGrad->SetPreconditioner(*amgH1);
+      pcgGrad->SetRelTol(1e-16);
+      pcgGrad->SetMaxIter(100000000);
+      pcgGrad->SetPrintLevel(-2);
+
+      pcgH1Vec = new CGSolver(MPI_COMM_WORLD);
+      pcgH1Vec->SetOperator(*H1VecLaplaceMat);
+      pcgH1Vec->SetPreconditioner(*amgVecH1);
+      pcgH1Vec->SetRelTol(1e-16);
+      pcgH1Vec->SetMaxIter(100000000);
+      pcgH1Vec->SetPrintLevel(-2);
+
+      delete H1FESpace; delete fecH1;
+      delete H1VecFESpace; delete fecH1Vec;
+
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smoother->Mult(x,y);
+
+      idMat->MultTranspose(x,*fH1Vec);
+      *uH1Vec = 0.0;
+      if (exactSolves) { pcgH1Vec->Mult(*fH1Vec, *uH1Vec); }
+      else { amgVecH1->Mult(*fH1Vec, *uH1Vec); }
+      idMat->Mult(1.0, *uH1Vec, 1.0, y);
+
+      gradMat->MultTranspose(x,*fGrad);
+      *uGrad = 0.0;
+      if (exactSolves) { pcgGrad->Mult(*fGrad, *uGrad); }
+      else { amgH1->Mult(*fGrad, *uGrad); }
+      gradMat->Mult(1.0, *uGrad, 1.0, y);
+
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool exactH1Solver = false;
+   bool spe10Coeff = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   kappa = freq * M_PI;
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+
+   if (dim !=4 || sdim != 4)
+   {
+      MPI_Finalize();
+      return 0;
+   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->ReorientTetMesh();
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (dim==4)
+   {
+      if (order==1) { fec = new ND1_4DFECollection; }
+      else { fec = new ND2_4DFECollection; }
+   }
+   else { fec = new ND_FECollection(order, dim); }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      VectorFunctionCoefficient f(sdim, f_exact);
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      x.ProjectCoefficient(E);
+
+      // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+      //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+      //     mass domain integrators.
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+      Coefficient *neg_beta = new ConstantCoefficient(-weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new CurlCurlIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      Solver *prec;
+      if (dim<=3) { prec = new HypreAMS(A, prec_fespace); }
+      else if (dim==4) { prec = new Curl4dPrec(&A, fespace, alpha, beta, neg_beta, ess_bdr, order, false); }
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(A);
+      pcg->SetRelTol(tol);
+      pcg->SetMaxIter(5000);
+      pcg->SetPrintLevel(1);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      int iter = pcg->GetNumIterations();
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double err = x.ComputeL2Error(E);
+         if (myid == 0)
+         {
+            cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 15. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //   {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //   }
+
+      //   // 16. Send the solution by socket to a GLVis server.
+      //   if (visualization)
+      //   {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //   }
+
+      delete pcg;
+      delete prec;
+      delete a;
+      delete alpha;
+      delete beta;
+      delete b;
+
+   }
+
+   // 17. Free the used memory.
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim==4)
+   {
+      E(0) =  sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(1) = -cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(M_PI*x(3));
+      E(2) =  cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(M_PI*x(3));
+      E(3) = -cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(M_PI*x(3));
+   }
+   else if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   //f_exact = E +  DivSkew P( curl E ), where P is the 4d permutation operator
+   if (dim==4)
+   {
+      f(0) =  (kappa+4.0*M_PI*M_PI)*sin(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(1) = -(kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*sin(M_PI*x(1))*cos(M_PI*x(2))*cos(
+                M_PI*x(3));
+      f(2) =  (kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*sin(M_PI*x(2))*cos(
+                 M_PI*x(3));
+      f(3) = -(kappa+4.0*M_PI*M_PI)*cos(M_PI*x(0))*cos(M_PI*x(1))*cos(M_PI*x(2))*sin(
+                M_PI*x(3));
+   }
+   else if (dim == 3)
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ex40.cpp

@@ -67,8 +67,6 @@ public:
    ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
       : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
 
-   using VectorCoefficient::Eval;
-
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
    void SetAlpha(real_t alpha_) { alpha = alpha_; }

examples/ex40p.cpp

@@ -67,8 +67,6 @@ public:
    ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
       : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
 
-   using VectorCoefficient::Eval;
-
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
    void SetAlpha(real_t alpha_) { alpha = alpha_; }

examples/ex4D_DivSkew.cpp

@@ -0,0 +1,782 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_DivSkew.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_DivSkew.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact_vec(const Vector &x, Vector &E);
+void E_exact(const Vector &, DenseMatrix &);
+void f_exact(const Vector &, DenseMatrix &);
+
+
+class DivSkew4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+   Coefficient *alpha_, *beta_;
+
+   //kernel operators
+   HypreParMatrix *P_d_HCurl_HDivSkew;
+
+
+   HypreParMatrix *P_H1_HCurl;
+   HypreParMatrix *H1_KernelMat;
+   HypreBoomerAMG *amgH1_Kernel;
+
+   //"image" operators
+   HypreParMatrix *P_H1_HDivSkew;
+   HypreParMatrix *H1_ImageMat;
+   HypreBoomerAMG *amgH1_Image;
+
+
+   HypreParMatrix *HCurlMat;
+   HypreSmoother * smootherDivSkew;
+   HypreSmoother * smootherCurl;
+
+   CGSolver *pcgKernel;
+   CGSolver *pcgImage;
+
+   Vector *f;
+   Vector *fKernel, *uKernel;
+   Vector *fImage, *uImage;
+   Vector *fCurl, *uCurl;
+
+   bool exactSolves;
+
+   FiniteElementCollection* fecHCurlKernel;
+   ParFiniteElementSpace *HCurlKernelFESpace;
+
+
+public:
+   ~DivSkew4dPrec()
+   {
+      delete pcgImage, pcgKernel;
+
+      delete f, fKernel, uKernel, fImage, uImage, fCurl, uCurl;
+
+      delete smootherCurl, HCurlMat;
+
+      delete P_d_HCurl_HDivSkew, P_H1_HDivSkew, P_H1_HCurl;
+
+      delete amgH1_Image, H1_ImageMat;
+      delete amgH1_Kernel, H1_KernelMat;
+
+      delete smootherDivSkew;
+
+      delete HCurlKernelFESpace, fecHCurlKernel;
+   }
+   DivSkew4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+                 Coefficient *alpha, Coefficient *beta,
+                 const Array<int> &essBnd, int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+      int orderIm=1;  //H1  --> H(divSkew)
+      int orderKer=orderKernel; //curl V --> H(divSkew)
+
+      smootherDivSkew = new HypreSmoother(*A, 16, 3);
+
+      Array<int> HDivSkew_essDof(fespace->GetVSize()); HDivSkew_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, HDivSkew_essDof);
+
+      //setup the H1 FESpace for the kernel
+      FiniteElementCollection* fecH1Kernel = new H1_FECollection(orderKer, 4);
+
+      ParFiniteElementSpace *H1KernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Kernel, dim, Ordering::byVDIM);
+      Array<int> H1Kernel_essDof(H1KernelFESpace->GetVSize()); H1Kernel_essDof = 0;
+      H1KernelFESpace->GetEssentialVDofs(essBnd, H1Kernel_essDof);
+
+
+      //setup the H(curl) FESpace for the kernel
+      if (orderKer==1) { fecHCurlKernel = new ND1_4DFECollection; }
+      else { fecHCurlKernel = new ND2_4DFECollection; }
+
+      HCurlKernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                     fecHCurlKernel);
+      Array<int> HCurlKernel_essDof(HCurlKernelFESpace->GetVSize());
+      HCurlKernel_essDof = 0;
+      HCurlKernelFESpace->GetEssentialVDofs(essBnd, HCurlKernel_essDof);
+
+
+      //setup the FESpace for the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1_ImageFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Vec, 6, Ordering::byVDIM);
+      Array<int> H1Image_essDof(H1_ImageFESpace->GetVSize()); H1Image_essDof = 0;
+      H1_ImageFESpace->GetEssentialVDofs(essBnd, H1Image_essDof);
+
+
+
+      //setup the H1 preconditioner for the kernel
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1KernelFESpace);
+      H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*beta_));
+      //      H1Varf->AddDomainIntegrator(new VectorMassIntegrator);
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof = 0; dof < H1Kernel_essDof.Size(); dof++)
+         if (H1Kernel_essDof[dof] < 0)
+         {
+            matH1.EliminateRowCol(dof);
+         }
+      H1_KernelMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1_Kernel = new HypreBoomerAMG(*H1_KernelMat);
+      amgH1_Kernel->SetSystemsOptions(dim);
+      amgH1_Kernel->SetPrintLevel(0);
+
+      //setup the H1 preconditioner for the image
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1_ImageFESpace);
+      VectorDiffusionIntegrator *alpha_integ = new VectorDiffusionIntegrator(*alpha_);
+      alpha_integ->SetVDim(6);
+      H1VecVarf->AddDomainIntegrator(alpha_integ);
+      VectorMassIntegrator *beta_integ = new VectorMassIntegrator(*beta);
+      beta_integ->SetVDim(6);
+      H1VecVarf->AddDomainIntegrator(beta_integ);
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Image_essDof.Size(); dof++) if (H1Image_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1_ImageMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgH1_Image = new HypreBoomerAMG(*H1_ImageMat);
+      amgH1_Image->SetSystemsOptions(6);
+      amgH1_Image->SetPrintLevel(0);
+
+
+      //setup the injection of H1 into H(curl)
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1KernelFESpace, HCurlKernelFESpace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Kernel_essDof);
+      for (int dof=0; dof<HCurlKernel_essDof.Size();
+           dof++) if (HCurlKernel_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      P_H1_HCurl = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+      //setup the injection of H1 into H(DivSkew)
+      ParDiscreteLinearOperator *disInterpolIm = new ParDiscreteLinearOperator(
+         H1_ImageFESpace, fespace);
+      disInterpolIm->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpolIm->Assemble();
+      disInterpolIm->Finalize();
+      SparseMatrix* smatIDIm = &(disInterpolIm->SpMat());
+      smatIDIm->EliminateCols(H1Image_essDof);
+      for (int dof=0; dof<HDivSkew_essDof.Size(); dof++) if (HDivSkew_essDof[dof]<0) { smatIDIm->EliminateRow(dof); }
+      P_H1_HDivSkew = disInterpolIm->ParallelAssemble();
+      delete disInterpolIm;
+
+
+      //setup the injection of the curl(H(curl)) into H(DivSkew)
+      ParDiscreteLinearOperator *disCurl = new ParDiscreteLinearOperator(
+         HCurlKernelFESpace, fespace);
+      disCurl->AddDomainInterpolator(new CurlInterpolator);
+      disCurl->Assemble();
+      disCurl->Finalize();
+      SparseMatrix* smatCurl = &(disCurl->SpMat());
+      smatCurl->EliminateCols(HCurlKernel_essDof);
+      for (int dof=0; dof<HDivSkew_essDof.Size(); dof++) if (HDivSkew_essDof[dof]<0) { smatCurl->EliminateRow(dof); }
+      P_d_HCurl_HDivSkew = disCurl->ParallelAssemble();
+      delete disCurl;
+
+      //setup the smoother for H(curl)
+      //      Coefficient *massC = new ConstantCoefficient(1.0);
+      //      Coefficient *CurlCurlC = new ConstantCoefficient(1.0);
+      ParBilinearForm *a_HCurl = new ParBilinearForm(HCurlKernelFESpace);
+      a_HCurl->AddDomainIntegrator(new CurlCurlIntegrator(*beta_));
+      //      a_HCurl->AddDomainIntegrator(new CurlCurlIntegrator(*CurlCurlC));
+      //      a_HCurl->AddDomainIntegrator(new VectorFEMassIntegrator(*massC));
+      a_HCurl->Assemble();
+      a_HCurl->Finalize();
+      SparseMatrix &matHCurl(a_HCurl->SpMat());
+      for (int dof=0; dof<HCurlKernel_essDof.Size();
+           dof++) if (HCurlKernel_essDof[dof]<0) { matHCurl.EliminateRowCol(dof); }
+      HCurlMat = a_HCurl->ParallelAssemble();
+      delete a_HCurl;
+      smootherCurl = new HypreSmoother(*HCurlMat, 16, 3);
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+      uKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+
+      fImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+      uImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+
+      fCurl = new Vector(HCurlKernelFESpace->GetTrueVSize());
+      uCurl = new Vector(HCurlKernelFESpace->GetTrueVSize());
+
+
+      amgH1_Kernel->Mult(*fKernel, *uKernel);
+      amgH1_Image->Mult(*fImage, *uImage);
+
+      pcgKernel = new CGSolver(MPI_COMM_WORLD);
+      pcgKernel->SetOperator(*H1_KernelMat);
+      pcgKernel->SetPreconditioner(*amgH1_Kernel);
+      pcgKernel->SetRelTol(1e-16);
+      pcgKernel->SetMaxIter(100000000);
+      pcgKernel->SetPrintLevel(-2);
+
+      pcgImage = new CGSolver(MPI_COMM_WORLD);
+      pcgImage->SetOperator(*H1_ImageMat);
+      pcgImage->SetPreconditioner(*amgH1_Image);
+      pcgImage->SetRelTol(1e-16);
+      pcgImage->SetMaxIter(100000000);
+      pcgImage->SetPrintLevel(-2);
+
+      delete H1KernelFESpace, fecH1Kernel;
+      delete H1_ImageFESpace, fecH1Vec;
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smootherDivSkew->Mult(x,y);
+
+      P_H1_HDivSkew->MultTranspose(x,*fImage);
+      *uImage = 0.0;
+      if (exactSolves) { pcgImage->Mult(*fImage, *uImage); }
+      else { amgH1_Image->Mult(*fImage, *uImage); }
+      P_H1_HDivSkew->Mult(1.0, *uImage, 1.0, y);
+
+      *uCurl = 0.0;
+      P_d_HCurl_HDivSkew->MultTranspose(x,*fCurl);
+
+      smootherCurl->Mult(*fCurl, *uCurl);
+
+      P_H1_HCurl->MultTranspose(*fCurl,*fKernel);
+      *uKernel = 0.0;
+      if (exactSolves) { pcgKernel->Mult(*fKernel, *uKernel); }
+      else { amgH1_Kernel->Mult(*fKernel, *uKernel); }
+      P_H1_HCurl->Mult(1.0, *uKernel, 1.0, *uCurl);
+
+      P_d_HCurl_HDivSkew->Mult(1.0, *uCurl, 1.0, y);
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/cube4d_96.MFEM";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool exactH1Solver = false;
+   bool spe10Coeff = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial order of the finite element space.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (verbose) { args.PrintOptions(cout); }
+
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   if (dim !=4 || sdim != 4)
+   {
+      MPI_Finalize();
+      return 0;
+   }
+
+   for (int i=0; i<sequ_ref_levels; i++) { mesh->UniformRefinement(); }
+   if (verbose) { mesh->PrintCharacteristics(); }
+
+   if (verbose) { cout << "now we partition the mesh..." << endl << endl; }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i=0; i<par_ref_levels; i++) { pmesh->UniformRefinement(); }
+
+   pmesh->PrintInfo(std::cout);
+   if (verbose) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (order==1) { fec = new DivSkew1_4DFECollection; }
+   //    else fec = new F2K1_4DFECollection;
+
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   fespace->SetUpdateOperatorType(Operator::Hypre_ParCSR);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   MatrixFunctionCoefficient f(sdim, f_exact);
+   MatrixFunctionCoefficient solMat(sdim, E_exact);
+   VectorFunctionCoefficient solVec(6, E_exact_vec);
+
+
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+
+      x.ProjectCoefficient(solVec);
+
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new MatFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      //   cout << x << endl;
+      //   x = 0.0;
+
+      // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+      //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+      //     mass domain integrators.
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFE_DivSkewMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation, etc.
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      //Define the preconditioner
+
+      if (myid == 0) { cout << "Set up the preconditioner" << endl; }
+      Solver *prec;
+      if (dim==4) { prec = new DivSkew4dPrec(&A, fespace, alpha, beta, ess_bdr, order, exactH1Solver); }
+
+      IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetOperator(A);
+      pcg->SetRelTol(tol);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(1);
+      pcg->SetPreconditioner(*prec);
+      pcg->Mult(B, X);
+
+      delete prec;
+
+      int iter = pcg->GetNumIterations();
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double error = 0.0;
+         for (int i = 0; i < fespace->GetNE(); i++)
+         {
+            const FiniteElement* fe = fespace->GetFE(i);
+            int fdof = fe->GetDof();
+            ElementTransformation* transf = fespace->GetElementTransformation(i);
+            DenseMatrix shape(fdof,dim*dim);
+
+            int intorder = 2*fe->GetOrder() + 1; // <----------
+            const IntegrationRule *ir;
+            ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+
+            Vector elSol(dim*dim);
+            DenseMatrix elSolMat(dim,dim);
+            DenseMatrix exactSol(dim,dim);
+            Vector exactSolVec(dim*dim);
+
+
+
+            Array<int> vdofs;
+            fespace->GetElementVDofs(i, vdofs);
+            for (int j = 0; j < ir->GetNPoints(); j++)
+            {
+               const IntegrationPoint &ip = ir->IntPoint(j);
+               transf->SetIntPoint(&ip);
+
+               fe->CalcVShape(*transf, shape);
+
+               elSol = 0.0;
+               for (int k = 0; k < fdof; k++)
+               {
+                  if (vdofs[k] >= 0)
+                  {
+                     for (int l=0; l<dim*dim; l++) { elSol(l) += shape(k,l)*x(vdofs[k]); }
+                  }
+                  else
+                  {
+                     for (int l=0; l<dim*dim; l++) { elSol(l) -= shape(k,l)*x(-1-vdofs[k]); }
+                  }
+               }
+               for (int k=0; k<dim; k++)
+                  for (int l=0; l<dim; l++)
+                  {
+                     elSolMat(k,l) = elSol(dim*k+l);
+                  }
+
+
+               solMat.Eval(exactSol,*transf, ip);
+               for (int k=0; k<dim; k++)
+                  for (int l=0; l<dim; l++)
+                  {
+                     exactSolVec(dim*k+l) = exactSol(k,l);
+                  }
+               elSol.Add(-1.0, exactSolVec);
+
+               error += ip.weight * fabs(transf->Weight()) * (elSol * elSol);
+            }
+         }
+         double globalError = 0.0;
+         MPI_Allreduce(&error, &globalError, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+         if (myid==0) { std::cout << "L2 error: " << sqrt(globalError) << std::endl; }
+
+
+      }
+
+      delete pcg;
+      delete a;
+      delete alpha;
+      delete beta;
+
+      delete b;
+   }
+
+   // 17. Free the used memory.
+
+
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+void E_exact_vec(const Vector &x, Vector &E)
+{
+   int dim = x.Size();
+
+   if (dim==4)
+   {
+      E.SetSize(6);
+
+      double s0 = sin(M_PI*x(0)), s1 = sin(M_PI*x(1)), s2 = sin(M_PI*x(2)),
+             s3 = sin(M_PI*x(3));
+      double c0 = cos(M_PI*x(0)), c1 = cos(M_PI*x(1)), c2 = cos(M_PI*x(2)),
+             c3 = cos(M_PI*x(3));
+
+      E(0) =  c0*c1*s2*s3;
+      E(1) = -c0*s1*c2*s3;
+      E(2) =  c0*s1*s2*c3;
+      E(3) =  s0*c1*c2*s3;
+      E(4) = -s0*c1*s2*c3;
+      E(5) =  s0*s1*c2*c3;
+   }
+}
+
+void E_exact(const Vector &x, DenseMatrix &E)
+{
+   int dim = x.Size();
+
+   E.SetSize(dim*dim);
+
+   if (dim==4)
+   {
+      Vector vecE; E_exact_vec(x, vecE);
+
+      E = 0.0;
+
+      E(0,1) = vecE(0);
+      E(0,2) = vecE(1);
+      E(0,3) = vecE(2);
+      E(1,2) = vecE(3);
+      E(1,3) = vecE(4);
+      E(2,3) = vecE(5);
+
+      E(1,0) =  -E(0,1);
+      E(2,0) =  -E(0,2);
+      E(3,0) =  -E(0,3);
+      E(2,1) =  -E(1,2);
+      E(3,1) =  -E(1,3);
+      E(3,2) =  -E(2,3);
+   }
+}
+
+
+
+//f_exact = E + 0.5 * P( curl DivSkew E ), where P is the 4d permutation operator
+void f_exact(const Vector &x, DenseMatrix &f)
+{
+   int dim = x.Size();
+
+   f.SetSize(dim,dim);
+
+   if (dim==4)
+   {
+      f = 0.0;
+
+      double s0 = sin(M_PI*x(0)), s1 = sin(M_PI*x(1)), s2 = sin(M_PI*x(2)),
+             s3 = sin(M_PI*x(3));
+      double c0 = cos(M_PI*x(0)), c1 = cos(M_PI*x(1)), c2 = cos(M_PI*x(2)),
+             c3 = cos(M_PI*x(3));
+
+      f(0,1) =  (1.0 + 1.0  * M_PI*M_PI)*c0*c1*s2*s3;
+      f(0,2) = -(1.0 + 0.0  * M_PI*M_PI)*c0*s1*c2*s3;
+      f(0,3) =  (1.0 + 1.0  * M_PI*M_PI)*c0*s1*s2*c3;
+      f(1,2) =  (1.0 - 1.0  * M_PI*M_PI)*s0*c1*c2*s3;
+      f(1,3) = -(1.0 + 0.0  * M_PI*M_PI)*s0*c1*s2*c3;
+      f(2,3) =  (1.0 + 1.0  * M_PI*M_PI)*s0*s1*c2*c3;
+
+      f(1,0) =  -f(0,1);
+      f(2,0) =  -f(0,2);
+      f(3,0) =  -f(0,3);
+      f(2,1) =  -f(1,2);
+      f(3,1) =  -f(1,3);
+      f(3,2) =  -f(2,3);
+   }
+}

examples/ex4p_4d.cpp

@@ -0,0 +1,800 @@
+//                       MFEM Example 4 - Parallel Version
+//
+// Compile with: make ex4p
+//
+// Sample runs:  mpirun -np 4 ex4p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex4p -m ../data/star.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/beam-hex-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
+//
+// Description:  This example code solves a simple 2D/3D H(div) diffusion
+//               problem corresponding to the second order definite equation
+//               -grad(alpha div F) + beta F = f with boundary condition F dot n
+//               = <given normal field>. Here, we use a given exact solution F
+//               and compute the corresponding r.h.s. f.  We discretize with
+//               Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of H(div) finite element
+//               spaces with the grad-div and H(div) vector finite element mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Bilinear form
+//               hybridization and static condensation are also illustrated.
+//
+//               We recommend viewing examples 1-3 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "./spe10_coeff.cpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int* LoadIterations(int NRows, int NCol)
+{
+   ifstream in("iter_div.txt");
+
+   //initialize
+   int *iters = new int[NCol*NRows];
+   for (int col = 0; col < NCol; col++)
+   {
+      for (int row = 0; row < NRows; row++)
+      {
+         iters[row*NCol+col] = -1;
+      }
+   }
+
+   if (!in)
+   {
+      cout << "Cannot open file.\n";
+      return iters;
+   }
+
+   for (int row = 0; row < NRows; row++)
+      for (int col = 0; col < NCol; col++)
+      {
+         if (in.eof())
+         {
+            in.close();
+            return iters;
+         }
+         in >> iters[row*NCol+col];
+      }
+
+
+   in.close();
+
+   return iters;
+}
+
+void putIterationsInArray(int iter, int row, int col, int NCol, int* iters)
+{
+   iters[row*NCol+col] = iter;
+}
+
+void WriteIterations(int *iters, int NRows, int NCol)
+{
+   ofstream out;
+   out.open("iter_div.txt",fstream::out);
+
+   if (!out)
+   {
+      cout << "Cannot open file.\n";
+      delete[] iters;
+
+      return;
+   }
+
+   for (int row = 0; row < NRows; row++)
+   {
+      for (int col = 0; col < NCol; col++)
+      {
+         out << iters[row*NCol+col] << "\t";
+      }
+      out << endl;
+   }
+   out.close();
+
+   delete[] iters;
+}
+
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+void F_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+
+
+
+class div4dPrec : public Solver
+{
+
+private:
+   HypreParMatrix *A;
+   ParFiniteElementSpace *fespace;
+
+   Coefficient *alpha_, *beta_;
+
+   //kernel operators
+   HypreParMatrix *P_d_HSkewDiv_Hdiv;
+
+   HypreParMatrix *P_H1_HDivSkew;
+   HypreParMatrix *H1_KernelMat;
+   HypreBoomerAMG *amgH1_Kernel;
+
+   //"image" operators
+   HypreParMatrix *P_H1_Hdiv;
+   HypreParMatrix *H1_ImageMat;
+   HypreBoomerAMG *amgH1_Image;
+
+   HypreParMatrix *HDivSkewMat;
+   HypreSmoother * smootherdiv;
+   HypreSmoother * smootherDivSkew;
+
+   CGSolver *pcgKernel;
+   CGSolver *pcgImage;
+
+   Vector *f;
+   Vector *fKernel, *uKernel;
+   Vector *fImage, *uImage;
+   Vector *fDivSkew, *uDivSkew;
+
+   FiniteElementCollection* fecHDivSkewKernel;
+   ParFiniteElementSpace *HDivSkewKernelFESpace;
+
+   bool exactSolves;
+
+public:
+   ~div4dPrec()
+   {
+      delete pcgImage;
+      delete pcgKernel;
+
+      delete uDivSkew, fDivSkew, uImage, fImage, uKernel, fKernel, f;
+
+      delete smootherDivSkew;
+      delete HDivSkewMat;
+
+      delete P_d_HSkewDiv_Hdiv;
+      delete P_H1_Hdiv;
+      delete P_H1_HDivSkew;
+
+      delete amgH1_Image, H1_ImageMat;
+      delete amgH1_Kernel, H1_KernelMat;
+
+      delete smootherdiv;
+
+      delete HDivSkewKernelFESpace;
+      delete fecHDivSkewKernel;
+   }
+   div4dPrec(HypreParMatrix *AUser, ParFiniteElementSpace *fespaceUser,
+             Coefficient *alpha, Coefficient *beta, const Array<int> &essBnd,
+             int orderKernel=1, bool exactSolvesUser=false)
+   {
+      A = AUser;
+      fespace = fespaceUser;
+      alpha_ = alpha;
+      beta_ = beta;
+
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+
+      exactSolves = exactSolvesUser;
+
+
+
+
+      int orderIm=1;  //H1  --> H(div)
+      int orderKer=orderKernel; //DivSkew V --> H(div)
+
+
+
+      smootherdiv = new HypreSmoother(*A, 16, 3);
+
+      Array<int> Hdiv_essDof(fespace->GetVSize()); Hdiv_essDof = 0;
+      fespace->GetEssentialVDofs(essBnd, Hdiv_essDof);
+
+
+
+
+      //setup the H1 FESpace for the kernel
+      FiniteElementCollection* fecH1Kernel;
+      if (orderKer==1) { fecH1Kernel = new LinearFECollection; }
+      else { fecH1Kernel = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1KernelFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Kernel, 6, Ordering::byVDIM);
+      Array<int> H1Kernel_essDof(H1KernelFESpace->GetVSize()); H1Kernel_essDof = 0;
+      H1KernelFESpace->GetEssentialVDofs(essBnd, H1Kernel_essDof);
+
+
+      //setup the H(DivSkew) FESpace for the kernel
+      if (orderKer==1) { fecHDivSkewKernel = new DivSkew1_4DFECollection; }
+      //       else fecHDivSkewKernel = new DivSkewFull1_4DFECollection;
+      HDivSkewKernelFESpace = new ParFiniteElementSpace(pmesh, fecHDivSkewKernel);
+      Array<int> HDivSkewKernel_essDof(HDivSkewKernelFESpace->GetVSize());
+      HDivSkewKernel_essDof = 0;
+      HDivSkewKernelFESpace->GetEssentialVDofs(essBnd, HDivSkewKernel_essDof);
+
+
+      //setup the FESpace for the H1 injection
+      FiniteElementCollection* fecH1Vec;
+      if (orderIm==1) { fecH1Vec = new LinearFECollection; }
+      else { fecH1Vec = new QuadraticFECollection; }
+      ParFiniteElementSpace *H1_ImageFESpace = new ParFiniteElementSpace(pmesh,
+                                                                         fecH1Vec, dim, Ordering::byVDIM);
+      Array<int> H1Image_essDof(H1_ImageFESpace->GetVSize()); H1Image_essDof = 0;
+      H1_ImageFESpace->GetEssentialVDofs(essBnd, H1Image_essDof);
+
+
+
+      //setup the H1 preconditioner for the kernel
+      ParBilinearForm* H1Varf = new ParBilinearForm(H1KernelFESpace);
+      //    H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_, 6));
+      //    H1Varf->AddDomainIntegrator(new VectorMassIntegrator(6, beta_));
+
+      H1Varf->AddDomainIntegrator(new VectorDiffusionIntegrator(*beta_, 6));
+      H1Varf->Assemble();
+      H1Varf->Finalize();
+      SparseMatrix &matH1(H1Varf->SpMat());
+      for (int dof=0; dof<H1Kernel_essDof.Size(); dof++) if (H1Kernel_essDof[dof]<0) { matH1.EliminateRowCol(dof); }
+      H1_KernelMat = H1Varf->ParallelAssemble();
+      delete H1Varf;
+      amgH1_Kernel = new HypreBoomerAMG(*H1_KernelMat);
+      amgH1_Kernel->SetSystemsOptions(6);
+
+      //setup the H1 preconditioner for the image
+      ParBilinearForm* H1VecVarf = new ParBilinearForm(H1_ImageFESpace);
+      H1VecVarf->AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_));
+      H1VecVarf->AddDomainIntegrator(new VectorMassIntegrator(-1, beta_));
+      H1VecVarf->Assemble();
+      H1VecVarf->Finalize();
+      SparseMatrix &matH1Vec(H1VecVarf->SpMat());
+      for (int dof=0; dof<H1Image_essDof.Size(); dof++) if (H1Image_essDof[dof]<0) { matH1Vec.EliminateRowCol(dof); }
+      H1_ImageMat = H1VecVarf->ParallelAssemble();
+      delete H1VecVarf;
+      amgH1_Image = new HypreBoomerAMG(*H1_ImageMat);
+      amgH1_Image->SetSystemsOptions(dim);
+
+
+      //setup the injection of H1 into H(DivSkew)
+      ParDiscreteLinearOperator *disInterpolIm = new ParDiscreteLinearOperator(
+         H1KernelFESpace, HDivSkewKernelFESpace);
+      disInterpolIm->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpolIm->Assemble();
+      disInterpolIm->Finalize();
+      SparseMatrix* smatIDIm = &(disInterpolIm->SpMat());
+      smatIDIm->EliminateCols(H1Kernel_essDof);
+      for (int dof=0; dof<HDivSkewKernel_essDof.Size();
+           dof++) if (HDivSkewKernel_essDof[dof]<0) { smatIDIm->EliminateRow(dof); }
+      P_H1_HDivSkew = disInterpolIm->ParallelAssemble();
+      delete disInterpolIm;
+
+      //setup the injection of H1 into H(div)
+      ParDiscreteLinearOperator *disInterpol = new ParDiscreteLinearOperator(
+         H1_ImageFESpace, fespace);
+      disInterpol->AddDomainInterpolator(new IdentityInterpolator);
+      disInterpol->Assemble();
+      disInterpol->Finalize();
+      SparseMatrix* smatID = &(disInterpol->SpMat());
+      smatID->EliminateCols(H1Image_essDof);
+      for (int dof=0; dof<Hdiv_essDof.Size(); dof++) if (Hdiv_essDof[dof]<0) { smatID->EliminateRow(dof); }
+      P_H1_Hdiv = disInterpol->ParallelAssemble();
+      delete disInterpol;
+
+
+
+
+      //setup the injection of the DivSkew(H(DivSkew)) into H(div)
+      ParDiscreteLinearOperator *disDivSkew = new ParDiscreteLinearOperator(
+         HDivSkewKernelFESpace, fespace);
+      disDivSkew->AddDomainInterpolator(new DivSkewInterpolator);
+      disDivSkew->Assemble();
+      disDivSkew->Finalize();
+      SparseMatrix* smatDivSkew= &(disDivSkew->SpMat());
+      smatDivSkew->EliminateCols(HDivSkewKernel_essDof);
+      for (int dof=0; dof<Hdiv_essDof.Size(); dof++) if (Hdiv_essDof[dof]<0) { smatDivSkew->EliminateRow(dof); }
+      P_d_HSkewDiv_Hdiv = disDivSkew->ParallelAssemble();
+      delete disDivSkew;
+
+
+      //setup the smoother for H(DivSkew)
+      ParBilinearForm *a_HDivSkew = new ParBilinearForm(HDivSkewKernelFESpace);
+      //    a_HDivSkew->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*alpha_));
+      //    a_HDivSkew->AddDomainIntegrator(new VectorFE_DivSkewMassIntegrator(*beta_));
+
+      a_HDivSkew->AddDomainIntegrator(new DivSkewDivSkewIntegrator(*beta_));
+
+      a_HDivSkew->Assemble();
+      a_HDivSkew->Finalize();
+      SparseMatrix &matHDivSkew(a_HDivSkew->SpMat());
+      for (int dof=0; dof<HDivSkewKernel_essDof.Size();
+           dof++) if (HDivSkewKernel_essDof[dof]<0) { matHDivSkew.EliminateRowCol(dof); }
+      HDivSkewMat = a_HDivSkew->ParallelAssemble();
+      delete a_HDivSkew;
+      smootherDivSkew = new HypreSmoother(*HDivSkewMat, 16, 3);
+
+
+
+      f = new Vector(fespace->GetTrueVSize());
+
+      fKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+      uKernel = new Vector(H1KernelFESpace->GetTrueVSize());
+
+      fImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+      uImage = new Vector(H1_ImageFESpace->GetTrueVSize());
+
+      fDivSkew = new Vector(HDivSkewKernelFESpace->GetTrueVSize());
+      uDivSkew = new Vector(HDivSkewKernelFESpace->GetTrueVSize());
+
+      amgH1_Kernel->Mult(*fKernel, *uKernel);
+      amgH1_Image->Mult(*fImage, *uImage);
+
+      pcgKernel = new CGSolver(MPI_COMM_WORLD);
+      pcgKernel->SetOperator(*H1_KernelMat);
+      pcgKernel->SetPreconditioner(*amgH1_Kernel);
+      pcgKernel->SetRelTol(1e-16);
+      pcgKernel->SetMaxIter(100000000);
+      pcgKernel->SetPrintLevel(-2);
+
+      pcgImage = new CGSolver(MPI_COMM_WORLD);
+      pcgImage->SetOperator(*H1_ImageMat);
+      pcgImage->SetPreconditioner(*amgH1_Image);
+      pcgImage->SetRelTol(1e-16);
+      pcgImage->SetMaxIter(100000000);
+      pcgImage->SetPrintLevel(-2);
+
+      delete H1_ImageFESpace;
+      delete H1KernelFESpace;
+      delete fecH1Kernel;
+      delete fecH1Vec;
+   }
+
+   void setExactSolve(bool exSol)
+   {
+      exactSolves = exSol;
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      smootherdiv->Mult(x,y);
+
+      P_H1_Hdiv->MultTranspose(x,*fImage);
+      *uImage = 0.0;
+      if (exactSolves) { pcgImage->Mult(*fImage, *uImage); }
+      else { amgH1_Image->Mult(*fImage, *uImage); }
+      P_H1_Hdiv->Mult(1.0, *uImage, 1.0, y);
+
+
+      *uDivSkew = 0.0;
+      P_d_HSkewDiv_Hdiv->MultTranspose(x,*fDivSkew);
+
+      smootherDivSkew->Mult(*fDivSkew, *uDivSkew);
+
+      P_H1_HDivSkew->MultTranspose(*fDivSkew,*fKernel);
+      *uKernel = 0.0;
+      if (exactSolves) { pcgKernel->Mult(*fKernel, *uKernel); }
+      else { amgH1_Kernel->Mult(*fKernel, *uKernel); }
+      P_H1_HDivSkew->Mult(1.0, *uKernel, 1.0, *uDivSkew);
+
+      P_d_HSkewDiv_Hdiv->Mult(1.0, *uDivSkew, 1.0, y);
+   }
+
+   virtual void SetOperator(const Operator &op) {};
+
+};
+
+
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   bool verbose = (myid==0);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool set_bc = true;
+   bool static_cond = false;
+   bool hybridization = false;
+   bool visualization = 1;
+   int sequ_ref_levels = 0;
+   int par_ref_levels = 0;
+   double tol = 1e-6;
+   double coeffWeight = 1.0;
+   bool spe10Coeff = false;
+   bool exactH1Solver = false;
+   bool standardCG = true;
+
+   int NExpo = 8;
+   int weightStart = -NExpo;
+   int weightEnd = NExpo;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&sequ_ref_levels, "-sr", "--seqrefinement",
+                  "Number of sequential refinement steps.");
+   args.AddOption(&par_ref_levels, "-pr", "--parrefinement",
+                  "Number of parallel refinement steps.");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
+                  "--no-hybridization", "Enable hybridization.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "A parameter.");
+   args.AddOption(&coeffWeight, "-c", "--coeffMass",
+                  "the weight for the mass term.");
+   args.AddOption(&exactH1Solver, "-exH1Sol", "--exactH1Solver", "-H1prec",
+                  "--H1preconditioner",
+                  "Use exact H1 solvers for the preconditioner.");
+   args.AddOption(&spe10Coeff, "-spe10", "--useSPE10Coeff", "-constCoeff",
+                  "--constCoeff",
+                  "Switch between the coefficients for the mass bilinear form.");
+   args.AddOption(&standardCG, "-sCG", "--stdCG", "-rCG", "--resCG",
+                  "Switch between standard PCG or recompute residuals in every step and use the residuals itself for the stopping criteria.");
+   args.AddOption(&weightStart, "-ws", "--weightStart",
+                  "the exponent for the starting weight (for the mass term).");
+   args.AddOption(&weightEnd, "-we", "--weightEnd",
+                  "the exponent for the weight at the end (for the mass term).");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume, as well as periodic meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      for (int l = 0; l < sequ_ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Raviart-Thomas finite elements of the specified order.
+   FiniteElementCollection *fec;
+   if (dim==4) { fec = new RT0_4DFECollection; }
+   else { fec = new RT_FECollection(order-1, dim); }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = set_bc ? 1 : 0;
+   if (pmesh->bdr_attributes.Size())
+   {
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary faces will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient F(sdim, F_exact);
+
+   for (int expo=weightStart; expo<=weightEnd; expo++)
+   {
+      double weight = pow(10.0,expo);
+      kappa = weight;
+
+      x.ProjectCoefficient(F);
+
+      VectorFunctionCoefficient f(sdim, f_exact);
+      ParLinearForm *b = new ParLinearForm(fespace);
+      b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+      b->Assemble();
+
+      // 10. Set up the parallel bilinear form corresponding to the H(div)
+      //     diffusion operator grad alpha div + beta I, by adding the div-div and
+      //     the mass domain integrators.
+
+      //    std::string permFile = "spe_perm.dat";
+      //    InversePermeabilityFunction::ReadPermeabilityFile(permFile, MPI_COMM_WORLD);
+
+      Coefficient *alpha = new ConstantCoefficient(1.0);
+      Coefficient *beta;
+      //    if(spe10Coeff) beta = new FunctionCoefficient(InversePermeabilityFunction::Norm2Permeability);
+      //    else
+      beta = new ConstantCoefficient(weight);
+
+      ParBilinearForm *a = new ParBilinearForm(fespace);
+      a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
+      a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
+
+      // 11. Assemble the parallel bilinear form and the corresponding linear
+      //     system, applying any necessary transformations such as: parallel
+      //     assembly, eliminating boundary conditions, applying conforming
+      //     constraints for non-conforming AMR, static condensation,
+      //     hybridization, etc.
+      FiniteElementCollection *hfec = NULL;
+      ParFiniteElementSpace *hfes = NULL;
+      if (static_cond)
+      {
+         a->EnableStaticCondensation();
+      }
+      else if (hybridization)
+      {
+         hfec = new DG_Interface_FECollection(order-1, dim);
+         hfes = new ParFiniteElementSpace(pmesh, hfec);
+         a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
+                                ess_tdof_list);
+      }
+      a->Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      HYPRE_Int glob_size = A.GetGlobalNumRows();
+      if (myid == 0)
+      {
+         cout << "Size of linear system: " << glob_size << endl;
+      }
+
+      // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
+      //     the 3D ADS preconditioners from hypre. If using hybridization, the
+      //     system is preconditioned with hypre's BoomerAMG.
+      Solver *prec = NULL;
+      if (hybridization) { prec = new HypreBoomerAMG(A); }
+      else
+      {
+         ParFiniteElementSpace *prec_fespace =
+            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+         if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
+         else if (dim==3) { prec = new HypreADS(A, prec_fespace); }
+         else if (dim==4) { prec = new div4dPrec(&A, fespace, alpha, beta, ess_bdr, order, exactH1Solver); }
+         else { prec = NULL; }
+      }
+
+      int iter = -1;
+      if (standardCG)
+      {
+         IterativeSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+         pcg->SetOperator(A);
+         pcg->SetRelTol(tol);
+         pcg->SetMaxIter(500);
+         pcg->SetPrintLevel(1);
+         pcg->SetPreconditioner(*prec);
+         pcg->Mult(B, X);
+
+         iter = pcg->GetNumIterations();
+
+         delete pcg;
+      }
+      else
+      {
+         HyprePCG *pcg = new HyprePCG(A);
+         pcg->SetTol(tol);
+         pcg->SetMaxIter(5000);
+         pcg->SetResidualConvergenceOptions(1,tol);
+         pcg->SetPrintLevel(2);
+         //          pcg->SetPreconditioner(*prec);
+         pcg->Mult(B, X);
+
+         pcg->GetNumIterations(iter);
+
+         delete pcg;
+      }
+
+      if (myid==0)
+      {
+         cout << "Weigth: " << weight << " " << iter << endl;
+
+         int *iters = LoadIterations(10, 2*NExpo+1);
+         putIterationsInArray(iter, sequ_ref_levels+par_ref_levels, expo+NExpo,
+                              2*NExpo+1, iters);
+         WriteIterations(iters, 10, 2*NExpo+1);
+      }
+
+
+      // 13. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 14. Compute and print the L^2 norm of the error.
+      {
+         double err = x.ComputeL2Error(F);
+         if (myid == 0)
+         {
+            cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+         }
+      }
+
+      // 15. Save the refined mesh and the solution in parallel. This output can
+      //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      //    {
+      //      ostringstream mesh_name, sol_name;
+      //      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      //      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      //
+      //      ofstream mesh_ofs(mesh_name.str().c_str());
+      //      mesh_ofs.precision(8);
+      //      pmesh->Print(mesh_ofs);
+      //
+      //      ofstream sol_ofs(sol_name.str().c_str());
+      //      sol_ofs.precision(8);
+      //      x.Save(sol_ofs);
+      //    }
+
+      // 16. Send the solution by socket to a GLVis server.
+      //    if (visualization)
+      //    {
+      //      char vishost[] = "localhost";
+      //      int  visport   = 19916;
+      //      socketstream sol_sock(vishost, visport);
+      //      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      //      sol_sock.precision(8);
+      //      sol_sock << "solution\n" << *pmesh << x << flush;
+      //    }
+
+      if (prec!=NULL) { delete prec; }
+      delete hfes;
+      delete hfec;
+      delete a;
+      delete alpha;
+      delete beta;
+
+      delete b;
+   }
+
+   // 17. Free the used memory.
+
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   if (dim==4)
+   {
+      double s0 = sin(M_PI*p(0)), s1 = sin(M_PI*p(1)), s2 = sin(M_PI*p(2)),
+             s3 = sin(M_PI*p(3));
+      double c0 = cos(M_PI*p(0)), c1 = cos(M_PI*p(1)), c2 = cos(M_PI*p(2)),
+             c3 = cos(M_PI*p(3));
+
+      F(0) = c0 * s1 * s2 * s3;
+      F(1) = s0 * c1 * s2 * s3;
+      F(2) = s0 * s1 * c2 * s3;
+      F(3) = s0 * s1 * s2 * c3;
+   }
+   else
+   {
+      double x = p(0);
+      double y = p(1);
+      // double z = (dim == 3) ? p(2) : 0.0;
+
+      F(0) = cos(kappa*x)*sin(kappa*y);
+      F(1) = cos(kappa*y)*sin(kappa*x);
+      if (dim == 3)
+      {
+         F(2) = 0.0;
+      }
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+   if (dim==4)
+   {
+      double s0 = sin(M_PI*p(0)), s1 = sin(M_PI*p(1)), s2 = sin(M_PI*p(2)),
+             s3 = sin(M_PI*p(3));
+      double c0 = cos(M_PI*p(0)), c1 = cos(M_PI*p(1)), c2 = cos(M_PI*p(2)),
+             c3 = cos(M_PI*p(3));
+
+      f(0) = c0 * s1 * s2 * s3;
+      f(1) = s0 * c1 * s2 * s3;
+      f(2) = s0 * s1 * c2 * s3;
+      f(3) = s0 * s1 * s2 * c3;
+
+      f *= (kappa + 4.0 * M_PI*M_PI);
+   }
+   else
+   {
+      double x = p(0);
+      double y = p(1);
+      // double z = (dim == 3) ? p(2) : 0.0;
+
+      double temp = 1 + 2*kappa*kappa;
+
+      f(0) = temp*cos(kappa*x)*sin(kappa*y);
+      f(1) = temp*cos(kappa*y)*sin(kappa*x);
+      if (dim == 3)
+      {
+         f(2) = 0;
+      }
+   }
+}

examples/ginkgo/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ginkgo/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/hiop/ex9p.cpp

@@ -96,7 +96,6 @@ public:
    {
       Vector w_glob(width);
       pfes.Dof_TrueDof_Matrix()->MultTranspose(w, w_glob);
-      w_glob.HostReadWrite(); // read+write -> can use w_glob(i) (non-const)
       for (int i = 0; i < width; i++) { grad(0, i) = w_glob(i); }
    }
 

examples/hiop/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/hiop/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ..
 MFEM_BUILD_DIR ?= ..
-MFEM_INSTALL_DIR ?= ../mfem
 SRC = $(if $(MFEM_DIR:..=),$(MFEM_DIR)/examples/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
@@ -26,7 +27,8 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
    ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
    ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p ex40p
+   ex37p ex39p ex40p \
+   ex1p_4d ex3p_4d ex4D_DivSkew
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
    ex22p ex24p ex25p ex26p ex34p ex35p

examples/moonolith/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/moonolith/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/petsc/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/petsc/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/pumi/makefile

@@ -12,10 +12,11 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
 SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/pumi/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

examples/spe10_coeff.cpp

@@ -0,0 +1,352 @@
+/*
+ * spe10_coeff.cpp
+ *
+ *  Created on: Aug 23, 2017
+ *      Author: neumueller
+ */
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class InversePermeabilityFunction
+{
+public:
+
+   enum SliceOrientation {NONE, XY, XZ, YZ};
+
+   static void SetNumberCells(int Nx_, int Ny_, int Nz_);
+   static void SetMeshSizes(double hx, double hy, double hz);
+   static void Set2DSlice(SliceOrientation o, int npos );
+
+   static void ReadPermeabilityFile(const std::string fileName);
+#ifdef MFEM_USE_MPI
+   static void ReadPermeabilityFile(const std::string fileName, MPI_Comm comm);
+#endif
+   static void SetConstantInversePermeability(double ipx, double ipy, double ipz);
+
+   template<class F>
+   static void Transform(const F & f)
+   {
+      for (int i = 0; i < 3*Nx*Ny*Nz; ++i)
+      {
+         inversePermeability[i] = f(inversePermeability[i]);
+      }
+   }
+
+   static void InversePermeability(const Vector & x, Vector & val);
+   static double PermeabilityXY(Vector &x);
+   static void NegativeInversePermeability(const Vector & x, Vector & val);
+   static void Permeability(const Vector & x, Vector & val);
+
+   static double Norm2Permeability(const Vector & x);
+
+   static double Norm2InversePermeability(const Vector & x);
+   static double Norm1InversePermeability(const Vector & x);
+   static double NormInfInversePermeability(const Vector & x);
+
+   static double InvNorm2(const Vector & x);
+   static double InvNorm1(const Vector & x);
+   static double InvNormInf(const Vector & x);
+
+
+   static void ClearMemory();
+
+private:
+   static int Nx;
+   static int Ny;
+   static int Nz;
+   static double hx;
+   static double hy;
+   static double hz;
+   static double * inversePermeability;
+
+   static SliceOrientation orientation;
+   static int npos;
+};
+
+
+void InversePermeabilityFunction::SetNumberCells(int Nx_, int Ny_, int Nz_)
+{
+   Nx = Nx_;
+   Ny = Ny_;
+   Nz = Nz_;
+}
+
+void InversePermeabilityFunction::SetMeshSizes(double hx_, double hy_,
+                                               double hz_)
+{
+   hx = hx_;
+   hy = hy_;
+   hz = hz_;
+}
+
+void InversePermeabilityFunction::Set2DSlice(SliceOrientation o, int npos_ )
+{
+   orientation = o;
+   npos = npos_;
+}
+
+void InversePermeabilityFunction::SetConstantInversePermeability(double ipx,
+                                                                 double ipy, double ipz)
+{
+   int compSize = Nx*Ny*Nz;
+   int size = 3*compSize;
+   inversePermeability = new double [size];
+   double *ip = inversePermeability;
+
+   for (int i(0); i < compSize; ++i)
+   {
+      ip[i] = ipx;
+      ip[i+compSize] = ipy;
+      ip[i+2*compSize] = ipz;
+   }
+
+}
+
+#ifdef MFEM_USE_MPI
+void InversePermeabilityFunction::ReadPermeabilityFile(const std::string
+                                                       fileName, MPI_Comm comm)
+{
+   int num_procs, myid;
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   StopWatch chrono;
+
+   chrono.Start();
+   if (myid == 0)
+   {
+      ReadPermeabilityFile(fileName);
+   }
+   else
+   {
+      inversePermeability = new double [3*Nx*Ny*Nz];
+   }
+   chrono.Stop();
+
+   if (myid==0)
+   {
+      std::cout<<"Permeability file read in " << chrono.RealTime() << ".s \n";
+   }
+
+   chrono.Clear();
+
+   chrono.Start();
+   MPI_Bcast(inversePermeability, 3*Nx*Ny*Nz, MPI_DOUBLE, 0, comm);
+   chrono.Stop();
+
+   if (myid==0)
+   {
+      std::cout<<"Permeability field distributed in " << chrono.RealTime() <<
+               ".s \n";
+   }
+
+}
+#endif
+
+void InversePermeabilityFunction::ReadPermeabilityFile(const std::string
+                                                       fileName)
+{
+   std::ifstream permfile(fileName.c_str());
+
+   if (!permfile.is_open())
+   {
+      std::cout << "Error in opening file " << fileName << "\n";
+      mfem_error("File do not exists");
+   }
+
+   inversePermeability = new double [3*Nx*Ny*Nz];
+   double *ip = inversePermeability;
+   double tmp;
+   for (int l = 0; l < 3; l++)
+   {
+      for (int k = 0; k < Nz; k++)
+      {
+         for (int j = 0; j < Ny; j++)
+         {
+            for (int i = 0; i < Nx; i++)
+            {
+               permfile >> *ip;
+               *ip = 1./(*ip);
+               ip++;
+            }
+            for (int i = 0; i < 60-Nx; i++)
+            {
+               permfile >> tmp;   // skip unneeded part
+            }
+         }
+         for (int j = 0; j < 220-Ny; j++)
+            for (int i = 0; i < 60; i++)
+            {
+               permfile >> tmp;   // skip unneeded part
+            }
+      }
+
+      if (l < 2) // if not processing Kz, skip unneeded part
+         for (int k = 0; k < 85-Nz; k++)
+            for (int j = 0; j < 220; j++)
+               for (int i = 0; i < 60; i++)
+               {
+                  permfile >> tmp;
+               }
+   }
+
+}
+
+void InversePermeabilityFunction::InversePermeability(const Vector & x,
+                                                      Vector & val)
+{
+   val.SetSize(3);
+
+   unsigned int i,j,k;
+
+   switch (orientation)
+   {
+      case NONE:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      case XY:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = npos;
+         break;
+      case XZ:
+         i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+         j = npos;
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      case YZ:
+         i = npos;
+         j = (int)floor(x[1]/hy/(1.+3e-16));
+         k = Nz-1-(int)floor(x[2]/hz/(1.+3e-16));
+         break;
+      default:
+      {
+         mfem_error("InversePermeabilityFunction::InversePermeability");
+      }
+   }
+
+
+   int NMax = 3*Nx*Ny*Nz-1;
+   if (Ny*Nx*k + Nx*j + i>NMax || Ny*Nx*k + Nx*j + i + Nx*Ny*Nz>NMax ||
+       Ny*Nx*k + Nx*j + i + 2*Nx*Ny*Nz>NMax)
+   {
+      cout << " the indicies are wrong!" << endl;
+      cout << i << " " << j << " " << k << endl;
+   }
+
+   val[0] = inversePermeability[Ny*Nx*k + Nx*j + i];
+   val[1] = inversePermeability[Ny*Nx*k + Nx*j + i + Nx*Ny*Nz];
+
+   if (orientation == NONE)
+   {
+      val[2] = inversePermeability[Ny*Nx*k + Nx*j + i + 2*Nx*Ny*Nz];
+   }
+
+}
+
+double InversePermeabilityFunction::PermeabilityXY(Vector &x)
+{
+   unsigned int i,j,k;
+
+   i = Nx-1-(int)floor(x[0]/hx/(1.+3e-16));
+   j = (int)floor(x[1]/hy/(1.+3e-16));
+   k = npos;
+
+   return 1./inversePermeability[Ny*Nx*k + Nx*j + i];
+}
+
+void InversePermeabilityFunction::NegativeInversePermeability(const Vector & x,
+                                                              Vector & val)
+{
+   InversePermeability(x,val);
+   val *= -1.;
+}
+
+
+void InversePermeabilityFunction::Permeability(const Vector & x, Vector & val)
+{
+   InversePermeability(x,val);
+
+   for (double * it = val.GetData(), *end = val.GetData()+val.Size(); it != end;
+        ++it )
+   {
+      (*it) = 1./ (*it);
+   }
+}
+
+double InversePermeabilityFunction::Norm2Permeability(const Vector & x)
+{
+   Vector val(3);
+   Permeability(x,val);
+   return val.Norml2();
+}
+
+
+double InversePermeabilityFunction::Norm2InversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Norml2();
+}
+
+double InversePermeabilityFunction::Norm1InversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Norml1();
+}
+
+double InversePermeabilityFunction::NormInfInversePermeability(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return val.Normlinf();
+}
+
+double InversePermeabilityFunction::InvNorm2(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Norml2();
+}
+
+double InversePermeabilityFunction::InvNorm1(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Norml1();
+}
+
+double InversePermeabilityFunction::InvNormInf(const Vector & x)
+{
+   Vector val(3);
+   InversePermeability(x,val);
+   return 1./val.Normlinf();
+}
+
+
+void InversePermeabilityFunction::ClearMemory()
+{
+   delete[] inversePermeability;
+}
+
+int InversePermeabilityFunction::Nx(60);
+int InversePermeabilityFunction::Ny(220);
+int InversePermeabilityFunction::Nz(85);
+double InversePermeabilityFunction::hx(20);
+double InversePermeabilityFunction::hy(10);
+double InversePermeabilityFunction::hz(2);
+double * InversePermeabilityFunction::inversePermeability(NULL);
+InversePermeabilityFunction::SliceOrientation
+InversePermeabilityFunction::orientation( InversePermeabilityFunction::NONE );
+int InversePermeabilityFunction::npos(-1);
+
+
+

examples/sundials/CMakeLists.txt

@@ -31,21 +31,11 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 add_custom_target(test_sundials
   ${CMAKE_CTEST_COMMAND} -R sundials USES_TERMINAL)
 
-# Add one executable per cpp file, adding "sundials_" as prefix so the CMake
-# target is unique from those in the non-SUNDIALS examples. Also sets
-# "test_sundials" as a target that depends on the given SUNDIALS examples.
+# Add one executable per cpp file, adding "sundials_" as prefix. Sets
+# "test_sundials" as a target that depends on the given examples.
 set(PFX sundials_)
 add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)
 
-# Remove "sundials_" prefix from exectuable name for consistency with GNU build
-# system.
-foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TARGET_NAME "${PFX}${SRC_FILENAME}")
-  string(REPLACE ${PFX} "" EXE_NAME ${TARGET_NAME})
-  set_target_properties(${TARGET_NAME} PROPERTIES OUTPUT_NAME ${EXE_NAME})
-endforeach()
-
 # Testing.
 # The SUNDIALS tests can be run separately using the target "test_sundials"
 # which builds the examples and runs:
@@ -61,10 +51,7 @@ if (MFEM_ENABLE_TESTING)
   set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
   set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
   set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
-  # Example 16: test ARKODE with implicit time stepping using mass form
-  set(EX16_COMMON_OPTS -s 15)
-  set(EX16_TEST_OPTS ${EX16_COMMON_OPTS})
-  set(EX16P_TEST_OPTS ${EX16_COMMON_OPTS})
+  # Example 16: use the default options
 
   # Add the tests: one test per source file.
   foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})

examples/sundials/ex10.cpp

@@ -1,9 +1,7 @@
 //                                MFEM Example 10
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex10             (GNU make)
-//    make sundials_ex10    (CMake)
+// Compile with: make ex10
 //
 // Sample runs:
 //    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 12 -dt 0.15 -vs 10

examples/sundials/ex10p.cpp

@@ -1,9 +1,7 @@
 //                       MFEM Example 10 - Parallel Version
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex10p            (GNU make)
-//    make sundials_ex10p   (CMake)
+// Compile with: make ex10p
 //
 // Sample runs:
 //    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 12 -dt 0.15 -vs 10

examples/sundials/ex16.cpp

@@ -1,21 +1,15 @@
 //                                MFEM Example 16
 //                             SUNDIALS Modification
 //
-// Compile with:
-//    make ex16             (GNU make)
-//    make sundials_ex16    (CMake)
+// Compile with: make ex16
 //
 // Sample runs:  ex16
 //               ex16 -m ../../data/inline-tri.mesh
 //               ex16 -m ../../data/disc-nurbs.mesh -tf 2
 //               ex16 -s 12 -a 0.0 -k 1.0
-//               ex16 -s 15 -a 0.0 -k 1.0
 //               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
-//               ex16 -s 11 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
 //               ex16 -s 9 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
-//               ex16 -s 12 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
 //               ex16 -s 10 -dt 1.0e-4 -tf 4.0e-2 -vs 40
-//               ex16 -s 13 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               ex16 -m ../../data/fichera-q2.mesh
 //               ex16 -m ../../data/escher.mesh
 //               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -43,102 +37,75 @@
 using namespace std;
 using namespace mfem;
 
-/** After spatial discretization, the conduction model is expressed as
+/** After spatial discretization, the conduction model can be written as:
  *
- *   M du/dt = - K(u) u
+ *     du/dt = M^{-1}(-Ku)
  *
  *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K(u) is the diffusion operator with diffusivity depending on u:
+ *  and K is the diffusion operator with diffusivity depending on u:
  *  (\kappa + \alpha u).
  *
- *  Class ConductionOperatorOperator represents the above ODE operator in the
- *  general form F(u, k, t) = G(u, t) where
- *
- *    1. F(u, du/dt, t) = du/dt            (ODE is expressed in EXPLICIT form)
- *       G(u, t) = - inv(M) K(u) u
- *    2. F(u, du/dt, t) = M du/dt          (ODE is expressed in IMPLICIT form)
- *       G(u, t) = - K(u) u
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
  */
 class ConductionOperator : public TimeDependentOperator
 {
+protected:
    FiniteElementSpace &fespace;
    Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
 
-   BilinearForm M;
-   SparseMatrix Mmat;
+   BilinearForm *M;
+   BilinearForm *K;
 
-   const real_t alpha, kappa;
-   std::unique_ptr<BilinearForm> K;
-   SparseMatrix Kmat;
-
-   std::unique_ptr<SparseMatrix> T; // T = M + gam K(u)
+   SparseMatrix Mmat, Kmat;
+   SparseMatrix *T; // T = M + dt K
 
    CGSolver M_solver; // Krylov solver for inverting the mass matrix M
    DSmoother M_prec;  // Preconditioner for the mass matrix M
 
-   CGSolver T_solver; // Implicit solver for T = M + gam K(u)
+   CGSolver T_solver; // Implicit solver for T = M + dt K
    DSmoother T_prec;  // Preconditioner for the implicit solver
 
+   double alpha, kappa;
+
    mutable Vector z; // auxiliary vector
 
 public:
+   ConductionOperator(FiniteElementSpace &f, double alpha, double kappa,
+                      const Vector &u);
 
-   ConductionOperator(FiniteElementSpace &f, const real_t alpha,
-                      const real_t kappa, const Vector &u,
-                      const Type &ode_expression_type);
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
 
-   // Compute K(u_n) for use as an approximation in - K(u) u
-   void SetConductionTensor(const Vector &u);
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
 
-   /** Compute G(u, t) as defined in the IMPLICIT expression form of the ODE
-    operator, i.e., @a v = - K(u_n) @a u. Note that K(u_n) is an
-      approximation to K(u). */
-   void ExplicitMult(const Vector &u, Vector &v) const override;
+   /// Custom Jacobian system solver for the SUNDIALS time integrators.
+   /** For the ODE system represented by ConductionOperator
 
-   /** Solve for k in F(u, k, t) = G(u, t) for either EXPLICIT or IMPLICIT
-       expression forms of the ODE operator, i.e., @a k = - inv(M) K(u_n) @a u.
-       Note that K(u_n) is an approximation to K(u). */
-   void Mult(const Vector &u, Vector &k) const override;
+           M du/dt = -K(u),
 
-   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t) for either EXPLICIT
-       or IMPLICIT expression forms of the ODE operator, i.e.,
-       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u . Note that K(u_n) is an
-       approximation to K(u). */
-   void ImplicitSolve(const real_t gam, const Vector &u, Vector &k) override;
+       this class facilitates the solution of linear systems of the form
 
-   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F for
-       either EXPLICIT or IMPLICIT expression forms of the ODE operator, i.e.,
-       [M - @a gam Jf(u)] dk = G - F, where Jf(u) is an approximation of the
-       Jacobian of -K(u) u. The approximation chosen here is Jf(u) = -K(u_n). */
-   int SUNImplicitSetup(const Vector &u, const Vector &fu, int jok, int *jcur,
-                        real_t gam) override;
+           (M + γK) y = M b,
 
-   /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing either
-        1. @a r = G - F = inv(M) f(u) - k       (EXPLICIT expression form)
-        1. @a r = G - F = f(u) - M k            (IMPLICIT expression form)
-       */
-   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
+       for given b, u (not used), and γ = GetTimeStep(). */
 
-   int SUNMassSetup() override;
+   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+                                int jok, int *jcur, double gamma);
 
-   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override;
+   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
 
-   int SUNMassMult(const Vector &x, Vector &v) override;
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
+
+   virtual ~ConductionOperator();
 };
 
-real_t InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 2.0;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
+double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
@@ -150,16 +117,16 @@ int main(int argc, char *argv[])
    int ref_levels = 2;
    int order = 2;
    int ode_solver_type = 9; // CVODE implicit BDF
-   real_t t_final = 0.5;
-   real_t dt = 1.0e-2;
-   real_t alpha = 1.0e-2;
-   real_t kappa = 0.5;
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
 
    // Relative and absolute tolerances for CVODE and ARKODE.
-   const real_t reltol = 1e-4, abstol = 1e-4;
+   const double reltol = 1e-4, abstol = 1e-4;
 
    int precision = 8;
    cout.precision(precision);
@@ -184,10 +151,7 @@ int main(int argc, char *argv[])
                   "9  - CVODE (implicit BDF),\n\t"
                   "10 - ARKODE (default explicit),\n\t"
                   "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default implicit),\n\t"
-                  "13 - ARKODE (default explicit with MFEM mass solve),\n\t"
-                  "14 - ARKODE (explicit Fehlberg-6-4-5 with MFEM mass solve),\n\t"
-                  "15 - ARKODE (default implicit with MFEM mass solve).");
+                  "12 - ARKODE (default impicit).");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -210,13 +174,16 @@ int main(int argc, char *argv[])
       args.PrintUsage(cout);
       return 1;
    }
+   if (ode_solver_type < 1 || ode_solver_type > 12)
+   {
+      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      return 3;
+   }
    args.PrintOptions(cout);
 
-   bool use_mass_solver = ode_solver_type >= 13;
-
    // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
-   std::unique_ptr<Mesh> mesh(new Mesh(mesh_file, 1, 1));
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
    // 3. Refine the mesh to increase the resolution. In this example we do
@@ -230,7 +197,7 @@ int main(int argc, char *argv[])
    // 4. Define the vector finite element space representing the current and the
    //    initial temperature, u_ref.
    H1_FECollection fe_coll(order, dim);
-   FiniteElementSpace fespace(mesh.get(), &fe_coll);
+   FiniteElementSpace fespace(mesh, &fe_coll);
 
    int fe_size = fespace.GetTrueVSize();
    cout << "Number of temperature unknowns: " << fe_size << endl;
@@ -244,17 +211,8 @@ int main(int argc, char *argv[])
    Vector u;
    u_gf.GetTrueDofs(u);
 
-   // 6. Initialize the conduction ODE operator and the visualization.
-   ConductionOperator::Type ode_expression_type;
-   if (use_mass_solver)
-   {
-      ode_expression_type = ConductionOperator::Type::IMPLICIT;
-   }
-   else
-   {
-      ode_expression_type = ConductionOperator::Type::EXPLICIT;
-   }
-   ConductionOperator oper(fespace, alpha, kappa, u, ode_expression_type);
+   // 6. Initialize the conduction operator and the visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -266,7 +224,7 @@ int main(int argc, char *argv[])
       u_gf.Save(osol);
    }
 
-   VisItDataCollection visit_dc("Example16", mesh.get());
+   VisItDataCollection visit_dc("Example16", mesh);
    visit_dc.RegisterField("temperature", &u_gf);
    if (visit)
    {
@@ -300,75 +258,52 @@ int main(int argc, char *argv[])
    }
 
    // 7. Define the ODE solver used for time integration.
-   real_t t = 0.0;
-   std::unique_ptr<ODESolver> ode_solver;
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKStepSolver *arkode = NULL;
    switch (ode_solver_type)
    {
       // MFEM explicit methods
-      case 1: ode_solver = std::make_unique<ForwardEulerSolver>(); break;
-      case 2: ode_solver = std::make_unique<RK2Solver>(0.5); break; // midpoint method
-      case 3: ode_solver = std::make_unique<RK3SSPSolver>(); break;
-      case 4: ode_solver = std::make_unique<RK4Solver>(); break;
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
       // MFEM implicit L-stable methods
-      case 5: ode_solver = std::make_unique<BackwardEulerSolver>(); break;
-      case 6: ode_solver = std::make_unique<SDIRK23Solver>(2); break;
-      case 7: ode_solver = std::make_unique<SDIRK33Solver>(); break;
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
       // CVODE
       case 8:
-      case 9:
-      {
-         int cvode_solver_type;
-         if (ode_solver_type == 8)
-         {
-            cvode_solver_type = CV_ADAMS;
-         }
-         else
-         {
-            cvode_solver_type = CV_BDF;
-         }
-         std::unique_ptr<CVODESolver> cvode(new CVODESolver(cvode_solver_type));
+         cvode = new CVODESolver(CV_ADAMS);
          cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          cvode->SetMaxStep(dt);
-         ode_solver = std::move(cvode);
-         break;
-      }
+         ode_solver = cvode; break;
+      case 9:
+         cvode = new CVODESolver(CV_BDF);
+         cvode->Init(oper);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
       // ARKODE
       case 10:
       case 11:
-      case 12:
-      case 13:
-      case 14:
-      case 15:
-      {
-         ARKStepSolver::Type arkode_solver_type;
-         if (ode_solver_type == 12 || ode_solver_type == 15)
-         {
-            arkode_solver_type = ARKStepSolver::IMPLICIT;
-         }
-         else
-         {
-            arkode_solver_type = ARKStepSolver::EXPLICIT;
-         }
-         std::unique_ptr<ARKStepSolver> arkode(
-            new ARKStepSolver(arkode_solver_type));
+         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11 || ode_solver_type == 14)
+         if (ode_solver_type == 11)
          {
             arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
          }
-         if (use_mass_solver)
-         {
-            arkode->UseMFEMMassLinearSolver(SUNFALSE);
-         }
-         ode_solver = std::move(arkode);
-         break;
-      }
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
+         arkode->Init(oper);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
    }
 
    // Initialize MFEM integrators, SUNDIALS integrators are initialized above
@@ -376,14 +311,8 @@ int main(int argc, char *argv[])
 
    // Since we want to update the diffusion coefficient after every time step,
    // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-   {
-      cvode->SetStepMode(CV_ONE_STEP);
-   }
-   else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-   {
-      arkode->SetStepMode(ARK_ONE_STEP);
-   }
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
 
    // 8. Perform time-integration (looping over the time iterations, ti, with a
    //    time-step dt).
@@ -394,7 +323,7 @@ int main(int argc, char *argv[])
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
    {
-      real_t dt_real = min(dt, t_final - t);
+      double dt_real = min(dt, t_final - t);
 
       // Note that since we are using the "one-step" mode of the SUNDIALS
       // solvers, they will, generally, step over the final time and will not
@@ -408,14 +337,8 @@ int main(int argc, char *argv[])
       if (last_step || (ti % vis_steps) == 0)
       {
          cout << "step " << ti << ", t = " << t << endl;
-         if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-         {
-            cvode->PrintInfo();
-         }
-         else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-         {
-            arkode->PrintInfo();
-         }
+         if (cvode) { cvode->PrintInfo(); }
+         if (arkode) { arkode->PrintInfo(); }
 
          u_gf.SetFromTrueDofs(u);
          if (visualization)
@@ -430,153 +353,137 @@ int main(int argc, char *argv[])
             visit_dc.Save();
          }
       }
-      oper.SetConductionTensor(u);
+      oper.SetParameters(u);
    }
    tic_toc.Stop();
    cout << "Done, " << tic_toc.RealTime() << "s." << endl;
 
    // 9. Save the final solution. This output can be viewed later using GLVis:
    //    "glvis -m ex16.mesh -g ex16-final.gf".
-   u_gf.Save("ex16-final.gf", precision);
+   {
+      ofstream osol("ex16-final.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete mesh;
 
    return 0;
 }
 
-ConductionOperator::ConductionOperator(FiniteElementSpace &fes,
-                                       const real_t alpha, const real_t kappa,
-                                       const Vector &u,
-                                       const Type &ode_expression_type)
-   : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), alpha(alpha), kappa(kappa), M(&fespace), z(height)
+ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
+     T(NULL), z(height)
 {
-   // specify a relative tolerance for all solves with MFEM integrators
-   const real_t rel_tol = 1e-8;
+   const double rel_tol = 1e-8;
 
-   M.AddDomainIntegrator(new MassIntegrator());
-   M.Assemble();
-   M.FormSystemMatrix(ess_tdof_list, Mmat);
+   M = new BilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble();
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
 
    M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   M_solver.SetRelTol(rel_tol);
    M_solver.SetAbsTol(0.0);
    M_solver.SetMaxIter(50);
    M_solver.SetPrintLevel(0);
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(Mmat);
 
+   alpha = al;
+   kappa = kap;
+
    T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   T_solver.SetRelTol(rel_tol);
    T_solver.SetAbsTol(0.0);
    T_solver.SetMaxIter(100);
    T_solver.SetPrintLevel(0);
    T_solver.SetPreconditioner(T_prec);
 
-   SetConductionTensor(u);
+   SetParameters(u);
 }
 
-void ConductionOperator::SetConductionTensor(const Vector &u)
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Kmat.Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, dt, Kmat);
+   T_solver.SetOperator(*T);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
 {
-   // Compute K(u_n).
    GridFunction u_alpha_gf(&fespace);
    u_alpha_gf.SetFromTrueDofs(u);
    for (int i = 0; i < u_alpha_gf.Size(); i++)
    {
       u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
    }
+
+   delete K;
+   K = new BilinearForm(&fespace);
+
    GridFunctionCoefficient u_coeff(&u_alpha_gf);
 
-   K = std::make_unique<BilinearForm>(&fespace);
    K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
    K->Assemble();
    K->FormSystemMatrix(ess_tdof_list, Kmat);
 }
 
-void ConductionOperator::ExplicitMult(const Vector &u, Vector &v) const
+int ConductionOperator::SUNImplicitSetup(const Vector &x,
+                                         const Vector &fx, int jok, int *jcur,
+                                         double gamma)
 {
-   // Compute - K(u_n) u.
-   Kmat.Mult(u, v);
-   v.Neg();
-}
-
-void ConductionOperator::Mult(const Vector &u, Vector &k) const
-{
-   // Compute - inv(M) K(u_n) u.
-   ExplicitMult(u, z);
-   M_solver.Mult(z, k);
-}
-
-void ConductionOperator::ImplicitSolve(const real_t gam, const Vector &u,
-                                       Vector &k)
-{
-   // Solve for k in M k = - K(u_n) [u + gam*k].
-   ExplicitMult(u, z);
-   T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
+   // Setup the ODE Jacobian T = M + gamma K.
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, gamma, Kmat);
    T_solver.SetOperator(*T);
-   T_solver.Mult(z, k);
+   *jcur = 1;
+   return (0);
 }
 
-int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
-                                         int jok, int *jcur, real_t gam)
+int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
 {
-   // Compute T = M + gamma K(u_n).
-   T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
-   T_solver.SetOperator(*T);
-   *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUNLS_SUCCESS;
+   // Solve the system A x = z => (M - gamma K) x = M b.
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, x);
+   return (0);
 }
 
-int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
-                                         real_t tol)
+ConductionOperator::~ConductionOperator()
 {
-   // Solve the system [M + gamma K(u_n)] dk = - K(u_n) u - M k.
-   // What value r is providing depends on the ODE expression form:
-   //   EXPLICIT form: r = -inv(M) K(u_n) u - k
-   //   IMPLICIT form: r = -K(u_n) u - M k
-   T_solver.SetRelTol(tol);
-   if (isExplicit())
+   delete T;
+   delete M;
+   delete K;
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5)
    {
-      Mmat.Mult(r, z);
-      T_solver.Mult(z, dk);
+      return 2.0;
    }
    else
    {
-      T_solver.Mult(r, dk);
-   }
-   if (T_solver.GetConverged())
-   {
-      return SUNLS_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
+      return 1.0;
    }
 }
-
-int ConductionOperator::SUNMassSetup()
-{
-   // Do nothing b/c mass solver was setup in constructor.
-   return SUNLS_SUCCESS;
-}
-
-int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
-{
-   // Solve the system M x = b.
-   M_solver.SetRelTol(tol);
-   M_solver.Mult(b, x);
-   if (M_solver.GetConverged())
-   {
-      return SUNLS_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
-   }
-}
-
-int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
-{
-   // Compute M x.
-   Mmat.Mult(x, v);
-   return SUNLS_SUCCESS;
-}
-

examples/sundials/ex16p.cpp

@@ -1,22 +1,16 @@
 //                         MFEM Example 16 - Parallel Version
 //                              SUNDIALS Modification
 //
-// Compile with:
-//    make ex16p            (GNU make)
-//    make sundials_ex16p   (CMake)
+// Compile with: make ex16p
 //
 // Sample runs:
 //     mpirun -np 4 ex16p
 //     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
 //     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
 //     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
-//     mpirun -np 4 ex16p -s 15 -a 0.0 -k 1.0
 //     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
-//     mpirun -np 4 ex16p -s 11 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
 //     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
-//     mpirun -np 8 ex16p -s 12 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
 //     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
-//     mpirun -np 4 ex16p -s 13 -dt 2.0e-4 -tf 4.0e-2
 //     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
 //     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
 //     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -44,102 +38,66 @@
 using namespace std;
 using namespace mfem;
 
-/** After spatial discretization, the conduction model is expressed as
+/** After spatial discretization, the conduction model can be written as:
  *
- *   M du/dt = - K(u) u
+ *     du/dt = M^{-1}(-Ku)
  *
  *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K(u) is the diffusion operator with diffusivity depending on u:
+ *  and K is the diffusion operator with diffusivity depending on u:
  *  (\kappa + \alpha u).
  *
- *  Class ConductionOperatorOperator represents the above ODE operator in the
- *  general form F(u, k, t) = G(u, t) where either
- *
- *    1. F(u, du/dt, t) = du/dt            (ODE is expressed in EXPLICIT form)
- *       G(u, t) = - inv(M) K(u) u
- *    2. F(u, du/dt, t) = M du/dt          (ODE is expressed in IMPLICIT form)
- *       G(u, t) = - K(u) u
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
  */
 class ConductionOperator : public TimeDependentOperator
 {
+protected:
    ParFiniteElementSpace &fespace;
    Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
 
-   ParBilinearForm M;
+   ParBilinearForm *M;
+   ParBilinearForm *K;
+
    HypreParMatrix Mmat;
-
-   const real_t alpha, kappa;
-   std::unique_ptr<BilinearForm> K;
    HypreParMatrix Kmat;
+   HypreParMatrix *T; // T = M + dt K
+   double current_dt;
 
-   std::unique_ptr<HypreParMatrix> T; // T = M + gam K(u)
+   CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
+   HypreSmoother M_prec; // Preconditioner for the mass matrix M
 
-   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
-   HypreSmoother M_prec;  // Preconditioner for the mass matrix M
+   CGSolver T_solver;    // Implicit solver for T = M + dt K
+   HypreSmoother T_prec; // Preconditioner for the implicit solver
 
-   CGSolver T_solver; // Implicit solver for T = M + gam K(u)
-   HypreSmoother T_prec;  // Preconditioner for the implicit solver
+   double alpha, kappa;
 
    mutable Vector z; // auxiliary vector
 
 public:
+   ConductionOperator(ParFiniteElementSpace &f, double alpha, double kappa,
+                      const Vector &u);
 
-   ConductionOperator(ParFiniteElementSpace &f, const real_t alpha,
-                      const real_t kappa, const Vector &u,
-                      const Type &ode_expression_type);
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
 
-   // Compute K(u_n) for use as an approximation in - K(u) u
-   void SetConductionTensor(const Vector &u);
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
 
-   /** Compute G(u, t) as defined in the IMPLICIT expression form of the ODE
-       operator, i.e., @a v = - K(u_n) @a u. Note that K(u_n) is an
-       approximation to K(u). */
-   void ExplicitMult(const Vector &u, Vector &v) const override;
+   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+                                int jok, int *jcur, double gamma);
 
-   /** Solve for k in F(u, k, t) = G(u, t) for either EXPLICIT or IMPLICIT
-       expression forms of the ODE operator, i.e., @a k = - inv(M) K(u_n) @a u.
-       Note that K(u_n) is an approximation to K(u). */
-   void Mult(const Vector &u, Vector &k) const override;
+   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
 
-   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t) for either EXPLICIT
-       or IMPLICIT expression forms of the ODE operator, i.e.,
-       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u . Note that K(u_n) is an
-       approximation to K(u). */
-   void ImplicitSolve(const real_t gam, const Vector &u, Vector &k) override;
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
 
-   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F for
-       either EXPLICIT or IMPLICIT expression forms of the ODE operator, i.e.,
-       [M - @a gam Jf(u)] dk = G - F, where Jf(u) is an approximation of the
-       Jacobian of -K(u) u. The approximation chosen here is Jf(u) = -K(u_n). */
-   int SUNImplicitSetup(const Vector &u, const Vector &fu, int jok, int *jcur,
-                        real_t gam) override;
-
-   /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing either
-        1. @a r = G - F = inv(M) f(u) - k       (EXPLICIT expression form)
-        1. @a r = G - F = f(u) - M k            (IMPLICIT expression form)
-       */
-   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
-
-   int SUNMassSetup() override;
-
-   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override;
-
-   int SUNMassMult(const Vector &x, Vector &v) override;
+   virtual ~ConductionOperator();
 };
 
-real_t InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 2.0;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
+double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
@@ -156,16 +114,16 @@ int main(int argc, char *argv[])
    int par_ref_levels = 1;
    int order = 2;
    int ode_solver_type = 9; // CVODE implicit BDF
-   real_t t_final = 0.5;
-   real_t dt = 1.0e-2;
-   real_t alpha = 1.0e-2;
-   real_t kappa = 0.5;
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
 
    // Relative and absolute tolerances for CVODE and ARKODE.
-   const real_t reltol = 1e-4, abstol = 1e-4;
+   const double reltol = 1e-4, abstol = 1e-4;
 
    int precision = 8;
    cout.precision(precision);
@@ -192,10 +150,7 @@ int main(int argc, char *argv[])
                   "9  - CVODE (implicit BDF),\n\t"
                   "10 - ARKODE (default explicit),\n\t"
                   "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default implicit),\n\t"
-                  "13 - ARKODE (default explicit with MFEM mass solve),\n\t"
-                  "14 - ARKODE (explicit Fehlberg-6-4-5 with MFEM mass solve),\n\t"
-                  "15 - ARKODE (default implicit with MFEM mass solve).");
+                  "12 - ARKODE (default impicit).");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -219,33 +174,40 @@ int main(int argc, char *argv[])
       return 1;
    }
 
-   if (Mpi::Root())
+   if (myid == 0)
    {
       args.PrintOptions(cout);
    }
 
-   bool use_mass_solver = ode_solver_type >= 13;
+   // check for valid ODE solver option
+   if (ode_solver_type < 1 || ode_solver_type > 12)
+   {
+      if (myid == 0)
+      {
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      }
+      return 1;
+   }
 
-   // 3. Define a parallel mesh by a partitioning of a serial mesh. Read the
-   //    serial mesh from the given mesh file on all processors. We can
+   // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code.
-   std::unique_ptr<ParMesh> pmesh;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the mesh in serial to increase the resolution. In this example
+   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
+   //    a command-line parameter.
+   for (int lev = 0; lev < ser_ref_levels; lev++)
    {
-      std::unique_ptr<Mesh> mesh(new Mesh(mesh_file, 1, 1));
-
-      // 4. Refine the mesh in serial to increase the resolution. In this example
-      //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-      //    a command-line parameter.
-      for (int lev = 0; lev < ser_ref_levels; lev++)
-      {
-         mesh->UniformRefinement();
-      }
-
-      // 5. Refine this mesh further in parallel to increase the resolution.
-      //    Once the parallel mesh is defined, the serial mesh can be deleted.
-      pmesh = std::make_unique<ParMesh>(MPI_COMM_WORLD, *mesh);
+      mesh->UniformRefinement();
    }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
    for (int lev = 0; lev < par_ref_levels; lev++)
    {
       pmesh->UniformRefinement();
@@ -253,9 +215,8 @@ int main(int argc, char *argv[])
 
    // 6. Define the vector finite element space representing the current and the
    //    initial temperature, u_ref.
-   int dim = pmesh->Dimension();
    H1_FECollection fe_coll(order, dim);
-   ParFiniteElementSpace fespace(pmesh.get(), &fe_coll);
+   ParFiniteElementSpace fespace(pmesh, &fe_coll);
 
    int fe_size = fespace.GlobalTrueVSize();
    if (myid == 0)
@@ -272,17 +233,8 @@ int main(int argc, char *argv[])
    Vector u;
    u_gf.GetTrueDofs(u);
 
-   // 8. Initialize the conduction ODE operator and the visualization.
-   ConductionOperator::Type ode_expression_type;
-   if (use_mass_solver)
-   {
-      ode_expression_type = ConductionOperator::Type::IMPLICIT;
-   }
-   else
-   {
-      ode_expression_type = ConductionOperator::Type::EXPLICIT;
-   }
-   ConductionOperator oper(fespace, alpha, kappa, u, ode_expression_type);
+   // 8. Initialize the conduction operator and the VisIt visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -297,7 +249,7 @@ int main(int argc, char *argv[])
       u_gf.Save(osol);
    }
 
-   VisItDataCollection visit_dc("Example16-Parallel", pmesh.get());
+   VisItDataCollection visit_dc("Example16-Parallel", pmesh);
    visit_dc.RegisterField("temperature", &u_gf);
    if (visit)
    {
@@ -341,76 +293,52 @@ int main(int argc, char *argv[])
    }
 
    // 9. Define the ODE solver used for time integration.
-   real_t t = 0.0;
-   std::unique_ptr<ODESolver> ode_solver;
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKStepSolver *arkode = NULL;
    switch (ode_solver_type)
    {
       // MFEM explicit methods
-      case 1: ode_solver = std::make_unique<ForwardEulerSolver>(); break;
-      case 2: ode_solver = std::make_unique<RK2Solver>(0.5); break; // midpoint method
-      case 3: ode_solver = std::make_unique<RK3SSPSolver>(); break;
-      case 4: ode_solver = std::make_unique<RK4Solver>(); break;
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
       // MFEM implicit L-stable methods
-      case 5: ode_solver = std::make_unique<BackwardEulerSolver>(); break;
-      case 6: ode_solver = std::make_unique<SDIRK23Solver>(2); break;
-      case 7: ode_solver = std::make_unique<SDIRK33Solver>(); break;
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
       // CVODE
       case 8:
-      case 9:
-      {
-         int cvode_solver_type;
-         if (ode_solver_type == 8)
-         {
-            cvode_solver_type = CV_ADAMS;
-         }
-         else
-         {
-            cvode_solver_type = CV_BDF;
-         }
-         std::unique_ptr<CVODESolver> cvode(
-            new CVODESolver(MPI_COMM_WORLD, cvode_solver_type));
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
          cvode->Init(oper);
          cvode->SetSStolerances(reltol, abstol);
          cvode->SetMaxStep(dt);
-         ode_solver = std::move(cvode);
-         break;
-      }
+         ode_solver = cvode; break;
+      case 9:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF);
+         cvode->Init(oper);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
       // ARKODE
       case 10:
       case 11:
-      case 12:
-      case 13:
-      case 14:
-      case 15:
-      {
-         ARKStepSolver::Type arkode_solver_type;
-         if (ode_solver_type == 12 || ode_solver_type == 15)
-         {
-            arkode_solver_type = ARKStepSolver::IMPLICIT;
-         }
-         else
-         {
-            arkode_solver_type = ARKStepSolver::EXPLICIT;
-         }
-         std::unique_ptr<ARKStepSolver> arkode(
-            new ARKStepSolver(MPI_COMM_WORLD, arkode_solver_type));
+         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11 || ode_solver_type == 14)
+         if (ode_solver_type == 11)
          {
             arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
          }
-         if (use_mass_solver)
-         {
-            arkode->UseMFEMMassLinearSolver(SUNFALSE);
-         }
-         ode_solver = std::move(arkode);
-         break;
-      }
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
+         arkode->Init(oper);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
    }
 
    // Initialize MFEM integrators, SUNDIALS integrators are initialized above
@@ -418,18 +346,12 @@ int main(int argc, char *argv[])
 
    // Since we want to update the diffusion coefficient after every time step,
    // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-   {
-      cvode->SetStepMode(CV_ONE_STEP);
-   }
-   else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-   {
-      arkode->SetStepMode(ARK_ONE_STEP);
-   }
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
 
    // 10. Perform time-integration (looping over the time iterations, ti, with a
    //     time-step dt).
-   if (Mpi::Root())
+   if (myid == 0)
    {
       cout << "Integrating the ODE ..." << endl;
    }
@@ -439,7 +361,7 @@ int main(int argc, char *argv[])
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
    {
-      real_t dt_real = min(dt, t_final - t);
+      double dt_real = min(dt, t_final - t);
 
       // Note that since we are using the "one-step" mode of the SUNDIALS
       // solvers, they will, generally, step over the final time and will not
@@ -455,14 +377,8 @@ int main(int argc, char *argv[])
          if (myid == 0)
          {
             cout << "step " << ti << ", t = " << t << endl;
-            if (CVODESolver* cvode = dynamic_cast<CVODESolver*>(ode_solver.get()))
-            {
-               cvode->PrintInfo();
-            }
-            else if (ARKStepSolver* arkode = dynamic_cast<ARKStepSolver*>(ode_solver.get()))
-            {
-               arkode->PrintInfo();
-            }
+            if (cvode) { cvode->PrintInfo(); }
+            if (arkode) { arkode->PrintInfo(); }
          }
 
          u_gf.SetFromTrueDofs(u);
@@ -479,38 +395,46 @@ int main(int argc, char *argv[])
             visit_dc.Save();
          }
       }
-      oper.SetConductionTensor(u);
+      oper.SetParameters(u);
    }
    tic_toc.Stop();
-   if (Mpi::Root())
+   if (myid == 0)
    {
       cout << "Done, " << tic_toc.RealTime() << "s." << endl;
    }
 
    // 11. Save the final solution in parallel. This output can be viewed later
    //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
-   u_gf.Save("ex16-final", precision);
+   {
+      ostringstream sol_name;
+      sol_name << "ex16-final." << setfill('0') << setw(6) << myid;
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   // 12. Free the used memory.
+   delete ode_solver;
+   delete pmesh;
 
    return 0;
 }
 
-ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
-                                       const real_t alpha, const real_t kappa,
-                                       const Vector &u,
-                                       const Type &ode_expression_type)
-   : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), alpha(alpha), kappa(kappa), M(&fespace),
-     M_solver(fes.GetComm()), T_solver(fes.GetComm()), z(height)
+ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
+     T(NULL),
+     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
 {
-   // specify a relative tolerance for all solves with MFEM integrators
-   const real_t rel_tol = 1e-8;
+   const double rel_tol = 1e-8;
 
-   M.AddDomainIntegrator(new MassIntegrator());
-   M.Assemble(0); // keep zeros to keep sparsity pattern of M and K the same
-   M.FormSystemMatrix(ess_tdof_list, Mmat);
+   M = new ParBilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble(0); // keep sparsity pattern of M and K the same
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
 
    M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   M_solver.SetRelTol(rel_tol);
    M_solver.SetAbsTol(0.0);
    M_solver.SetMaxIter(100);
    M_solver.SetPrintLevel(0);
@@ -518,118 +442,97 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(Mmat);
 
+   alpha = al;
+   kappa = kap;
+
    T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol); // will be overwritten with SUNDIALS integrators
+   T_solver.SetRelTol(rel_tol);
    T_solver.SetAbsTol(0.0);
    T_solver.SetMaxIter(100);
    T_solver.SetPrintLevel(0);
    T_solver.SetPreconditioner(T_prec);
 
-   SetConductionTensor(u);
+   SetParameters(u);
 }
 
-void ConductionOperator::SetConductionTensor(const Vector &u)
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Kmat.Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, dt, Kmat);
+   T_solver.SetOperator(*T);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
+}
+
+int ConductionOperator::SUNImplicitSetup(const Vector &x,
+                                         const Vector &fx, int jok, int *jcur,
+                                         double gamma)
+{
+   // Setup the ODE Jacobian T = M + gamma K.
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, gamma, Kmat);
+   T_solver.SetOperator(*T);
+   *jcur = 1;
+   return (0);
+}
+
+int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
+{
+   // Solve the system A x = z => (M - gamma K) x = M b.
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, x);
+   return (0);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
 {
-   // Compute K(u_n).
    ParGridFunction u_alpha_gf(&fespace);
    u_alpha_gf.SetFromTrueDofs(u);
    for (int i = 0; i < u_alpha_gf.Size(); i++)
    {
       u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
    }
+
+   delete K;
+   K = new ParBilinearForm(&fespace);
+
    GridFunctionCoefficient u_coeff(&u_alpha_gf);
 
-   K = std::make_unique<ParBilinearForm>(&fespace);
    K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
-   K->Assemble(0); // keep zeros to keep sparsity pattern of M and K the same
+   K->Assemble(0); // keep sparsity pattern of M and K the same
    K->FormSystemMatrix(ess_tdof_list, Kmat);
 }
 
-void ConductionOperator::ExplicitMult(const Vector &u, Vector &v) const
+ConductionOperator::~ConductionOperator()
 {
-   // Compute - K(u_n) u.
-   Kmat.Mult(u, v);
-   v.Neg();
+   delete T;
+   delete M;
+   delete K;
 }
 
-void ConductionOperator::Mult(const Vector &u, Vector &k) const
+double InitialTemperature(const Vector &x)
 {
-   // Compute - inv(M) K(u_n) u.
-   ExplicitMult(u, z);
-   M_solver.Mult(z, k);
-}
-
-void ConductionOperator::ImplicitSolve(const real_t gam, const Vector &u,
-                                       Vector &k)
-{
-   // Solve for k in M k = - K(u_n) [u + gam*k].
-   ExplicitMult(u, z);
-   T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
-   T_solver.SetOperator(*T);
-   T_solver.Mult(z, k);
-}
-
-int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
-                                         int jok, int *jcur, real_t gam)
-{
-   // Compute T = M + gamma K(u_n).
-   T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
-   T_solver.SetOperator(*T);
-   *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUNLS_SUCCESS;
-}
-
-int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
-                                         real_t tol)
-{
-   // Solve the system [M + gamma K(u_n)] dk = - K(u_n) u - M k.
-   // What value r is providing depends on the ODE expression form:
-   //   EXPLICIT form: r = -inv(M) K(u_n) u - k
-   //   IMPLICIT form: r = -K(u_n) u - M k
-   T_solver.SetRelTol(tol);
-   if (isExplicit())
+   if (x.Norml2() < 0.5)
    {
-      Mmat.Mult(r, z);
-      T_solver.Mult(z, dk);
+      return 2.0;
    }
    else
    {
-      T_solver.Mult(r, dk);
-   }
-   if (T_solver.GetConverged())
-   {
-      return SUNLS_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
+      return 1.0;
    }
 }
-
-int ConductionOperator::SUNMassSetup()
-{
-   // Do nothing b/c mass solver was setup in constructor.
-   return SUNLS_SUCCESS;
-}
-
-int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
-{
-   // Solve the system M x = b.
-   M_solver.SetRelTol(tol);
-   M_solver.Mult(b, x);
-   if (M_solver.GetConverged())
-   {
-      return SUNLS_SUCCESS;
-   }
-   else
-   {
-      return SUNLS_CONV_FAIL;
-   }
-}
-
-int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
-{
-   // Compute M x.
-   Mmat.Mult(x, v);
-   return SUNLS_SUCCESS;
-}