Include "mpi-ext.h" only if using OpenMPI.

pointers in MPI calls, regardless of the result of the
CUDA-aware MPI check.

.appveyor.yml

@@ -1,13 +1,10 @@
 version: '{build}'
 
 # https://www.appveyor.com/docs/build-environment/#build-worker-images
-image: Visual Studio 2019
+image: Visual Studio 2017
 
 install:
 
-# Start from outside clone directory
-- cd ..
-
 # Install MS-MPI
 - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe'
 - MSMpiSetup.exe -unattend
@@ -18,51 +15,35 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%
 
-# Set MSMPI environment variables needed for CMake detection
-- set MSMPI_LIB32=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86
-- set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
-- set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
-
-# Install METIS, use MFEM's mirror because the original source server is often
-# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
-- ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
+# Install METIS
+- ps: Start-FileDownload 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
-- cmake --build build --target metis -j 4
-- set METIS_PATH=%cd%
+- cmake --build build
 - cd ..
 
 # Install hypre
-- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
-- 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
-- cd hypre-2.19.0/src
-- cmake -H. -Bbuild
-- cmake --build build -j 4
+- ps: Start-FileDownload 'https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz'
+- 7z x hypre-2.10.0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2.10.0b
+- cmake -Hsrc -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+# - cmake -Hsrc -Bbuild -DCMAKE_BUILD_TYPE=Release -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- cmake --build build
 - cmake --build build --target install
-- cd ../..
-
-# Return to clone directory
-- cd %APPVEYOR_BUILD_FOLDER%
+- cd ..
 
 # MFEM
 before_build:
-- cmake . -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
-# - cmake . -Bbuild_serial -DMFEM_USE_MPI=FALSE
-- cmake . -Bbuild_shared_serial -DMFEM_USE_MPI=FALSE -DBUILD_SHARED_LIBS=ON
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 
 build_script:
-- cmake --build build_parallel --config Debug -j 4
-# - cmake --build build_serial --config Release -j 4
-# - cmake --build build_serial --target exec --config Release -j 4
-- cmake --build build_shared_serial --config Release -j 4
-- cmake --build build_shared_serial --target exec --config Release -j 4
+- cmake --build build_parallel
+- cmake --build build_serial
 
 after_build:
-# - cd build_serial
-# - ctest -C Release --output-on-failure
-# - cd ..
-- cd build_shared_serial
-- ctest -C Release --output-on-failure
+# - cmake --build build_parallel --target check
+- cmake --build build_serial --target check

.binder/environment.yml

@@ -1,12 +0,0 @@
-# extends https://github.com/jupyterhub/repo2docker/blob/main/repo2docker/buildpacks/conda/environment.yml
-# see https://mybinder.readthedocs.io/en/latest/using/config_files.html#environment-yml-install-a-conda-environment
-channels:
-- conda-forge
-dependencies:
-- xeus-cling=0.13.0
-- xwidgets=0.26.0
-# NOTE: it is possible that these are not needed for the lab frontend
-- widgetsnbextension=3.5.1
-- pip
-- pip:
-  - glvis==0.3.2

.binder/postBuild

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# cling is installed here (in bin) and will look in {dir}/include and {dir}/lib
-# without extra intervention (jk it doesn't look in {dir}/lib unless something
-# has been #included from {dir}/include first...)
-install_dir=/srv/conda/envs/notebook
-mkdir -p $install_dir
-
-# build and install mfem, which is the directory we start in
-make serial SHARED=YES -j8
-make install PREFIX=$install_dir
-
-# install xeus-glvis
-git clone https://github.com/GLVis/xeus-glvis.git
-pushd xeus-glvis
-make install prefix=$install_dir
-popd
-
-# install jupyter-lab extension
-jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build
-jupyter labextension install glvis-jupyter
-
-# fixup kernelspec, we could probably do this from sh but ¯\_(ツ)_/¯
-python .binder/update_kernel_env.py

.binder/update_kernel_env.py

@@ -1,14 +0,0 @@
-# Update the LD_LIBRARY_PATH of the C++14 kernel so it can find mfem without
-# extra pragma cling statements
-
-import json
-
-kernelspec = "/srv/conda/envs/notebook/share/jupyter/kernels/xcpp14/kernel.json"
-
-with open(kernelspec, "r") as f:
-    obj = json.load(f)
-
-obj["env"] = {"LD_LIBRARY_PATH": "/srv/conda/envs/notebook/lib"}
-
-with open(kernelspec, "w") as f:
-    json.dump(obj, f)

.codecov.yml

@@ -1,32 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  precision: 2
-  round: nearest
-  range: "0...100"
-  status:
-    patch:
-      default:
-        target: auto
-        threshold: 0%
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        informational: true
-        only_pulls: true
-    project:
-      default:
-        target: auto   # compares coverage to the previous base commit
-        threshold: 1%  # allows variations around the target
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        only_pulls: true
-
-github_checks:
-  annotations: false
-
-comment: false

.github/actions/sanitize/config/action.yml

@@ -1,154 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: Sanitizer Config
-description: Sets up environment variables for MFEM sanitizer workflow
-
-inputs:
-  DEBUG:
-    description: If true, use intermediate caches to speed up the workflow
-                 by reusing previous builds.
-    default: false
-
-  REPOSITORY:
-    description: Repository to checkout
-    default: mfem/mfem
-
-  BRANCH:
-    description: Branch to checkout
-    default: ubsan
-
-  CLANG_VER:
-    description: CLANG version to use
-    default: 18
-
-  # https://github.com/llvm/llvm-project/releases
-  LLVM_VER:
-    description: LLVM version to use
-    default: 19.1.7
-
-  # https://github.com/hypre-space/hypre/releases
-  HYPRE_VER:
-    description: HYPRE version to use
-    default: 2.19.0
-
-  METIS_VER:
-    description: METIS version to use
-    default: 4.0.3
-
-  CTEST:
-    description: CTest command to use
-    default: ctest -j --test-load $(nproc)
-                      --schedule-random
-                      --stop-on-failure --output-on-failure
-                      --test-dir
-
-  # https://clang.llvm.org/docs/AddressSanitizer.html
-  ASAN_OPTIONS:
-    default: detect_leaks=1,
-             strict_init_order=1,
-             strict_string_checks=1,
-             check_initialization_order=1,
-             detect_stack_use_after_return=1
-  ASAN_CXXFLAGS:
-    default: -fsanitize=address
-             -fsanitize-address-use-after-scope
-  ASAN_LDFLAGS:
-    default: -fsanitize=address
-
-  # https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
-  UBSAN_OPTIONS:
-    default: halt_on_error=1, print_stacktrace=1
-  UBSAN_CXXFLAGS:
-    default: -fsanitize=undefined
-  UBSAN_LDFLAGS:
-    default: -fsanitize=undefined
-
-  # https://clang.llvm.org/docs/MemorySanitizer.html
-  MSAN_OPTIONS:
-    default: "poison_in_dtor=1"
-  MSAN_CXXFLAGS:
-    default: -fsanitize=memory
-             -fsanitize-memory-track-origins
-             -fsanitize-memory-use-after-dtor
-  MSAN_LDFLAGS:
-    default: -fsanitize=memory
-
-  LSAN_DIR:
-    description: LSAN suppression directory
-    default: lsan
-
-  LSAN_FILE:
-    description: LSAN suppression file
-    default: lsan.supp
-
-  NO_FLAGS:
-    description: If true, do not set any CXXFLAGS or LDFLAGS.
-    default: false
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Env (Inputs)
-      run: |
-        echo DEBUG=${{inputs.DEBUG}} >> $GITHUB_ENV
-        echo REPOSITORY=${{inputs.REPOSITORY}} >> $GITHUB_ENV
-        echo BRANCH=${{inputs.BRANCH}} >> $GITHUB_ENV
-        echo CLANG_VER=${{inputs.CLANG_VER}} >> $GITHUB_ENV
-        echo LLVM_VER=${{inputs.LLVM_VER}} >> $GITHUB_ENV
-        echo HYPRE_VER=${{inputs.HYPRE_VER}} >> $GITHUB_ENV
-        echo METIS_VER=${{inputs.METIS_VER}} >> $GITHUB_ENV
-        echo CTEST=${{inputs.CTEST}} >> $GITHUB_ENV
-        echo ASAN_OPTIONS=${{inputs.ASAN_OPTIONS}} >> $GITHUB_ENV
-        echo UBSAN_OPTIONS=${{inputs.UBSAN_OPTIONS}} >> $GITHUB_ENV
-        echo MSAN_OPTIONS=${{inputs.MSAN_OPTIONS}} >> $GITHUB_ENV
-        echo LSAN_DIR=${{inputs.LSAN_DIR}} >> $GITHUB_ENV
-        echo LSAN_FILE=${{inputs.LSAN_FILE}} >> $GITHUB_ENV
-        echo ASAN_CXXFLAGS=${{inputs.ASAN_CXXFLAGS}} >> $GITHUB_ENV
-        echo ASAN_LDFLAGS=${{inputs.ASAN_LDFLAGS}} >> $GITHUB_ENV
-        echo UBSAN_CXXFLAGS=${{inputs.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
-        echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
-        echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
-        echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (dir)
-      run: |
-        echo LLVM_DIR=${{github.workspace}}/llvm >> $GITHUB_ENV
-        echo HYPRE_DIR=hypre-${{inputs.HYPRE_VER}} >> $GITHUB_ENV
-        echo METIS_DIR=metis-${{inputs.METIS_VER}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (bis)
-      run: |
-        echo CC=clang-${{inputs.CLANG_VER}} >> $GITHUB_ENV
-        echo CXX=clang++-${{inputs.CLANG_VER}} >> $GITHUB_ENV
-        echo LLVM_INC=${{env.LLVM_DIR}}/include/c++/v1 >> $GITHUB_ENV
-        echo LLVM_LIB=${{env.LLVM_DIR}}/lib >> $GITHUB_ENV
-        echo HYPRE_TGZ=v${{inputs.HYPRE_VER}}.tar.gz >> $GITHUB_ENV
-        echo METIS_TGZ=metis-${{inputs.METIS_VER}}.tar.gz >> $GITHUB_ENV
-        LSAN_SUPPRESSIONS="${{github.workspace}}/${{inputs.LSAN_DIR}}/${{inputs.LSAN_FILE}}"
-        echo "LSAN_OPTIONS=suppressions=$LSAN_SUPPRESSIONS" >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (ter)
-      if: ${{ inputs.NO_FLAGS != 'true' }}
-      run: |
-        echo LLVM_CXXFLAGS=-stdlib=libc++ -I${{env.LLVM_INC}} -Isystem${{env.LLVM_INC}} >> $GITHUB_ENV
-        echo LLVM_LDFLAGS=-L${{env.LLVM_LIB}} -lc++abi -Wl,-rpath,${{env.LLVM_LIB}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (quater)
-      if: ${{ inputs.NO_FLAGS != 'true' }}
-      run: |
-        echo CXXFLAGS=${{env.LLVM_CXXFLAGS}} >> $GITHUB_ENV
-        echo LDFLAGS=${{env.LLVM_LDFLAGS}} >> $GITHUB_ENV
-      shell: bash

.github/actions/sanitize/mfem/action.yml

@@ -1,91 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: 'MFEM Compilation'
-description: 'MFEM Compilation'
-
-inputs:
-  par:
-    description: 'Whether to build for parallel (true/false)'
-    default: false
-  sanitizer:
-    description: 'Sanitizer to use (asan, msan, ubsan)'
-    default: asan
-
-runs:
-  using: 'composite'
-  steps:
-    - uses: ./.github/actions/sanitize/config
-
-    - uses: actions/cache@v4
-      if: ${{env.DEBUG == 'true'}}
-      id: debug
-      with:
-        path: mfem/build
-        key: build-${{inputs.par}}-${{inputs.sanitizer}}
-
-    - uses: ./.github/actions/sanitize/setup
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      with:
-        par: ${{inputs.par}}
-        sanitizer: ${{inputs.sanitizer}}
-
-    - name: Build with ASAN
-      if: inputs.sanitizer == 'asan'
-      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.ASAN_CXXFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Build with MSAN
-      if: inputs.sanitizer == 'msan'
-      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MSAN_CXXFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Build with UBSAN
-      if: inputs.sanitizer == 'ubsan'
-      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - uses: mfem/github-actions/build-mfem@v2.5
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      env:
-        CXXFLAGS: ${{env.CXXFLAGS}}
-        LDFLAGS: ${{env.LDFLAGS}}
-      with:
-        mpi: ${{inputs.par == 'false' && 'seq' || 'par'}}
-        mfem-dir: mfem
-        os: ${{runner.os}}
-        library-only: true
-        build-system: cmake
-        hypre-dir: ${{env.HYPRE_DIR}}
-        metis-dir: ${{env.METIS_DIR}}
-        config-options: >-
-          -GNinja
-          -DMPICXX=${{env.CXX}}
-          -DCMAKE_CXX_STANDARD=17
-          -DMFEM_USE_MEMALLOC=OFF
-          -DCMAKE_BUILD_TYPE=Release
-          -DCMAKE_VERBOSE_MAKEFILE=ON
-          -DCMAKE_CXX_COMPILER=${{env.CXX}}
-          -DCMAKE_CXX_FLAGS_RELEASE='-g -O1 -fno-omit-frame-pointer'
-
-    - name: Delete object files
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      working-directory: mfem/build
-      run: find . -type f -name '*.o' -delete
-      shell: bash
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: build-${{inputs.par}}-${{inputs.sanitizer}}
-        path: mfem/build
-        if-no-files-found: error
-        retention-days: 1
-        overwrite: false

.github/actions/sanitize/mpi/action.yml

@@ -1,33 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: 'Install MPI'
-description: 'Installs MPI and set up its environment variables'
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Install
-      run: sudo apt-get install openmpi-bin libopenmpi-dev
-      shell: bash
-
-    - name: Env
-      run: |
-        echo PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe >> $GITHUB_ENV
-        echo MPI_INC=$(mpicxx --showme:compile) >> $GITHUB_ENV
-        echo MPI_LIB=$(mpicxx --showme:link) >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (bis)
-      run: |
-        echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
-        echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV
-      shell: bash

.github/actions/sanitize/restore/action.yml

@@ -1,71 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: 'Restore state'
-description: 'Restore state to be able to run checks, tests'
-
-inputs:
-  par:
-    description: 'Whether to build for parallel (true/false)'
-    default: false
-  sanitizer:
-    description: 'Sanitizer to use (asan, msan, ubsan)'
-    default: asan
-  cache-path:
-    description: 'path to what needs to be restored'
-    default: none
-  cache-skip:
-    description: 'Skip cache restoration'
-    default: false
-
-outputs:
-  cache-hit:
-    description: 'Output from a specific step'
-    value: ${{steps.debug.outputs.cache-hit}}
-
-runs:
-  using: 'composite'
-  steps:
-    - uses: ./.github/actions/sanitize/config
-
-    - uses: actions/cache@v4
-      if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
-      id: debug
-      with:
-        path: ${{inputs.cache-path}}
-        key: ${{github.job}}-${{inputs.par}}-${{inputs.sanitizer}}
-
-    - uses: ./.github/actions/sanitize/setup
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      with:
-        par: ${{inputs.par}}
-        sanitizer: ${{inputs.sanitizer}}
-
-    - uses: actions/download-artifact@v4
-      with:
-        name: build-${{inputs.par}}-${{inputs.sanitizer}}
-        path: mfem/build
-
-    - name: Ninja Patch
-      working-directory: mfem/build
-      run: |
-        sed -i -e 's/CXX_STATIC_LIBRARY_LINKER__mfem_Release.*/CUSTOM_COMMAND/' build.ninja
-        sed -i -e '/build tests\/unit\/all:/ s/tests\/unit\/[^ ]*unit_tests[^ ]*//g' build.ninja
-        sed -i -e '/^add_test(\[=\[\(unit_tests\|punit_tests\)\]=\]/ s/)/ "--input-file .\/list-test-names-${{matrix.tag}}" "--min-duration 1")/' tests/unit/CTestTestfile.cmake
-      shell: bash
-
-    - name: Copy Data
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      working-directory: mfem/build
-      run: |
-        ninja cmake_object_order_depends_target_unit_tests
-        cp -pR ../tests/unit/data tests/unit
-      shell: bash

.github/actions/sanitize/setup/action.yml

@@ -1,64 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: 'Setup state'
-description: 'Sets up the state to be able to run build & run'
-
-inputs:
-  par:
-    description: 'Whether to build for parallel (true/false)'
-    default: false
-  sanitizer:
-    description: 'Sanitizer to use (asan, msan, ubsan)'
-    default: asan
-
-runs:
-  using: 'composite'
-  steps:
-    - uses: actions/cache/restore@v4  # Cache for LLVM libcxx
-      with:
-        path: ${{env.LLVM_DIR}}
-        fail-on-cache-miss: true
-        key: build-libcxx-${{env.LLVM_VER}}-${{inputs.sanitizer}}
-
-    - uses: ./.github/actions/sanitize/mpi
-      if: ${{inputs.par == 'true'}}
-
-    - uses: actions/cache/restore@v4  # Cache for Hypre
-      if: ${{inputs.par == 'true'}}
-      with:
-        path: ${{env.HYPRE_DIR}}
-        fail-on-cache-miss: true
-        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
-
-    - uses: actions/cache/restore@v4  # Cache for Metis
-      if: ${{inputs.par == 'true'}}
-      with:
-        path: ${{env.METIS_DIR}}
-        fail-on-cache-miss: true
-        key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
-
-    - name: Hypre/Metis links
-      if: ${{inputs.par == 'true'}}
-      run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
-      shell: bash
-
-    - uses: actions/cache/restore@v4  # Cache for LSAN suppression file
-      with:
-        path: ${{env.LSAN_DIR}}
-        fail-on-cache-miss: true
-        key: build-lsan-suppression-file
-
-    - uses: actions/checkout@v4  # Checkout the repository
-      with:
-        path: mfem
-        # ref: ${{env.BRANCH}}
-        # repository: ${{env.REPOSITORY}}

.github/workflows/README.md

@@ -1,67 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains the GitHub CI scripts for MFEM.
-
-Note that some of these scripts use the shared MFEM GitHub Actions from the external mfem/github-actions repository:
-
-  <https://github.com/mfem/github-actions>
-
-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
-
-The current CI workflows are:
-
-## `repo-check.yml`
-
-Runs a number of static repository-level sanity checks.
-
-- `file-headers-check` checks copyright date, license, etc. using the `--copyright`, `--license` and `--release` options of the `config/githooks/pre-push` script.
-
-- `code-style` checks the code style using the `--style` option of the `config/githooks/pre-push` script.
-
-- `documentation` checks the documentation build using the `tests/scripts/documentation` script.
-
-- `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
-
-## `mfem-analysis.yml` (`build-analysis`)
-
-Checks if the code builds and satisfies minimal requirements.
-
-- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
-
-## `builds-and-tests.yml`
-
-Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
-
-Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
-
-- `mfem/github-actions/build-hypre`
-- `mfem/github-actions/build-metis`
-- `mfem/github-actions/build-mfem`
-- `mfem/github-actions/upload-coverage`
-
-## Sanitizer Workflow for MFEM Verification
-
-This workflow validates MFEM unit tests, examples, and miniapps using sanitizer tools.
-
-- `sanitizers.yml` orchestrates:
-  - Building and caching dependencies: HYPRE, METIS, LSAN suppression file, and LLVM libcxx.
-  - Launching fine-grained jobs for serial (ASAN, MSAN, UBSAN) and parallel (ASAN, UBSAN) sanitizers.
-- `sanitize-tests.yml` is a reusable workflow accepting `par` mode (`true` for parallel) and `sanitizer` (ASAN, MSAN, or UBSAN) as inputs. It executes the following jobs:
-  - **Build**: Compiles the MFEM library with specified parallel and sanitizer settings.
-  - **Check**: Runs verification checks.
-  - Parallel jobs to test the following: **Examples**, **Miniapps** and **Unit tests**
-
-The workflow leverages composite actions in `.github/actions/sanitize/`:
-
-- `config`: Centralizes settings for the sanitizer workflow.
-- `mfem`: Manages the MFEM library build process.
-- `mpi`: Installs MPI and applies additional compilation flags.
-- `restore`: Restores the testing environment state.
-- `setup`: Builds or restores cached dependencies.

.github/workflows/build-container.yml

@@ -1,85 +0,0 @@
-name: "Docker"
-
-on:
-  # Always have a base image ready to go - this is a nightly build
-  schedule:
-    - cron: 0 3 * * *
-
-  # Allow manual trigger of a build
-  workflow_dispatch:
-
-  # On push to main we build and deploy images
-  push:
-    branches:
-      - master
-
-  # Publish packages on release
-  release:
-    types: [published]
-
-jobs:
-  build:
-    if: github.repository == 'mfem/mfem' # Don't run in forks
-    permissions:
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-        # Dockerfiles to build, a matrix supports future expanded builds
-        container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
-                    ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]
-
-    runs-on: ubuntu-latest
-    name: Build
-    steps:
-      - name: Run Actions Cleaner
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: 'true'
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          remove-codeql: 'true'
-          remove-docker-images: 'true'
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # It's easier to reference named variables than indexes of the matrix
-      - name: Set Environment
-        env:
-          dockerfile: ${{ matrix.container[0] }}
-          uri: ${{ matrix.container[1] }}
-        run: |
-          echo "dockerfile=$dockerfile" >> $GITHUB_ENV
-          echo "uri=$uri" >> $GITHUB_ENV
-
-      - name: Pull previous layers for cache
-        run: docker pull ${uri}:latest || echo "No container to pull"
-
-      - name: Build Container
-        run: |
-           container=$uri:latest
-           docker build -f ${dockerfile} -t ${container} .
-           echo "container=$container" >> $GITHUB_ENV
-
-      - name: GHCR Login
-        if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Deploy
-        if: (github.event_name != 'pull_request')
-        run: |
-            docker push ${container}
-
-      - name: Tag and Push Release
-        if: (github.event_name == 'release')
-        run: |
-            tag=${GITHUB_REF#refs/tags/}
-            echo "Tagging and releasing ${uri}:${tag}"
-            docker tag ${uri}:latest ${uri}:${tag}
-            docker push ${uri}:${tag}

.github/workflows/builds-and-tests.yml

@@ -1,381 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# In this CI section, we build different variants of mfem and run test on them.
-name: "Tests"
-
-# Github actions can use the default "GITHUB_TOKEN". By default, this token
-# is set to have permissive access. However, this is not a good practice
-# security-wise. Here we use an external action, so we restrict the
-# permission to the minimum required.
-# When the 'permissions' is set, all the scopes not mentioned are set to the
-# most restrictive setting. So the following is enough.
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
-  METIS_TOP_DIR: metis-4.0.3
-  MFEM_TOP_DIR: mfem
-
-# Note for future improvements:
-#
-# We cannot reuse cached dependencies and have to build them for each target
-# although they could be shared sometimes. That's because Github cache Action
-# has no read-only mode. But there is a PR ready for this
-# (https://github.com/actions/cache/pull/489)
-
-jobs:
-  builds-and-tests:
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        target: [dbg, opt]
-        mpi: [seq, par]
-        build-system: [make, cmake]
-        hypre-target: [int32]
-        precision: [fp64]
-        enzyme: [false]
-        exclude:
-          - os: ubuntu-latest
-            build-system: cmake
-          - os: macos-latest
-            build-system: cmake
-          - os: windows-latest
-            build-system: make
-        # 'include' allows us to:
-        # - Add a variable to all jobs without creating a new matrix dimension.
-        #   Codecov is defined that way.
-        # - Add a new combination.
-        #   'build-system: cmake' and 'hypre-target: int64'
-        #
-        # Note: we will gather coverage info for any non-debug run except the
-        # CMake build.
-        include:
-          - target: dbg
-            codecov: NO
-          - target: opt
-            codecov: YES
-          - os: ubuntu-latest
-            target: dbg
-            config-opts: "CPPFLAGS+=-Og"
-          - os: macos-latest
-            codecov: NO
-          - os: windows-latest
-            codecov: NO
-            # config-opts: '-G "Ninja Multi-Config"'
-          - os: windows-latest
-            target: opt
-            mpi: par
-            config-opts: "-DBUILD_SHARED_LIBS=ON"
-            # config-opts: '-DBUILD_SHARED_LIBS=ON -G "Ninja Multi-Config"'
-          - os: ubuntu-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: cmake
-            hypre-target: int32
-            precision: fp64
-            config-opts: "-DCMAKE_INSTALL_PREFIX=../cmake-install"
-            # This option can be set to pass additional configuration options to
-            # the MFEM configuration command.
-            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
-          - os: ubuntu-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int64
-            precision: fp64
-          - os: ubuntu-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int32
-            precision: fp32
-          - os: macos-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int32
-            precision: fp32
-          - os: macos-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int32
-            precision: fp64
-            enzyme: true
-            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme) LDFLAGS=-L$LLVM_PREFIX/lib/c++
-
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}
-
-    runs-on: ${{ matrix.os }}
-
-    continue-on-error: ${{ matrix.enzyme && true || false }}
-
-    steps:
-      # Fix 'No space left on device' errors for Ubuntu builds.
-      - name: Run Actions Cleaner
-        if: matrix.os == 'ubuntu-latest'
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: "true"
-          remove-android: "true"
-
-      # Checkout MFEM in "mfem" subdirectory. Final path:
-      # /home/runner/work/mfem/mfem/mfem
-      # Note: Done now to access "install-hypre" and "install-metis" actions.
-      - name: checkout mfem
-        uses: actions/checkout@v4
-        with:
-          path: ${{ env.MFEM_TOP_DIR }}
-          # Fetch the complete history for codecov to access commits ID
-          fetch-depth: 0
-
-      - name: Windows environment - PowerShell [debug]
-        if: matrix.os == 'windows-latest'
-        run: |
-          ls env: | fl
-
-      - name: Windows environment - Bash [debug]
-        if: matrix.os == 'windows-latest'
-        run: |
-          env
-        shell: bash
-
-      # For info on Xcode see:
-      #  - https://github.com/actions/runner-images/issues/12541
-      #  - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
-      - name: Xcode version setup (MacOS)
-        if: matrix.os == 'macos-latest'
-        run: |
-          XCODE_PATH="/Applications/Xcode_16.4.app"
-          echo "> sudo xcode-select -s ${XCODE_PATH}"
-          sudo xcode-select -s ${XCODE_PATH}
-          echo "> g++ -v"
-          g++ -v
-          echo "> clang++ -v"
-          clang++ -v
-
-      # Only get MPI if defined for the job.
-      # TODO: It would be nice to have only one step, e.g. with a dedicated
-      # action, but I (@adrienbernede) don't see how at the moment.
-      - name: get MPI (Linux)
-        if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install openmpi-bin libopenmpi-dev
-          export OMPI_MCA_rmaps_base_oversubscribe=1
-
-      - name: get lcov (Linux)
-        if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install lcov
-
-      # Keep the following section in case we need it again in the future,
-      # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-      # - name: Set up Homebrew
-      #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-      #   uses: Homebrew/actions/setup-homebrew@master
-
-      - name: get MPI (MacOS)
-        if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install openmpi
-
-      - name: get lcov (MacOS)
-        if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install lcov
-
-      - name: get MPI (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        uses: mpi4py/setup-mpi@v1.1.4
-
-      # Get Hypre through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache hypre
-        id: hypre-cache
-        if: matrix.mpi == 'par'
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
-
-      - name: get hypre
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: make
-          precision: ${{ matrix.precision }}
-
-      - name: get hypre (Windows)
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: cmake
-          precision: ${{ matrix.precision }}
-
-      # Get Metis through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache metis
-        id: metis-cache
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
-
-      - name: install metis
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
-        with:
-          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
-          dir: ${{ env.METIS_TOP_DIR }}
-
-      - name: cache vcpkg (Windows)
-        id: vcpkg-cache
-        if: matrix.os == 'windows-latest'
-        uses: actions/cache@v4
-        with:
-          path: vcpkg_cache
-          key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
-
-      - name: prepare vcpkg binary cache location (Windows)
-        if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p vcpkg_cache
-
-      - name: install metis (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        run: |
-          vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
-
-      # It's usually fine to build the above TPLs with a different compiler.
-      #
-      - name: install Enzyme (macOS w/ Enzyme)
-        if: matrix.enzyme && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew update
-          brew install enzyme
-          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
-          LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
-          echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
-          echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
-          echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
-
-      # MFEM build and test
-      - name: build
-        uses: mfem/github-actions/build-mfem@v2.5
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        with:
-          os: ${{ matrix.os }}
-          target: ${{ matrix.target }}
-          codecov: ${{ matrix.codecov }}
-          mpi: ${{ matrix.mpi }}
-          build-system: ${{ matrix.build-system }}
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: ${{ env.MFEM_TOP_DIR }}
-          precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }}
-          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
-
-      # Run checks (and only checks) on debug targets
-      - name: checks
-        if: matrix.build-system == 'make' && matrix.target == 'dbg'
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }}  && make check
-
-      # Note: 'tests' include the unit tests
-      - name: tests
-        if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make test
-
-      - name: cmake checks
-        if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
-        run: |
-          CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
-        shell: bash
-
-      - name: cmake unit tests (Ubuntu)
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
-
-      - name: cmake tests
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          cd ${{ env.MFEM_TOP_DIR }}/build && \
-          ctest --output-on-failure -C ${CTEST_CONFIG} || \
-          ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
-
-      - name: make install
-        if: matrix.build-system == 'make'
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make install
-
-      - name: cmake install
-        if: matrix.build-system == 'cmake'
-        run: |
-          CONFIG="Release"
-          [[ ${{ matrix.target }} == 'dbg' ]] && CONFIG="Debug"
-          TARGET="install"
-          [[ ${{ matrix.os }} == 'windows-latest' ]] && TARGET="INSTALL"
-          cd ${{ env.MFEM_TOP_DIR }} && \
-          cmake --build build --target ${TARGET} --config ${CONFIG}
-        shell: bash
-
-      # Code coverage (process and upload reports)
-      - name: codecov
-        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.5
-        with:
-          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-          project_dir: ${{ env.MFEM_TOP_DIR }}
-          directories: "fem general linalg mesh"

.github/workflows/codeql-analysis.yml

@@ -1,73 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "Static Analysis"
-
-on:
-  push:
-    branches: ["master", "next"]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: ["master"]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'cpp' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-
-          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-          # queries: security-extended,security-and-quality
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-      #   If the Autobuild fails above, remove it and uncomment the following three lines.
-      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-      # - run: |
-      #   echo "Run, Build Application using script"
-      #   ./location_of_script_within_repo/buildscript.sh
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2

.github/workflows/mfem-analysis.yml

@@ -1,100 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Build Analysis"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  COVERAGE_ENV: mfem-coverage
-
-jobs:
-  gitignore:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: checkout MFEM
-        uses: actions/checkout@v4
-        with:
-          path: mfem
-
-      - name: Get MPI (Linux)
-        run: |
-          sudo apt-get install openmpi-bin libopenmpi-dev
-          export OMPI_MCA_rmaps_base_oversubscribe=1
-
-      - name: Cache Hypre Install
-        id: hypre-cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5
-
-      - name: Get Hypre
-        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: int32
-
-      - name: Cache Metis Install
-        id: metis-cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
-
-      - name: Install Metis
-        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
-
-      # MFEM build and test
-      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.5
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          codecov: NO
-          mpi: par
-          build-system: make
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: mfem
-
-      - name: test (no clean)
-        run: |
-          cd mfem && make test-noclean
-
-      - name: gitignore
-        run: |
-          cd mfem/tests/scripts
-          ./runtest gitignore

.github/workflows/repo-check.yml

@@ -1,149 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Checks"
-
-permissions:
-  actions: write
-
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-# This workflow is run on pushes to any branch in the MFEM repo (with or without
-# PRs), as well as on updates to PRs from forks. In particular, we do not
-# duplicate work by running on both pushes and updates to local PRs. We do that
-# by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
-# and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
-# github.repository").
-
-jobs:
-  file-headers-check:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v4
-
-      - name: copyright check
-        id: copyright
-        run: |
-          ./config/githooks/pre-push --copyright
-
-        continue-on-error: true
-
-      - name: license check
-        id: license
-        run: |
-          ./config/githooks/pre-push --license
-        continue-on-error: true
-
-      - name: release check
-        id: release
-        run: |
-          ./config/githooks/pre-push --release
-        continue-on-error: true
-
-      - name: math check
-        id: math
-        run: |
-          ./config/githooks/pre-push --math
-        continue-on-error: true
-
-      - name: wrap-up
-        if: |
-          steps.copyright.outcome != 'success' ||
-          steps.license.outcome != 'success' ||
-          steps.release.outcome != 'success' ||
-          steps.math.outcome != 'success'
-        run: |
-          if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-            echo "copyright check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-            echo "license check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-            echo "release check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.math.outcome }}" != "success" ]]; then
-            echo "math check failed, unroll log for details"
-          fi
-          exit 1
-
-  code-style:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v4
-
-      - name: get astyle
-        run: |
-          sudo apt-get install astyle
-
-      - name: style check
-        run: |
-          ./config/githooks/pre-push --style
-
-  documentation:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v4
-
-      - name: get doxygen and graphviz
-        run: |
-          sudo apt-get install doxygen graphviz
-
-      - name: update doxygen config file
-        run: |
-          cd doc
-          doxygen -u CodeDocumentation.conf.in
-
-      - name: build documentation
-        run: |
-          cd tests/scripts
-          ./runtest documentation
-
-  branch-history:
-    if: |
-      github.ref != 'refs/heads/next' &&
-      github.ref != 'refs/heads/master' &&
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: branch-history
-        run: |
-          # We override origin to make sure we point to the main repo.
-          # This is to have consistent test results on PRs from forks.
-          git remote remove origin
-          git remote add origin https://github.com/mfem/mfem.git
-          git checkout -b gh-actions-branch-history
-          ./config/githooks/pre-push --history

.github/workflows/sanitize-build-hypre.yml

@@ -1,39 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: build-hypre
-on:
-  workflow_call:
-jobs:
-  build-hypre:
-    runs-on: ubuntu-latest
-    name: 2.19.0
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.HYPRE_DIR}}
-          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
-      - name: Setup
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: ./.github/actions/sanitize/mpi
-      - name: Build
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
-        with:
-          archive: ${{env.HYPRE_TGZ}}
-          dir: ${{env.HYPRE_DIR}}
-          target: int32
-          precision: fp64
-          build-system: make

.github/workflows/sanitize-build-libcxx.yml

@@ -1,76 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: build-libcxx
-on:
-  workflow_call:
-jobs:
-  build-llvm-libcxx:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        sanitizer: [asan, msan, ubsan]
-        include:
-          - sanitizer: asan
-            llvm_use_sanitizer: "Address"
-          - sanitizer: msan
-            llvm_use_sanitizer: "MemoryWithOrigins"
-          - sanitizer: ubsan
-            llvm_use_sanitizer: "Undefined"
-    name: ${{matrix.sanitizer}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-        with:
-          NO_FLAGS: true
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.LLVM_DIR}}
-          key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}
-      - name: Clone
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: >
-          git clone --filter=blob:none --depth=1
-          --branch llvmorg-${{env.LLVM_VER}}
-          --no-checkout https://github.com/llvm/llvm-project.git llvm-project
-      - name: Checkout
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        working-directory: llvm-project
-        run: |
-          git sparse-checkout set --cone
-          git checkout llvmorg-${{env.LLVM_VER}}
-          git sparse-checkout set cmake llvm/cmake runtimes libcxx libcxxabi
-      - name: Mkdir
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: mkdir ${{env.LLVM_DIR}}
-      - name: CMake
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        working-directory: ${{env.LLVM_DIR}}
-        run: >
-          VERBOSE=1
-          cmake -GNinja ../llvm-project/runtimes/
-          -DCMAKE_C_COMPILER=${{env.CC}}
-          -DCMAKE_CXX_COMPILER=${{env.CXX}}
-          -DCMAKE_BUILD_TYPE=RelWithDebInfo
-          -DCMAKE_INSTALL_PREFIX=/usr
-          -DLLVM_USE_SANITIZER=${{matrix.llvm_use_sanitizer}}
-          -DLLVM_BUILD_32_BITS=OFF
-          -DLIBCXXABI_USE_LLVM_UNWINDER=OFF
-          -DLLVM_INCLUDE_TESTS=OFF
-          -DLIBCXX_INCLUDE_TESTS=OFF
-          -DLIBCXX_INCLUDE_BENCHMARKS=OFF
-          -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi'
-      - name: Build
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        working-directory: ${{env.LLVM_DIR}}
-        run: cmake --build . -- cxx cxxabi

.github/workflows/sanitize-build-lsan.yml

@@ -1,38 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: build-file-lsan
-on:
-  workflow_call:
-jobs:
-  build-file-lsan:
-    runs-on: ubuntu-latest
-    name: lsan.supp
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.LSAN_DIR}}
-          key: build-lsan-suppression-file
-      - name: Setup
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p ${{env.LSAN_DIR}}
-          cat << EOF > ${{env.LSAN_DIR}}/${{env.LSAN_FILE}}
-          leak:libevent_core-2.1.so
-          leak:ompi_mpi_finalize
-          leak:ompi_mpi_init
-          leak:PMPI_Init
-          leak:strdup
-          EOF

.github/workflows/sanitize-build-metis.yml

@@ -1,36 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: build-metis
-on:
-  workflow_call:
-jobs:
-  build-metis:
-    runs-on: ubuntu-latest
-    name: 4.0.3
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.METIS_DIR}}
-          key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
-      - name: Setup
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: ./.github/actions/sanitize/mpi
-      - name: Build
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
-        with:
-          archive: ${{env.METIS_TGZ}}
-          dir: ${{env.METIS_DIR}}

.github/workflows/sanitize-tests.yml

@@ -1,197 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: Sanitize
-on:
-  workflow_call:
-    inputs:
-      par:
-        description: 'Whether to build for parallel (true/false)'
-        required: false
-        default: false
-        type: boolean
-      sanitizer:
-        description: 'Sanitizer to use (asan, msan, ubsan)'
-        required: true
-        default: asan
-        type: string
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/mfem
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-
-  check:
-    needs: [build]
-    runs-on: ubuntu-latest
-    env:
-      ex: ${{inputs.par && 'ex1p' || 'ex1'}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/examples/${{env.ex}}
-      - name: MFEM Check
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v check
-
-  examples:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      exclude: ${{inputs.par && '-E "_ser"' || ''}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/examples/ex1
-      - name: Build Examples
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v examples
-      - name: Test Examples
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} examples ${{env.exclude}} --show-only
-          ${{env.CTEST}} examples ${{env.exclude}}
-
-  miniapps:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      exclude: ${{inputs.par && '-E "_ser"' || ''}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/miniapps/meshing/minimal-surface
-      - name: Build Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v miniapps
-      - name: Test Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} miniapps ${{env.exclude}} --show-only
-          ${{env.CTEST}} miniapps ${{env.exclude}}
-
-  tests-miniapps:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      run: ${{inputs.par && '-R "_cpu_np"' || ''}}
-      exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/tests/unit/sedov_tests_cpu
-      - name: Build Tests Unit Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v tests/unit/all
-      - name: Run Tests Unit Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}} --show-only
-          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}}
-
-  tests-unit-build:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
-      - name: Build Unit Tests
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v ${{env.unit_tests}}
-      - name: Delete object files
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build/tests/unit
-        run: find . -type f -name '*.o' -delete
-      - uses: actions/upload-artifact@v4
-        with:
-          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
-          path: mfem/build/tests/unit/${{env.unit_tests}}
-          if-no-files-found: error
-          retention-days: 1
-          overwrite: false
-
-  tests-unit-run:
-    needs: [tests-unit-build]
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        tag: [0, 1, 2, 3]
-    name: tests-unit-run-${{matrix.tag}}
-    env:
-      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
-      np: ${{inputs.par && '_np=2' || ''}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
-      - uses: actions/download-artifact@v4
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        with:
-          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
-          path: mfem/build/tests/unit
-      - name: Split Unit Tests
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build/tests/unit
-        run: |
-          chmod 755 ${{env.unit_tests}}
-          ./${{env.unit_tests}} --list-test-names-only | tail -n +2 > list-test-names
-          shuf list-test-names -o list-test-names
-          split --verbose -n l/4 -d -a 1 list-test-names list-test-names-
-      - name: Cat Unit Tests ${{matrix.tag}}
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build/tests/unit
-        run: cat list-test-names-${{matrix.tag}}
-      - name: Run Unit Tests ${{matrix.tag}}
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}" --show-only
-          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}"

.github/workflows/sanitizers.yml

@@ -1,73 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-name: Sanitizers
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches: ["master", "next"]
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{github.workflow}}-${{github.ref}}
-  cancel-in-progress: true
-
-jobs:
-
-  # Build steps for dependencies
-  build-hypre:
-    uses: ./.github/workflows/sanitize-build-hypre.yml
-
-  build-metis:
-    uses: ./.github/workflows/sanitize-build-metis.yml
-
-  build-lsan:
-    uses: ./.github/workflows/sanitize-build-lsan.yml
-
-  build-libcxx:
-    uses: ./.github/workflows/sanitize-build-libcxx.yml
-
-  # Serial sanitizers: asan, msan, ubsan
-  seq-asan:
-    needs: [build-libcxx]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      sanitizer: asan
-
-  seq-msan:
-    needs: [build-libcxx]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      sanitizer: msan
-
-  seq-ubsan:
-    needs: [build-libcxx]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      sanitizer: ubsan
-
-  # Parallel sanitizers: asan, ubsan
-  par-asan:
-    needs: [build-libcxx, build-hypre, build-metis]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      par: true
-      sanitizer: asan
-  par-ubsan:
-    needs: [build-libcxx, build-hypre, build-metis]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      par: true
-      sanitizer: ubsan

.github/workflows/stale.yml

@@ -1,75 +0,0 @@
-# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
-# For more information, see: https://github.com/actions/stale
-name: Mark stale issues and pull requests
-
-on:
-  workflow_dispatch:
-  schedule:
-  - cron: '0 0 * * *'
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: ':warning: This issue has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
-        days-before-stale: 30
-        days-before-close: 7
-        stale-issue-label: 'stale'
-        stale-pr-label: 'stale'
-        operations-per-run: 500
-        exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
-        exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"
-
-  # Stale action for PRs with "in-review" label.
-  stale-in-review-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 150 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "in-review"
-        days-before-pr-stale: 150
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500
-
-  # Stale action for PRs with "WIP" label.
-  stale-wip-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 300 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "WIP"
-        days-before-pr-stale: 300
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500

.github/workflows/trigger-pymfem-ci.yml

@@ -1,31 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Trigger PyMFEM CI"
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  trigger-pymfem:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Send POST request to trigger PyMFEM CI
-      run: |
-        curl -L \
-             -X POST \
-             -H "Accept: application/vnd.github+json" \
-             -H "Authorization: Bearer ${{ secrets.PYMFEM_CI_TOKEN }}" \
-             -H "X-GitHub-Api-Version: 2022-11-28" \
-             https://api.github.com/repos/mfem/pymfem/actions/workflows/build-and-test-dispatch.yml/dispatches \
-             -d '{"ref":"master", "inputs":{"test_options":"fast"}}'

.gitignore

@@ -8,44 +8,26 @@
 
 # Object and library files
 *.o
-*.o.tmp
 /libmfem.*
-/miniapps/common/libmfem-common.*
 
 # CMake generated files
 CMakeCache.txt
 CMakeFiles/
 
-# Clangd server cache
-*.cache*
-
-#vscode settings
-/.vscode/
-
 # Backup files
 *~
 
-# clangd index
-/.cache/
-
 # Default install location
 /mfem/
 
-# Typical build directory
-/build/
-
 # Generated files in main directory, config/ and docs/
 /deps.mk
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
-config/user.cmake
-config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
-doc/undoc.log
-doc/warnings.log
 
 # Temporary files created by the tests.
 *.stderr
@@ -59,23 +41,16 @@ doc/warnings.log
 
 # Example and miniapp binaries and outputs
 
-examples/ex[0-9]
-examples/ex[0-9]p
+examples/ex[1-9]
+examples/ex[1-9]p
 examples/ex1[04-9]
 examples/ex1[0-9]p
-examples/ex2[0-9]
-examples/ex2[0-9]p
-examples/ex3[0-9]
-examples/ex3[0-9]p
-examples/ex4[0-9]
-examples/ex4[0-9]p
 
 examples/refined.mesh
 examples/displaced.mesh
 examples/mesh.*
 examples/ex5.mesh
 examples/Example5*
-examples/ParaView
 examples/Example9*
 examples/Example15*
 examples/Example16*
@@ -83,114 +58,30 @@ examples/sphere_refined.*
 examples/sol.*
 examples/sol_u.*
 examples/sol_p.*
-examples/sol_r.*
-examples/sol_i.*
-examples/sol_z.*
-examples/ex6p-checkpoint.*
-examples/order.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
 examples/ex9-final.*
-examples/ex41.mesh
-examples/ex41-mesh.*
-examples/ex41-init.*
-examples/ex41-final.*
 examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
 examples/mode_*
-examples/mode_deriv_*
-examples/ex5-p-*.bp
-examples/ex9-p-*.bp
-examples/ex12-p-*.bp
-examples/ex16-p-*.bp
 examples/ex16.mesh
 examples/ex16-mesh.*
 examples/ex16-init.*
 examples/ex16-final.*
+examples/vortex-mesh.*
+examples/vortex.mesh
+examples/vortex-?-init.*
+examples/vortex-?-final.*
 examples/deformation.*
 examples/pressure.*
-examples/ex20.dat
-examples/ex20p_?????.dat
-examples/gnuplot_ex20.inp
-examples/gnuplot_ex20p.inp
-examples/ex21*.mesh
-examples/ex21*.sol
-examples/ex21p_*.*
-examples/ex23-*.gf
-examples/ex23*.mesh
-examples/Example23*
-examples/ex25.mesh
-examples/ex25-*.gf
-examples/ex25p-*.*
-examples/ex28_*
-examples/ex28p_*
-examples/flux.*
-examples/dsol.*
-examples/cond.*
-examples/cond_j.*
-examples/cond_mesh.*
-examples/port_mesh.*
-examples/port_mode.*
-
-examples/euler-*
-
-examples/amgx/ex1
-examples/amgx/ex1p
-examples/amgx/.logamgx
-examples/amgx/refined.mesh
-examples/amgx/sol.gf
-examples/amgx/mesh.*
-examples/amgx/sol.*
-
-examples/caliper/ex1
-examples/caliper/ex1p
-examples/caliper/refined.mesh
-examples/caliper/sol.gf
-examples/caliper/mesh.*
-examples/caliper/sol.*
-
-examples/ginkgo/ex1
-examples/ginkgo/refined.mesh
-examples/ginkgo/sol.gf
-
-examples/hiop/ex9
-examples/hiop/ex9p
-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
-
-examples/petsc/ex[1-69]p
-examples/petsc/ex1[0-1]p
-examples/petsc/mesh.*
-examples/petsc/sol.*
-examples/petsc/sol_p.*
-examples/petsc/sol_u.*
-examples/petsc/Example5*
-examples/petsc/ex9.mesh
-examples/petsc/ex9-mesh.*
-examples/petsc/ex9-init.*
-examples/petsc/ex9-final.*
-examples/petsc/Example9*
-examples/petsc/deformed.*
-examples/petsc/velocity.*
-examples/petsc/elastic_energy.*
-examples/petsc/mode_*
-
-examples/pumi/ex[12]
-examples/pumi/ex[16]p
-examples/pumi/refined.mesh
-examples/pumi/sol.gf
-examples/pumi/mesh.*
-examples/pumi/sol.*
-examples/pumi/displaced.mesh
 
 examples/sundials/ex9
 examples/sundials/ex1[06]
 examples/sundials/ex9p
 examples/sundials/ex1[06]p
+
 examples/sundials/ex9.mesh
 examples/sundials/ex9-mesh.*
 examples/sundials/ex9-init.*
@@ -205,287 +96,72 @@ examples/sundials/ex16-init.*
 examples/sundials/ex16-final.*
 examples/sundials/Example16*
 
-examples/superlu/ex1p
-examples/superlu/mesh.*
-examples/superlu/sol.*
+examples/petsc/ex[1-69]p
+examples/petsc/ex10p
 
-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
+examples/petsc/mesh.*
+examples/petsc/sol.*
+examples/petsc/sol_p.*
+examples/petsc/sol_u.*
+examples/petsc/Example5*
+examples/petsc/ex9-mesh.*
+examples/petsc/ex9-init.*
+examples/petsc/ex9-final.*
+examples/petsc/Example9*
+examples/petsc/deformed.*
+examples/petsc/velocity.*
+examples/petsc/elastic_energy.*
 
-miniapps/dfem/dfem-minimal-surface
-miniapps/dfem/dfem-minimal-surface-output
+examples/pumi/ex1
+examples/pumi/ex[126]p
+
+examples/pumi/refined.mesh
+examples/pumi/sol.gf
+examples/pumi/mesh.*
+examples/pumi/sol.*
+examples/pumi/displaced.mesh
 
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
-miniapps/electromagnetics/lorentz
+
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
-miniapps/electromagnetics/Joule_[0-9]*
-miniapps/electromagnetics/Lorentz_[0-9]*
-miniapps/electromagnetics/Lorentz.dat
-
-miniapps/fluids/schrodinger-flow/schrodinger_flow
-miniapps/fluids/schrodinger-flow/pschrodinger_flow
-
-miniapps/gslib/field-diff
-miniapps/gslib/field-interp
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-miniapps/gslib/schwarz_ex1
-miniapps/gslib/schwarz_ex1p
-miniapps/gslib/interpolated.gf
-miniapps/gslib/particles_redist
+miniapps/electromagnetics/Joule_*
 
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
-miniapps/meshing/toroid
-miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
-miniapps/meshing/extruder
-miniapps/meshing/fit-node-position
-miniapps/meshing/trimmer
-miniapps/meshing/reflector
-miniapps/meshing/ref321
-miniapps/meshing/mesh-bounding-boxes
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
-miniapps/meshing/pmesh-fitting
-miniapps/meshing/minimal-surface
-miniapps/meshing/pminimal-surface
-miniapps/meshing/polar-nc
-miniapps/meshing/mesh-quality
-miniapps/meshing/hpref
-miniapps/meshing/phpref
+
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
-miniapps/meshing/toroid-*.mesh
-miniapps/meshing/twist-*.mesh
-miniapps/meshing/mesh-explorer.mesh*
+miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
-miniapps/meshing/mesh-explorer-visit*
-miniapps/meshing/mesh-explorer-paraview/
 miniapps/meshing/shaper.mesh
-miniapps/meshing/extruder.mesh
-miniapps/meshing/trimmer.mesh
-miniapps/meshing/reflected.mesh
-miniapps/meshing/ref321.mesh
-miniapps/meshing/sol.gf
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
-miniapps/meshing/polar-nc.mesh
-miniapps/meshing/mesh.*
-miniapps/meshing/order.*
-miniapps/meshing/sol.*
-miniapps/meshing/refined.mesh
-miniapps/meshing/bounding-box*
-miniapps/meshing/jacobian-determinant*
-
-miniapps/mtop/ParaView/
-miniapps/mtop/mtop_test_iso_elasticity
-
-miniapps/autodiff/paradiff
-miniapps/autodiff/seqadiff
-miniapps/autodiff/seqtest
-miniapps/autodiff/par_example
-miniapps/autodiff/seq_example
-miniapps/autodiff/seq_test
-miniapps/autodiff/Example/*
-
-miniapps/fluids/navier/navier_mms
-miniapps/fluids/navier/navier_kovasznay
-miniapps/fluids/navier/navier_kovasznay_vs
-miniapps/fluids/navier/navier_tgv
-miniapps/fluids/navier/navier_shear
-miniapps/fluids/navier/navier_3dfoc
-miniapps/fluids/navier/navier_turbchan
-miniapps/fluids/navier/navier_cht
-miniapps/fluids/navier/navier_bifurcation
-miniapps/fluids/navier/Navier_Bifurcation_[0-9]*
-miniapps/fluids/navier/ParaView
-miniapps/fluids/navier/tgv_out*.txt
-miniapps/fluids/navier/*_output
-
-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex3
-miniapps/nurbs/nurbs_ex5
-miniapps/nurbs/nurbs_ex10
-miniapps/nurbs/nurbs_ex10p
-miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_ex24
-miniapps/nurbs/nurbs_solenoidal
-miniapps/nurbs/nurbs_printfunc
-miniapps/nurbs/nurbs_patch_ex1
-miniapps/nurbs/nurbs_curveint
-miniapps/nurbs/nurbs_surface
-miniapps/nurbs/refined.mesh
-miniapps/nurbs/mesh.*
-miniapps/nurbs/sol_?.gf
-miniapps/nurbs/sol.*
-miniapps/nurbs/mode_*
-miniapps/nurbs/Example1*
-miniapps/nurbs/Example3*
-miniapps/nurbs/Example5*
-miniapps/nurbs/Solenoidal*
-miniapps/nurbs/ParaView
-miniapps/nurbs/sin-fit.mesh
-miniapps/nurbs/ex5.mesh
-miniapps/nurbs/exsol.mesh
-miniapps/nurbs/CurveInt
-miniapps/nurbs/nurbs_naca_cmesh
-miniapps/nurbs/naca-cmesh.mesh
-miniapps/nurbs/glvis_naca-cmesh.mesh
-miniapps/nurbs/Naca_cmesh
-miniapps/nurbs/nurbs_mesh_info
-miniapps/nurbs/k*_*.dat
-miniapps/nurbs/*-Surface.mesh
-miniapps/nurbs/*.mesh
-miniapps/nurbs/*.sol
-miniapps/nurbs/deformed.*
-miniapps/nurbs/elastic_energy.*
-miniapps/nurbs/velocity.*
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
+
 miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
 
-miniapps/shifted/distance
-miniapps/shifted/ParaViewDistance
-miniapps/shifted/ParaViewLSF
-miniapps/shifted/extrapolate
-miniapps/shifted/ParaViewExtrapolate
-miniapps/shifted/diffusion
-miniapps/shifted/diffusion.mesh
-miniapps/shifted/diffusion.gf
-miniapps/shifted/ParaViewDiffusion
-miniapps/shifted/lsf_integral
-
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
-miniapps/tools/compare-dc
-miniapps/tools/gridfunction-bounds
-miniapps/tools/lor-transfer
-miniapps/tools/plor-transfer
-miniapps/tools/get-values
-miniapps/tools/tmop-check-metric
-miniapps/tools/tmop-metric-magnitude
-miniapps/tools/nodal-transfer
-miniapps/tools/ParaView
-miniapps/tools/gridfunc_*
-miniapps/tools/mesh_*
 
-miniapps/toys/automata
-miniapps/toys/life
-miniapps/toys/mandel
-miniapps/toys/rubik
-miniapps/toys/snake
-miniapps/toys/lissajous
-miniapps/toys/mondrian
-miniapps/toys/spiral
-miniapps/toys/snake-init.mesh
-miniapps/toys/snake-user.mesh
-miniapps/toys/snake-joined.mesh
-miniapps/toys/snake-c*.mesh
-miniapps/toys/automata.gf
-miniapps/toys/automata.mesh
-miniapps/toys/rubik-init.mesh
-miniapps/toys/mandel.mesh
-miniapps/toys/life.gf
-miniapps/toys/life.mesh
-miniapps/toys/lissajous.mesh
-miniapps/toys/lissajous.gf
-miniapps/toys/mondrian.mesh
-
-miniapps/solvers/block-solvers
-miniapps/solvers/lor_solvers
-miniapps/solvers/plor_solvers
-miniapps/solvers/lor_elast
-miniapps/solvers/ParaView
-miniapps/solvers/mesh.*
-miniapps/solvers/sol.*
-
-miniapps/hdiv-linear-solver/darcy
-miniapps/hdiv-linear-solver/grad_div
-
-miniapps/parelag/MultilevelHcurlHdivSolver
-miniapps/parelag/*.mesh
-
-miniapps/multidomain/multidomain
-miniapps/multidomain/multidomain_nd
-miniapps/multidomain/multidomain_rt
-miniapps/hooke/hooke
-
-miniapps/dpg/diffusion
-miniapps/dpg/pdiffusion
-miniapps/dpg/convection-diffusion
-miniapps/dpg/pconvection-diffusion
-miniapps/dpg/acoustics
-miniapps/dpg/pacoustics
-miniapps/dpg/maxwell
-miniapps/dpg/pmaxwell
-miniapps/dpg/ParaView
-
-miniapps/spde/generate_random_field
-miniapps/spde/ParaView
-
-miniapps/tribol/contact-patch-test
-
-miniapps/diag-smoothers/abs-l1-jacobi
-miniapps/diag-smoothers/mg-abs-l1-jacobi
-
-miniapps/contact/contact
-miniapps/contact/ParaView
-
-# Unit test binary and outputs
-tests/unit/output_meshes
-tests/unit/unit_tests
-tests/unit/punit_tests
-tests/unit/gpu_unit_tests
-tests/unit/pgpu_unit_tests
-tests/unit/sedov_tests_*
-tests/unit/psedov_tests_*
-tests/unit/tmop_pa_tests_*
-tests/unit/ptmop_pa_tests_*
-tests/unit/ceed_tests
-tests/unit/debug_device_tests
-tests/unit/parallel_in_serial.mesh
-tests/unit/parallel_in_serial.gf
-
-# Benchmark binaries
-tests/benchmarks/bench_ceed
-tests/benchmarks/bench_tmop
-tests/benchmarks/bench_vector
-tests/benchmarks/bench_virtuals
-
-# Test script output
-tests/scripts/*.err
-tests/scripts/*.out
-tests/scripts/*.msg
-
-# Other tests
-tests/convergence/rates
-tests/convergence/prates
-tests/par-mesh-format/ex1p
-
-# VPATH builds
-build-*/*
-
-# PETSc automated build
-petsc-build/*
-pkg.gitcommit
-
-# Jupyter Notebook Checkpoints
-.ipynb_checkpoints
-
-# emacs tag file
-TAGS
-
-# vs code
-.vscode
+miniapps/nurbs/ex1
+miniapps/nurbs/ex1p
+miniapps/nurbs/ex11p
+miniapps/nurbs/refined.mesh
+miniapps/nurbs/mesh.*
+miniapps/nurbs/sol.*
+miniapps/nurbs/mode_*
+miniapps/nurbs/Example1*

.gitlab-ci.yml

@@ -1,558 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# DESCRIPTION:
-###############################################################################
-# General GitLab pipelines configurations for supercomputers and Linux clusters
-# at Lawrence Livermore National Laboratory (LLNL).
-# This entire pipeline is LLNL-specific
-#
-# Important note: This file is a template provided by llnl/radiuss-shared-ci.
-# Remains to set variable values, change the reference to the radiuss-shared-ci
-# repo, opt-in and out optional features. The project can then extend it with
-# additional stages.
-#
-# In addition, each project should copy over and complete:
-# - .gitlab/custom-jobs-and-variables.yml
-# - .gitlab/subscribed-pipelines.yml
-#
-# The jobs should be specified in a file local to the project,
-# - .gitlab/jobs/${CI_MACHINE}.yml
-# or generated (see LLNL/Umpire for an example).
-###############################################################################
-#   MAP OF GITLAB CI
-#######################
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# File dependencies: direct, through jobs, through variables
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# .gitlab-ci.yml
-#  ├── .build-and-test  [job]
-#  │    ├── .gitlab/custom-jobs-and-variables.yml
-#  │    │    ├── .custom_job          [job]
-#  │    │    ├── .reproducer_vars     [job]
-#  │    │    ├── .report_job_success  [job]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
-#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
-#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
-#  │    │    ├── .report_job_failure  [job]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
-#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
-#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
-#  │    │    └── JOB_CMD              [var]
-#  │    │         └── tests/gitlab/build_and_test            [script]
-#  │    │              └── tests/gitlab/get_mfem_uberenv     [script]
-#  │    ├── <radiuss-shared-ci>/pipelines/matrix.yml  [conditional]
-#  │    │    ├── .on_matrix               [job]
-#  │    │    ├── .matrix_reproducer_init  [job]
-#  │    │    ├── .matrix_reproducer_vars  [job]
-#  │    │    ├── .matrix_reproducer_job   [job]
-#  │    │    ├── .matrix_job_command      [job]
-#  │    │    └── .job_on_matrix           [job]
-#  │    ├── <radiuss-shared-ci>/pipelines/dane.yml  [conditional]
-#  │    │    ├── .on_dane               [job]
-#  │    │    ├── .dane_reproducer_init  [job]
-#  │    │    ├── .dane_reproducer_vars  [job]
-#  │    │    ├── .dane_reproducer_job   [job]
-#  │    │    ├── .dane_job_command      [job]
-#  │    │    ├── .job_on_dane           [job]
-#  │    │    ├── allocate_resources     [job]
-#  │    │    └── release_resources      [job]
-#  │    ├── <radiuss-shared-ci>/pipelines/tioga.yml  [conditional]
-#  │    │    ├── .on_tioga               [job]
-#  │    │    ├── .tioga_reproducer_init  [job]
-#  │    │    ├── .tioga_reproducer_vars  [job]
-#  │    │    ├── .tioga_reproducer_job   [job]
-#  │    │    ├── .tioga_job_command      [job]
-#  │    │    ├── .job_on_tioga           [job]
-#  │    │    ├── allocate_resources      [job]
-#  │    │    └── release_resources       [job]
-#  │    ├── <artifact>/matrix-jobs.yml  [conditional, from 'generate-job-lists']
-#  │    │    ├── .gitlab/jobs/matrix.yml
-#  │    │    │    ├── .matrix_reproducer_vars      [job]
-#  │    │    │    ├── setup                        [job]
-#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │    │    │    ├── opt_mpi_cuda_gcc             [job]
-#  │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc  [job]
-#  │    │    └── .gitlab/jobs/matrix-reports.yml  [used conditionally]
-#  │    │         ├── report_job_success
-#  │    │         └── report_job_failure
-#  │    ├── <artifact>/dane-jobs.yml    [conditional, from 'generate-job-lists']
-#  │    │    ├── .gitlab/jobs/dane.yml
-#  │    │    │    ├── .dane_reproducer_vars    [job]
-#  │    │    │    ├── setup                    [job]
-#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │    │    │    ├── debug_ser_gcc_10         [job]
-#  │    │    │    ├── debug_par_gcc_10         [job]
-#  │    │    │    ├── opt_ser_gcc_10           [job]
-#  │    │    │    ├── opt_par_gcc_10           [job]
-#  │    │    │    ├── opt_par_gcc_10_sundials  [job]
-#  │    │    │    ├── opt_par_gcc_10_petsc     [job]
-#  │    │    │    └── opt_par_gcc_10_pumi      [job]
-#  │    │    └── .gitlab/jobs/dane-reports.yml    [used conditionally]
-#  │    │         ├── report_job_success
-#  │    │         └── report_job_failure
-#  │    └── <artifact>/tioga-jobs.yml   [conditional, from 'generate-job-lists']
-#  │         ├── .gitlab/jobs/tioga.yml
-#  │         │    ├── .tioga_reproducer_vars  [job]
-#  │         │    ├── setup                   [job]
-#  │         │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │         │    └── cce_16_0_1              [job]
-#  │         └── .gitlab/jobs/tioga-reports.yml   [used conditionally]
-#  │              ├── report_job_success
-#  │              └── report_job_failure
-#  └── .gitlab/subscribed-pipelines.yml
-#       ├── .machine-check         [job]
-#       ├── generate-job-lists     [job]
-#       ├── dane-up-check          [job]
-#       ├── dane-build-and-test    [job]
-#       ├── dane-baseline          [job]
-#       │    └── .gitlab/dane-baseline.yml
-#       │         ├── .on_dane                       [job]
-#       │         ├── baselinecheck_mfem_intel_dane  [job]
-#       │         │    └── .gitlab/scripts/baseline            [script]
-#       │         ├── cleanup                        [job]
-#       │         ├── report_baseline                [job]
-#       │         │    ├── .gitlab/scripts/safe_create_rundir  [script]
-#       │         │    └── .gitlab/scripts/git_try_to_push     [script]
-#       │         ├── baselinepublish_mfem_dane      [job]
-#       │         │    └── .gitlab/scripts/rebaseline          [script]
-#       │         ├── .gitlab/custom-jobs-and-variables.yml
-#       │         │    └── <same as above: see .gitlab-ci.yml/.build-and-test>
-#       │         └── .gitlab/configs/setup-baseline.yml
-#       │              └── setup_baseline            [job]
-#       ├── tioga-up-check         [job]
-#       ├── tioga-build-and-test   [job]
-#       ├── matrix-up-check        [job]
-#       └── matrix-build-and-test  [job]
-#
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# File tree hierarchy with file contents highlights
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# In addition to the files in the MFEM repo, the Gitlab CI uses files from the
-# radiuss/radiuss-shared-ci project, see below, after the <mfem-root> tree.
-#
-# <mfem root>
-#  ├── .gitlab-ci.yml  [this file]
-#  │    ├── <jobs>
-#  │    │    └── .build-and-test
-#  │    ├── <included files>
-#  │    │    ├── .gitlab/subscribed-pipelines.yml
-#  │    │    ├── .gitlab/custom-jobs-and-variables.yml  [by ".build-and-test"]
-#  │    │    ├── <artifact>  [by ".build-and-test"]
-#  │    │    │    ├── artifact: '${CI_MACHINE}-jobs.yml'
-#  │    │    │    └── job:      'generate-job-lists'
-#  │    │    └── <external>  [by ".build-and-test"]
-#  │    │         ├── project: 'radiuss/radiuss-shared-ci'
-#  │    │         ├── ref:     'v2025.09.1'
-#  │    │         └── file:    'pipelines/${CI_MACHINE}.yml'
-#  │    └── <defined variables>
-#  │         ├── CUSTOM_CI_BUILDS_DIR
-#  │         ├── USER_CI_TOP_DIR
-#  │         ├── SHARED_REPOS_DIR
-#  │         ├── AUTOTEST_ROOT
-#  │         ├── MFEM_DATA_DIR
-#  │         ├── AUTOTEST
-#  │         ├── AUTOTEST_COMMIT
-#  │         ├── REBASELINE
-#  │         ├── GITHUB_PROJECT_NAME
-#  │         └── GITHUB_PROJECT_ORG
-#  ├── .gitlab
-#  │    ├── configs
-#  │    │    └── setup-baseline.yml
-#  │    │         ├── <jobs>
-#  │    │         │    └── setup_baseline
-#  │    │         └── <used variables>
-#  │    │              ├── MACHINE_NAME
-#  │    │              ├── REBASELINE
-#  │    │              ├── AUTOTEST
-#  │    │              ├── AUTOTEST_COMMIT
-#  │    │              ├── BUILD_ROOT
-#  │    │              ├── TPLS_REPO
-#  │    │              ├── TESTS_REPO
-#  │    │              ├── AUTOTEST_ROOT
-#  │    │              └── AUTOTEST_REPO
-#  │    ├── jobs
-#  │    │    ├── matrix-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_matrix
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    ├── matrix.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── .matrix_reproducer_vars
-#  │    │    │    │    ├── setup
-#  │    │    │    │    ├── opt_mpi_cuda_gcc
-#  │    │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc
-#  │    │    │    ├── <used jobs>
-#  │    │    │    │    ├── .reproducer_vars
-#  │    │    │    │    ├── .on_matrix
-#  │    │    │    │    └── .job_on_matrix
-#  │    │    │    ├── <included and used files>
-#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │    │    └── <defined variables>
-#  │    │    │         └── SPEC
-#  │    │    ├── dane-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_dane
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    ├── dane.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── .dane_reproducer_vars
-#  │    │    │    │    ├── setup
-#  │    │    │    │    ├── debug_ser_gcc_10
-#  │    │    │    │    ├── debug_par_gcc_10
-#  │    │    │    │    ├── opt_ser_gcc_10
-#  │    │    │    │    ├── opt_par_gcc_10
-#  │    │    │    │    ├── opt_par_gcc_10_sundials
-#  │    │    │    │    ├── opt_par_gcc_10_petsc
-#  │    │    │    │    └── opt_par_gcc_10_pumi
-#  │    │    │    ├── <used jobs>
-#  │    │    │    │    ├── .reproducer_vars
-#  │    │    │    │    ├── .on_dane
-#  │    │    │    │    └── .job_on_dane
-#  │    │    │    ├── <included and used files>
-#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │    │    └── <defined variables>
-#  │    │    │         ├── SPEC
-#  │    │    │         └── THREADS
-#  │    │    ├── tioga-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_tioga
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    └── tioga.yml
-#  │    │         ├── <jobs>
-#  │    │         │    ├── .tioga_reproducer_vars
-#  │    │         │    ├── setup
-#  │    │         │    └── opt_mpi_rocm_hypre_rocm
-#  │    │         ├── <used jobs>
-#  │    │         │    ├── .reproducer_vars
-#  │    │         │    ├── .on_tioga
-#  │    │         │    └── .job_on_tioga
-#  │    │         ├── <included and used files>
-#  │    │         │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │         └── <defined variables>
-#  │    │              ├── SPEC
-#  │    │              └── THREADS
-#  │    ├── scripts
-#  │    │    ├── baseline
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── BASELINE_TEST
-#  │    │    │         ├── SYS_TYPE
-#  │    │    │         ├── MACHINE_NAME
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── ARTIFACTS_DIR
-#  │    │    │         ├── BUILD_ROOT
-#  │    │    │         └── TPLS_DIR
-#  │    │    ├── git_try_to_push
-#  │    │    ├── rebaseline
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── ARTIFACTS_DIR
-#  │    │    │         ├── SYS_TYPE
-#  │    │    │         ├── BUILD_ROOT
-#  │    │    │         ├── MACHINE_NAME
-#  │    │    │         └── CI_PIPELINE_ID
-#  │    │    ├── report_build_and_test
-#  │    │    │    ├── <used files>
-#  │    │    │    │    ├── .gitlab/scripts/safe_create_rundir
-#  │    │    │    │    └── .gitlab/scripts/git_try_to_push
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── AUTOTEST_ROOT
-#  │    │    │         ├── CI_COMMIT_REF_SLUG
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── CI_PIPELINE_URL
-#  │    │    │         ├── AUTOTEST_COMMIT
-#  │    │    │         └── CI_MACHINE
-#  │    │    └── safe_create_rundir
-#  │    ├── custom-jobs-and-variables.yml
-#  │    │    ├── <jobs>
-#  │    │    │    ├── .custom_job
-#  │    │    │    ├── .reproducer_vars
-#  │    │    │    ├── .report_job_success
-#  │    │    │    └── .report_job_failure
-#  │    │    ├── <used files>
-#  │    │    │    ├── tests/gitlab/build_and_test  [in JOB_CMD]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test [by .report_job_*]
-#  │    │    ├── <defined variables>
-#  │    │    │    ├── JOB_CMD
-#  │    │    │    ├── BUILD_ROOT
-#  │    │    │    ├── ALLOC_NAME
-#  │    │    │    ├── TPLS_REPO
-#  │    │    │    ├── TESTS_REPO
-#  │    │    │    ├── AUTOTEST_REPO
-#  │    │    │    ├── MFEM_DATA_REPO
-#  │    │    │    ├── ARTIFACTS_DIR: artifacts
-#  │    │    │    ├── SLURM_OVERLAP: 1
-#  │    │    │    ├── DANE_SHARED_ALLOC
-#  │    │    │    ├── DANE_JOB_ALLOC
-#  │    │    │    ├── TIOGA_SHARED_ALLOC
-#  │    │    │    ├── TIOGA_JOB_ALLOC
-#  │    │    │    └── MATRIX_JOB_ALLOC
-#  │    │    └── <used variables>
-#  │    │         ├── SPEC
-#  │    │         ├── BUILD_ROOT
-#  │    │         └── ...
-#  │    ├── dane-baseline.yml
-#  │    │    ├── <jobs>
-#  │    │    │    ├── .on_dane
-#  │    │    │    ├── baselinecheck_mfem_intel_dane
-#  │    │    │    ├── cleanup
-#  │    │    │    ├── report_baseline
-#  │    │    │    └── baselinepublish_mfem_dane
-#  │    │    ├── <included and used files>
-#  │    │    │    ├── .gitlab/custom-jobs-and-variables.yml
-#  │    │    │    ├── .gitlab/configs/setup-baseline.yml
-#  │    │    │    ├── .gitlab/scripts/rebaseline
-#  │    │    │    ├── .gitlab/scripts/baseline
-#  │    │    │    └── .gitlab/scripts/git_try_to_push
-#  │    │    ├── <defined variables>
-#  │    │    │    ├── BASELINE_TEST: baseline
-#  │    │    │    ├── MACHINE_NAME: dane
-#  │    │    │    ├── TPLS_DIR
-#  │    │    │    └── export MFEM_TEST_NP
-#  │    │    └── <used variables>
-#  │    │         ├── ON_DANE
-#  │    │         ├── AUTOTEST       [defined by .gitlab-ci.yml]
-#  │    │         ├── BUILD_ROOT     [defined by custom-jobs-and-variables.yml]
-#  │    │         ├── TPLS_DIR       [defined by this file]
-#  │    │         ├── ARTIFACTS_DIR  [defined by custom-jobs-and-variables.yml]
-#  │    │         ├── MACHINE_NAME   [defined by this file]
-#  │    │         ├── AUTOTEST_COMMIT  [defined by .gitlab-ci.yml]
-#  │    │         ├── AUTOTEST_ROOT  [defined by .gitlab-ci.yml]
-#  │    │         ├── BASELINE_TEST  [defined by this file]
-#  │    │         └── REBASELINE     [defined by .gitlab-ci.yml]
-#  │    └── subscribed-pipelines.yml
-#  │         ├── <jobs>
-#  │         │    ├── .machine-check
-#  │         │    ├── generate-job-lists
-#  │         │    ├── dane-up-check
-#  │         │    ├── dane-build-and-test
-#  │         │    ├── dane-baseline
-#  │         │    ├── tioga-up-check
-#  │         │    ├── tioga-build-and-test
-#  │         │    ├── matrix-up-check
-#  │         │    └── matrix-build-and-test
-#  │         ├── <used jobs>
-#  │         │    └── .build-and-test  [from ".gitlab-ci.yml"]
-#  │         ├── <included files>
-#  │         │    └── .gitlab/dane-baseline.yml  [by "dane-baseline"]
-#  │         └── <used variables>
-#  │              ├── GITHUB_PROJECT_ORG
-#  │              ├── GITHUB_PROJECT_NAME
-#  │              ├── AUTOTEST
-#  │              ├── AUTOTEST_COMMIT
-#  │              └── REBASELINE
-#  └── tests
-#       ├── gitlab
-#       │    ├── build_and_test
-#       │    │    ├── <builds and tests a given MFEM spec with uberenv>
-#       │    │    ├── <used files>
-#       │    │    │    ├── tests/uberenv/uberenv.py        [deps mode, cloned]
-#       │    │    │    └── tests/gitlab/get_mfem_uberenv   [deps mode]
-#       │    │    └── <used variables>
-#       │    │         ├── SYS_TYPE
-#       │    │         ├── THREADS      [num. parallel jobs to build MFEM]
-#       │    │         ├── MODULE_LIST  [modules to load]
-#       │    │         ├── CI_JOB_ID
-#       │    │         ├── USE_DEV_SHM
-#       │    │         ├── SPACK_DEBUG
-#       │    │         ├── DEBUG_MODE
-#       │    │         ├── REGISTRY_TOKEN
-#       │    │         ├── CI_REGISTRY_USER  (defined by Gitlab)
-#       │    │         ├── USER
-#       │    │         ├── CI_REGISTRY_IMAGE (defined by Gitlab)
-#       │    │         └── CI_JOB_TOKEN      (defined by Gitlab)
-#       │    ├── build_and_test_setup
-#       │    │    ├── <updates MFEM_DATA_REPO and AUTOTEST_REPO using locks>
-#       │    │    └── <used variables>
-#       │    │         ├── MFEM_DATA_REPO
-#       │    │         ├── SHARED_REPOS_DIR
-#       │    │         ├── AUTOTEST_REPO
-#       │    │         └── AUTOTEST_ROOT
-#       │    └── get_mfem_uberenv
-#       │         ├── <github.com/mfem/mfem-uberenv.git -> tests/uberenv>
-#       │         └── <defines the uberenv hash to use>
-#       └── uberenv  [cloned by tests/gitlab/get_mfem_uberenv]
-#            └── uberenv.py
-#
-# <root of radiuss/radiuss-shared-ci, ref: 'v2025.09.1'>
-#  └── pipelines
-#       ├── matrix.yml
-#       │    ├── <jobs>
-#       │    │    ├── .on_matrix
-#       │    │    ├── .matrix_reproducer_init
-#       │    │    ├── .matrix_reproducer_vars
-#       │    │    ├── .matrix_reproducer_job
-#       │    │    ├── .matrix_job_command
-#       │    │    └── .job_on_matrix
-#       │    ├── <used jobs>
-#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#       │    └── <used variables>
-#       │         ├── ON_MATRIX
-#       │         ├── ADVANCED_JOB
-#       │         ├── ALL_TARGETS
-#       │         ├── SYS_TYPE
-#       │         ├── LLNL_SERVICE_USER
-#       │         ├── USER
-#       │         ├── GITHUB_PROJECT_NAME
-#       │         ├── GITHUB_PROJECT_ORG
-#       │         ├── MATRIX_JOB_ALLOC
-#       │         └── JOB_CMD
-#       ├── dane.yml
-#       │    ├── <jobs>
-#       │    │    ├── .on_dane
-#       │    │    ├── .dane_reproducer_init
-#       │    │    ├── .dane_reproducer_vars
-#       │    │    ├── .dane_reproducer_job
-#       │    │    ├── .dane_job_command
-#       │    │    ├── .job_on_dane
-#       │    │    ├── allocate_resources
-#       │    │    └── release_resources
-#       │    ├── <used jobs>
-#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#       │    ├── <defined variables>
-#       │    │    └── export JOBID
-#       │    └── <used variables>
-#       │         ├── ON_DANE
-#       │         ├── ADVANCED_JOB
-#       │         ├── ALL_TARGETS
-#       │         ├── SYS_TYPE
-#       │         ├── LLNL_SERVICE_USER
-#       │         ├── USER
-#       │         ├── GITHUB_PROJECT_NAME
-#       │         ├── GITHUB_PROJECT_ORG
-#       │         ├── DANE_JOB_ALLOC
-#       │         ├── JOB_CMD
-#       │         ├── JOBID
-#       │         ├── ALLOC_NAME
-#       │         └── DANE_SHARED_ALLOC
-#       └── tioga.yml
-#            ├── <jobs>
-#            │    ├── .on_tioga
-#            │    ├── .tioga_reproducer_init
-#            │    ├── .tioga_reproducer_vars
-#            │    ├── .tioga_reproducer_job
-#            │    ├── .tioga_job_command
-#            │    ├── .job_on_tioga
-#            │    ├── allocate_resources
-#            │    └── release_resources
-#            ├── <used jobs>
-#            │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#            ├── <defined variables>
-#            │    └── PROXY
-#            └── <used variables>
-#                 ├── ON_TIOGA
-#                 ├── ADVANCED_JOB
-#                 ├── ALL_TARGETS
-#                 ├── SYS_TYPE
-#                 ├── LLNL_SERVICE_USER
-#                 ├── USER
-#                 ├── GITHUB_PROJECT_NAME
-#                 ├── GITHUB_PROJECT_ORG
-#                 ├── TIOGA_JOB_ALLOC
-#                 ├── JOB_CMD
-#                 ├── PROXY
-#                 ├── ALLOC_NAME
-#                 └── TIOGA_SHARED_ALLOC
-
-###############################################################################
-# We define the following GitLab pipeline variables:
-variables:
-##### LC GITLAB CONFIGURATION
-  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
-
-##### PROJECT VARIABLES
-  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
-  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
-  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
-  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-# AUTOTEST: enable (ON/YES) or disable (any other value) test reporting. See
-# also AUTOTEST_COMMIT.
-  AUTOTEST: "OFF"
-# AUTOTEST_COMMIT: used only when AUTOTEST is set to ON/YES.
-# * If AUTOTEST_COMMIT is set to ON/YES, reporting jobs will commit their
-#   files to the MFEM/autotest repo.
-# * If AUTOTEST_COMMIT is NOT set to ON/YES, reporting jobs will NOT commit
-#   their files to the MFEM/autotest repo. Instead they will just show the
-#   contents of the report files and remove them.
-  AUTOTEST_COMMIT: "ON"
-# REBASELINE:
-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "OFF"
-
-##### SHARED_CI CONFIGURATION
-# Required information about GitHub repository
-  GITHUB_PROJECT_NAME: "mfem"
-  GITHUB_PROJECT_ORG: "MFEM"
-# Override the pattern describing branches that will skip the "draft PR filter
-# test".  Add protected branches here. See default value in
-# preliminary-ignore-draft-pr.yml.
-#  ALWAYS_RUN_PATTERN: ""
-
-###############################################################################
-##### High level stages
-# We organize the test-pipelines stage with sub-pipelines. Each sub-pipeline
-# corresponds to a test batch on a given machine.
-stages:
-  - prerequisites
-  - test-pipelines
-
-###############################################################################
-# Template for jobs triggering a build-and-test sub-pipeline:
-.build-and-test:
-  stage: test-pipelines
-  variables:
-    # Explicitly pass down values that are not always propagated to child
-    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
-    # interface (project variables).
-    # Note: in some cases, this does not work as expected, e.g. when the
-    # variable is not re-defined in the web interface; in such cases, the child
-    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
-    # AUTOTEST is undefined, even though there is a default value in
-    # .gitlab-ci.yml.
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include:
-      - local: '.gitlab/custom-jobs-and-variables.yml'
-      - project: 'radiuss/radiuss-shared-ci'
-        ref: 'v2025.09.1'
-        file: 'pipelines/${CI_MACHINE}.yml'
-      - artifact: '${CI_MACHINE}-jobs.yml'
-        job: 'generate-job-lists'
-    strategy: depend
-    forward:
-      pipeline_variables: true
-
-###############################################################################
-include:
-  # Sets ID tokens for every job using `default:`
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-  # [Optional] checks preliminary to running the actual CI test
-  #- project: 'radiuss/radiuss-shared-ci'
-  #  ref: 'v2025.09.1'
-  #  file: 'preliminary-ignore-draft-pr.yml'
-  # pipelines subscribed by the project
-  - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/README.md

@@ -1,116 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains most of the GitLab CI configuration. MFEM runs both PR
-and nightly testing on GitLab.
-
-# Structure
-
-## Top level
-
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo. This
-file only defines three stages, a prerequisites one, and two main stages in
-which we trigger several sub-pipelines.
-
-We use sub-pipelines to isolate the test for one combination of `machine`
-and `test type`.
-
-Machines typically include:
-
-* Dane: Intel Sapphire Rapids
-* Matrix: Intel Sapphire Rapids + Nvidia H100 GPU
-* Tioga: AMD MI250X GPU
-
-Test types include:
-
-* Build and test: Spack driven build of dependencies, mfem build, mfem
-  test
-* Baseline: Script driven build of dependencies, thorough testing
-
-⚠️ The sub-pipeline design allows to add a new machine or a new test type without
-altering the scheduling, execution and displaying of the others.
-
-## Sub-pipelines
-
-### build-and-test
-
-The build-and-test sub-pipelines leverage RADIUSS Shared CI to share most of
-the CI implementation. RADIUSS Shared CI provides a shared CI infrastructure
-vetted on most LC systems of interest and efficiently leveraging each machine
-scheduler to increase CI throughput. The maintenance of RADIUSS Shared CI is
-shared among several RADIUSS projects.
-
-Jobs for the build-and-test sub-pipelines are defined in the jobs directory.
-Because build-and-test jobs leverage Uberenv and Spack to build the
-dependencies automatically, the jobs essentially consists in a `spack spec`
-defined in the jobs files, and some scheduling parameters defined in the
-`.gitlab/custom-jobs-and-variables.yml` file.
-
-Build-and-test jobs all run the `tests/gitlab/build_and_test` script.
-
-The build-and-test pipelines are controlled by the
-`.gitlab/subscribed-pipelines.yml` which defines which machines to run on and
-implements additional features like machine availability check, and job list
-generation.
-
-### baseline
-
-Baseline sub-pipelines are described by files with names reflecting the
-machine it runs on, e.g. `dane-baseline`.
-
-Those files define the *stages* and the *jobs* for the sub-pipeline. They
-also contain any configuration that cannot be shared. For the most part
-though, the configuration is shared and is placed in `.gitlab/configs`.
-
-We try to keep scripts out of the CI config and share them among similar
-jobs. They are gathered in `.gitlab/scripts`.
-
-## Scripts
-
-Scripts specific to the CI only are in `.gitlab/scripts`. It is best practice
-to keep scripts outside the CI configuration (no bash scripts embedded in a
-yaml file) because it helps with readability, maintenance and also with
-transition to another CI system.
-
-⚠️ Most of the scripts there are driven by environment variables and do not have a
-usage function. This should be improved.
-
-
-# More testing
-
-## Adding a new target to a build-and-test pipeline
-
-`build-and-test` pipelines rely on Spack to install dependencies. Spack is
-driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to a specific commit in Spack and isolate its configuration so that it is
-not influenced by the user environment. More documentation about this can be
-found in `tests/gitlab`.
-
-In the end, the MFEM target for which to build the dependencies is expressed
-with a spack spec of MFEM, within the limits permitted by the MFEM spack
-package.
-
-In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on Dane for example resumes to:
-
-```yaml
-<job_name>:
-  variables:
-    SPEC: "<spack_spec>"
-  extends: .job_on_dane
-```
-
-The remaining and non trivial work is to make sure this spec is working. To
-test a spec before adding it, or reproduce a CI configuration, please refer to
-`tests/gitlab/reproduce-ci-jobs-interactively.md`.
-
-⚠️ It is assumed that the spack spec applies to `mfem@develop`. That's why in the
-CI all the specs start with the compiler or the variants to apply to mfem. The
-mechanism still works with a full spec.

.gitlab/configs/setup-baseline.yml

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The setup_baseline job doesn't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO:
-# updating tests and tpls is not necessary anymore since pipelines are now
-# using unique directories so repo are never shared with another pipeline. This
-# is not memory efficient (we keep a lot of data), hence this reminder.
-setup_baseline:
-  tags:
-    - shell
-    - dane
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup ${BUILD_ROOT}/tpls and ${BUILD_ROOT}/tests:
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "REBASELINE = ${REBASELINE}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "BUILD_ROOT ${BUILD_ROOT}"
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
-    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
-    - cd tpls && git pull && cd ..
-    - cd tests && git pull origin && cd ..
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/custom-jobs-and-variables.yml

@@ -1,132 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
-# We define the following GitLab pipeline variables:
-variables:
-# Set the build-and-test command.
-# Nested variables are allowed and useful to customize the job command. We
-# protect variables with quotes so that their value may remain a string even if
-# they contain whitespaces.
-  JOB_CMD:
-    value: tests/gitlab/build_and_test --spec \"${SPEC}\" --data-dir ${MFEM_DATA_DIR} --data
-# The path to the shared resource between all jobs in the 'dane-baseline'
-# pipeline. For example, external repositories like 'tests' and 'tpls' are
-# cloned here. Also, 'tpls' is built once for all targets, so that build happens
-# here. The BUILD_ROOT is unique to the pipeline, preventing any form of
-# concurrency with other pipelines. This directory is removed by the 'cleanup'
-# stage in the 'dane-baseline' pipeline.
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${CI_MACHINE}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's dane and tioga, the 'build-and-test' pipelines creates only one
-# allocation shared among jobs in the pipeline in order to save time and
-# resources. This allocation has to be uniquely named so that we are sure to
-# retrieve it and avoid collisions.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipelines:
-# - TPLS_REPO and TESTS_REPO are used only by the 'dane-baseline' pipeline
-# - AUTOTEST_REPO is used by all pipelines
-# - MFEM_DATA_REPO is used only by the 'build-and-test' pipelines
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts:
-# - ARTIFACTS_DIR is only used by the 'dane-baseline' pipeline
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
-
-# Dane
-# Arguments for top level allocation
-  DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1"
-# Arguments for job level allocation
-# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
-  DANE_JOB_ALLOC: "--reservation=ci --overlap --nodes=1"
-
-# Tioga
-# Arguments for top level allocation
-  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=45m --nodes=1"
-# Arguments for job level allocation
-  TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
-
-# Matrix
-# Arguments for top level allocation
-  MATRIX_SHARED_ALLOC: "-p pdebug --exclusive --time=45 --nodes=1 -G 4"
-# Arguments for job level allocation
-# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
-  MATRIX_JOB_ALLOC: "--overlap --nodes=1"
-
-# Configuration shared by build and test jobs specific to this project.
-# Not all configuration can be shared. Here projects can fine tune the
-# CI behavior.
-# See Umpire for an example (export junit test reports).
-.custom_job:
-  artifacts:
-    reports:
-
-# Note: this part is not used by the 'dane-baseline' pipeline.
-# FIXME: BUILD_ROOT, TPLS_REPO, TESTS_REPO are not needed here.
-#        Also, the definition of SHARED_REPOS_DIR is wrong.
-.reproducer_vars:
-  script:
-    - |
-      echo -e "
-      # Variables \n
-      export SPEC=\"${SPEC//\"/\\\"}\" \n
-      # Directories \n
-      export BUILD_ROOT=\"\${working_dir}\" \n
-      export SHARED_REPOS_DIR=\"\${BUILD_ROOT}/..\" \n
-      export MFEM_DATA_DIR=\"\${SHARED_REPOS_DIR}/mfem-data\" \n
-      # Repositories \n
-      export TPLS_REPO=\"${TPLS_REPO//\"/\\\"}\" \n
-      export TESTS_REPO=\"${TESTS_REPO//\"/\\\"}\" \n
-      export AUTOTEST_REPO=\"${AUTOTEST_REPO//\"/\\\"}\" \n
-      export MFEM_DATA_REPO=\"${MFEM_DATA_REPO//\"/\\\"}\" \n
-      # Setup directories \n
-      ./tests/gitlab/build_and_test_setup \n
-      # Using the CI build cache is optional and requires a token. Set it like so: \n
-      # export REGISTRY_TOKEN=\"<your token here>\" \n"
-      #
-
-# Jobs report
-.report_job_success:
-  script:
-    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test SUCCESS
-  rules:
-    - when: on_success
-
-.report_job_failure:
-  script:
-    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test FAILURE
-  rules:
-    - when: on_failure
-
-# Keep the following for debugging purposes: renaming this job from
-# '.show_variables' to 'show_variables' will insert this debug job at the
-# beginning of all child pipelines.
-.show_variables:
-  tags: [shell, oslic]
-  variables:
-    GIT_STRATEGY: none
-  stage: .pre
-  script:
-    - |
-      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-      echo "AUTOTEST=${AUTOTEST}"
-      echo "AUTOTEST_COMMIT=${AUTOTEST_COMMIT}"
-      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-      # Fail the job on purpose to prevent the rest of the pipeline from running
-      false

.gitlab/dane-baseline.yml

@@ -1,161 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-variables:
-  BASELINE_TEST: baseline
-  MACHINE_NAME: dane
-
-stages:
-  - setup
-  - baseline_check
-  - baseline_report
-  - cleanup
-  - baseline_publish
-
-.on_dane:
-  tags:
-    - shell
-    - dane
-  rules:
-    # Don't run dane jobs if...
-    - if: '$ON_DANE == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    # Note: in some cases, the content of AUTOTEST can be '${AUTOTEST}', so we
-    # need to treat that value as the default value of 'OFF'.
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "ON" && $AUTOTEST != "YES"'
-      when: never
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-baselinecheck_mfem_intel_dane:
-  extends: [.on_dane]
-  stage: baseline_check
-  variables:
-    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
-    # when call the runtest script in MFEM test repo.
-    # Note: the value must be consistent with the setup performed in
-    #       .gitlab/configs/setup-baseline.yml.
-    TPLS_DIR: ${BUILD_ROOT}/tpls
-  script:
-    - echo "AUTOTEST=$AUTOTEST"
-    - echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-    - echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
-    - echo ${BUILD_ROOT}
-    - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests, dane has 224 threads/node:
-    - export MFEM_TEST_NP=192
-    # The next script uses the following environment variables:
-    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
-    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
-    - .gitlab/scripts/baseline
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-cleanup:
-  extends: .on_dane
-  stage: cleanup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - echo "BUILD_ROOT=${BUILD_ROOT}"
-    - rm -rf "${BUILD_ROOT}" || true
-
-report_baseline:
-  extends: [.on_dane]
-  stage: baseline_report
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # ----------------------
-        cd ${AUTOTEST_ROOT}/autotest || \
-          { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-        mkdir -p ${MACHINE_NAME}
-        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
-        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
-          >> ${rundir}/pipeline.txt
-        # We create an autotest-email.html file, because that's how we signal
-        # that there was an error / diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
-          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
-        fi
-        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        # Note: in some cases, the content of AUTOTEST_COMMIT can be
-        # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
-        # value of 'ON'.
-        if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
-          AUTOTEST_COMMIT="ON"
-        fi
-        if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
-          git pull && \
-          git add ${rundir} && \
-          git commit -m "${msg}" && \
-          ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-        else
-          for file in ${rundir}/*; do
-            echo "------------------------------"
-            echo "Content of '$file'"
-            echo "******************************"
-            cat $file
-            echo "******************************"
-          done
-          rm -rf ${rundir} || true
-        fi
-        err=$?
-        # ----------------------
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-baselinepublish_mfem_dane:
-  extends: [.on_dane]
-  stage: baseline_publish
-  rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "ON"'
-    - if: '$REBASELINE == "ON"'
-      when: manual
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${PWD}
-    - echo ${ARTIFACTS_DIR}
-    - ls -lA ${ARTIFACTS_DIR}
-    - .gitlab/scripts/rebaseline
-
-include:
-  - local: .gitlab/custom-jobs-and-variables.yml
-  - local: .gitlab/configs/setup-baseline.yml

.gitlab/jobs/dane-reports.yml

@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_dane, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_dane, .report_job_failure]
-  stage: jobs-stage-3

.gitlab/jobs/dane.yml

@@ -1,87 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define MFEM specific variables.
-.dane_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-# TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY
-#       cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_dane
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs, we can duplicate them here to override description and
-# add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-.mfem_job_on_dane:
-  extends: .job_on_dane
-  stage: jobs-stage-2
-  variables:
-    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
-    THREADS: 28
-
-debug_ser_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +debug~mpi"
-
-debug_par_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +debug+mpi"
-
-opt_ser_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 ~mpi"
-
-opt_par_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1"
-
-opt_par_gcc_10_sundials:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +sundials"
-
-opt_par_gcc_10_petsc:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-
-opt_par_gcc_10_pumi:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +pumi"

.gitlab/jobs/matrix-reports.yml

@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_matrix, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_matrix, .report_job_failure]
-  stage: jobs-stage-3

.gitlab/jobs/matrix.yml

@@ -1,65 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define UMPIRE specific variables.
-.matrix_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_matrix
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs , we can duplicate them here to override description and add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-.mfem_job_on_matrix:
-  extends: .job_on_matrix
-  stage: jobs-stage-2
-  variables:
-    # We run 2 jobs on 1 node that has 112 threads
-    THREADS: 48
-    # These modules need to be consistent with the uberenv configurations:
-    MODULE_LIST: "gcc/10.3.1-magic cuda/12.9.1"
-
-allocate_resources:
-  timeout: 4h
-
-opt_mpi_cuda_gcc:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90"
-
-opt_mpi_cuda_hypre_cuda_gcc:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"

.gitlab/jobs/tioga-reports.yml

@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_tioga, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_tioga, .report_job_failure]
-  stage: jobs-stage-3
-

.gitlab/jobs/tioga.yml

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define UMPIRE specific variables.
-.tioga_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_tioga
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs , we can duplicate them here to override description and add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-# Build and test jobs, simply provide a spec
-
-#.tioga_job_command:
-#  script:
-#    - echo PROXY="${PROXY}"
-#    - echo TIOGA_JOB_ALLOC="${TIOGA_JOB_ALLOC}"
-#    - "printf '#!/bin/bash\n%s\n' \"${JOB_CMD}\" > flux_script.sh"
-#    - cat flux_script.sh
-#    - ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${TIOGA_JOB_ALLOC} flux_script.sh )
-#    - rm -f flux_script.sh
-
-.mfem_job_on_tioga:
-  extends: .job_on_tioga
-  stage: jobs-stage-2
-  variables:
-    # We run 1 job on 1 node that has 64 threads
-    THREADS: 64
-
-opt_mpi_rocm_hypre_rocm:
-  extends: .mfem_job_on_tioga
-  variables:
-    SPEC: "%rocmcc@=6.3.1 +rocm amdgpu_target=gfx90a ^hypre+rocm"
-
-# cce_16_0_1:
-#   extends: .mfem_job_on_tioga
-#   variables:
-#     SPEC: "%cce@=16.0.1"

.gitlab/scripts/baseline

@@ -1,76 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# locals
-glob_err=${BASELINE_TEST}.err
-base=${BASELINE_TEST}-${SYS_TYPE}
-if [[ "${MACHINE_NAME}" == "dane" ]]; then
-  base="${BASELINE_TEST}-${MACHINE_NAME}"
-fi
-base_diff=${base}.diff
-base_patch=${base}.patch
-base_out=${base}.out
-artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
-
-# prepare
-cd ${BUILD_ROOT} || \
-  { echo "Invalid BUILD_ROOT=$BUILD_ROOT"; exit 1; }
-ln -snf ${CI_PROJECT_DIR} mfem
-cd tests
-[[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
-mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
-
-# run
-if [[ "${MACHINE_NAME}" == "dane" ]]; then
-  salloc --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-else
-  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
-  exit 1
-fi
-status="$?"
-
-# post
-mkdir ${artifacts_path}
-
-if [[ -f ${BASELINE_TEST}.out ]]; then
-  cp ${BASELINE_TEST}.out ${artifacts_path}
-fi
-if [[ -s ${glob_err} ]]; then
-  echo "ERROR during ${BASELINE_TEST} execution"
-  echo "Here is the ${glob_err} file content"
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  status=1
-fi
-if [[ -f ${base_patch} ]]; then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]; then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-# base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]; then
-  echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
-  cat ${base_diff}
-  cp ${base_diff} ${artifacts_path}/${base_diff}
-  status=1
-fi
-if [[ $status -eq 0 ]]; then
-  echo "${BASELINE_TEST}: PASSED"
-else
-  echo "${BASELINE_TEST}: FAILED"
-fi
-exit $status

.gitlab/scripts/git_try_to_push

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Try to push to the remote 5 times. If the push fails, and the local and remote
-# have diverged, then pull from the remote to merge changes, and try pushing
-# again. If some other failure happens
-for i in {1..5}; do
-   git push origin master && exit 0
-   # Wait for 20 seconds in case someone else is pushing to the remote
-   # concurrently
-   sleep 20
-   # Fetch any updates from the remote
-   git remote update
-   # Get the latest commit on the local branch
-   LOCAL=$(git rev-parse @)
-   # Get the latest commit on the remote
-   REMOTE=$(git rev-parse @{u})
-   # Get the common ancestor
-   BASE=$(git merge-base @ @{u})
-   # Have the local and remote diverged?
-   if [[ $LOCAL != $REMOTE && $LOCAL != $BASE && $REMOTE != $BASE ]]; then
-      git pull
-      if [[ $? == 0 ]]; then
-         continue
-      else
-         exit 1 # Something else went wrong trying to pull
-      fi
-   fi
-done
-
-exit 1 # Did not succeed in 5 attempts

.gitlab/scripts/rebaseline

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# There will be collision between corona and dane baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
-#ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}-${MACHINE_NAME}
-
-PATCH_FILE=${ARTIFACT_PATH}.patch
-FULL_FILE=${ARTIFACT_PATH}.out
-DIFF_FILE=${ARTIFACT_PATH}.diff
-
-# There will be collision between corona and dane baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-SAVED_NAME=baseline-${SYS_TYPE}.saved
-#SAVED_NAME=baseline-${SYS_TYPE}-${MACHINE_NAME}.saved
-
-cd ${BUILD_ROOT}/tests
-
-if [[ ! -f "${DIFF_FILE}" ]]
-then
-    echo "Nothing to be done: no relevant change in baseline"
-    exit 0
-elif [[ -f "${PATCH_FILE}" ]]
-then
-    patch "${SAVED_NAME}" < "${PATCH_FILE}"
-elif [[ -f "${FULL_FILE}" ]]
-then
-    cp "${FULL_FILE}" "${SAVED_NAME}"
-else
-    echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
-    exit 1
-fi
-
-git add "${SAVED_NAME}"
-git commit -m "${SYS_TYPE} (${MACHINE_NAME}) rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
-git push origin master

.gitlab/scripts/report_build_and_test

@@ -1,118 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-function info_msg ()
-{
-  echo "[Information:] ${1}"
-}
-
-function error_msg ()
-{
-  echo "[Error:] ${1}"
-}
-
-# Perform a report while holding a lock file to prevent concurrency on
-# the destination.
-# Usage:
-# locked_clone <report_function> <lock_name>
-function locked_report ()
-{
-  if ! command -v flock
-  then
-    error_msg "Required command 'flock' not found"
-    exit 1
-  fi
-
-  info_msg "Will report ${1} while holding a lock in ${2}"
-
-  ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..."
-    # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
-    # try every 5 seconds; we may want to add a counter for the number of
-    # retries to interrupt a potential infinite loop
-    while ! flock -n 9; do sleep 5; done
-    date; info_msg "Acquired lock on '${PWD}/${2}.lock'"
-
-    report ${1}
-    err=$?
-
-    # sleep for a period to allow NFS to propagate the above changes;
-    # clearly, there is no guarantee that other NFS clients will see the
-    # changes even after the timeout
-    sleep 10
-    exit $err
-  ) 9> ${2}.lock
-}
-
-function report ()
-{
-  if [[ "${1}" == "SUCCESS" ]]
-  then
-    info_msg "All the ${MACHINE_NAME} jobs passed"
-    status_msg="The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL."
-  elif [[ "${1}" == "FAILURE" ]]
-  then
-    info_msg "At least one failure on ${MACHINE_NAME}"
-    status_msg="Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED."
-  else
-    error_msg "Unknown status: ${1} ... aborting"
-    exit 1
-  fi
-
-  cd ${AUTOTEST_ROOT}/autotest || \
-    { error_msg "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-  mkdir -p ${MACHINE_NAME}
-
-  rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-  rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-  printf "%s\n" "${status_msg}" \
-    "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-  msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-  if [[ "${1}" == "FAILURE" ]]
-  then
-    # Create 'autotest-email.html' to indicate failure:
-    cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-  fi
-
-  # Note: in some cases, the content of AUTOTEST_COMMIT can be
-  # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
-  # value of 'ON'.
-  if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
-    AUTOTEST_COMMIT="ON"
-  fi
-  if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
-    git pull && \
-    git add ${rundir} && \
-    git commit -m "${msg}" && \
-    ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-  else
-    for file in ${rundir}/*; do
-      echo "------------------------------"
-      echo "Content of '$file'"
-      echo "******************************"
-      cat $file
-      echo "******************************"
-    done
-    rm -rf ${rundir} || true
-  fi
-}
-
-export MACHINE_NAME=${CI_MACHINE}
-info_msg "MACHINE_NAME is ${MACHINE_NAME}"
-info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}"
-info_msg "AUTOTEST=$AUTOTEST"
-info_msg "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-
-cd ${AUTOTEST_ROOT} && locked_report ${1} autotest

.gitlab/scripts/safe_create_rundir

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# This script takes a seed for a directory name and appends it with a counter
-# incremented until it can create a new directory with it.
-
-# Usage:
-#
-# Expects 1 argument: a string that is use as a seed for the directory name.
-#
-# > rundir="desired_name"
-# > rundir=$(./safe_create_rundir $rundir)
-
-set -o errexit
-set -o nounset
-
-rundir=${1:-""}
-if [[ -z ${rundir} ]]; then
-  >&2 echo "The script expects a string as argument for directory creation."
-  exit 1
-fi
-
-if ! mkdir ${rundir}; then
-  n=1
-  while ! mkdir ${rundir}_${n}
-  do
-    n=$((n+1))
-  done
-  rundir=${rundir}_${n}
-fi
-
-echo $rundir

.gitlab/subscribed-pipelines.yml

@@ -1,130 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The template job to test whether a machine is up.
-# Expects CI_MACHINE defined to machine name.
-.machine-check:
-  stage: prerequisites
-  tags: [shell, oslic]
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - |
-      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
-      then
-        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
-        false && \
-        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
-             --header 'Content-Type: application/json' \
-             --header "authorization: Bearer ${GITHUB_TOKEN}" \
-             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
-        exit 1
-      fi
-
-###
-# Trigger a build-and-test pipeline for a machine.
-# Comment the jobs for machines you don’t need.
-###
-
-# One job to generate the job list for all the subpipelines
-generate-job-lists:
-  stage: prerequisites
-  tags: [shell, oslic]
-  variables:
-    LOCAL_JOBS_PATH: ".gitlab/jobs"
-  script:
-    - |
-      echo "AUTOTEST=$AUTOTEST"
-      echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-      echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
-    - |
-      cat ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/dane-reports.yml >> dane-jobs.yml
-      fi
-    - |
-      cat ${LOCAL_JOBS_PATH}/matrix.yml > matrix-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/matrix-reports.yml >> matrix-jobs.yml
-      fi
-    - |
-      cat ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/tioga-reports.yml >> tioga-jobs.yml
-      fi
-  artifacts:
-    paths:
-      - dane-jobs.yml
-      - matrix-jobs.yml
-      - tioga-jobs.yml
-
-
-# DANE
-dane-up-check:
-  variables:
-    CI_MACHINE: "dane"
-  extends: [.machine-check]
-
-dane-build-and-test:
-  variables:
-    CI_MACHINE: "dane"
-  needs: [dane-up-check, generate-job-lists]
-  extends: [.build-and-test]
-
-# DANE, MFEM Specific
-dane-baseline:
-  stage: test-pipelines
-  variables:
-    # Explicitly pass down values that are not always propagated to child
-    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
-    # interface (project variables).
-    # Note: in some cases, this does not work as expected, e.g. when the
-    # variable is not re-defined in the web interface; in such cases, the child
-    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
-    # AUTOTEST is undefined, even though there is a default value in
-    # .gitlab-ci.yml.
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/dane-baseline.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs: [dane-up-check]
-
-
-# TIOGA
-tioga-up-check:
-  variables:
-    CI_MACHINE: "tioga"
-  extends: [.machine-check]
-
-tioga-build-and-test:
-  variables:
-    CI_MACHINE: "tioga"
-  needs: [tioga-up-check, generate-job-lists]
-  extends: [.build-and-test]
-
-
-# Matrix
-matrix-up-check:
-  variables:
-    CI_MACHINE: "matrix"
-  extends: [.machine-check]
-
-matrix-build-and-test:
-  variables:
-    CI_MACHINE: "matrix"
-  needs: [matrix-up-check, generate-job-lists]
-  extends: [.build-and-test]

.travis.yml

@@ -0,0 +1,263 @@
+sudo: false
+
+language: cpp
+
+matrix:
+  include:
+    #
+    # Linux
+    #
+    - os: linux
+      compiler: gcc
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+    #
+    - os: linux
+      compiler: gcc
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+    #
+    - os: linux
+      compiler: gcc
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=2
+      cache:
+        directories:
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
+    #
+    - os: linux
+      compiler: gcc
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=2
+      cache:
+        directories:
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
+    #
+    # Mac OS X
+    #
+    - os: osx
+      # osx_image: xcode7.3
+      compiler: clang
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+    #
+    - os: osx
+      # osx_image: xcode7.3
+      compiler: clang
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+    #
+    - os: osx
+      # osx_image: xcode7.3
+      compiler: clang
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        directories:
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
+    #
+    - os: osx
+      # osx_image: xcode7.3
+      compiler: clang
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        directories:
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
+
+
+before_install:
+   # No addon for brew yet, have to install OSX packages this way.
+   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+   #      brew install open-mpi;
+   #   fi
+
+   # On Mac OS X, build and cache OpenMPI 2.1.1:
+   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
+           mkdir -p $HOME/builds && cd $HOME/builds &&
+           wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2 &&
+           tar jxf openmpi-2.1.1.tar.bz2 &&
+           mkdir openmpi-build && cd openmpi-build &&
+           ../openmpi-2.1.1/configure --prefix=$HOME/local-cached &&
+           make -j3 all && make install;
+        fi;
+        PATH=$HOME/local-cached/bin:$PATH;
+        cd $TRAVIS_BUILD_DIR;
+     fi
+
+   # Update environment to find g++ 4.9 installation first.
+   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
+   #      mkdir -p latest-gcc-symlinks;
+   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
+   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
+   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
+   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
+   #   fi
+
+   # Install tool to upload code coverage reports to coveralls.io
+   - if [ "$CODECOV" == "YES" ]; then
+        export PYTHONUSERBASE=$HOME/local;
+        pip install --user cpp-coveralls;
+        pip install --user pyyaml;
+        PATH=$HOME/local/bin:$PATH;
+     fi
+
+install:
+   # Set MPI compilers, print compiler version
+   - if [ $MPI == "YES" ]; then
+        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+           export MPICH_CC="$CC";
+           export MPICH_CXX="$CXX";
+        else
+           export OMPI_CC="$CC";
+           export OMPI_CXX="$CXX";
+           mpic++ --showme:version;
+        fi;
+        mpic++ -v;
+     else
+        $CXX -v;
+     fi
+
+   # Back out of the mfem directory to install the libraries
+   - cd ..
+
+   # hypre
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e hypre-2.10.0b/src/hypre/lib/libHYPRE.a ]; then
+           wget https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz --no-check-certificate;
+           rm -rf hypre-2.10.0b;
+           tar xvzf hypre-2.10.0b.tar.gz;
+           cd hypre-2.10.0b/src;
+           ./configure --disable-fortran --without-fei CC=mpicc CXX=mpic++;
+           make -j3;
+           cd ../..;
+        else
+            echo "Reusing cached hypre-2.10.0b/";
+        fi;
+     else
+        echo "Serial build, not using hypre";
+     fi
+
+   # METIS
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e metis-4.0/libmetis.a ]; then
+           wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz;
+           tar xvzf metis-4.0.3.tar.gz;
+           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
+           rm -rf metis-4.0;
+           mv metis-4.0.3 metis-4.0;
+        else
+           echo "Reusing cached metis-4.0/";
+        fi;
+     fi
+
+script:
+   # Compiler
+   - if [ $MPI == "YES" ]; then
+        export MYCXX=mpic++;
+     else
+        export MYCXX="$CXX";
+     fi
+
+   # Print the compiler version
+   - $MYCXX -v
+
+   # Set some variables
+   - cd $TRAVIS_BUILD_DIR;
+     CPPFLAGS="";
+     SKIP_TEST_DIRS="";
+     if [ "$CODECOV" == "YES" ]; then
+        CPPFLAGS="--coverage -g";
+     fi;
+     if [ "$CXX" == "clang++" ]; then
+        export MFEM_PERF_SW=clang;
+     fi
+
+   # Configure the library
+   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG MFEM_CXX="$MYCXX"
+        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
+   # Show the configuration
+   - make info
+   # Build the library
+   - make -j3
+   # Build the examples and the miniapps
+   - make -j3 all
+   # Run tests
+   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
+
+after_success:
+   - if [ "$CODECOV" == "YES" ]; then
+        coveralls --include fem --include general --include linalg --include
+           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
+     fi

CITATION.cff

@@ -1,66 +0,0 @@
-cff-version: 1.2.0
-message: "If you use MFEM, please cite it as follows."
-authors:
-- family-names: "MFEM Team"
-title: "MFEM: Modular Finite Element Methods [Software]"
-doi: 10.11578/dc.20171025.1248
-url: "https://mfem.org"
-preferred-citation:
-  type: article
-  authors:
-  - family-names: "Anderson"
-    given-names: "Robert"
-    orcid: "https://orcid.org/0000-0002-3508-9944"
-  - family-names: "Andrej"
-    given-names: "Julian"
-    orcid: "https://orcid.org/0000-0001-7661-4840"
-  - family-names: "Barker"
-    given-names: "Andrew"
-    orcid: "https://orcid.org/0000-0003-3572-911X"
-  - family-names: "Bramwell"
-    given-names: "Jamie"
-  - family-names: "Camier"
-    given-names: "Jean-Sylvain"
-    orcid: "https://orcid.org/0000-0003-2421-1999"
-  - family-names: "Cerveny"
-    given-names: "Jakub"
-    orcid: "https://orcid.org/0000-0003-4231-2531"
-  - family-names: "Dobrev"
-    given-names: "Veselin"
-    orcid: "https://orcid.org/0000-0003-1793-5622"
-  - family-names: "Dudouit"
-    given-names: "Yohann"
-    orcid: "https://orcid.org/0000-0001-5831-561X"
-  - family-names: "Fisher"
-    given-names: "Aaron"
-  - family-names: "Kolev"
-    given-names: "Tzanio"
-    orcid: "https://orcid.org/0000-0002-2810-3090"
-  - family-names: "Pazner"
-    given-names: "Will"
-    orcid: "https://orcid.org/0000-0003-4885-2934"
-  - family-names: "Stowell"
-    given-names: "Mark"
-    orcid: "https://orcid.org/0000-0002-5389-7435"
-  - family-names: "Tomov"
-    given-names: "Vladimir"
-    orcid: "https://orcid.org/0000-0002-1846-6816"
-  - family-names: "Akkerman"
-    given-names: "Ido"
-    orcid: "https://orcid.org/0000-0002-5937-0300"
-  - family-names: "Dahm"
-    given-names: "Johann"
-    orcid: "https://orcid.org/0000-0001-9657-3564"
-  - family-names: "Medina"
-    given-names: "David"
-  - family-names: "Zampini"
-    given-names: "Stefano"
-    orcid: "https://orcid.org/0000-0002-0435-0433"
-  doi: "10.1016/j.camwa.2020.06.009"
-  journal: "Computers \\& Mathematics with Applications"
-  month: 1
-  start: 42 # First page number
-  end: 74 # Last page number
-  title: "MFEM: A Modular Finite Element Methods Library"
-  volume: 81
-  year: 2021

CMakeLists.txt

@@ -1,37 +1,23 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
 
-# The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-# Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
-# Version 3.8 or newer is required for direct CUDA support.
-# Version 3.12 or newer is required for setting maximum policy version.
-cmake_minimum_required(VERSION 3.12.0...4.0.0)
-message(STATUS "CMake version: ${CMAKE_VERSION}")
+cmake_minimum_required(VERSION 2.8.11)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
-# Require C++17 and disable compiler-specific extensions
-set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use.")
-set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL
-  "Force the use of the chosen C++ standard.")
-set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Enable C++ standard extensions.")
-
 # Load user settings before the defaults - this way the defaults will not
 # overwrite the user set options. If the user has not set all options, we still
 # have the defaults.
-include("${USER_CONFIG}" OPTIONAL RESULT_VARIABLE USER_CONFIG_LOADED)
-if (USER_CONFIG_LOADED)
-  set(USER_CONFIG_LOADED "LOADED")
-endif()
-message(STATUS "Loading USER_CONFIG = ${USER_CONFIG} (${USER_CONFIG_LOADED})")
+message(STATUS "(optional) USER_CONFIG = ${USER_CONFIG}")
+include("${USER_CONFIG}" OPTIONAL)
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/defaults.cmake")
 
 # Allow overwriting of the compiler by setting CXX/MPICXX on the command line or
@@ -59,7 +45,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.9.1)
+set(${PROJECT_NAME}_VERSION 3.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -74,7 +60,8 @@ set(XSDK_ENABLE_C OFF)
 set(XSDK_ENABLE_Fortran OFF)
 
 # Check if we need to enable C or Fortran.
-if (MFEM_USE_CONDUIT OR
+if (CMAKE_VERSION VERSION_LESS 3.2 OR
+    MFEM_USE_CONDUIT OR
     MFEM_USE_SIDRE OR
     MFEM_USE_PETSC)
    # This seems to be needed by:
@@ -84,113 +71,16 @@ if (MFEM_USE_CONDUIT OR
    #  * find_package(PETSc REQUIRED)
    set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
+if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# Ginkgo requires C++17:
-if ((MFEM_USE_GINKGO) AND ("${CMAKE_CXX_STANDARD}" LESS "17"))
-  set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use." FORCE)
-# Google Benchmark, SUNDIALS, STRUMPACK, Tribol, RAJA and Umpire require C++14:
-elseif ((MFEM_USE_BENCHMARK OR
-         MFEM_USE_SUNDIALS OR
-         MFEM_USE_STRUMPACK OR
-         MFEM_USE_TRIBOL OR
-         MFEM_USE_RAJA OR
-         MFEM_USE_UMPIRE) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
-  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to use." FORCE)
-endif()
 
 # Include xSDK default CMake file.
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 
-# Path to MFEM's CMake modules and utilities.
-set(MFEM_CMAKE_PATH ${PROJECT_SOURCE_DIR}/config)
-set(CMAKE_MODULE_PATH ${MFEM_CMAKE_PATH}/cmake/modules)
-
-# Load MFEM CMake utilities.
-include(MfemCmakeUtilities)
-
 # Enable languages.
 enable_language(CXX)
-if (MINGW)
-   # MinGW GCC does not expose the functions jn/_jn, yn/_yn (used in Example
-   # 25/25p) unless we use '-std=gnu++11':
-   set(CMAKE_CXX_EXTENSIONS ON
-      CACHE BOOL "Enable C++ standard extensions." FORCE)
-endif()
-if (MFEM_USE_CUDA)
-   if (MFEM_USE_HIP)
-      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
-   endif()
-   # CUDAToolkit was added in CMake 3.17, so we require at least CMake 3.17 when
-   # CUDA is enabled:
-   if (CMAKE_VERSION VERSION_LESS 3.17.0)
-      message(FATAL_ERROR "CUDA support requires CMake >= 3.17")
-   endif()
-   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
-   if (NOT CMAKE_CUDA_HOST_COMPILER)
-      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
-   if (NOT CMAKE_CUDA_ARCHITECTURES)
-      # make CUDA_ARCH resemble the same form as CMAKE_CUDA_ARCHITECTURES
-      string(REPLACE "sm_" "" CUDA_ARCH_TMP "${CUDA_ARCH}")
-      string(REPLACE "," ";" CUDA_ARCH "${CUDA_ARCH_TMP}")
-      set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH}")
-   else()
-      set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
-   endif()
-   message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
-   enable_language(CUDA)
-   if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      # backup try to detect if this is clang or nvcc
-      if(CMAKE_CUDA_COMPILER MATCHES "nvcc$")
-         # nvcc
-         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-         set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-         if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-             OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-             OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            set(CUDA_FLAGS "-arch=${CMAKE_CUDA_ARCHITECTURES} ${CUDA_FLAGS}")
-         else()
-            # build -gencode sequence for multiple architectures
-            foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-               set(CUDA_FLAGS
-                   "-gencode arch=compute_${ENTRY},code=sm_${ENTRY} ${CUDA_FLAGS}")
-            endforeach()
-         endif()
-      else()
-         # build cuda-gpu-arch sequence for multiple architectures
-         # does not support all/all-major/native
-         foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-            set(CUDA_FLAGS "-cuda-gpu-arch=sm_${ENTRY} ${CUDA_FLAGS}")
-         endforeach()
-      endif()
-   else()
-      # TODO: all, native, all-major require CMake 3.24+
-      # backport support for CMake 3.18 to 3.24
-      if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
-         # nvcc
-         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-         set(CUDA_FLAGS
-             "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-      endif()
-   endif()
-   set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING
-      "CUDA standard to use.")
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON CACHE BOOL
-      "Force the use of the chosen CUDA standard.")
-   set(CMAKE_CUDA_EXTENSIONS OFF CACHE BOOL "Enable CUDA standard extensions.")
-   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
-   find_package(CUDAToolkit REQUIRED)
-   set(CUSPARSE_FOUND TRUE)
-   set(CUBLAS_FOUND TRUE)
-   # Initialize CUSPARSE_LIBRARIES and CUBLAS_LIBRARIES:
-   mfem_culib_set_libraries(CUSPARSE cusparse)
-   mfem_culib_set_libraries(CUBLAS cublas)
-endif()
-
 if (XSDK_ENABLE_C)
    enable_language(C)
 endif()
@@ -201,6 +91,13 @@ endif()
 # Suppress warnings about MACOSX_RPATH
 set(CMAKE_MACOSX_RPATH OFF CACHE BOOL "")
 
+# CMake needs to know where to find things
+set(MFEM_CMAKE_PATH ${PROJECT_SOURCE_DIR}/config)
+set(CMAKE_MODULE_PATH ${MFEM_CMAKE_PATH}/cmake/modules)
+
+# Load MFEM CMake utilities.
+include(MfemCmakeUtilities)
+
 string(TOUPPER "${PROJECT_NAME}" PROJECT_NAME_UC)
 mfem_version_to_int(${${PROJECT_NAME}_VERSION} ${PROJECT_NAME_UC}_VERSION)
 set(${PROJECT_NAME_UC}_VERSION_STRING ${${PROJECT_NAME}_VERSION})
@@ -219,19 +116,6 @@ endif()
 # Process configuration options
 #-------------------------------------------------------------------------------
 
-# MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
-if (MFEM_PRECISION MATCHES "^(double|Double|DOUBLE)$")
-  set(MFEM_USE_SINGLE OFF)
-  set(MFEM_USE_DOUBLE ON)
-elseif (MFEM_PRECISION MATCHES "^(single|Single|SINGLE)$")
-  set(MFEM_USE_SINGLE ON)
-  set(MFEM_USE_DOUBLE OFF)
-else()
-  message(FATAL_ERROR " *** Invalid floating-point precision: "
-    "MFEM_PRECISION = ${MFEM_PRECISION}")
-endif()
-message(STATUS "Floating-point precision: MFEM_PRECISION = ${MFEM_PRECISION}")
-
 # MFEM_DEBUG
 if (CMAKE_BUILD_TYPE MATCHES "Debug|debug|DEBUG")
   set(MFEM_DEBUG ON)
@@ -239,93 +123,10 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# Shared build on Windows
-if (WIN32 AND BUILD_SHARED_LIBS)
-  # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
-  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON CACHE BOOL
-      "Automatically export symbols for shared Windows DLL build")
-  # Place all .dll and .exe files in the same sub-directory, 'bin/<config>', of
-  # the build directory, so that the .exe files can find the .dll files at
-  # runtime:
-  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
-  # Note: CMake tests are run from the build-tree sub-directory where the test
-  # is defined, e.g. <build-dir>/tests/unit, so hard coded paths to meshes still
-  # work fine.
-  # Not strictly necessary:
-  # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
-  # set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
-endif()
-
-# AMD HIP
-if (MFEM_USE_HIP)
-  if (NOT CMAKE_HIP_ARCHITECTURES)
-    if (HIP_ARCH)
-      set(CMAKE_HIP_ARCHITECTURES CACHE STRING "HIP targets to compile for" "${HIP_ARCH}")
-      set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for" FORCE)
-    endif()
-  else()
-    set(HIP_ARCH CACHE STRING "HIP targets to compile for" "${CMAKE_HIP_ARCHITECTURES}")
-    set(GPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "HIP targets to compile for" FORCE)
-  endif()
-  message(STATUS "Using HIP architecture: ${CMAKE_HIP_ARCHITECTURES}")
-  if (ROCM_PATH)
-    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
-  endif()
-  find_package(HIP REQUIRED)
-  find_package(HIPBLAS REQUIRED)
-  find_package(HIPSPARSE REQUIRED)
-endif()
-
-# OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
-  endif()
-  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
-  if (OPENMP_FOUND)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    if (MFEM_USE_CUDA)
-      if(MFEM_CUDA_COMPILER_IS_NVCC)
-        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-      else()
-        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${OpenMP_CXX_FLAGS}")
-      endif()
-    endif()
-  endif()
-endif()
-
-# Warn user if deprecated FETCH_TPLS is provided
-if (DEFINED FETCH_TPLS)
-  message(STATUS "Setting MFEM_FETCH_TPLS to user-provided value of FETCH_TPLS (i.e., MFEM_FETCH_TPLS=${FETCH_TPLS})")
-  set (MFEM_FETCH_TPLS FETCH_TPLS)
-  message(DEPRECATION "The use of FETCH_TPLS is deprecated and will be removed in future verison. Please use MFEM_FETCH_TPLS instead.")
-endif()
-
-  # Umpire (must be included before hypre, so hypre can use it if needed)
-if (MFEM_USE_UMPIRE)
-   # umpire uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
-   if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
-      cmake_policy(SET CMP0146 OLD)
-   endif()
-   find_package(UMPIRE REQUIRED)
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
   set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
-  if (MFEM_MPIEXEC)
-    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
-  endif()
-  if (MFEM_MPIEXEC_NP)
-    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
-  endif()
   # Parallel MFEM depends on hypre
   find_package(HYPRE REQUIRED)
   set(MFEM_HYPRE_VERSION ${HYPRE_VERSION})
@@ -336,13 +137,9 @@ if (MFEM_USE_MPI)
       message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
     endif()
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
-    if (MFEM_USE_SLEPC)
-      find_package(SLEPc REQUIRED config)
-      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
-    endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU MUMPS PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -355,17 +152,8 @@ if (MFEM_USE_METIS)
   find_package(METIS REQUIRED)
 endif()
 
-if (MFEM_USE_GINKGO)
-  find_package(Ginkgo REQUIRED)
-  if (Ginkgo_FOUND)
-    get_target_property(Ginkgo_INCLUDE_DIRS
-      Ginkgo::ginkgo INTERFACE_INCLUDE_DIRECTORIES)
-    set(Ginkgo_LIBRARIES Ginkgo::ginkgo)
-  endif()
-endif()
-
-# zlib
-if (MFEM_USE_ZLIB)
+# GZSTREAM -> zlib
+if (MFEM_USE_GZSTREAM)
   find_package(ZLIB REQUIRED)
 endif()
 
@@ -381,6 +169,15 @@ if (MFEM_USE_LAPACK)
   find_package(LAPACK REQUIRED)
 endif()
 
+# OpenMP
+if (MFEM_USE_OPENMP)
+  if (MFEM_THREAD_SAFE)
+    find_package(OpenMP REQUIRED)
+  else()
+    message(FATAL_ERROR " *** MFEM_USE_OPENMP requires MFEM_THREAD_SAFE=ON.")
+  endif()
+endif()
+
 # SuiteSparse (before SUNDIALS which may depend on KLU)
 if (MFEM_USE_SUITESPARSE)
   find_package(SuiteSparse REQUIRED
@@ -389,20 +186,17 @@ endif()
 
 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
-  set(SUNDIALS_COMPONENTS CVODES ARKODE KINSOL NVector_Serial)
-  if (MFEM_USE_MPI)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Parallel NVector_MPIPlusX)
+  if (NOT MFEM_USE_MPI)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODE ARKODE KINSOL)
+  else()
+    find_package(SUNDIALS REQUIRED
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODE ARKODE KINSOL)
   endif()
-  if (MFEM_USE_CUDA)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
-  endif()
-  if (MFEM_USE_HIP)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
-  endif()
-  # The Core component was added in SUNDIALS v7, so we treat it as optional in
-  # order to support older versions.
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS}
-    OPTIONAL_COMPONENTS Core)
+endif()
+
+# Mesquite
+if (MFEM_USE_MESQUITE)
+  find_package(Mesquite REQUIRED)
 endif()
 
 # SuperLU_DIST can only be enabled in parallel
@@ -414,16 +208,6 @@ if (MFEM_USE_SUPERLU)
   endif()
 endif()
 
-# MUMPS can only be enabled in parallel
-if (MFEM_USE_MUMPS)
-  if (MFEM_USE_MPI)
-    find_package(MUMPS REQUIRED mumps_common pord)
-    set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})
-  else()
-    message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
-  endif()
-endif()
-
 # STRUMPACK can only be enabled in parallel
 if (MFEM_USE_STRUMPACK)
   if (MFEM_USE_MPI)
@@ -433,21 +217,16 @@ if (MFEM_USE_STRUMPACK)
   endif()
 endif()
 
+# Gecko
+if (MFEM_USE_GECKO)
+  find_package(Gecko REQUIRED)
+endif()
+
 # GnuTLS
 if (MFEM_USE_GNUTLS)
   find_package(_GnuTLS REQUIRED)
 endif()
 
-# GSLIB
-if (MFEM_USE_GSLIB)
-  find_package(GSLIB REQUIRED)
-endif()
-
-# HDF5
-if (MFEM_USE_HDF5)
-  find_package(HDF5 REQUIRED)
-endif()
-
 # NetCDF
 if (MFEM_USE_NETCDF)
   find_package(NetCDF REQUIRED)
@@ -458,36 +237,20 @@ if (MFEM_USE_MPFR)
   find_package(MPFR REQUIRED)
 endif()
 
-if (MFEM_USE_CEED)
-  find_package(libCEED REQUIRED)
-endif()
-
-if (MFEM_USE_AMGX)
-  find_package(AMGX REQUIRED)
-endif()
-
-if (MFEM_USE_MAGMA)
-  find_package(MAGMA REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
-    find_package(Conduit REQUIRED conduit relay blueprint)
-endif()
-
-if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms)
+    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
 
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
-  find_package(Axom REQUIRED Axom)
+  find_package(Axom REQUIRED Sidre SLIC axom_utils)
 endif()
 
 # PUMI
 if (MFEM_USE_PUMI)
   # If PUMI_DIR was specified, only link to that directory,
   # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.2.6 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
     CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
   if (SCOREC_FOUND)
     # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
@@ -499,99 +262,6 @@ if (MFEM_USE_PUMI)
   endif()
 endif()
 
-# Moonolith
-if(MFEM_USE_MOONOLITH)
-  find_package(ParMoonolith REQUIRED)
-  if(ParMoonolith_FOUND)
-    get_target_property(
-      MOONOLITH_INCLUDE_DIRS ParMoonolith::par_moonolith
-      INTERFACE_INCLUDE_DIRECTORIES)
-    set(MOONOLITH_FOUND TRUE)
-    set(MOONOLITH_LIBRARIES ParMoonolith::par_moonolith)
-    message(
-      STATUS
-      "MOONOLITH_LIBRARIES=${MOONOLITH_LIBRARIES}, MOONOLITH_INCLUDE_DIRS=${MOONOLITH_INCLUDE_DIRS}")
-  endif()
-endif()
-
-# HiOp optimizer
-if (MFEM_USE_HIOP)
-  find_package(HIOP REQUIRED)
-  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
-endif()
-
-# CoDiPack package
-if (MFEM_USE_CODIPACK)
-  find_package(CODIPACK REQUIRED)
-  # find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
-endif()
-
-# OCCA
-if (MFEM_USE_OCCA)
-   find_package(OCCA REQUIRED)
-endif()
-
-# RAJA
-if (MFEM_USE_RAJA)
-   # RAJA uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
-   if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
-      cmake_policy(SET CMP0146 OLD)
-   endif()
-   find_package(RAJA REQUIRED)
-endif()
-
-# GOOGLE-BENCHMARK
-if (MFEM_USE_BENCHMARK)
-   find_package(Benchmark REQUIRED)
-endif()
-
-# Caliper
-if (MFEM_USE_CALIPER)
-  find_package(Caliper REQUIRED)
-endif()
-
-# Algoim
-if (MFEM_USE_ALGOIM)
-  find_package(Algoim REQUIRED)
-endif()
-
-# ADIOS2 for parallel I/O
-if (MFEM_USE_ADIOS2)
-   find_package(ADIOS2 REQUIRED)
-endif()
-
-# MKL CPardiso
-if (MFEM_USE_MKL_CPARDISO)
-  if (MFEM_USE_MPI)
-   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
-  endif()
-endif()
-
-# MKL Pardiso
-if (MFEM_USE_MKL_PARDISO)
-   find_package(MKL_PARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64)
-endif()
-
-# PARELAG
-if (MFEM_USE_PARELAG)
-   find_package(PARELAG REQUIRED)
-endif()
-
-# Tribol
-if (MFEM_USE_TRIBOL)
-  if (MFEM_USE_MPI)
-    find_package(Tribol REQUIRED tribol redecomp)
-  else()
-    message(FATAL_ERROR " *** Tribol requires that MPI be enabled.")
-  endif()
-endif()
-
-if (MFEM_USE_ENZYME)
-    find_package(Enzyme REQUIRED HINTS ${ENZYME_DIR})
-    message(STATUS "Enzyme found in ${ENZYME_DIR}.")
-    set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -610,59 +280,41 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
   endif()
 endif()
 
-# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
-# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
-#   Target "mfem" links to target "Threads::Threads" but the target was not
-#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
-#   an ALIAS target is missing?
-# Call Stack (most recent call first):
-#   CMakeLists.txt:474 (mfem_add_library)
-#
-# NOTE: We need to figure out which TPL library adds the dependency on
-#       "Threads::Threads" and call the next line only when that TPL library is
-#       enabled.  -V. Dobrev
-find_package(Threads REQUIRED)
-
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
-    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB HDF5
-    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
-    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
-    ALGOIM ENZYME CUDA::cudart)
-
-# Add all created targets and *_FOUND libraries in the variables TPL_TARGETS and
-# TPL_LIBRARIES, respectively.
-set(TPL_TARGETS)
+set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
+    POSIXCLOCKS MFEMBacktrace ZLIB)
+# Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
 foreach(TPL IN LISTS MFEM_TPLS)
-  if (${TPL}_FOUND OR TARGET ${TPL})
+  if (${TPL}_FOUND)
     message(STATUS "MFEM: using package ${TPL}")
-    if (TARGET ${TPL})
-      list(APPEND TPL_TARGETS ${TPL})
-    else()
-      list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
-      list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
-    endif()
+    list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
+    list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
   endif()
 endforeach(TPL)
-
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
+include_directories(${TPL_INCLUDE_DIRS})
+
+if (OPENMP_FOUND)
+  message(STATUS "MFEM: using package OpenMP")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+endif()
 
-message(STATUS "MFEM shared library: BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}")
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
 
+# Windows specific
+set(_USE_MATH_DEFINES ${WIN32})
+
 #-------------------------------------------------------------------------------
 # Define and configure the MFEM library
 #-------------------------------------------------------------------------------
@@ -671,103 +323,41 @@ message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
 set(SOURCES "")
 set(HEADERS "")
 set(MFEM_SOURCE_DIRS general linalg mesh fem)
-
-if(MFEM_USE_MOONOLITH)
-  set(MFEM_SOURCE_DIRS ${MFEM_SOURCE_DIRS} fem/moonolith)
-endif()
-
 foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
   add_subdirectory(${DIR})
 endforeach()
-
-if (MFEM_USE_CUDA)
-  set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
-endif()
-
 add_subdirectory(config)
 set(MASTER_HEADERS
   ${PROJECT_SOURCE_DIR}/mfem.hpp
   ${PROJECT_SOURCE_DIR}/mfem-performance.hpp)
 
-# Installation options (we use GNUInstallDirs but prefer lib by default)
-set(MFEM_USE_GNUINSTALLDIRS OFF CACHE BOOL
-  "Use CMAKE_INSTALL_LIBDIR as defined by the GNUInstallDirs CMake module.")
-if (NOT MFEM_USE_GNUINSTALLDIRS)
-  mfem_cache_path(CMAKE_INSTALL_LIBDIR "lib" "Object code libraries (lib)")
-endif()
-include(GNUInstallDirs)
-mfem_cache_path(INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}"
-  "Relative or absolute path for installing header files.")
-mfem_cache_path(INSTALL_BIN_DIR "${CMAKE_INSTALL_BINDIR}"
-  "Relative or absolute path for installing the binaries.")
-mfem_cache_path(INSTALL_LIB_DIR "${CMAKE_INSTALL_LIBDIR}"
-  "Relative or absolute path for installing the library.")
-mfem_cache_path(INSTALL_SHARE_DIR "${CMAKE_INSTALL_DATAROOTDIR}"
-  "Relative or absolute path for installing shared data.")
-# other options: "share/mfem/cmake", "lib/mfem/cmake"
-mfem_cache_path(INSTALL_CMAKE_DIR "${INSTALL_LIB_DIR}/cmake/mfem"
-  "Relative or absolute path for installing cmake config files.")
-
-mfem_path_to_fullpath("${INSTALL_LIB_DIR}" "${CMAKE_INSTALL_PREFIX}" _lib_path)
+set(_lib_path "${CMAKE_INSTALL_PREFIX}/lib")
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON CACHE BOOL "")
-if(NOT DEFINED CMAKE_INSTALL_RPATH)
-  set(CMAKE_INSTALL_RPATH "${_lib_path}")
-endif()
-if(NOT DEFINED CMAKE_INSTALL_NAME_DIR)
-  set(CMAKE_INSTALL_NAME_DIR "${_lib_path}")
-endif()
-
-# Variables used when generating _config.hpp, and config.mk
-set(MFEM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
+set(CMAKE_INSTALL_RPATH "${_lib_path}" CACHE PATH "")
+set(CMAKE_INSTALL_NAME_DIR "${_lib_path}" CACHE PATH "")
 
 # Declaring the library
-mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
-target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
+add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
-if (TPL_TARGETS)
-  add_dependencies(mfem ${TPL_TARGETS})
+if (CMAKE_VERSION VERSION_GREATER 2.8.11)
+  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+else()
+  target_link_libraries(mfem ${TPL_LIBRARIES})
 endif()
 if (MINGW)
-  target_link_libraries(mfem PRIVATE ws2_32)
+  target_link_libraries(mfem ws2_32)
 endif()
-if (MSVC)
-  target_compile_options(mfem PUBLIC "/wd4819")
-endif()
-message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
-target_include_directories(mfem
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    # The INSTALL_INTERFACE is handled below
-    ${TPL_INCLUDE_DIRS})
-if (MFEM_USE_MPI)
-  if (MPI_CXX_COMPILE_FLAGS)
-    separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-      "${MPI_CXX_COMPILE_FLAGS}")
-    target_compile_options(mfem PUBLIC ${MPI_CXX_COMPILE_ARGS})
-  endif()
-  if (MPI_CXX_LINK_FLAGS)
-    target_link_libraries(mfem PUBLIC ${MPI_CXX_LINK_FLAGS})
-  endif()
-endif()
-if (MFEM_USE_ENZYME)
-  target_link_libraries(mfem PUBLIC ClangEnzymeFlags)
-endif()
-
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 
-# If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
-# inside the build directory.
+# If building out-of-source, define MFEM_BUILD_DIR to point to the build
+# directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
   target_compile_definitions(mfem PRIVATE
-    "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
+    "MFEM_BUILD_DIR=${PROJECT_BINARY_DIR}")
 endif()
 
 # Generate configuration file in the build directory: config/_config.hpp.
-set(MFEM_SHARED_BUILD ${BUILD_SHARED_LIBS})
 configure_file(
   "${PROJECT_SOURCE_DIR}/config/cmake/config.hpp.in"
   "${PROJECT_BINARY_DIR}/config/_config.hpp")
@@ -778,61 +368,26 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
   foreach(Header mfem.hpp mfem-performance.hpp)
     message(STATUS
       "Writing substitute header --> \"${Header}\"")
-    file(WRITE "${PROJECT_BINARY_DIR}/${Header}.tmp"
+    file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
 "// Auto-generated file.
-#define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
+#define MFEM_BUILD_DIR ${PROJECT_BINARY_DIR}
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
-
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
-      "${PROJECT_BINARY_DIR}/${Header}.tmp"
-      "${PROJECT_BINARY_DIR}/${Header}"
-    )
-    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
-      "${PROJECT_BINARY_DIR}/${Header}.tmp"
-    )
-
     # This version will be installed in the top include directory:
-    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
 "// Auto-generated file.
 #include \"mfem/${Header}\"
 ")
-
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
-    )
-    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
-    )
-
   endforeach()
 endif()
 
-set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")
-
 #-------------------------------------------------------------------------------
-# Examples, miniapps, benchmarks and testing
+# Examples, miniapps, and testing
 #-------------------------------------------------------------------------------
 
-# Enable testing and benchmarks if required
+# Enable testing if required
 if (MFEM_ENABLE_TESTING)
   enable_testing()
-  set(MFEM_ALL_TESTS_TARGET_NAME tests)
-  add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
-  add_subdirectory(tests EXCLUDE_FROM_ALL)
-
-  if (MFEM_USE_BENCHMARK)
-    # Create a target for all benchmarks and, optionally, enable it.
-    set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-    add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                    ${MFEM_ENABLE_BENCHMARKS})
-    if (MFEM_ENABLE_BENCHMARKS)
-      add_subdirectory(tests/benchmarks) #install benchmarks if enabled
-    else()
-      add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
-    endif()
-  endif()
 endif()
 
 # Define a target that all examples and miniapps will depend on.
@@ -842,40 +397,28 @@ add_custom_target(${MFEM_EXEC_PREREQUISITES_TARGET_NAME})
 # Create a target for all examples and, optionally, enable it.
 set(MFEM_ALL_EXAMPLES_TARGET_NAME examples)
 add_mfem_target(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${MFEM_ENABLE_EXAMPLES})
-if (MFEM_ENABLE_EXAMPLES)
-  add_subdirectory(examples) #install examples if enabled
-else()
-  add_subdirectory(examples EXCLUDE_FROM_ALL)
-endif()
+add_subdirectory(examples EXCLUDE_FROM_ALL)
 
 # Create a target for all miniapps and, optionally, enable it.
 set(MFEM_ALL_MINIAPPS_TARGET_NAME miniapps)
 add_mfem_target(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_ENABLE_MINIAPPS})
-if (MFEM_ENABLE_MINIAPPS)
-  add_subdirectory(miniapps) #install miniapps if enabled
-else()
-  add_subdirectory(miniapps EXCLUDE_FROM_ALL)
-endif()
+add_subdirectory(miniapps EXCLUDE_FROM_ALL)
 
 # Target to build all executables, i.e. everything.
 add_custom_target(exec)
 add_dependencies(exec
-  ${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-  ${MFEM_ALL_EXAMPLES_TARGET_NAME}
-  ${MFEM_ALL_MINIAPPS_TARGET_NAME}
-  ${MFEM_ALL_TESTS_TARGET_NAME})
+  ${MFEM_ALL_EXAMPLES_TARGET_NAME} ${MFEM_ALL_MINIAPPS_TARGET_NAME})
 # Here, we want to "add_dependencies(test exec)". However, dependencies for
 # 'test' (and other built-in targets) can not be added with add_dependencies():
 #  - https://gitlab.kitware.com/cmake/cmake/issues/8438
 #  - https://cmake.org/Bug/view.php?id=8438
 
 # Add a target to copy the mfem data directory to the build directory
-# Implementable as a single copy_directory_if_different command w/ CMake >= 3.26
-file(GLOB DATA_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/data/*)
-add_custom_target(copy_data
-  COMMAND ${CMAKE_COMMAND} -E make_directory data
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${DATA_FILES} data
-  COMMENT "Syncing the data directory ...")
+add_custom_command(OUTPUT data_is_copied
+  COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/data data
+  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
+  COMMENT "Copying the data directory ...")
+add_custom_target(copy_data DEPENDS data_is_copied)
 # Add 'copy_data' as a prerequisite for all executables, if the source and the
 # build directories are not the same.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
@@ -883,17 +426,16 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()
 
 # Add 'check' target - quick test
-set(MFEM_CHECK_TARGET_NAME ${MFEM_CUSTOM_TARGET_PREFIX}check)
 if (NOT MFEM_USE_MPI)
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
-    ${CMAKE_CTEST_COMMAND} -R \"^ex1_ser\" -C ${CMAKE_CFG_INTDIR}
+  add_custom_target(check
+    ${CMAKE_CTEST_COMMAND} -R '^ex1_ser' -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1)
+  add_dependencies(check ex1)
 else()
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
-    ${CMAKE_CTEST_COMMAND} -R \"^ex1p\" -C ${CMAKE_CFG_INTDIR}
+  add_custom_target(check
+    ${CMAKE_CTEST_COMMAND} -R '^ex1p' -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1p)
+  add_dependencies(check ex1p)
 endif()
 
 #-------------------------------------------------------------------------------
@@ -906,10 +448,13 @@ add_subdirectory(doc)
 #-------------------------------------------------------------------------------
 
 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
-
-target_include_directories(mfem BEFORE
-  PUBLIC
-    $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)
+set(INSTALL_INCLUDE_DIR include
+  CACHE PATH "Relative path for installing header files.")
+set(INSTALL_LIB_DIR lib
+  CACHE PATH "Relative path for installing the library.")
+# other options: "share/mfem/cmake", "lib/mfem/cmake"
+set(INSTALL_CMAKE_DIR lib/cmake/mfem
+  CACHE PATH "Relative path for installing cmake config files.")
 
 # The 'install' target will not depend on 'all'.
 # set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)
@@ -919,9 +464,7 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Development)
 # Install the library
 install(TARGETS ${PROJECT_NAME}
   EXPORT ${PROJECT_NAME_UC}Targets
-  RUNTIME DESTINATION ${INSTALL_BIN_DIR}
-  LIBRARY DESTINATION ${INSTALL_LIB_DIR}
-  ARCHIVE DESTINATION ${INSTALL_LIB_DIR})
+  DESTINATION ${INSTALL_LIB_DIR})
 
 # Install the master headers
 foreach(Header mfem.hpp mfem-performance.hpp)
@@ -930,25 +473,11 @@ foreach(Header mfem.hpp mfem-performance.hpp)
 endforeach()
 install(FILES ${MASTER_HEADERS} DESTINATION ${INSTALL_INCLUDE_DIR}/mfem)
 
-# Install the headers (except common miniapp which is installed from its subdir)
+# Install the headers; currently, the miniapps headers are excluded
 install(DIRECTORY ${MFEM_SOURCE_DIRS}
   DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
   FILES_MATCHING PATTERN "*.hpp")
 
-# Install the okl files
-if (MFEM_USE_OCCA)
-  install(DIRECTORY ${MFEM_SOURCE_DIRS}
-    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "*.okl")
-endif()
-
-# Install the libCEED files
-if (MFEM_USE_CEED)
-  install(DIRECTORY ${MFEM_SOURCE_DIRS}
-    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/integrators/*/*.h")
-endif()
-
 # Install ${HEADERS}
 # ---
 # foreach (HDR ${HEADERS})
@@ -959,8 +488,10 @@ endif()
 
 # Install the configuration header files
 install(FILES ${PROJECT_BINARY_DIR}/config/_config.hpp
-  ${PROJECT_SOURCE_DIR}/config/config.hpp
-  ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
+  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config
+  RENAME config.hpp)
+
+install(FILES ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
   DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config)
 
 # Package the whole thing up nicely
@@ -975,8 +506,14 @@ export(TARGETS ${PROJECT_NAME}
 # TODO: How do we register the install-tree? Replacing the build-tree?
 export(PACKAGE ${PROJECT_NAME})
 
+# Extract the include directories required to use MFEM
+get_target_property(MFEM_TPL_INCLUDE_DIRS mfem INCLUDE_DIRECTORIES)
+if (NOT MFEM_TPL_INCLUDE_DIRS)
+  set(MFEM_TPL_INCLUDE_DIRS "")
+endif()
+
 # This is the build-tree version
-set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${PROJECT_BINARY_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
   ${CMAKE_CURRENT_BINARY_DIR}/MFEMConfig.cmake
@@ -984,7 +521,7 @@ configure_package_config_file(config/cmake/MFEMConfig.cmake.in
   PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR)
 
 # This is the version that will be installed
-set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${INSTALL_LIB_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
   ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/MFEMConfig.cmake
@@ -1006,14 +543,3 @@ install(FILES
 # Install the export set for use with the install-tree
 install(EXPORT ${PROJECT_NAME_UC}Targets
     DESTINATION ${INSTALL_CMAKE_DIR})
-
-# Install the data directory if present, i.e. if the copy_data target is built
-install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
-  DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
-
-#-------------------------------------------------------------------------------
-# Create 'config.mk' from 'config.mk.in' for the build and install locations and
-# define install rules for 'config.mk' and 'test.mk'
-#-------------------------------------------------------------------------------
-
-mfem_export_mk_files()

CODE_OF_CONDUCT.md

@@ -1,133 +0,0 @@
-# MFEM Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at mfem@llnl.gov.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident. Anyone involved in the reported behavior will recuse
-themselves from the investigation and decision making about the resolution of
-the complaint.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -1,51 +1,39 @@
 <p align="center">
-<a href="https://mfem.org/"><img alt="mfem" src="https://mfem.org/img/logo-300.png"></a>
+<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
 </p>
 
 <p align="center">
-<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/blob/master/COPYRIGHT"><img alt="License" src="https://img.shields.io/badge/License-LGPL--2.1-brightgreen.svg"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
 
 
 # How to Contribute
 
-The MFEM team welcomes contributions at all levels: bugfixes; code improvements;
-simplifications; new mesh, discretization or solver capabilities; improved
-documentation; new examples and miniapps; HPC performance improvements; etc.
+The MFEM team welcomes contributions at all levels: bugfixes; code
+improvements; simplifications; new mesh, discretization or solver
+capabilities; improved documentation; new examples and miniapps;
+HPC performance improvements; ...
 
-MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license.
-
-Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
-in the MFEM community, you agree to abide by its rules.
-
-If you plan on contributing to MFEM, consider reviewing the
-[issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
-already exists for your desired feature or the bug you ran into. Use a pull
-request (PR) toward the `mfem:master` branch to propose your contribution. If
-you are planning significant code changes or have questions, you may want to
-open an [issue](https://github.com/mfem/mfem/issues) before issuing a PR. In
-addition to technical contributions, we are also interested in your results and
-[simulation images](https://mfem.org/gallery/), which you can share via a pull
-request in the [mfem/web](https://github.com/mfem/web) repo.
+Use a pull request (PR) toward the `mfem:master` branch to propose your
+contribution. If you are planning significant code changes, or have any
+questions, you can also open an [issue](https://github.com/mfem/mfem/issues)
+before issuing a PR.  We also welcome your [simulation
+images](http://mfem.org/gallery/), which you can submit via a pull request in
+[mfem/web](https://github.com/mfem/web).
 
 See the [Quick Summary](#quick-summary) section for the main highlights of our
 GitHub workflow. For more details, consult the following sections and refer
 back to them before issuing pull requests:
 
-- [Quick Summary](#quick-summary)
 - [Code Overview](#code-overview)
 - [GitHub Workflow](#github-workflow)
   - [MFEM Organization](#mfem-organization)
   - [New Feature Development](#new-feature-development)
   - [Developer Guidelines](#developer-guidelines)
   - [Pull Requests](#pull-requests)
-  - [MFEM PR Rules](#mfem-pr-rules)
   - [Pull Request Checklist](#pull-request-checklist)
   - [Master/Next Workflow](#masternext-workflow)
   - [Releases](#releases)
@@ -57,7 +45,7 @@ back to them before issuing pull requests:
 Contributing to MFEM requires knowledge of Git and, likely, finite elements. If
 you are new to Git, see the [GitHub learning
 resources](https://help.github.com/articles/git-and-github-learning-resources/).
-To learn more about the finite element method, see our [FEM page](https://mfem.org/fem).
+To learn more about the finite element method, see our [FEM page](http://mfem.org/fem).
 
 *By submitting a pull request, you are affirming the [Developer's Certificate of
 Origin](#developers-certificate-of-origin-11) at the end of this file.*
@@ -69,18 +57,8 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
   development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
   with regards to documentation and code styling.
-- Please do not commit large/binary files to the central repository (use a fork
-  instead).
-- Pull requests should be issued toward `mfem:master`. Make sure
-  to check the items off the [Pull Request Checklist](#pull-request-checklist) and
-  follow the [MFEM PR Rules](#mfem-pr-rules).
-- When your contribution is fully working and ready to be reviewed, add
-  the `ready-for-review` label.
-- PRs are treated similarly to journal submission with an "editor" assigning two
-  reviewers to evaluate the changes.
-- The reviewers have 3 weeks to evaluate the PR and work with the author to
-  fix issues and implement improvements.
-- During review there should be no force pushes/rewriting history in the branch.
+- Pull requests  should be issued toward `mfem:master`. Make sure
+  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -90,169 +68,98 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
 
 ### Code Overview
 
-#### Source code structure:
+- The MFEM library uses object-orient design principles which reflect, in code,
+  the independent mathematical concepts of meshing, linear algebra and finite
+  element spaces and operators.
 
-The MFEM library uses object-oriented design principles which reflect, in code,
-the independent mathematical concepts of meshing, linear algebra and finite
-element spaces and operators.
-
-The MFEM source code has the following structure:
-
-```
+- The MFEM source code has the following structure:
+  ```
   .
   ├── config
-  │   ├── cmake
-  │   ├── docker
-  │   ├── githooks
-  │   └── vcpkg
+  │   └── cmake
+  │       └── modules
   ├── data
   ├── doc
+  │   └── web
+  │       └── examples
   ├── examples
-  │   ├── amgx
-  │   ├── caliper
-  │   ├── ginkgo
-  │   ├── hiop
-  │   ├── jupyter
-  │   ├── moonolith
   │   ├── petsc
   │   ├── pumi
-  │   ├── sundials
-  |   └── superlu
+  │   └── sundials
   ├── fem
-  │   ├── ceed
-  │   ├── dfem
-  │   ├── eltrans
-  │   ├── fe
-  │   ├── gslib
-  │   ├── integ
-  │   ├── lor
-  │   ├── moonolith
-  │   ├── qinterp
-  │   └── tmop
-  │   |   ├── assemble
-  │   |   ├── metrics
-  │   |   ├── mult
-  │   |   └── tools
   ├── general
   ├── linalg
-  │   ├── batched
-  │   └── simd
   ├── mesh
-  │   └── submesh
-  ├── miniapps
-  │   ├── adjoint
-  │   ├── autodiff
-  │   ├── common
-  │   ├── contact
-  │   ├── dfem
-  │   ├── dpg
-  │   ├── electromagnetics
-  │   ├── fluids
-  │   │   ├── navier
-  │   │   └── schrodinger-flow
-  │   ├── gslib
-  │   ├── hdiv-linear-solver
-  │   ├── hooke
-  │   ├── meshing
-  │   ├── mtop
-  │   ├── multidomain
-  │   ├── nurbs
-  │   ├── parelag
-  │   ├── performance
-  │   ├── shifted
-  │   ├── solvers
-  │   ├── spde
-  │   ├── tools
-  │   ├── toys
-  │   └── tribol
-  └── tests
-      ├── benchmarks
-      ├── convergence
-      ├── gitlab
-      ├── mem_manager
-      ├── par-mesh-format
-      ├── scripts
-      └── unit
-```
+  └── miniapps
+      ├── common
+      ├── electromagnetics
+      ├── meshing
+      ├── nurbs
+      ├── performance
+      └── tools
+  ```
 
-#### Main directories and classes
-
-The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
-classes implementing the finite element, mesh and linear algebra concepts
-respectively.
+- The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
+  classes implementing the finite element, mesh and linear algebra concepts
+  respectively.
 
 - The main mesh classes are:
-  + [`Mesh`](https://docs.mfem.org/html/classmfem_1_1Mesh.html)
-  + [`NCMesh`](https://docs.mfem.org/html/classmfem_1_1NCMesh.html)
-  + [`Element`](https://docs.mfem.org/html/classmfem_1_1Element.html)
-  + [`ElementTransformation`](https://docs.mfem.org/html/classmfem_1_1ElementTransformation.html)
+  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
 
 - The main finite element classes are:
-  + [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
-  + [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
-  + [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
 
 - The main linear algebra classes and sources are
-  + [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
-  + [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
-  + Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)
+  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
 
-#### Parallel implementation
-
-Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
-mesh for example is just a serial mesh on each task plus the information on
-shared geometric entities between different tasks. The parallel source files
-have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
+- Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
+  mesh for example is just a serial mesh on each task plus the information on
+  shared geometric entities between different tasks. The parallel source files
+  have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 
 - The main parallel classes are
-  + [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
-  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
-  + [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
-  + [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
-  + [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
-  + [`HypreParMatrix`](https://docs.mfem.org/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://docs.mfem.org/html/classmfem_1_1HypreParVector.html)
-  + [`HypreSolver`](https://docs.mfem.org/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://docs.mfem.org/html/hypre_8hpp.html)
+  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
 
-#### GPU and general device support
-
-GPU and multi-core CPU support is based on device kernels supporting different
-backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
-device/host memory manager.
-
-- The main device-relevant classes and sources are:
-  + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
-  + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
-  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
-
-#### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
   communication, error handling, arrays, (Boolean) tables, timing, etc.
+
 - The `config/` directory contains build-related files, both for the plain
   Makefile and the CMake build options.
+
 - The `doc/` directory contains configuration for the Doxygen code documentation
-  that can either be built locally or browsed online at https://docs.mfem.org.
+  that can either be build locally, or browsed online at
+  http://mfem.github.io/doxygen/html/index.html.
 
-#### Examples and tests
-- `examples` and `miniapps` respectively gather simple and more fully-featured
-  demonstrations of the usage on MFEM. They both rely on `data/` for the
-  collection of meshes.
-- The `tests/` directory contains a unit test suite and will later contain more
-  tests that run example codes.
+- The `data/` directory contains a collection of small mesh files, that are used
+  in the simple example codes and more fully-featured mini applications in the
+  `examples/` and `miniapps/` directories.
 
-See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
-website.
+- See also the [code overview](http://mfem.org/code-overview/) section on the
+  MFEM website.
 
 ## GitHub Workflow
 
-The MFEM GitHub organization: https://github.com/mfem, is the main developer hub
-for the MFEM project.
+The GitHub organization, https://github.com/mfem, is the main developer hub for
+the MFEM project.
 
-If you plan to make contributions or would like to stay up-to-date with changes
+If you plan to make contributions or will like to stay up-to-date with changes
 in the code, *we strongly encourage you to [join the MFEM organization](#mfem-organization)*.
 
 This will simplify the workflow (by providing you additional permissions), and
@@ -261,31 +168,34 @@ will allow us to reach you directly with project announcements.
 
 ### MFEM Organization
 
-#### Getting started (GitHub)
-Before you can start, you need a GitHub account, here are a few suggestions:
-  + Create the account at: [github.com/join](https://github.com/join).
-  + For easy identification, please add your full name and maybe a picture of you at:
-    https://github.com/settings/profile.
+- Before you can start, you need a GitHub account, here are a few suggestions:
+  + Create the account at: github.com/join.
+  + For easy identification, please add your name and maybe a picture of you at: https://github.com/settings/profile.
   + To receive notification, set a primary email at: https://github.com/settings/emails.
   + For password-less pull/push over SSH, add your SSH keys at: https://github.com/settings/keys.
 
-#### Joining
 - [Contact us](#contact-information) for an invitation to join the MFEM GitHub
-  organization. You will receive an invitation email, which you can directly accept.
+  organization.
+
+- You should receive an invitation email, which you can directly accept.
   Alternatively, *after logging into GitHub*, you can accept the invitation at
   the top of https://github.com/mfem.
+
 - Consider making your membership public by going to https://github.com/orgs/mfem/people
-  and clicking on the organization visibility drop box next to your name.
+  and clicking on the organization visibility dropbox next to your name.
+
 - Project discussions and announcements will be posted at
   https://github.com/orgs/mfem/teams/everyone.
 
-#### Structure
 - The MFEM source code is in the [mfem](https://github.com/mfem/mfem)
   repository.
+
 - The website and corresponding documentation are in the
   [web](https://github.com/mfem/web) repository.
+
 - The [PyMFEM](https://github.com/mfem/PyMFEM) repository contains a Python
   wrapper for MFEM.
+
 - The [data](https://github.com/mfem/data) repository contains additional
   (large) datafiles for MFEM.
 
@@ -293,9 +203,9 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 ### New Feature Development
 
 - A new feature should be important enough that at least one person, the
-  author, is willing to work on it and be its champion.
+  proposer, is willing to work on it and be its champion.
 
-- The author creates a branch for the new feature (with suffix `-dev`), off
+- The proposer creates a branch for the new feature (with suffix `-dev`), off
   the `master` branch, or another existing feature branch, for example:
 
   ```
@@ -338,13 +248,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   - Lean code base is easier to understand by new collaborators.
   - New features should be added only if they are necessary or generally useful.
   - Introduction of language constructions not currently used in MFEM should be
-    justified and generally avoided (to maintain portability to various systems
-    and compilers, including early access hardware).
+    justified and generally avoided (so we can build on cutting-edge systems).
   - We prefer basic C++ and the C++03 standard, to keep the code readable by
     a large audience and to make sure it compiles anywhere.
 
 - *Keep the code general and reasonably efficient*
-  - The main goal is fast prototyping for research and application development.
+  - Main goal is fast prototyping for research.
   - When in doubt, generality wins over efficiency.
   - Respect the needs of different users (current and/or future).
 
@@ -357,27 +266,15 @@ Before you can start, you need a GitHub account, here are a few suggestions:
     change the code by default.
 
 - Code specifics
-  - All new public, protected, and private classes, methods, data members, and
-    functions have Doxygen-style documentation in source comments.
-  - Math formulas can be included in the Doxygen comments either with standard
-    LaTeX (`$..$` and `$$..$$`) for portions that need detailed explanation, or with
-    [Unicode](https://www.unicodeit.net/) or plain text when short or readable description is preferable.
-  - In addition to arguments and functionality, documentation should include the
-    current limitations of the code, any background information that is
-    implicitly assumed in the implementation, and the ownership and lifetime
-    of data.
+  - All significant new classes, methods and functions have Doxygen-style
+    documentation in source comments.
   - Consistent code styling is enforced with `make style` in the top-level
     directory. This requires [Artistic Style](http://astyle.sourceforge.net) (we
-    specifically use version 3.1). See also the file `config/mfem.astylerc`.
+    specifically use version 2.05.1). See also the file `config/mfem.astylerc`.
   - Use `mfem::out` and `mfem::err` instead of `std::cout` and `std::cerr` in
     internal library code. (You can use `std` in examples and miniapps.)
   - When manually resolving conflicts during a merge, make sure to mention the
     conflicted files in the commit message.
-  - All significant new features and changes should be documented in CHANGELOG.
-  - New examples and miniapps should have documentation on the MFEM webpage.
-  - The general floating-point type `real_t` should be used, rather than
-    `float` or `double`, except in special cases where only one is possible.
-
 
 ### Pull Requests
 
@@ -399,125 +296,14 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 
      `[DISCUSS] Hybridized DG [hdg-dev]`
 
-- If the PR is still a work in progress, add the `WIP` label to it and
-  optionally the `[WIP]` prefix in the title.
-
 - Add a description, appropriate labels and assign yourself to the PR. The MFEM
   team will add reviewers as appropriate.
 
 - List outstanding TODO items in the description, see PR #222 for an example.
 
-- When your contribution is fully working and ready to be reviewed, add
-  the `ready-for-review` label.
-
-- PRs are treated similarly to journal submission with an "editor" assigning
-  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
-  the PR and work with the author to implement improvements and fix issues.
-
-- Once the `ready-for-review` label has been applied and reviewers have been
-  assigned, the PR is considered under review. To help with the review process
-  there should be no force pushes/rewriting history in the branch.
-
-- After approval, the PR is [tested](#masternext-workflow) for a week with
-  other approved PRs in the `mfem:next` branch.
-
-- Consider manually running the tests in `tests/scripts` before merging in
-  `mfem:next`, see the [README](tests/scripts/README) file in that directory
-  for more details.
-
-- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
-  builds at the end of the PR. These should generally run clean, so address any
-  errors as soon as possible. Please ask if you are unsure how to do that.
-
-- Note that some tests, such as the `branch-history` check in GitHub Actions
-  are safeguards that are allowed to fail in certain cases.
-
-- Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in GitHub Actions enforce MFEM-specific rules which are explained in
-  the error messages and the `tests/scripts` directory.
-
-- Also note that the tests `branch-history` and `repos-checks` found in GitHub
-  Actions can be triggered automatically before each push using git hooks. See
-  the [git hooks README](config/githooks/README.md) for a detailed explanation.
-
-- If triggered, track the status of the LLNL GitLab tests. If failing, ask
-  one of the _LLNL developers_ for details.
-
-
-### MFEM PR Rules
-
-The Pull Request (PR) approval process in MFEM is similar to the approval of papers in a peer-reviewed journal. In particular:
-
-1. There is an MFEM board of "editors" that evaluates new PRs and assigns "reviewers" for each PR.
-
-2. The assigned reviewers are responsible to carefully review and test the proposed PR.
-
-3. A PR can be (manually) merged in the *next* branch only if 2 of the assigned reviewers have approved it and it has passed internal testing. This merge can be performed by any of the assigned reviewers or by any of the editors.
-
-4. A PR can be merged in the *master* branch only if it has been tested successfully for a week in *next* and an editor has (optionally) taken a final look. This merge can be performed only by one of the editors.
-
-#### Responsibilities of Editors
-
-The current list of MFEM editors is:
-
-- [@v-dobrev](https://github.com/v-dobrev) (Veselin Dobrev)
-- [@tzanio](https://github.com/tzanio) (Tzanio Kolev)
-- [@pazner](https://github.com/pazner) (Will Pazner)
-- [@mlstowell](https://github.com/mlstowell) (Mark Stowell)
-
-**The responsibilities of the editors are:**
-
-1. To assign appropriate milestone and labels for new PRs, e.g. *bugfix*, *minor*, *api-change*, *high-impact*, etc.
-
-2. To assign at least 2 reviewers for new PRs. An editor can also be a reviewer. The editor, reviewers, and author should be listed as "Assignees" on the GitHub PR page. After assignment, the `in-review` label should be added.
-
-3. To complete the initial PR evaluation and assignments in a timely manner: 1 week from submission.
-
-4. To assist reviewers when they need help with their reviews (but also to stay out of the way when they don't).
-
-5. To remind the reviewers about timely completion of their review.
-
-6. To take a final look and complete the PR merge in *master*. The final look step is optional and shouldn't take more than 3 days.
-
-7. The assignment of bugfixes should be expedited proportional to their importance, e.g. in some cases the editor can assign much shorter review window.
-
-#### Responsibilities of Reviewers
-
-Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their area of expertise.
-
-**The responsibilities of the reviewers are:**
-
-1. To let the editors know if the proposed assignment is not a good match for them.
-
-2. To communicate with the PR author, provide feedback and work with them to resolve issues.
-
-3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
-
-3. To seek help from the editors in case of difficulties.
-
-4. To complete the review in a timely manner: 3 weeks from assignment.
-
-5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
-
-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
-
-7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
-
-#### Responsibilities of Authors
-
-Authors should clearly indicate when a PR is ready for review (before that the PR should be marked as `Draft` or `[WIP]`).
-
-**The responsibilities of the authors are:**
-
-1. To follow the instructions and PR checklist in the `CONTRIBUTING.md` document in the MFEM repository.
-
-2. To respond to reviewer feedback in a timely manner.
-
-3. Authors are encouraged to perform testing and inform the reviewers about the results.
-
-4. Authors can use the "Reviewers" section of the GitHub PR page to suggest reviewers, but the "Assignees" section will show who the editor has assigned to do the reviews.
-
-5. To indicate when the PR is ready for review by adding the `ready-for-review` label.
+- Track the Travis CI and Appveyor [continuous integration](#automated-testing)
+  builds at the end of the PR. These should run clean, so address any errors as
+  soon as possible.
 
 
 ### Pull Request Checklist
@@ -530,18 +316,14 @@ Before a PR can be merged, it should satisfy the following:
     - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
     - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
 - [ ] Update `INSTALL`:
-    - [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
-    - [ ] Have the version ranges for any required or optional libraries changed?
+    - [ ] Has a new optional library been added? (*Make sure the external library is licensed under LGPL, not GPL!*)
     - [ ] Does `make` or `cmake` have a new target?
-    - [ ] Did the requirements or the installation process change? *(rare)*
-- [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.github`
-    - [ ] `.appveyor.yml`
+    - [ ] Did the requirements or the installation process change? *(rare)*.
 - [ ] Update `.gitignore`:
-    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
+    - [ ] Check if `make distclean; git status` shows any files that are generated from the source but we don't want to track in the repository.
     - [ ] Add new patterns (just for the new files above) and re-run the above test.
 - [ ] New examples:
-    - [ ] All sample runs at the top of the example source file work.
+    - [ ] All sample runs at the top of the example work.
     - [ ] Update `examples/makefile`:
       - [ ] Add the example code to the appropriate `SEQ_EXAMPLES` and `PAR_EXAMPLES` variables.
       - [ ] Add any files generated by it to the `clean` target.
@@ -550,16 +332,13 @@ Before a PR can be merged, it should satisfy the following:
       - [ ] Add the example code to the `ALL_EXE_SRCS` variable.
       - [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
    - [ ] List the new example in `doc/CodeDocumentation.dox`.
-   - [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
    - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
       - [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
       - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
       - [ ] In `examples.md`, list the example under the appropriate categories, add new categories if necessary.
       - [ ] Add a short description of the example in the "Extensive Examples" section of `features.md`.
 - [ ] New miniapps:
-   - [ ] All sample runs at the top of the miniapp source file work.
-   - [ ] Add to internal testing repo, if sample runs should be included in nightly tests [internally](#tests-at-llnl).
-   - [ ] Exclude long sample runs from automated testing, with `* ` (one space) before the command.
+   - [ ] All sample runs at the top of the miniapp work.
    - [ ] Update top-level `makefile` and `makefile` in corresponding miniapp directory.
    - [ ] Add the miniapp binary and any files generated by it to the top-level `.gitignore` file.
    - [ ] Update CMake build system:
@@ -567,27 +346,27 @@ Before a PR can be merged, it should satisfy the following:
       - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
       - [ ] Consider adding a new test for the new miniapp.
    - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
    - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
      - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
      - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
      - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
-   - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
-   - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
-   - [ ] Any new functions should include descriptions of their intended use e.g. for internal use only, user-facing, etc., along with references to example code whenever possible/appropriate.
+   - [ ] All significant new classes, methods and functions have Doxygen-style documentation in source comments.
    - [ ] Consider adding new sample runs in existing examples to highlight the new capability.
    - [ ] Consider saving cool simulation pictures with the new capability in the Confluence gallery (LLNL only) or submitting them, via pull request, to the gallery section of the `mfem/web` repo.
-   - [ ] If this is a major new feature, consider mentioning it in the short summary inside `README` *(rare)*.
+   - [ ] If this is a major new feature, consider mentioning in the short summary inside `README` *(rare)*.
    - [ ] List major new classes in `doc/CodeDocumentation.dox` *(rare)*.
 - [ ] Update this checklist, if the new pull request affects it.
-- [ ] Run `make unittest` to make sure all unit tests pass.
-- [ ] Run the tests in `tests/scripts`.
+- [ ] (LLNL only) Clone the `tests` repository and run the following tests, see `mfem/tests/README.md`:
+   - [ ] `compilers`
+   - [ ] `memcheck`
+   - [ ] `unit-test`
+   - [ ] `documentation`
 - [ ] (LLNL only) After merging:
-   - [ ] Update internal tests to include the new features.
-
+   - [ ] Regenerate `README.html` files from companion documentation pull requests.
+   - [ ] Update the `baseline` and `compiler` tests, add new tests if necessary.
+   - [ ] Consider updating the script `mfem/tests/sample-runs` (`sample-runs-serial` and `sample-runs-parallel`).
 
 ### Master/Next Workflow
 
@@ -646,7 +425,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   (replace `3.3.2` with current release):
   - Rename the current `next` branch to `next-pre-v3.3.2`.
   - Create a new `next` branch starting from the `v3.3.2` release.
-  - Local copies of `next` can then be updated with `git fetch origin next && git checkout -B next origin/next`.
+  - Local copies of `next` can then be updated with `git checkout -B next origin/next`.
 
 ### Release Checklist
 
@@ -655,14 +434,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - [ ] `makefile`
     - [ ] `CMakeLists.txt`
     - [ ] `doc/CodeDocumentation.conf.in`
-- [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
-- [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.github`
-    - [ ] `.appveyor.yml`
-- [ ] Update the `CHANGELOG` to organize all release contributions
-- [ ] Review the whole source code once over
-- [ ] Ask MFEM-based applications to test the pre-release branch
-- [ ] Test on additional platforms and compilers
+- [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:
 
   ```
@@ -671,10 +443,10 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   ```
 - [ ] Create the release tarball and push to `mfem/releases`.
 - [ ] Recreate the `next` branch as described in previous section.
-- [ ] Update and push documentation to `mfem/doxygen`. Update the `README.md` file and the `html` link in the `mfem/doxygen` repo.
-- [ ] Update URL shortlinks:
-    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
-    - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
+- [ ] Update and push documentation  to `mfem/doxygen`.
+- [ ] Update URL shorlinks:
+    - [ ] Create a shortlink at [https://goo.gl/](https://goo.gl/) for the release tarball, e.g. http://mfem.github.io/releases/mfem-3.1.tgz.
+    - [ ] (LLNL only) Add and commit the new shorlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
     - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
 - [ ] Update website in `mfem/web` repo:
     - Update version and shortlinks in `src/index.md` and `src/download.md`.
@@ -683,9 +455,6 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 
 ## LLNL Workflow
 
-
-### Mirroring on Bitbucket
-
 - The GitHub `master` and `next` branches are mirrored to the LLNL institutional
   Bitbucket repository as `gh-master` and `gh-next`.
 
@@ -704,41 +473,22 @@ MFEM uses a `master`/`next`-branch workflow as described below:
      your own risk.
 
 
-### Mirroring on GitLab
-
-- MFEM repository is also mirrored on the LLNL GitLab instance, in a
-  semi-automated manner.
-
-- This instance is meant to complete CI testing with tests on Livermore
-  Computing systems. GitLab pipeline status is reported in the corresponding
-  GitHub pull request.
-
-- In GitLab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
-
-- No change to the MFEM repo can be made on this instance.
-
 ## Automated Testing
 
 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.
 
-In addition, developers can set local git hooks to run some quick checks on
-commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
-directory.
-
-
 ### Linux and Mac smoke tests
-We use GitHub Actions to drive the default tests on the `master` and `next`
-branches. See the `.github/workflows` files and the logs at
-[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+We use Travis CI to drive the default tests on the `master` and `next`
+branches. See the `.travis` file and the logs at
+[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).
 
-Testing using GitHub Actions should be kept lightweight, as there is a time
+Testing using Travis CI should be kept lightweight, as there is a 50 minute time
 constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
 
 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.
 
-
 ### Windows smoke test
 We use Appveyor to test building with the MS Visual C++ compiler in a Windows
 environment, as well as to test the CMake build. See the `.appveyor` file and the
@@ -748,17 +498,12 @@ build logs at
 CMake is used to generate the MSVC Project files and drive the build.  A release
 and debug build is performed with a simple run of `ex1` to verify the executable.
 
-
 ### Tests at LLNL
+At LLNL, we mirror the `master` and `next` branches internally (to `gh-master`
+and `gh-next`) and run longer nightly tests via cron. On the weekends, a more
+extensive test is run which extracts and executes all the different sample runs
+from each example.
 
-- We mirror the `master` and `next` branches internally (to `gh-master` and
-  `gh-next`) and run longer nightly tests via cron. On the weekends, a more
-  extensive test is run which extracts and executes all the different sample
-  runs from each example and most miniapps.
-
-- We also mirror PRs on the LLNL GitLab instance. PR mirroring can only be
-  triggered by _LLNL developers_, but test status is publicly available. Only
-  _LLNL developers_ can access the detailed test report.
 
 ## Contact Information
 

COPYRIGHT

@@ -0,0 +1,64 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+COPYRIGHT
+
+The following copyright applies to each file in the MFEM distribution, unless
+otherwise stated in the file:
+
+Copyright (c) 2010, Lawrence Livermore National Security, LLC.  Produced at the
+Lawrence Livermore National Laboratory under Contract No. DE-AC52-07NA27344 with
+the DOE.
+
+Neither the United States Government nor Lawrence Livermore National Security,
+LLC nor any of their employees, makes any warranty, express or implied, or
+assumes any liability or responsibility for the accuracy, completeness, or
+usefulness of any information, apparatus, product, or process disclosed, or
+represents that its use would not infringe privately-owned rights.
+
+Also, reference herein to any specific commercial products, process, or services
+by trade name, trademark, manufacturer or otherwise does not necessarily
+constitute or imply its endorsement, recommendation, or favoring by the United
+States Government or Lawrence Livermore National Security, LLC. The views and
+opinions of authors expressed herein do not necessarily state or reflect those
+of the United States Government or Lawrence Livermore National Security, LLC,
+and shall not be used for advertising or product endorsement purposes.
+
+
+LICENSE
+
+MFEM is free software; you can redistribute it and/or modify it under the terms
+of the GNU Lesser General Public License (as published by the Free Software
+Foundation) version 2.1 dated February 1999, with the following EXCEPTIONS:
+
+   1. Subclasses of MFEM classes do not constitute a derivative work.
+
+   2. Static linking of applications to the MFEM library does not constitute a
+      derivative work and does not require the author to provide source code for
+      the application, use the shared MFEM library, or link their applications
+      against a user-supplied version of MFEM.
+
+MFEM is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the terms and conditions of the GNU Lesser General
+Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this library (file LICENSE); if not, write to the Free Software Foundation,
+Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+
+CONTACT INFORMATION
+
+The software is released under LLNL-CODE-443211. Please see
+
+                   http://mfem.org
+
+for more information and source code availability.

LICENSE

@@ -1,29 +1,504 @@
-BSD 3-Clause License
+		  GNU LESSER GENERAL PUBLIC LICENSE
+		       Version 2.1, February 1999
 
-Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC
-All rights reserved.
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
 
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
 
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
+			    Preamble
 
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard.  To achieve this, non-free programs must be
+allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+		  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+  
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+			    NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
 
-* Neither the name of the copyright holder nor the names of its
-  contributors may be used to endorse or promote products derived from
-  this software without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

NOTICE

@@ -1,33 +0,0 @@
-This work was produced under the auspices of the U.S. Department of Energy by
-Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344.
-
-This work was prepared as an account of work sponsored by an agency of
-the United States Government. Neither the United States Government nor
-Lawrence Livermore National Security, LLC, nor any of their employees
-makes any warranty, expressed or implied, or assumes any legal liability
-or responsibility for the accuracy, completeness, or usefulness of any
-information, apparatus, product, or process disclosed, or represents that
-its use would not infringe privately owned rights.
-
-Reference herein to any specific commercial product, process, or service
-by trade name, trademark, manufacturer, or otherwise does not necessarily
-constitute or imply its endorsement, recommendation, or favoring by the
-United States Government or Lawrence Livermore National Security, LLC.
-
-The views and opinions of authors expressed herein do not necessarily
-state or reflect those of the United States Government or Lawrence
-Livermore National Security, LLC, and shall not be used for advertising
-or product endorsement purposes.
-
-Inclusion of external software:
-
-This project distributes the sources of several external software products with
-their own respective licenses which can be found in their code and attached
-license files. These software products and their licenses are as follows:
-
-* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
-* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
-* Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
-* Picojson (fem/picojson.h) -- Custom 2-clause license
-* TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
-* Zstr (general/zstr.hpp) -- MIT license

README

@@ -0,0 +1,67 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               http://mfem.org
+
+MFEM is a modular parallel C++ library for finite element methods. Its goal is
+to enable the research and development of scalable finite element discretization
+and solver algorithms through general finite element abstractions, accurate and
+flexible visualization, and tight integration with the hypre library.
+
+* For building instructions, see the file INSTALL, or type "make help".
+
+* Copyright and licensing information can be found in the file COPYRIGHT.
+
+* The best starting point for new users interested in MFEM's features is the
+  interactive documentation in examples/README.html.
+
+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.
+
+Conceptually, MFEM can be viewed as a finite element toolbox that provides the
+building blocks for developing finite element algorithms in a manner similar to
+that of MATLAB for linear algebra methods. In particular, MFEM provides support
+for arbitrary high-order H1-conforming, discontinuous (L2), H(div)-conforming,
+H(curl)-conforming and NURBS finite element spaces in 2D and 3D, as well as many
+bilinear, linear and nonlinear forms defined on them. It enables the quick
+prototyping of various finite element discretizations, including Galerkin
+methods, mixed finite elements, Discontinuous Galerkin (DG), isogeometric
+analysis, hybridization and Discontinuous Petrov-Galerkin (DPG) approaches.
+
+MFEM includes classes for dealing with a wide range of mesh types: triangular,
+quadrilateral, tetrahedral and hexahedral, as well as surface and topologically
+periodical meshes. It has general support for mesh refinement, including local
+conforming and non-conforming (AMR) adaptive refinement. Arbitrary element
+transformations, allowing for high-order mesh elements with curved boundaries,
+are also supported.
+
+MFEM is commonly used as a "finite element to linear algebra translator", since
+it can take a problem described in terms of finite element-type objects, and
+produce the corresponding linear algebra vectors and sparse matrices. In order
+to facilitate this, MFEM uses compressed sparse row (CSR) sparse matrix storage
+and includes simple smoothers and Krylov solvers, such as PCG, MINRES and GMRES,
+as well as support for sequential sparse direct solvers from the SuiteSparse
+library. Nonlinear solvers (the Newton method), eigensolvers (LOBPCG), and
+several explicit and implicit Runge-Kutta time integrators are also available.
+
+MFEM supports MPI-based parallelism throughout the library, and can readily be
+used as a scalable unstructured finite element problem generator. MFEM-based
+applications require minimal changes to transition from a serial to a
+high-performing parallel version of the code, where they can take advantage of
+the integrated scalable linear solvers from the hypre library. Comprehensive
+support for other external packages, e.g. PETSc and SUNDIALS is also included,
+giving access to many additional linear and nonlinear solvers, preconditioners,
+time integrators, etc.
+
+For examples of using MFEM, see the examples/ and miniapps/ directories, as well
+as the OpenGL visualization tool GLVis which is available at http://glvis.org.
+
+This project is released under the LGPL v2.1 license with static linking
+exception. See files COPYRIGHT and LICENSE file for full details.
+
+LLNL Release Number: LLNL-CODE-443211
+DOI: 10.11578/dc.20171025.1248

README.md

@@ -1,76 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-[MFEM](https://mfem.org) is a modular parallel C++ library for finite element
-methods. Its goal is to enable high-performance scalable finite element
-discretization research and application development on a wide variety of
-platforms, ranging from laptops to supercomputers.
-
-We welcome contributions and feedback from the community. Please see the file
-[CONTRIBUTING.md](CONTRIBUTING.md) for additional details about our development
-process.
-
-* For building instructions, see the file [INSTALL](INSTALL), or type "make help".
-
-* Copyright and licensing information can be found in files [LICENSE](LICENSE) and [NOTICE](NOTICE).
-
-* The best starting point for new users interested in MFEM's features is to
-  review the examples and miniapps at https://mfem.org/examples.
-
-* Instructions for learning with Docker are in [config/docker](config/docker).
-
-Conceptually, MFEM can be viewed as a finite element toolbox that provides the
-building blocks for developing finite element algorithms in a manner similar to
-that of MATLAB for linear algebra methods. In particular, MFEM provides support
-for arbitrary high-order H1-conforming, discontinuous (L2), H(div)-conforming,
-H(curl)-conforming and NURBS finite element spaces in 2D and 3D, as well as many
-bilinear, linear and nonlinear forms defined on them. It enables the quick
-prototyping of various finite element discretizations, including Galerkin
-methods, mixed finite elements, Discontinuous Galerkin (DG), isogeometric
-analysis, hybridization and Discontinuous Petrov-Galerkin (DPG) approaches.
-
-MFEM includes classes for dealing with a wide range of mesh types: triangular,
-quadrilateral, tetrahedral and hexahedral, as well as surface and topologically
-periodical meshes. It has general support for mesh refinement, including local
-conforming and non-conforming (AMR) adaptive refinement. Arbitrary element
-transformations, allowing for high-order mesh elements with curved boundaries,
-are also supported.
-
-When used as a "finite element to linear algebra translator", MFEM can take a
-problem described in terms of finite element-type objects, and produce the
-corresponding linear algebra vectors and fully or partially assembled operators,
-e.g. in the form of global sparse matrices or matrix-free operators. The library
-includes simple smoothers and Krylov solvers, such as PCG, MINRES and GMRES, as
-well as support for sequential sparse direct solvers from the SuiteSparse
-library. Nonlinear solvers (the Newton method), eigensolvers (LOBPCG), and
-several explicit and implicit Runge-Kutta time integrators are also available.
-
-MFEM supports MPI-based parallelism throughout the library, and can readily be
-used as a scalable unstructured finite element problem generator. Starting with
-version 4.0, MFEM offers support for GPU acceleration, and programming models,
-such as CUDA, HIP, OCCA, RAJA and OpenMP. MFEM-based applications require
-minimal changes to switch from a serial to a highly-performant MPI-parallel
-version of the code, where they can take advantage of the integrated linear
-solvers from the hypre library. Comprehensive support for other external
-packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
-additional linear and nonlinear solvers, preconditioners, time integrators, etc.
-
-For examples of using MFEM, see the [examples/](examples) and [miniapps/](miniapps)
-directories, as well as the OpenGL visualization tool GLVis which is available
-at https://glvis.org.
-
-## License
-
-MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license. See [LICENSE](LICENSE) and [NOTICE](NOTICE) for
-details.
-
-SPDX-License-Identifier: BSD-3-Clause <br>
-LLNL Release Number: LLNL-CODE-806117 <br>
-DOI: 10.11578/dc.20171025.1248

backends/all.hpp

@@ -0,0 +1,27 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_ALL_HPP
+#define MFEM_BACKENDS_ALL_HPP
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_BACKENDS
+
+#include "base/backend.hpp"
+
+#ifdef MFEM_USE_OCCA
+#include "occa/backend.hpp"
+#endif
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -0,0 +1,213 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ARRAY_HPP
+#define MFEM_BACKENDS_BASE_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "layout.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+/// Extension to the template class Array<T>
+class PArray : public RefCounted
+{
+protected:
+   /// Layout with shared ownership (smart pointer)
+   DLayout layout;
+
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const = 0;
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size) = 0;
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   virtual void *DoPullData(void *buffer, std::size_t item_size) = 0;
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   virtual void DoFill(const void *value_ptr, std::size_t item_size) = 0;
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size) = 0;
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and item_size. */
+   virtual void DoAssign(const PArray &src, std::size_t item_size) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief The @a layout parameter will be reference counted and therefore it
+       should be dynamically allocated. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PArray(PLayout &p_layout)
+      : layout(&p_layout)
+   {
+      MFEM_ASSERT(layout && layout->HasEngine(), "invalid layout");
+   }
+
+   virtual ~PArray() { }
+
+   /// Get the current size of the array.
+   std::size_t Size() const { return layout->Size(); }
+
+   /// Get the current layout of the array.
+   PLayout &GetLayout() const { return *layout; }
+
+   /// TODO
+   /// Note: we cannot use static_cast for class PArray.
+   template <typename derived_t>
+   derived_t &As() { return dynamic_cast<derived_t&>(*this); }
+
+   /// TODO
+   /// Note: we cannot use static_cast for class PArray.
+   template <typename derived_t>
+   const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename T>
+   DArray Clone(bool copy_data, T **buffer) const
+   { return DArray(DoClone(copy_data, (void**)buffer, sizeof(T))); }
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   template <typename T>
+   int Resize(PLayout &new_layout, T **buffer)
+   { return DoResize(new_layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for Resize(*layout, buffer).
+   /** This method is useful for updating the array after its layout is changed
+       externally. */
+   template <typename T>
+   int Update(T **buffer)
+   { return DoResize(*layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for layout->Resize(new_size) followed by Update()
+   template <typename T>
+   int Resize(std::size_t new_size, T **buffer)
+   { layout->Resize(new_size); return Update(buffer); }
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   template <typename T>
+   T *PullData(T *buffer)
+   { return Size() ? (T*)DoPullData((void*)buffer, sizeof(T)) : NULL; }
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   template <typename T>
+   void Fill(const T &value) { if (Size()) { DoFill(&value, sizeof(T)); } }
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   template <typename T>
+   void PushData(const T *src_buffer)
+   { if (Size()) { DoPushData(src_buffer, sizeof(T)); } }
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and entry type. */
+   template <typename T>
+   void Assign(const PArray &src) { if (Size()) { DoAssign(src, sizeof(T)); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ARRAY_HPP

backends/base/backend.hpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BACKEND_HPP
+#define MFEM_BACKENDS_BASE_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+#include <string>
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+/// TODO
+class Backend
+{
+public:
+   /// TODO
+   virtual ~Backend() { }
+
+   /// TODO
+   virtual bool Supports(const std::string &engine_spec) const = 0;
+
+   /// TODO
+   virtual Engine *Create(const std::string &engine_spec) = 0;
+
+#ifdef MFEM_USE_MPI
+   /// TODO
+   virtual Engine *Create(MPI_Comm comm, const std::string &engine_spec) = 0;
+#endif
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BACKEND_HPP

backends/base/bilinearform.hpp

@@ -0,0 +1,72 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+#define MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+
+namespace mfem
+{
+
+class Vector;
+class OperatorHandle;
+class BilinearForm;
+
+/// TODO: doxygen
+class PBilinearForm : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   BilinearForm *bform;
+
+public:
+   /// TODO: doxygen
+   PBilinearForm(const Engine &e, BilinearForm &bf)
+      : engine(&e), bform(&bf) { }
+
+   /// Virtual destructor
+   virtual ~PBilinearForm() { }
+
+   /// Get the associated Engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble() = 0;
+
+   /// TODO: doxygen
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A) = 0;
+
+   /// TODO: doxygen
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior) = 0;
+
+   /// TODO: doxygen
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b,
+                                   Vector &x) = 0;
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BILINEARFORM_HPP

backends/base/engine.cpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+namespace mfem
+{
+
+Engine::Engine(Backend *b, int n_mem, int n_workers)
+   : backend(b),
+#ifdef MFEM_USE_MPI
+     comm(MPI_COMM_NULL),
+#endif
+     num_mem_res(n_mem),
+     num_workers(n_workers),
+     memory_resources(new MemoryResource*[num_mem_res]()),
+     workers_weights(new double[num_workers]()),
+     workers_mem_res(new int[num_workers]())
+{
+   // Note: all arrays are value-initialized with zeros.
+}
+
+Engine::~Engine()
+{
+   delete [] workers_mem_res;
+   delete [] workers_weights;
+   for (int i = 0; i < num_mem_res; i++)
+   {
+      delete memory_resources[i];
+   }
+   delete [] memory_resources;
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/engine.hpp

@@ -0,0 +1,190 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ENGINE_HPP
+#define MFEM_BACKENDS_BASE_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "memory_resource.hpp"
+#include "smart_pointers.hpp"
+#include "utils.hpp"
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+// Forward declarations.
+class Backend;
+template <typename T> class Array;
+class Operator;
+class FiniteElementSpace;
+class LinearForm;
+class BilinearForm;
+class MixedBilinearForm;
+class NonlinearForm;
+
+
+/// In parallel, each MPI rank will usually create a single engine.
+class Engine : public RefCounted
+{
+protected:
+   Backend *backend; ///< Backend that created the engine. Not owned.
+
+#ifdef MFEM_USE_MPI
+   MPI_Comm comm; ///< Associated MPI communicator (may be MPI_COMM_NULL).
+#endif
+
+   /// Number of memory resources used by the Engine.
+   int num_mem_res;
+   /// Number of workers used by the Engine.
+   int num_workers;
+
+   /// Memory resources used by the engine - array of pointers.
+   /** Both the array and the entries are owned. */
+   MemoryResource **memory_resources;
+
+   /// Relative computational speed of the workers. Owned.
+   double *workers_weights;
+
+   /// For each worker, which memory resource it uses.
+   int *workers_mem_res;
+
+public:
+   /// TODO: doxygen
+   Engine(Backend *b, int n_mem, int n_workers);
+
+   /// TODO: doxygen
+   virtual ~Engine();
+
+
+   /**
+       @name Machine resources interface
+    */
+   ///@{
+
+#ifdef MFEM_USE_MPI
+   /// Get the associated MPI_Comm
+   MPI_Comm GetComm() const { return comm; }
+#endif
+
+   /// TODO
+   int GetNumMemRes() const { return num_mem_res; }
+
+   /// TODO
+   MemoryResource &GetMemRes(int idx) const { return *memory_resources[idx]; }
+
+   /// TODO
+   int GetNumWorkers() const { return num_workers; }
+
+   /// TODO
+   const double *GetWorkersWeights() const { return workers_weights; }
+
+   /// TODO
+   const int *GetWorkersMemRes() const { return workers_mem_res; }
+
+   ///@}
+   // End: Machine resources interface
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   // TODO: Asynchronous execution in this class ...
+
+   /// Allocate and return a new layout for the given @a size.
+   /** The layout decomposition (in the case of multiple workers) is determined
+       automatically by the Engine using a deterministic algorithm: calls to
+       this method with the same @a size will produce the same result, as long
+       as the Engine remains unmodified between the calls.
+
+       The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+   */
+   virtual DLayout MakeLayout(std::size_t size) const = 0;
+
+   /// Allocate and return a new layout for the given worker decomposition.
+   /** The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+
+       The @a offsets should satisfy: offsets.Size() == number of workers + 1,
+       offsets[0] == 0, and offsets[i] <= offsets[i+1], for i: 0 <= i < number
+       of workers. */
+   virtual DLayout MakeLayout(const Array<std::size_t> &offsets) const = 0;
+
+   // Note: There may be other ways to construct layouts in the future, e.g.
+   //       block-vector layouts, or multi-vector layouts.
+
+   /// TODO
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const = 0;
+
+   /// Allocate and return a new vector using the given @a layout.
+   /** The returned object is a smart pointer that will automatically deallocate
+       the vector.
+
+       TODO: Produce an error if memory allocation fails?
+
+       Only layouts returned by this Engine are guaranteed to be supported.
+       Using a type that is not supported will produce an error. */
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const = 0;
+
+   /// TODO: doxygen
+   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const = 0;
+
+   /// TODO: doxygen
+   virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;
+
+
+   // Question: How do we construct coefficients?
+
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const MixedBilinearForm &mbl_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const NonlinearForm &nl_form) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ENGINE_HPP

backends/base/fespace.hpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_FE_SPACE_HPP
+#define MFEM_BACKENDS_BASE_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+class QuadratureSpace;
+
+/// TODO: doxygen
+class PFiniteElementSpace : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   mfem::FiniteElementSpace *fes;
+
+public:
+   /// TODO: doxygen
+   PFiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace)
+      : engine(&e), fes(&fespace) { }
+
+   /// Virtual destructor
+   virtual ~PFiniteElementSpace() { }
+
+   /// Get the associated engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Return the associated mfem::FiniteElementSpace
+   mfem::FiniteElementSpace *GetFESpace() const { return fes; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   /**
+       @name Virtual interface: finite element space functionality
+    */
+   ///@{
+
+   /// TODO
+   /** Return the operator mapping T-vectors to L-vectors. If a NULL pointer is
+       returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetProlongationOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to T-vectors that extracts the
+       subset of all true dofs, i.e. no assembly is performed. If a NULL pointer
+       is returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetRestrictionOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       values of a GridFunction as a QuadratureFunction on the given
+       QuadratureSpace. If the returned pointer is NULL, then the mapping is the
+       identity. */
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       _reference element_ gradients of a GridFunction as a QuadratureFunction
+       on the given QuadratureSpace. */
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_FE_SPACE_HPP

backends/base/layout.hpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_LAYOUT_HPP
+#define MFEM_BACKENDS_BASE_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "smart_pointers.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic layout (array/vector layout descriptor)
+class PLayout : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   std::size_t size;
+
+   template <typename DObject>
+   struct Maker
+   {
+      template <typename entry_t>
+      static DObject MakeNew(PLayout &layout);
+   };
+
+public:
+   explicit PLayout(std::size_t s = 0) : engine(NULL), size(s) { }
+
+   explicit PLayout(const Engine &e, std::size_t s = 0)
+      : engine(&e), size(s) { }
+
+   virtual ~PLayout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size) { size = new_size; }
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets)
+   { MFEM_ABORT("method not supported"); }
+
+   ///@}
+   // End: Virtual interface
+
+   /// Layouts without engine cannot create DArray, DVector, etc.
+   bool HasEngine() const { return engine != NULL; }
+
+   /// TODO: doxygen
+   const Engine &GetEngine() const { return *engine; }
+
+   /// TODO: doxygen
+   std::size_t Size() const { return size; }
+
+   /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+   /// TODO: doxygen
+   template <typename DObject, typename entry_t>
+   DObject Make()
+   {
+      MFEM_ASSERT(HasEngine(), "this method requires an Engine");
+      return Maker<DObject>::template MakeNew<entry_t>(*this);
+   }
+};
+
+template <> struct PLayout::Maker<DArray>
+{
+   template <typename entry_t> static DArray MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeArray(layout, sizeof(entry_t)); }
+};
+
+template <> struct PLayout::Maker<DVector>
+{
+   template <typename entry_t> static DVector MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeVector(layout, ScalarId<entry_t>::value); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_LAYOUT_HPP

backends/base/memory_resource.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "../../general/error.hpp"
+
+#include <cstdlib>
+#include <cstring>
+#include <cerrno>
+
+namespace mfem
+{
+
+void *NewDeleteMemoryResource::DoAllocate(std::size_t bytes,
+                                          std::size_t alignment)
+{
+   void *p = ::operator new[](bytes);
+   MFEM_VERIFY(!alignment || (std::size_t)(p) % alignment == 0,
+               "invalid alignment");
+   return p;
+}
+
+void NewDeleteMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                           std::size_t alignment)
+{
+   ::operator delete[](p);
+}
+
+
+void *AlignedMemoryResource::DoAllocate(std::size_t bytes,
+                                        std::size_t alignment)
+{
+   void *p;
+   if (!alignment) { alignment = sizeof(long double); }
+   MFEM_VERIFY(posix_memalign(&p, alignment, bytes) == 0,
+               "error in posix_memalign(): " << strerror(errno));
+   return p;
+}
+
+void AlignedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                         std::size_t alignment)
+{
+   free(p);
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/memory_resource.hpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+#define MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include <cstddef>
+
+namespace mfem
+{
+
+/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
+class MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment) = 0;
+   virtual void DoDeallocate(void* p, std::size_t bytes,
+                             std::size_t alignment) = 0;
+
+public:
+   // Implicitly defined default & copy constructors
+
+   /// Virtual destructor.
+   virtual ~MemoryResource() { }
+
+   /// If alignment == 0, use default alignment.
+   void *Allocate(std::size_t bytes, std::size_t alignment = 0)
+   { return DoAllocate(bytes, alignment); }
+
+   /// If alignment == 0, use default alignment.
+   void Deallocate(void *p, std::size_t bytes, std::size_t alignment = 0)
+   { DoDeallocate(p, bytes, alignment); }
+};
+
+
+/** @brief Dynamic host memory resource using operator new[](std::size_t) for
+    allocation and operator delete[](void*) for deallocation. */
+class NewDeleteMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+
+/** @brief Dynamic host memory resource using posix_memalign() for aligned
+    allocation and free() for deallocation. */
+class AlignedMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP

backends/base/smart_pointers.hpp

@@ -0,0 +1,234 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+#define MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "utils.hpp"
+#include "../../general/error.hpp"
+#include <cstddef>
+
+// #define MFEM_TRACE_SHARED_PTR
+#ifdef MFEM_TRACE_SHARED_PTR
+#include "../../general/globals.hpp"
+#endif
+
+namespace mfem
+{
+
+/// Base class for classes with simple reference counting.
+/** Reference counting is performed by the class SharedPtr. */
+class RefCounted
+{
+private:
+   mutable unsigned ref_count;
+
+   /// Only class SharedPtr can access ref_count.
+   template <typename T> friend class SharedPtr;
+
+public:
+   RefCounted() : ref_count(0) { }
+
+   /** @brief Prevent SharedPtr objects from deleting this object by
+       incrementing the reference counter by one. */
+   void DontDelete() const { ++ref_count; }
+};
+
+
+/** @brief Smart pointer class that manages objects of type T derived from class
+    RefCounted. */
+/** This class is generally meant to work with dynamically allocated object,
+    specifically objects allocated with operator new(). It will invoke operator
+    delete() to destroy the managed object when its reference counter reaches
+    zero. This behavior can be overriden by calling RefCounted::DontDelete() to
+    ensure that an object will not be deleted by a SharedPtr that holds a
+    pointer to it.
+    @note This class is NOT thread-safe and does not support circular ownership.
+ */
+template <typename T>
+class SharedPtr
+{
+public:
+   typedef T stored_type;
+
+private:
+   T *ptr;
+
+   void Init(T *new_ptr)
+   {
+      ptr = new_ptr;
+      if (ptr) { ++ptr->RefCounted::ref_count; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+   void Destroy()
+   {
+      MFEM_ASSERT(!ptr || ptr->RefCounted::ref_count >= 1, "invalid use");
+      if (ptr && --ptr->RefCounted::ref_count == 0) { delete ptr; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+
+public:
+   SharedPtr() : ptr(NULL)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]: ptr = " << ptr << '\n';
+#endif
+   }
+
+   SharedPtr(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.ptr);
+   }
+
+   template <typename U>
+   SharedPtr(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.Get());
+   }
+
+   explicit SharedPtr(T *p)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(p);
+   }
+
+   ~SharedPtr()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+   }
+
+   SharedPtr &operator=(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.ptr); return *this;
+   }
+
+   template <typename U>
+   SharedPtr &operator=(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.Get()); return *this;
+   }
+
+   T &operator*() const { return *ptr; }
+   T *operator->() const { return ptr; }
+
+   operator bool() const { return ptr; }
+   bool operator!() const { return !ptr; }
+
+   template <typename U>
+   bool operator==(const SharedPtr<U> &other) const
+   { return ptr == other.Get(); }
+   template <typename U>
+   bool operator!=(const SharedPtr<U> &other) const
+   { return ptr != other.Get(); }
+
+   // Comparison to any type convertible to void *, e.g. the type of NULL.
+   template <typename U>
+   bool operator==(const U &p) const { return ptr == (void*) p; }
+   template <typename U>
+   bool operator!=(const U &p) const { return ptr != (void*) p; }
+
+   T *Get() const { return ptr; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t *As() const { return util::As<derived_t>(ptr); }
+
+   unsigned UseCount() const { return ptr ? ptr->RefCounted::ref_count : 0; }
+
+   void Reset()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+      ptr = NULL;
+   }
+
+   /// The type U* needs to be implicitly convertible to T*
+   template <typename U>
+   void Reset(U *new_ptr)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      if (ptr != new_ptr) { Destroy(); Init(new_ptr); }
+   }
+
+   void Swap(SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      std::swap(ptr, other.ptr);
+   }
+};
+
+
+template <class T>
+inline void Swap(SharedPtr<T> &a, SharedPtr<T> &b) { a.Swap(b); }
+
+
+class PLayout;
+typedef SharedPtr<PLayout> DLayout;
+
+class PArray;
+typedef SharedPtr<PArray> DArray;
+
+class PVector;
+typedef SharedPtr<PVector> DVector;
+
+class PFiniteElementSpace;
+typedef SharedPtr<PFiniteElementSpace> DFiniteElementSpace;
+
+class PBilinearForm;
+typedef SharedPtr<PBilinearForm> DBilinearForm;
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_SMART_POINTERS_HPP

backends/base/utils.hpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_UTILS_HPP
+#define MFEM_BACKENDS_BASE_UTILS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/error.hpp"
+
+namespace mfem
+{
+
+namespace util
+{
+
+//
+// Inline methods
+//
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *As(base_t *base_obj)
+{
+   MFEM_ASSERT(dynamic_cast<derived_t*>(base_obj) != NULL,
+               "invalid object type");
+   return static_cast<derived_t*>(base_obj);
+}
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *Is(base_t *base_obj)
+{
+   return dynamic_cast<derived_t*>(base_obj);
+}
+
+} // namespace mfem::util
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_UTILS_HPP

backends/base/vector.hpp

@@ -0,0 +1,153 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_VECTOR_HPP
+#define MFEM_BACKENDS_BASE_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic vector - array of scalars.
+class PVector : virtual public PArray
+{
+protected:
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries specified by @a buffer_type_id
+       which should be a constant defined by the `value` field in a
+       specialization of the template class mfem::ScalarId.
+
+       Returns NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer), if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const = 0;
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const = 0;
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief Create a PVector. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PVector(PLayout &p_layout)
+      : PArray(p_layout) { }
+
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+   // TODO: Multi-vector interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries of type @a scalar_t.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename scalar_t>
+   DVector Clone(bool copy_data, scalar_t **buffer) const
+   {
+      return DVector(DoVectorClone(copy_data, (void**)buffer,
+                                   ScalarId<scalar_t>::value));
+   }
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   template <typename scalar_t>
+   scalar_t DotProduct(const PVector &x) const
+   {
+      scalar_t result;
+      DoDotProduct(x, &result, ScalarId<scalar_t>::value);
+      return result;
+   }
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   template <typename scalar_t>
+   void Axpby(const scalar_t &a, const PVector &x,
+              const scalar_t &b, const PVector &y)
+   { if (Size()) { DoAxpby(&a, x, &b, y, ScalarId<scalar_t>::value); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_VECTOR_HPP

backends/occa/DiffusionIntegrator.okl

@@ -0,0 +1,66 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  COEFF_ARGS      : Code that passes required arguments to the kernel
+  COEFF           : Code that computes the coefficient
+  ================================================
+
+  [MISSING]
+  - Add support to auto-pick @dim and use @idxOrder on stack arrays
+  |   double a[2][2];
+  |   a[0][1]; <-- regular index
+  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
+  - Add support for @idxOrder to change indexing order after allocation
+  |   double a[2][2] @idxOrder(0,1);
+  |   a(0,1) -> a[1][0]
+  |   @set(a, idxOrder(1,0));
+  |   a(0,1) -> a[0][1]
+  - Add support to iterate over loop depending on mode
+  |   for(i; @inner) {
+  |     for(0 < j < N) {} <-- ++j or j += block?
+  |   }
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/tensor/cpu.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/simplex/cpu.okl"
+#  endif
+#endif

backends/occa/MassIntegrator.okl

@@ -0,0 +1,66 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  COEFF_ARGS      : Code that passes required arguments to the kernel
+  COEFF           : Code that computes the coefficient
+  ================================================
+
+  [MISSING]
+  - Add support to auto-pick @dim and use @idxOrder on stack arrays
+  |   double a[2][2];
+  |   a[0][1]; <-- regular index
+  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
+  - Add support for @idxOrder to change indexing order after allocation
+  |   double a[2][2] @idxOrder(0,1);
+  |   a(0,1) -> a[1][0]
+  |   @set(a, idxOrder(1,0));
+  |   a(0,1) -> a[0][1]
+  - Add support to iterate over loop depending on mode
+  |   for(i; @inner) {
+  |     for(0 < j < N) {} <-- ++j or j += block?
+  |   }
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://mass/tensor/cpu.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://mass/simplex/cpu.okl"
+#  endif
+#endif

backends/occa/VectorMassIntegrator.okl

@@ -0,0 +1,38 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  CONST_COEFF     : If the coefficient is constant, pass it
+  .                   as a define
+  ================================================
+
+  [MISSING]
+  See kernels/DiffusionIntegrator.okl
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifndef OCCA_USING_GPU
+#    include "mfem-occa://vmass/tensor/cpu.okl"
+#  endif
+#endif

backends/occa/array.cpp

@@ -0,0 +1,121 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+PArray *Array::DoClone(bool copy_data, void **buffer,
+                       std::size_t item_size) const
+{
+   Array *new_array = new Array(OccaLayout(), item_size);
+   if (copy_data)
+   {
+      new_array->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_array->GetBuffer();
+   }
+   return new_array;
+}
+
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
+               "new_layout is not an OCCA Layout");
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   int err = OccaResize(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = GetBuffer();
+   }
+   return err;
+}
+
+void *Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   if (buffer)
+   {
+      slice.copyTo(buffer);
+   }
+   return buffer;
+}
+
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   switch (item_size)
+   {
+      case sizeof(int8_t):
+         OccaFill(*(const int8_t *)value_ptr);
+         break;
+      case sizeof(int16_t):
+         OccaFill(*(const int16_t *)value_ptr);
+         break;
+      case sizeof(int32_t):
+         OccaFill(*(const int32_t *)value_ptr);
+         break;
+      // case sizeof(int64_t):
+      //    OccaFill(*(const int64_t *)value_ptr);
+      //    break;
+      case sizeof(double):
+         OccaFill(*(const double *)value_ptr);
+         break;
+      // case sizeof(::occa::double2):
+      //    OccaFill(*(const ::occa::double2 *)value_ptr);
+      //    break;
+      default:
+         MFEM_ABORT("item_size = " << item_size << " is not supported");
+   }
+}
+
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (slice.getDevice().hasSeparateMemorySpace() || slice.ptr() != src_buffer)
+   {
+      slice.copyFrom(src_buffer);
+   }
+}
+
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   // Note: static_cast can not be used here since PArray is a virtual base
+   //       class.
+   const Array *source = dynamic_cast<const Array *>(&src);
+   MFEM_ASSERT(source != NULL, "invalid source Array type");
+   OccaAssign(*source);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/array.hpp

@@ -0,0 +1,175 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ARRAY_HPP
+#define MFEM_BACKENDS_OCCA_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "layout.hpp"
+#include "../base/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Array : public virtual PArray
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   // Always true: Size()*item_size == slice.size() <= data.size()
+   mutable ::occa::memory data, slice;
+
+   //
+   // Virtual interface
+   //
+
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void *DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   //
+   // Auxiliary methods
+   //
+
+   inline void *GetBuffer() const;
+
+public:
+   Array(const Engine &e)
+      : PArray(*(new Layout(e, 0))),
+        data(e.Alloc(0)),
+        slice(data)
+   { }
+
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        data(lt.OccaEngine().Alloc(lt.Size()*item_size)),
+        slice(data)
+   { }
+
+   virtual ~Array() { }
+
+   inline void MakeRef(Array &master);
+
+   Layout &OccaLayout() const { return layout->As<Layout>(); }
+
+   const Engine &OccaEngine() const { return OccaLayout().OccaEngine(); }
+
+   ::occa::memory &OccaMem() { return slice; }
+   const ::occa::memory &OccaMem() const { return slice; }
+
+   inline int OccaResize(Layout *lt, std::size_t item_size);
+
+   inline int OccaResize(std::size_t new_size, std::size_t item_size);
+
+   template <typename T>
+   inline void OccaFill(const T val);
+
+   inline void OccaAssign(const Array &src);
+
+   inline void OccaPush(const void *src);
+};
+
+
+//
+// Inline methods
+//
+
+inline void *Array::GetBuffer() const
+{
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   return NULL;
+}
+
+inline int Array::OccaResize(Layout *lt, std::size_t item_size)
+{
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   const std::size_t new_bytes = lt->Size()*item_size;
+   if (data.size() < new_bytes ||
+       data.getDevice() != lt->OccaEngine().GetDevice())
+   {
+      data = lt->OccaEngine().Alloc(new_bytes);
+      slice = data;
+      // If memory allocation fails - an exception is thrown.
+   }
+   else if (slice.size() != new_bytes)
+   {
+      slice = data.slice(0, new_bytes);
+   }
+   return 0;
+}
+
+inline void Array::MakeRef(Array &master)
+{
+   layout = master.layout;
+   data = master.data;
+   slice = master.slice;
+}
+
+inline int Array::OccaResize(std::size_t new_size, std::size_t item_size)
+{
+   Layout &ol = OccaLayout();
+   ol.OccaResize(new_size);
+   return OccaResize(&ol, item_size);
+}
+
+template <typename T>
+inline void Array::OccaFill(const T val)
+{
+   ::occa::linalg::operator_eq<T>(slice, val);
+}
+
+inline void Array::OccaAssign(const Array &src)
+{
+   if (slice != src.slice && slice.size() != 0)
+   {
+      MFEM_ASSERT(slice.size() == src.slice.size(), "");
+      slice.copyFrom(src.slice);
+   }
+}
+
+inline void Array::OccaPush(const void *src)
+{
+   if (slice.size() != 0)
+   {
+      slice.copyFrom(src);
+   }
+}
+
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ARRAY_HPP

backends/occa/backend.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Backend::Supports(const std::string &engine_spec) const
+{
+   // TODO: check if 'engine_spec' is valid OCCA string.
+   return true;
+}
+
+mfem::Engine *Create(const std::string &engine_spec)
+{
+   return new Engine(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+   return new Engine(comm, engine_spec);
+}
+#endif
+
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/backend.hpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BACKEND_HPP
+#define MFEM_BACKENDS_OCCA_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+// Only the Backend and Engine classes should be exposed through "backend.hpp"
+#include "../base/backend.hpp"
+#include "engine.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Backend : public mfem::Backend
+{
+public:
+   virtual ~Backend();
+
+   virtual bool Supports(const std::string &engine_spec) const;
+
+   virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BACKEND_HPP

backends/occa/bilinearform.cpp

@@ -0,0 +1,538 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/bilinearform.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
+   Operator(ofespace_->OccaVLayout()),
+   localX(ofespace_->OccaEVLayout()),
+   localY(ofespace_->OccaEVLayout())
+{
+   Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
+}
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                                   FiniteElementSpace *otestFESpace_) :
+   Operator(otrialFESpace_->OccaVLayout(),
+            otestFESpace_->OccaVLayout()),
+   localX(otrialFESpace_->OccaEVLayout()),
+   localY(otestFESpace_->OccaEVLayout())
+{
+   Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
+}
+
+void OccaBilinearForm::Init(const Engine &e,
+                            FiniteElementSpace *otrialFESpace_,
+                            FiniteElementSpace *otestFESpace_)
+{
+   engine.Reset(&e);
+
+   otrialFESpace = otrialFESpace_;
+   trialFESpace  = otrialFESpace_->GetFESpace();
+
+   otestFESpace = otestFESpace_;
+   testFESpace  = otestFESpace_->GetFESpace();
+
+   mesh = trialFESpace->GetMesh();
+
+   const int elements = GetNE();
+
+   const int trialVDim = trialFESpace->GetVDim();
+
+   const int trialLocalDofs = otrialFESpace->GetLocalDofs();
+   const int testLocalDofs  = otestFESpace->GetLocalDofs();
+
+   // First-touch policy when running with OpenMP
+   if (GetDevice().mode() == "OpenMP")
+   {
+      const std::string &okl_path = OccaEngine().GetOklPath();
+      ::occa::kernel initLocalKernel =
+         GetDevice().buildKernel(okl_path + "utils.okl",
+                                 "InitLocalVector");
+
+      const std::size_t sd = sizeof(double);
+      const uint64_t trialEntries = sd * (elements * trialLocalDofs);
+      const uint64_t testEntries  = sd * (elements * testLocalDofs);
+      for (int v = 0; v < trialVDim; ++v)
+      {
+         const uint64_t trialOffset = v * trialEntries;
+         const uint64_t testOffset  = v * testEntries;
+
+         initLocalKernel(elements, trialLocalDofs,
+                         localX.OccaMem().slice(trialOffset, trialEntries));
+         initLocalKernel(elements, testLocalDofs,
+                         localY.OccaMem().slice(testOffset, testEntries));
+      }
+   }
+}
+
+int OccaBilinearForm::BaseGeom() const
+{
+   return mesh->GetElementBaseGeometry();
+}
+
+int OccaBilinearForm::GetDim() const
+{
+   return mesh->Dimension();
+}
+
+int64_t OccaBilinearForm::GetNE() const
+{
+   return mesh->GetNE();
+}
+
+Mesh& OccaBilinearForm::GetMesh() const
+{
+   return *mesh;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+int64_t OccaBilinearForm::GetTrialNDofs() const
+{
+   return trialFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTestNDofs() const
+{
+   return testFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTrialVDim() const
+{
+   return trialFESpace->GetVDim();
+}
+
+int64_t OccaBilinearForm::GetTestVDim() const
+{
+   return testFESpace->GetVDim();
+}
+
+const FiniteElement& OccaBilinearForm::GetTrialFE(const int i) const
+{
+   return *(trialFESpace->GetFE(i));
+}
+
+const FiniteElement& OccaBilinearForm::GetTestFE(const int i) const
+{
+   return *(testFESpace->GetFE(i));
+}
+
+// Adds new Domain Integrator.
+void OccaBilinearForm::AddDomainIntegrator(OccaIntegrator *integrator,
+                                           const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, DomainIntegrator);
+}
+
+// Adds new Boundary Integrator.
+void OccaBilinearForm::AddBoundaryIntegrator(OccaIntegrator *integrator,
+                                             const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryIntegrator);
+}
+
+// Adds new interior Face Integrator.
+void OccaBilinearForm::AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, InteriorFaceIntegrator);
+}
+
+// Adds new boundary Face Integrator.
+void OccaBilinearForm::AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryFaceIntegrator);
+}
+
+// Adds Integrator based on OccaIntegratorType
+void OccaBilinearForm::AddIntegrator(OccaIntegrator *integrator,
+                                     const ::occa::properties &props,
+                                     const OccaIntegratorType itype)
+{
+   if (integrator == NULL)
+   {
+      std::stringstream error_ss;
+      error_ss << "OccaBilinearForm::";
+      switch (itype)
+      {
+         case DomainIntegrator      : error_ss << "AddDomainIntegrator";       break;
+         case BoundaryIntegrator    : error_ss << "AddBoundaryIntegrator";     break;
+         case InteriorFaceIntegrator: error_ss << "AddInteriorFaceIntegrator"; break;
+         case BoundaryFaceIntegrator: error_ss << "AddBoundaryFaceIntegrator"; break;
+      }
+      error_ss << " (...):\n"
+               << "  Integrator is NULL";
+      const std::string error = error_ss.str();
+      mfem_error(error.c_str());
+   }
+   integrator->SetupIntegrator(*this, baseKernelProps + props, itype);
+   integrators.push_back(integrator);
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialProlongation() const
+{
+   return otrialFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestProlongation() const
+{
+   return otestFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialRestriction() const
+{
+   return otrialFESpace->GetRestrictionOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestRestriction() const
+{
+   return otestFESpace->GetRestrictionOperator();
+}
+
+void OccaBilinearForm::Assemble()
+{
+   // [MISSING] Find geometric information that is needed by intergrators
+   //             to share between integrators.
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->Assemble();
+   }
+}
+
+void OccaBilinearForm::FormLinearSystem(const mfem::Array<int> &constraintList,
+                                        mfem::Vector &x, mfem::Vector &b,
+                                        mfem::Operator *&Aout,
+                                        mfem::Vector &X, mfem::Vector &B,
+                                        int copy_interior)
+{
+   FormOperator(constraintList, Aout);
+   InitRHS(constraintList, x, b, Aout, X, B, copy_interior);
+}
+
+void OccaBilinearForm::FormOperator(const mfem::Array<int> &constraintList,
+                                    mfem::Operator *&Aout)
+{
+   const mfem::Operator *trialP = GetTrialProlongation();
+   const mfem::Operator *testP  = GetTestProlongation();
+   mfem::Operator *rap = this;
+
+   if (trialP)
+   {
+      rap = new RAPOperator(*testP, *this, *trialP);
+   }
+
+   Aout = new OccaConstrainedOperator(rap, constraintList,
+                                      rap != this);
+}
+
+void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
+                               mfem::Vector &x, mfem::Vector &b,
+                               mfem::Operator *A,
+                               mfem::Vector &X, mfem::Vector &B,
+                               int copy_interior)
+{
+   // FIXME: move these kernels to the Backend?
+   static ::occa::kernelBuilder get_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_get_subvector",
+
+         "const int dof_i = v2[i];"
+         "v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i - 1];",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}");
+
+   static ::occa::kernelBuilder set_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_set_subvector",
+         "const int dof_i = v2[i];"
+         "if (dof_i >= 0) { v0[dof_i]      = v1[i]; }"
+         "else            { v0[-dof_i - 1] = -v1[i]; }",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}");
+
+   const mfem::Operator *P = GetTrialProlongation();
+   const mfem::Operator *R = GetTrialRestriction();
+
+   if (P)
+   {
+      // Variational restriction with P
+      B.Resize(P->InLayout());
+      P->MultTranspose(b, B);
+      X.Resize(R->OutLayout());
+      R->Mult(x, X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      X.MakeRef(x);
+      B.MakeRef(b);
+   }
+
+   if (!copy_interior && constraintList.Size() > 0)
+   {
+      ::occa::kernel get_subvector_kernel =
+         get_subvector_builder.build(GetDevice());
+      ::occa::kernel set_subvector_kernel =
+         set_subvector_builder.build(GetDevice());
+
+      const Array &constrList = constraintList.Get_PArray()->As<Array>();
+      Vector subvec(constrList.OccaLayout());
+
+      get_subvector_kernel(constraintList.Size(),
+                           subvec.OccaMem(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           constrList.OccaMem());
+
+      X.Fill(0.0);
+
+      set_subvector_kernel(constraintList.Size(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           subvec.OccaMem(),
+                           constrList.OccaMem());
+   }
+
+   // FIXME: add case for HypreParMatrix here
+   OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
+   if (cA)
+   {
+      cA->EliminateRHS(X.Get_PVector()->As<Vector>(),
+                       B.Get_PVector()->As<Vector>());
+   }
+   else
+   {
+      mfem_error("OccaBilinearForm::InitRHS expects an OccaConstrainedOperator");
+   }
+}
+
+// Matrix vector multiplication.
+void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
+{
+   otrialFESpace->GlobalToLocal(x, localX);
+   localY.OccaFill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultAdd(localX, localY);
+   }
+
+   otestFESpace->LocalToGlobal(localY, y);
+}
+
+// Matrix transpose vector multiplication.
+void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
+{
+   otestFESpace->GlobalToLocal(x, localX);
+   localY.OccaFill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultTransposeAdd(localX, localY);
+   }
+
+   otrialFESpace->LocalToGlobal(localY, y);
+}
+
+void OccaBilinearForm::OccaRecoverFEMSolution(const mfem::Vector &X,
+                                              const mfem::Vector &b,
+                                              mfem::Vector &x)
+{
+   const mfem::Operator *P = this->GetTrialProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+}
+
+// Frees memory bilinear form.
+OccaBilinearForm::~OccaBilinearForm()
+{
+   // Make sure all integrators free their data
+   IntegratorVector::iterator it = integrators.begin();
+   while (it != integrators.end())
+   {
+      delete *it;
+      ++it;
+   }
+}
+
+
+void BilinearForm::InitOccaBilinearForm()
+{
+   // Init 'obform' using 'bform'
+   MFEM_ASSERT(bform != NULL, "");
+   MFEM_ASSERT(obform == NULL, "");
+
+   FiniteElementSpace &ofes =
+      bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
+   obform = new OccaBilinearForm(&ofes);
+
+   // Transfer domain integrators
+   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
+   for (int i = 0; i < dbfi.Size(); i++)
+   {
+      std::string integ_name(dbfi[i]->Name());
+      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
+      ConstantCoefficient *const_coeff =
+         dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      GridFunctionCoefficient *gridfunc_coeff =
+         dynamic_cast<GridFunctionCoefficient*>(scal_coeff);
+      // TODO: other types of coefficients ...
+
+      OccaCoefficient *ocoeff = NULL;
+      if (const_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(),
+                                      const_coeff->constant);
+      }
+      else if (gridfunc_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(),
+                                      *gridfunc_coeff->GetGridFunction(), true);
+      }
+      else if (!scal_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(), 1.0);
+      }
+      else
+      {
+         MFEM_ABORT("Coefficient type not supported");
+      }
+
+      OccaIntegrator *ointeg = NULL;
+      if (integ_name == "(undefined)")
+      {
+         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+      }
+      else if (integ_name == "mass")
+      {
+         ointeg = new OccaMassIntegrator(*ocoeff);
+      }
+      else if (integ_name == "diffusion")
+      {
+         ointeg = new OccaDiffusionIntegrator(*ocoeff);
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+
+      // NOTE: The integrators copy ocoeff, so it can be deleted here so there
+      //       is no memory leak.
+      delete ocoeff;
+
+      const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
+      if (ir) { ointeg->SetIntegrationRule(*ir); }
+
+      obform->AddDomainIntegrator(ointeg);
+   }
+
+   // TODO: other types of integrators ...
+}
+
+bool BilinearForm::Assemble()
+{
+   if (obform == NULL) { InitOccaBilinearForm(); }
+
+   obform->Assemble();
+
+   return true; // --> host assembly is not needed
+}
+
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      mfem::Operator *Aout = NULL;
+      obform->FormOperator(ess_tdof_list, Aout);
+      A.Reset(Aout);
+   }
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+}
+
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A,
+                                    mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   obform->InitRHS(ess_tdof_list, x, b, A.Ptr(), X, B, copy_interior);
+}
+
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X,
+                                      const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   obform->OccaRecoverFEMSolution(X, b, x);
+}
+
+BilinearForm::~BilinearForm()
+{
+   delete obform;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilinearform.hpp

@@ -0,0 +1,213 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+#define MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+enum OccaIntegratorType
+{
+   DomainIntegrator       = 0,
+   BoundaryIntegrator     = 1,
+   InteriorFaceIntegrator = 2,
+   BoundaryFaceIntegrator = 3
+};
+
+class OccaIntegrator;
+
+
+/** Class for bilinear form - "Matrix" with associated FE space and
+    BLFIntegrators. */
+class OccaBilinearForm : public Operator
+{
+   friend class OccaIntegrator;
+
+protected:
+   typedef std::vector<OccaIntegrator*> IntegratorVector;
+
+   SharedPtr<const Engine> engine;
+
+   // State information
+   mutable mfem::Mesh *mesh;
+
+   mutable FiniteElementSpace *otrialFESpace;
+   mutable mfem::FiniteElementSpace *trialFESpace;
+
+   mutable FiniteElementSpace *otestFESpace;
+   mutable mfem::FiniteElementSpace *testFESpace;
+
+   IntegratorVector integrators;
+
+   // Device data
+   ::occa::properties baseKernelProps;
+
+   // The input and output vectors are mapped to local nodes for efficient
+   // operations. In other words, they are E-vectors.
+   // The size is: (number of elements) * (nodes in element) * (vector dim)
+   mutable Vector localX, localY;
+
+public:
+   OccaBilinearForm(FiniteElementSpace *ofespace_);
+
+   OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                    FiniteElementSpace *otestFESpace_);
+
+   void Init(const Engine &e,
+             FiniteElementSpace *otrialFESpace_,
+             FiniteElementSpace *otestFESpace_);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   // Useful mesh Information
+   int BaseGeom() const;
+   int GetDim() const;
+   int64_t GetNE() const;
+
+   mfem::Mesh& GetMesh() const;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   // Useful FE information
+   int64_t GetTrialNDofs() const;
+   int64_t GetTestNDofs() const;
+
+   int64_t GetTrialVDim() const;
+   int64_t GetTestVDim() const;
+
+   const mfem::FiniteElement& GetTrialFE(const int i) const;
+   const mfem::FiniteElement& GetTestFE(const int i) const;
+
+   // Adds new Domain Integrator.
+   void AddDomainIntegrator(OccaIntegrator *integrator,
+                            const ::occa::properties &props =
+                               ::occa::properties());
+
+   // Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(OccaIntegrator *integrator,
+                              const ::occa::properties &props =
+                                 ::occa::properties());
+
+   // Adds new interior Face Integrator.
+   void AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds new boundary Face Integrator.
+   void AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds Integrator based on OccaIntegratorType
+   void AddIntegrator(OccaIntegrator *integrator,
+                      const ::occa::properties &props,
+                      const OccaIntegratorType itype);
+
+   virtual const mfem::Operator *GetTrialProlongation() const;
+   virtual const mfem::Operator *GetTestProlongation() const;
+
+   virtual const mfem::Operator *GetTrialRestriction() const;
+   virtual const mfem::Operator *GetTestRestriction() const;
+
+   // Assembles the form i.e. sums over all domain/bdr integrators.
+   virtual void Assemble();
+
+   void FormLinearSystem(const mfem::Array<int> &constraintList,
+                         mfem::Vector &x, mfem::Vector &b,
+                         mfem::Operator *&Aout,
+                         mfem::Vector &X, mfem::Vector &B,
+                         int copy_interior = 0);
+
+   void FormOperator(const mfem::Array<int> &constraintList,
+                     mfem::Operator *&Aout);
+
+   void InitRHS(const mfem::Array<int> &constraintList,
+                mfem::Vector &x, mfem::Vector &b,
+                mfem::Operator *Aout,
+                mfem::Vector &X, mfem::Vector &B,
+                int copy_interior = 0);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   void OccaRecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                               mfem::Vector &x);
+
+   // Destroys bilinear form.
+   ~OccaBilinearForm();
+};
+
+
+/// TODO: doxygen
+class BilinearForm : public mfem::PBilinearForm
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::BilinearForm *bform;
+   OccaBilinearForm *obform;
+
+   // Called from Assemble() if obform is NULL to initialize obform.
+   void InitOccaBilinearForm();
+
+public:
+   /// TODO: doxygen
+   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf), obform(NULL) { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm();
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method mfem::BilinearForm::Assemble() of
+       the associated mfem::BilinearForm, #bform.
+       @returns True, if the host assembly should NOT be performed. */
+   virtual bool Assemble();
+
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A,
+                                 mfem::Vector &X, mfem::Vector &B,
+                                 int copy_interior);
+
+   virtual void RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                   mfem::Vector &x);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP

backends/occa/bilininteg.cpp

@@ -0,0 +1,954 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "bilininteg.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, OccaDofQuadMaps> OccaDofQuadMaps::AllDofQuadMaps;
+
+OccaGeometry OccaGeometry::Get(::occa::device device,
+                               FiniteElementSpace &ofespace,
+                               const mfem::IntegrationRule &ir,
+                               const int flags)
+{
+   OccaGeometry geom;
+
+   mfem::Mesh &mesh = *(ofespace.GetMesh());
+   if (!mesh.GetNodes())
+   {
+      mesh.SetCurvature(1, false, -1, mfem::Ordering::byVDIM);
+   }
+   mfem::GridFunction &nodes = *(mesh.GetNodes());
+   const mfem::FiniteElementSpace &fespace = *(nodes.FESpace());
+   const mfem::FiniteElement &fe = *(fespace.GetFE(0));
+
+   const int dims     = fe.GetDim();
+   const int elements = fespace.GetNE();
+   const int numDofs  = fe.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   MFEM_ASSERT(dims == mesh.SpaceDimension(), "");
+
+   geom.meshNodes.allocate(device,
+                           dims, numDofs, elements);
+
+   const mfem::Table &e2dTable = fespace.GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+   nodes.Pull();
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int dof = 0; dof < numDofs; ++dof)
+      {
+         const int gid = elementMap[dof + numDofs*e];
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            geom.meshNodes(dim, dof, e) = nodes[fespace.DofToVDof(gid,dim)];
+         }
+      }
+   }
+   geom.meshNodes.keepInDevice();
+
+   if (flags & Jacobian)
+   {
+      geom.J.allocate(device,
+                      dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.J.allocate(device, 1);
+   }
+   if (flags & JacobianInv)
+   {
+      geom.invJ.allocate(device,
+                         dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.invJ.allocate(device, 1);
+   }
+   if (flags & JacobianDet)
+   {
+      geom.detJ.allocate(device,
+                         numQuad, elements);
+   }
+   else
+   {
+      geom.detJ.allocate(device, 1);
+   }
+
+   geom.J.stopManaging();
+   geom.invJ.stopManaging();
+   geom.detJ.stopManaging();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::GetSimplexMaps(device, fe, ir);
+
+   ::occa::properties props;
+   props["defines/NUM_DOFS"] = numDofs;
+   props["defines/NUM_QUAD"] = numQuad;
+   props["defines/STORE_JACOBIAN"]     = (flags & Jacobian);
+   props["defines/STORE_JACOBIAN_INV"] = (flags & JacobianInv);
+   props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
+
+   const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
+   ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
+                                            stringWithDim("InitGeometryInfo",
+                                                          fe.GetDim()),
+                                            props);
+   init(elements,
+        maps.dofToQuadD,
+        geom.meshNodes,
+        geom.J, geom.invJ, geom.detJ);
+
+   return geom;
+}
+
+OccaDofQuadMaps::OccaDofQuadMaps() :
+   hash() {}
+
+OccaDofQuadMaps::OccaDofQuadMaps(const OccaDofQuadMaps &maps)
+{
+   *this = maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::operator = (const OccaDofQuadMaps &maps)
+{
+   hash = maps.hash;
+   dofToQuad   = maps.dofToQuad;
+   dofToQuadD  = maps.dofToQuadD;
+   quadToDof   = maps.quadToDof;
+   quadToDofD  = maps.quadToDofD;
+   quadWeights = maps.quadWeights;
+   return *this;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &fespace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *fespace.GetFE(0),
+              *fespace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &fe,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device, fe, fe, ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &trialFESpace,
+                                      const FiniteElementSpace &testFESpace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *trialFESpace.GetFE(0),
+              *testFESpace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &trialFE,
+                                      const mfem::FiniteElement &testFE,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return (dynamic_cast<const mfem::TensorBasisElement*>(&trialFE)
+           ? GetTensorMaps(device, trialFE, testFE, ir, transpose)
+           : GetSimplexMaps(device, trialFE, testFE, ir, transpose));
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &fe,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   return GetTensorMaps(device,
+                        fe, fe,
+                        ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &trialFE,
+                                                const mfem::FiniteElement &testFE,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   const mfem::TensorBasisElement &trialTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(trialFE);
+   const mfem::TensorBasisElement &testTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(testFE);
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Tensor"
+      << "O1:"  << trialFE.GetOrder()
+      << "O2:"  << testFE.GetOrder()
+      << "BT1:" << trialTFE.GetBasisType()
+      << "BT2:" << testTFE.GetBasisType()
+      << "Q:"   << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QTensorMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QTensorMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QTensorMaps(::occa::device device,
+                                                  const mfem::FiniteElement &fe,
+                                                  const mfem::IntegrationRule &ir,
+                                                  const bool transpose)
+{
+   const mfem::TensorBasisElement &tfe =
+      dynamic_cast<const mfem::TensorBasisElement&>(fe);
+
+   const mfem::Poly_1D::Basis &basis = tfe.GetBasis1D();
+   const int order = fe.GetOrder();
+   // [MISSING] Get 1D dofs
+   const int dofs = order + 1;
+   const int dims = fe.GetDim();
+
+   // Create the dof -> quadrature point map
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+   const int quadPoints = ir1D.GetNPoints();
+   const int quadPoints2D = quadPoints*quadPoints;
+   const int quadPoints3D = quadPoints2D*quadPoints;
+   const int quadPointsND = ((dims == 1) ? quadPoints :
+                             ((dims == 2) ? quadPoints2D : quadPoints3D));
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           quadPoints, dofs);
+   maps.dofToQuadD.allocate(device,
+                            quadPoints, dofs);
+
+   double *quadWeights1DData = NULL;
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                quadPointsND);
+      quadWeights1DData = new double[quadPoints];
+   }
+
+   mfem::Vector d2q(dofs);
+   mfem::Vector d2qD(dofs);
+   for (int q = 0; q < quadPoints; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir1D.IntPoint(q);
+      basis.Eval(ip.x, d2q, d2qD);
+      if (transpose)
+      {
+         quadWeights1DData[q] = ip.weight;
+      }
+      for (int d = 0; d < dofs; ++d)
+      {
+         maps.dofToQuad(q, d)  = d2q[d];
+         maps.dofToQuadD(q, d) = d2qD[d];
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+
+   if (transpose)
+   {
+      for (int q = 0; q < quadPointsND; ++q)
+      {
+         const int qx = q % quadPoints;
+         const int qz = q / quadPoints2D;
+         const int qy = (q - qz*quadPoints2D) / quadPoints;
+         double w = quadWeights1DData[qx];
+         if (dims > 1)
+         {
+            w *= quadWeights1DData[qy];
+         }
+         if (dims > 2)
+         {
+            w *= quadWeights1DData[qz];
+         }
+         maps.quadWeights[q] = w;
+      }
+      maps.quadWeights.keepInDevice();
+      delete [] quadWeights1DData;
+   }
+
+   return maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &fe,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   return GetSimplexMaps(device,
+                         fe, fe,
+                         ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &trialFE,
+                                                 const mfem::FiniteElement &testFE,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Simplex"
+      << "O1:" << trialFE.GetOrder()
+      << "O2:" << testFE.GetOrder()
+      << "Q:"  << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QSimplexMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QSimplexMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QSimplexMaps(::occa::device device,
+                                                   const mfem::FiniteElement &fe,
+                                                   const mfem::IntegrationRule &ir,
+                                                   const bool transpose)
+{
+   const int dims = fe.GetDim();
+   const int numDofs = fe.GetDof();
+   const int numQuad = ir.GetNPoints();
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           numQuad, numDofs);
+   maps.dofToQuadD.allocate(device,
+                            dims, numQuad, numDofs);
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                numQuad);
+   }
+
+   mfem::Vector d2q(numDofs);
+   mfem::DenseMatrix d2qD(numDofs, dims);
+   for (int q = 0; q < numQuad; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir.IntPoint(q);
+      if (transpose)
+      {
+         maps.quadWeights[q] = ip.weight;
+      }
+      fe.CalcShape(ip, d2q);
+      fe.CalcDShape(ip, d2qD);
+      for (int d = 0; d < numDofs; ++d)
+      {
+         const double w = d2q[d];
+         maps.dofToQuad(q, d) = w;
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            const double wD = d2qD(d, dim);
+            maps.dofToQuadD(dim, q, d) = wD;
+         }
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+   if (transpose)
+   {
+      maps.quadWeights.keepInDevice();
+   }
+
+   return maps;
+}
+
+//---[ Integrator Defines ]-----------
+std::string stringWithDim(const std::string &s, const int dim)
+{
+   std::string ret = s;
+   ret += ('0' + (char) dim);
+   ret += 'D';
+   return ret;
+}
+
+int closestWarpBatchTo(const int value)
+{
+   return ((value + 31) / 32) * 32;
+}
+
+int closestMultipleWarpBatch(const int multiple, const int maxSize)
+{
+   if (multiple > maxSize)
+   {
+      return maxSize;
+   }
+   int batch = (32 / multiple);
+   int minDiff = 32 - (multiple * batch);
+   for (int i = 64; i <= maxSize; i += 32)
+   {
+      const int newDiff = i - (multiple * (i / multiple));
+      if (newDiff < minDiff)
+      {
+         batch = (i / multiple);
+         minDiff = newDiff;
+      }
+   }
+   return batch;
+}
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   SetProperties(fespace, fespace, ir, props);
+}
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   props["defines/TRIAL_VDIM"] = trialFESpace.GetVDim();
+   props["defines/TEST_VDIM"]  = testFESpace.GetVDim();
+   props["defines/NUM_DIM"]    = trialFESpace.GetDim();
+
+   if (trialFESpace.hasTensorBasis())
+   {
+      SetTensorProperties(trialFESpace, testFESpace, ir, props);
+   }
+   else
+   {
+      SetSimplexProperties(trialFESpace, testFESpace, ir, props);
+   }
+}
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   SetTensorProperties(fespace, fespace, ir, props);
+}
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   const int trialDofs1D = trialFE.GetOrder() + 1;
+   const int testDofs1D  = testFE.GetOrder() + 1;
+   const int quad1D  = ir1D.GetNPoints();
+   int trialDofsND = trialDofs1D;
+   int testDofsND  = testDofs1D;
+   int quadND  = quad1D;
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 1;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   for (int d = 1; d <= 3; ++d)
+   {
+      if (d > 1)
+      {
+         trialDofsND *= trialDofs1D;
+         testDofsND  *= testDofs1D;
+         quadND *= quad1D;
+      }
+      props["defines"][stringWithDim("NUM_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("NUM_QUAD_", d)] = quadND;
+
+      props["defines"][stringWithDim("TRIAL_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("TEST_DOFS_" , d)] = testDofsND;
+   }
+
+   // 1D Defines
+   const int m1InnerBatch = 32 * ((quad1D + 31) / 32);
+   props["defines/A1_ELEMENT_BATCH"]       = closestMultipleWarpBatch(quad1D, 512);
+   props["defines/M1_OUTER_ELEMENT_BATCH"] = closestMultipleWarpBatch(m1InnerBatch,
+                                                                      512);
+   props["defines/M1_INNER_ELEMENT_BATCH"] = m1InnerBatch;
+
+   // 2D Defines
+   props["defines/A2_ELEMENT_BATCH"] = 1;
+   props["defines/A2_QUAD_BATCH"]    = 1;
+   props["defines/M2_ELEMENT_BATCH"] = 32;
+
+   // 3D Defines
+   const int a3QuadBatch = closestMultipleWarpBatch(quadND, 512);
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(a3QuadBatch, 512);
+   props["defines/A3_QUAD_BATCH"]    = a3QuadBatch;
+}
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   SetSimplexProperties(fespace, fespace, ir, props);
+}
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad = ir.GetNPoints();
+   const int maxDQ   = std::max(std::max(trialDofs, testDofs), numQuad);
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 0;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   // 2D Defines
+   const int quadBatch = closestWarpBatchTo(numQuad);
+   props["defines/A2_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A2_QUAD_BATCH"]    = quadBatch;
+   props["defines/M2_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+
+   // 3D Defines
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A3_QUAD_BATCH"]    = quadBatch;
+   props["defines/M3_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+}
+
+
+//---[ Base Integrator ]--------------
+OccaIntegrator::OccaIntegrator(const Engine &e)
+   : engine(&e),
+     bform(),
+     mesh(),
+     otrialFESpace(),
+     otestFESpace(),
+     trialFESpace(),
+     testFESpace(),
+     itype(DomainIntegrator),
+     ir(NULL),
+     hasTensorBasis(false) { }
+
+OccaIntegrator::~OccaIntegrator() {}
+
+void OccaIntegrator::SetupMaps()
+{
+   maps = OccaDofQuadMaps::Get(GetDevice(),
+                               *otrialFESpace,
+                               *otestFESpace,
+                               *ir);
+
+   mapsTranspose = OccaDofQuadMaps::Get(GetDevice(),
+                                        *otestFESpace,
+                                        *otrialFESpace,
+                                        *ir);
+}
+
+FiniteElementSpace& OccaIntegrator::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaIntegrator::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+void OccaIntegrator::SetIntegrationRule(const mfem::IntegrationRule &ir_)
+{
+   ir = &ir_;
+}
+
+const mfem::IntegrationRule& OccaIntegrator::GetIntegrationRule() const
+{
+   return *ir;
+}
+
+OccaDofQuadMaps& OccaIntegrator::GetDofQuadMaps()
+{
+   return maps;
+}
+
+void OccaIntegrator::SetupIntegrator(OccaBilinearForm &bform_,
+                                     const ::occa::properties &props_,
+                                     const OccaIntegratorType itype_)
+{
+   MFEM_ASSERT(engine == &bform_.OccaEngine(), "");
+   bform     = &bform_;
+   mesh      = &(bform_.GetMesh());
+
+   otrialFESpace = &(bform_.GetTrialOccaFESpace());
+   otestFESpace  = &(bform_.GetTestOccaFESpace());
+
+   trialFESpace = &(bform_.GetTrialFESpace());
+   testFESpace  = &(bform_.GetTestFESpace());
+
+   hasTensorBasis = otrialFESpace->hasTensorBasis();
+
+   props = props_;
+   itype = itype_;
+
+   if (ir == NULL)
+   {
+      SetupIntegrationRule();
+   }
+
+   SetupMaps();
+
+   SetProperties(*otrialFESpace,
+                 *otestFESpace,
+                 *ir,
+                 props);
+
+   Setup();
+}
+
+OccaGeometry OccaIntegrator::GetGeometry(const int flags)
+{
+   return OccaGeometry::Get(GetDevice(), *otrialFESpace, *ir, flags);
+}
+
+::occa::kernel OccaIntegrator::GetAssembleKernel(const ::occa::properties
+                                                 &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("Assemble", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetMultAddKernel(const ::occa::properties &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("MultAdd", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetKernel(const std::string &kernelName,
+                                         const ::occa::properties &props)
+{
+   const std::string filename = GetName() + ".okl";
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   return GetDevice().buildKernel(okl_path + filename,
+                                  kernelName,
+                                  props);
+}
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+OccaDiffusionIntegrator::OccaDiffusionIntegrator(const OccaCoefficient &coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaDiffusionIntegrator::~OccaDiffusionIntegrator() {}
+
+
+std::string OccaDiffusionIntegrator::GetName()
+{
+   return "DiffusionIntegrator";
+}
+
+void OccaDiffusionIntegrator::SetupIntegrationRule()
+{
+   const FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   ir = &mfem::DiffusionIntegrator::GetRule(trialFE, testFE);
+}
+
+void OccaDiffusionIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaDiffusionIntegrator::Assemble()
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+
+   const int dims = fe.GetDim();
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.OccaResize(symmDims * quadraturePoints * elements,
+                                sizeof(double));
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaDiffusionIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   // Note: x and y are E-vectors
+
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+OccaMassIntegrator::OccaMassIntegrator(const OccaCoefficient &coeff_) :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaMassIntegrator::~OccaMassIntegrator() {}
+
+std::string OccaMassIntegrator::GetName()
+{
+   return "MassIntegrator";
+}
+
+void OccaMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaMassIntegrator::Assemble()
+{
+   if (assembledOperator.Size())
+   {
+      return;
+   }
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaMassIntegrator::SetOperator(Vector &v)
+{
+   assembledOperator = v;
+}
+
+void OccaMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Vector Mass Integrator ]--------------
+OccaVectorMassIntegrator::OccaVectorMassIntegrator(const OccaCoefficient &
+                                                   coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaVectorMassIntegrator::~OccaVectorMassIntegrator() {}
+
+std::string OccaVectorMassIntegrator::GetName()
+{
+   return "VectorMassIntegrator";
+}
+
+void OccaVectorMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaVectorMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaVectorMassIntegrator::Assemble()
+{
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaVectorMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilininteg.hpp

@@ -0,0 +1,323 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+#define MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "coefficient.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaGeometry
+{
+public:
+   ::occa::array<double> meshNodes;
+   ::occa::array<double> J, invJ, detJ;
+
+   // byVDIM  -> [x y z x y z x y z]
+   // byNodes -> [x x x y y y z z z]
+   static const int Jacobian    = (1 << 0);
+   static const int JacobianInv = (1 << 1);
+   static const int JacobianDet = (1 << 2);
+
+   static OccaGeometry Get(::occa::device device,
+                           FiniteElementSpace &ofespace,
+                           const IntegrationRule &ir,
+                           const int flags = (Jacobian    |
+                                              JacobianInv |
+                                              JacobianDet));
+};
+
+class OccaDofQuadMaps
+{
+private:
+   // Reuse dof-quad maps
+   static std::map<std::string, OccaDofQuadMaps> AllDofQuadMaps;
+   std::string hash;
+
+public:
+   // Local stiffness matrices (B and B^T operators)
+   ::occa::array<double, ::occa::dynamic> dofToQuad, dofToQuadD; // B
+   ::occa::array<double, ::occa::dynamic> quadToDof, quadToDofD; // B^T
+   ::occa::array<double> quadWeights;
+
+   OccaDofQuadMaps();
+   OccaDofQuadMaps(const OccaDofQuadMaps &maps);
+   OccaDofQuadMaps& operator = (const OccaDofQuadMaps &maps);
+
+   // [[x y] [x y] [x y]]
+   // [[x y z] [x y z] [x y z]]
+   // mfem::GridFunction* mfem::Mesh::GetNodes() { return Nodes; }
+
+   // mfem::FiniteElementSpace *Nodes->FESpace()
+   // 25
+   // 1D [x x x x x x]
+   // 2D [x y x y x y]
+   // GetVdim()
+   // 3D ordering == byVDIM  -> [x y z x y z x y z x y z x y z x y z]
+   //    ordering == byNODES -> [x x x x x x y y y y y y z z z z z z]
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &fespace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &fe,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &trialFESpace,
+                               const FiniteElementSpace &testFESpace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &trialFE,
+                               const mfem::FiniteElement &testFE,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &fe,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &trialFE,
+                                         const mfem::FiniteElement &testFE,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QTensorMaps(::occa::device device,
+                                           const mfem::FiniteElement &fe,
+                                           const mfem::IntegrationRule &ir,
+                                           const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &fe,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &trialFE,
+                                          const mfem::FiniteElement &testFE,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QSimplexMaps(::occa::device device,
+                                            const mfem::FiniteElement &fe,
+                                            const mfem::IntegrationRule &ir,
+                                            const bool transpose = false);
+};
+
+//---[ Define Methods ]---------------
+std::string stringWithDim(const std::string &s, const int dim);
+int closestWarpBatch(const int multiple, const int maxSize);
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+//---[ Base Integrator ]--------------
+class OccaIntegrator
+{
+protected:
+   SharedPtr<const Engine> engine;
+
+   OccaBilinearForm *bform;
+   mfem::Mesh *mesh;
+
+   FiniteElementSpace *otrialFESpace;
+   FiniteElementSpace *otestFESpace;
+
+   mfem::FiniteElementSpace *trialFESpace;
+   mfem::FiniteElementSpace *testFESpace;
+
+   ::occa::properties props;
+   OccaIntegratorType itype;
+
+   const IntegrationRule *ir;
+   bool hasTensorBasis;
+   OccaDofQuadMaps maps;
+   OccaDofQuadMaps mapsTranspose;
+
+public:
+   OccaIntegrator(const Engine &e);
+   virtual ~OccaIntegrator();
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   virtual std::string GetName() = 0;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   void SetIntegrationRule(const mfem::IntegrationRule &ir_);
+   const mfem::IntegrationRule& GetIntegrationRule() const;
+
+   OccaDofQuadMaps& GetDofQuadMaps();
+
+   void SetupMaps();
+
+   virtual void SetupIntegrationRule() = 0;
+
+   virtual void SetupIntegrator(OccaBilinearForm &bform_,
+                                const ::occa::properties &props_,
+                                const OccaIntegratorType itype_);
+
+   virtual void Setup() = 0;
+
+   virtual void Assemble() = 0;
+   /// This method works on E-vectors!
+   virtual void MultAdd(Vector &x, Vector &y) = 0;
+
+   virtual void MultTransposeAdd(Vector &x, Vector &y)
+   {
+      mfem_error("OccaIntegrator::MultTransposeAdd() is not overloaded!");
+   }
+
+   OccaGeometry GetGeometry(const int flags = (OccaGeometry::Jacobian    |
+                                               OccaGeometry::JacobianInv |
+                                               OccaGeometry::JacobianDet));
+
+   ::occa::kernel GetAssembleKernel(const ::occa::properties &props);
+   ::occa::kernel GetMultAddKernel(const ::occa::properties &props);
+
+   ::occa::kernel GetKernel(const std::string &kernelName,
+                            const ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+class OccaDiffusionIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaDiffusionIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaDiffusionIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+class OccaMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   void SetOperator(Vector &v);
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+//---[ Vector Mass Integrator ]--------------
+class OccaVectorMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaVectorMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaVectorMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP

backends/occa/coefficient.cpp

@@ -0,0 +1,357 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "coefficient.hpp"
+#include "bilininteg.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+//---[ Parameter ]------------
+OccaParameter::~OccaParameter() {}
+
+void OccaParameter::Setup(OccaIntegrator &integ,
+                          ::occa::properties &props) {}
+
+::occa::kernelArg OccaParameter::KernelArgs()
+{
+   return ::occa::kernelArg();
+}
+//====================================
+
+
+//---[ Include Parameter ]------------
+OccaIncludeParameter::OccaIncludeParameter(const std::string &filename_) :
+   filename(filename_) {}
+
+OccaParameter* OccaIncludeParameter::Clone()
+{
+   return new OccaIncludeParameter(filename);
+}
+
+void OccaIncludeParameter::Setup(OccaIntegrator &integ,
+                                 ::occa::properties &props)
+{
+   props["headers"].asArray() += "#include " + filename;
+}
+//====================================
+
+
+//---[ Source Parameter ]------------
+OccaSourceParameter::OccaSourceParameter(const std::string &source_) :
+   source(source_) {}
+
+OccaParameter* OccaSourceParameter::Clone()
+{
+   return new OccaSourceParameter(source);
+}
+
+void OccaSourceParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   props["headers"].asArray() += source;
+}
+//====================================
+
+//---[ Vector Parameter ]-------
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr("") {}
+
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const std::string &attr_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr(attr_) {}
+
+OccaParameter* OccaVectorParameter::Clone()
+{
+   return new OccaVectorParameter(name, v, attr, useRestrict);
+}
+
+void OccaVectorParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   args += "const double *";
+   if (useRestrict)
+   {
+      args += " restrict ";
+   }
+   args += name;
+   if (attr.size())
+   {
+      args += ' ';
+      args += attr;
+   }
+   args += ",\n";
+}
+
+::occa::kernelArg OccaVectorParameter::KernelArgs()
+{
+   return ::occa::kernelArg(v.OccaMem());
+}
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
+                                                     const Engine &e,
+                                                     mfem::GridFunction &gf_,
+                                                     const bool useRestrict_)
+   : name(name_),
+     gf(gf_),
+     gfQuad(e),
+     useRestrict(useRestrict_) {}
+
+OccaParameter* OccaGridFunctionParameter::Clone()
+{
+   OccaGridFunctionParameter *param =
+      new OccaGridFunctionParameter(name, gfQuad.OccaEngine(), gf, useRestrict);
+   param->gfQuad.MakeRef(gfQuad);
+   return param;
+}
+
+void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
+                                      ::occa::properties &props)
+{
+
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   if (useRestrict)
+   {
+      args += "@restrict ";
+   }
+   args += "const double *";
+   args += name;
+   args += " @dim(NUM_QUAD, numElements),\n";
+
+   FiniteElementSpace &f = gf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
+   ToQuad(integ.GetIntegrationRule(), f, gf.Get_PVector()->As<Vector>(), gfQuad);
+}
+
+::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
+{
+   return gfQuad.OccaMem();
+}
+//====================================
+
+
+//---[ Coefficient ]------------------
+OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = value;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
+                                 const bool useRestrict) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = "(u(q, e))";
+   AddGridFunction("u", gf, useRestrict);
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const char *source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const OccaCoefficient &coeff) :
+   engine(coeff.engine),
+   integ(NULL),
+   name(coeff.name),
+   coeffValue(coeff.coeffValue)
+{
+
+   const int paramCount = (int) coeff.params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params.push_back(coeff.params[i]->Clone());
+   }
+}
+
+OccaCoefficient::~OccaCoefficient()
+{
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      delete params[i];
+   }
+}
+
+OccaCoefficient& OccaCoefficient::SetName(const std::string &name_)
+{
+   name = name_;
+   return *this;
+}
+
+void OccaCoefficient::Setup(OccaIntegrator &integ_,
+                            ::occa::properties &props_)
+{
+   integ = &integ_;
+
+   const int paramCount = (int) params.size();
+   props_["defines"][name + "_ARGS"] = "";
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params[i]->Setup(integ_, props_);
+   }
+   props_["defines"][name] = coeffValue;
+
+   props = props_;
+}
+
+OccaCoefficient& OccaCoefficient::Add(OccaParameter *param)
+{
+   params.push_back(param);
+   return *this;
+}
+
+OccaCoefficient& OccaCoefficient::IncludeHeader(const std::string &filename)
+{
+   return Add(new OccaIncludeParameter(filename));
+}
+
+OccaCoefficient& OccaCoefficient::IncludeSource(const std::string &source)
+{
+   return Add(new OccaSourceParameter(source));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const std::string &attr,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, attr, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
+                                                  mfem::GridFunction &gf,
+                                                  const bool useRestrict)
+{
+   MFEM_ASSERT(engine->CheckVector(gf.Get_PVector()) &&
+               engine->CheckFESpace(gf.FESpace()->Get_PFESpace()),
+               "invalid device GridFunction");
+   return Add(new OccaGridFunctionParameter(name_, *engine, gf, useRestrict));
+}
+
+bool OccaCoefficient::IsConstant()
+{
+   return coeffValue.isNumber();
+}
+
+double OccaCoefficient::GetConstantValue()
+{
+   if (!IsConstant())
+   {
+      mfem_error("OccaCoefficient is not constant");
+   }
+   return coeffValue.number();
+}
+
+Vector OccaCoefficient::Eval()
+{
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   mfem::FiniteElementSpace &fespace = integ->GetTrialFESpace();
+   const mfem::IntegrationRule &ir   = integ->GetIntegrationRule();
+
+   const int elements = fespace.GetNE();
+   const int numQuad  = ir.GetNPoints();
+
+   Vector quadCoeff(*(new Layout(OccaEngine(), numQuad * elements)));
+   Eval(quadCoeff);
+   return quadCoeff;
+}
+
+void OccaCoefficient::Eval(Vector &quadCoeff)
+{
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   static ::occa::kernelBuilder builder =
+      ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
+                                      "CoefficientEval");
+
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   const int elements = integ->GetTrialFESpace().GetNE();
+
+   ::occa::properties kernelProps = props;
+   if (name != "COEFF")
+   {
+      kernelProps["defines/COEFF"]      = name;
+      kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
+   }
+
+   ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
+   evalKernel(elements, *this, quadCoeff.OccaMem());
+}
+
+OccaCoefficient::operator ::occa::kernelArg ()
+{
+   ::occa::kernelArg kArg;
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      kArg.add(params[i]->KernelArgs());
+   }
+   return kArg;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/coefficient.hpp

@@ -0,0 +1,287 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+#define MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaIntegrator;
+
+
+class OccaParameter
+{
+public:
+   virtual ~OccaParameter();
+
+   virtual OccaParameter* Clone() = 0;
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+
+
+//---[ Include Parameter ]------------
+class OccaIncludeParameter : public OccaParameter
+{
+private:
+   std::string filename;
+
+public:
+   OccaIncludeParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Source Parameter ]------------
+class OccaSourceParameter : public OccaParameter
+{
+private:
+   std::string source;
+
+public:
+   OccaSourceParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Define Parameter ]------------
+template <class TM>
+class OccaDefineParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   TM value;
+
+public:
+   OccaDefineParameter(const std::string &name_,
+                       const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaDefineParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      props["defines"][name] = value;
+   }
+};
+//====================================
+
+
+//---[ Variable Parameter ]-----------
+template <class TM>
+class OccaVariableParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   const TM &value;
+
+public:
+   OccaVariableParameter(const std::string &name_,
+                         const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaVariableParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      std::string &args = (props["defines/COEFF_ARGS"]
+                           .asString()
+                           .string());
+      // const TM name,\n"
+      args += "const ";
+      args += ::occa::primitiveinfo<TM>::name;
+      args += ' ';
+      args += name;
+      args += ",\n";
+   }
+
+   virtual ::occa::kernelArg KernelArgs()
+   {
+      return ::occa::kernelArg(value);
+   }
+};
+//====================================
+
+
+//---[ Vector Parameter ]-------
+class OccaVectorParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   Vector v;
+   bool useRestrict;
+   std::string attr;
+
+public:
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const bool useRestrict_ = false);
+
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const std::string &attr_,
+                       const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+class OccaGridFunctionParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   mfem::GridFunction &gf;
+   Vector gfQuad;
+   bool useRestrict;
+
+public:
+   OccaGridFunctionParameter(const std::string &name_,
+                             const Engine &e,
+                             mfem::GridFunction &gf_,
+                             const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ Coefficient ]------------------
+// [MISSING]
+// Needs to know about the integrator's
+//   - fespace
+//   - ir
+// Step where parameters that need the ir get called for setup
+// For example, GridFunction (d, e) -> (q, e)
+class OccaCoefficient
+{
+private:
+   SharedPtr<const Engine> engine;
+
+   OccaIntegrator *integ;
+
+   std::string name;
+   ::occa::json coeffValue;
+
+   ::occa::properties props;
+   std::vector<OccaParameter*> params;
+
+public:
+   OccaCoefficient(const Engine &e, const double value = 1.0);
+   OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
+                   const bool useRestrict = false);
+   OccaCoefficient(const Engine &e, const std::string &source);
+   OccaCoefficient(const Engine &e, const char *source);
+   ~OccaCoefficient();
+
+   OccaCoefficient(const OccaCoefficient &coeff);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   OccaCoefficient& SetName(const std::string &name_);
+
+   void Setup(OccaIntegrator &integ_,
+              ::occa::properties &props_);
+
+   OccaCoefficient& Add(OccaParameter *param);
+
+   OccaCoefficient& IncludeHeader(const std::string &filename);
+   OccaCoefficient& IncludeSource(const std::string &source);
+
+   template <class TM>
+   OccaCoefficient& AddDefine(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaDefineParameter<TM>(name_, value));
+   }
+
+   template <class TM>
+   OccaCoefficient& AddVariable(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaVariableParameter<TM>(name_, value));
+   }
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const bool useRestrict = false);
+
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const std::string &attr,
+                              const bool useRestrict = false);
+
+   OccaCoefficient& AddGridFunction(const std::string &name_,
+                                    mfem::GridFunction &gf,
+                                    const bool useRestrict = false);
+
+   bool IsConstant();
+   double GetConstantValue();
+
+   Vector Eval();
+   void Eval(Vector &quadCoeff);
+
+   operator ::occa::kernelArg ();
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_COEFFICIENT_HPP

backends/occa/defines.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_OCCA_DEFINES
+#define MFEM_OCCA_DEFINES
+
+#ifndef USING_TENSOR_OPS
+#  define USING_TENSOR_OPS 0
+#endif
+
+#ifdef OCCA_USING_GPU
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(I0, I1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(I0, I1, I2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(I0, I1, I2, I3)
+#else
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(0, 1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(0, 1, 2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(0, 1, 2, 3)
+#endif
+
+#ifndef COEFF
+#  define COEFF 1.0
+#  define COEFF_ARGS
+#endif
+
+#if USING_TENSOR_OPS
+#  include "mfem-occa://defines/tensor.okl"
+#else
+#  include "mfem-occa://defines/simplex.okl"
+#endif
+
+#endif

backends/occa/defines/simplex.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#define USING_LOW_ORDER 1
+#define USING_HI_ORDER  0
+
+typedef double* DofToQuad_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* QuadToDof_t @dim(NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD2D_t @dim(2, NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD3D_t @dim(3, NUM_DOFS, NUM_QUAD);
+
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD, numElements);
+
+typedef double* DLocal_t @dim(NUM_DOFS, numElements);
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements);
+#else
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+#endif
+
+typedef int* DLocalMap_t @dim(NUM_DOFS, numElements);

backends/occa/defines/tensor.okl

@@ -0,0 +1,85 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#if NUM_QUAD_1D < NUM_DOFS_1D
+#  define NUM_MAX_1D NUM_DOFS_1D
+#else
+#  define NUM_MAX_1D NUM_QUAD_1D
+#endif
+
+#define NUM_MAX_2D (NUM_MAX_1D * NUM_MAX_1D)
+
+#define NUM_QUAD_DOFS_1D (NUM_QUAD_1D * NUM_DOFS_1D)
+
+#define QUAD_2D_ID(X, Y) (X + ((Y) * NUM_QUAD_1D))
+#define DOFS_2D_ID(X, Y) (X + ((Y) * NUM_DOFS_1D))
+
+#define QUAD_3D_ID(X, Y, Z) (X + ((Y) * NUM_QUAD_1D) + ((Z) * NUM_QUAD_2D))
+#define DOFS_3D_ID(X, Y, Z) (X + ((Y) * NUM_DOFS_1D) + ((Z) * NUM_DOFS_2D))
+
+#if NUM_MAX_1D < 8
+#  define USING_LOW_ORDER 1
+#  define USING_HI_ORDER  0
+#else
+#  define USING_LOW_ORDER 0
+#  define USING_HI_ORDER  1
+#endif
+
+#define M1_ELEMENT_BATCHES (M1_OUTER_ELEMENT_BATCH * M1_INNER_ELEMENT_BATCH)
+
+typedef double* DofToQuad_t @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+typedef double* QuadToDof_t @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+typedef double* Jacobian_t   @dim(NUM_DIM, NUM_DIM, numElements);
+typedef double* Jacobian1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD_2D, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD_3D, numElements);
+
+typedef double* SymmOperator1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD_2D, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD_3D, numElements);
+
+typedef double* DLocal_t   @dim(NUM_DOFS, numElements);
+typedef double* DLocal1D_t @dim(NUM_DOFS_1D, numElements);
+typedef double* DLocal2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DLocal3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QLocal_t   @dim(NUM_QUAD, numElements);
+typedef double* QLocal1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* QLocal2D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QLocal3D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+#else
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(4,0,1,2,3);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(4,0,1,2,3);
+#endif
+
+typedef int* DLocalMap_t   @dim(NUM_DOFS, numElements);
+typedef int* DLocalMap1D_t @dim(NUM_DOFS_1D, numElements);
+typedef int* DLocalMap2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef int* DLocalMap3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);

backends/occa/diffusion/simplex/cpu.okl

@@ -0,0 +1,168 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double *quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator2D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator3D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        const double gradZ2 = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)) +
+                       (gradZ2 * quadToDofD(2, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -0,0 +1,182 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY);
+      }
+    }
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          // FIXME: s_gradX and s_gradY are @shared used outside of @inner
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+    @shared double s_gradZ[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        s_gradZ[q] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+      }
+    }
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)) +
+                    (s_gradZ[q] * quadToDofD(2, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/cpu.okl

@@ -0,0 +1,370 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian1D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator1D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] = 0;
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const double s = solIn(dx, e);
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qx] += s * dofToQuadD(qx, dx);
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] *= oper(qx, e);
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        const double gradX = grad[qx];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(dx, e) += gradX * quadToDofD(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator2D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][2];
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qy][qx][0] = 0;
+          grad[qy][qx][1] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double gradX[NUM_QUAD_1D][2];
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          gradX[qx][0] = 0;
+          gradX[qx][1] = 0;
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const double s = solIn(dx, dy, e);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] += s * dofToQuad(qx, dx);
+            gradX[qx][1] += s * dofToQuadD(qx, dx);
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double wy  = dofToQuad(qy, dy);
+          const double wDy = dofToQuadD(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qy][qx][0] += gradX[qx][1] * wy;
+            grad[qy][qx][1] += gradX[qx][0] * wDy;
+          }
+        }
+      }
+
+      // Calculate Dxy, xDy in plane
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const int q = QUAD_2D_ID(qx, qy);
+          const double O11 = oper(0, q, e);
+          const double O12 = oper(1, q, e);
+          const double O22 = oper(2, q, e);
+
+          const double gradX = grad[qy][qx][0];
+          const double gradY = grad[qy][qx][1];
+
+          grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+          grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double gradX[NUM_DOFS_1D][2];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          gradX[dx][0] = 0;
+          gradX[dx][1] = 0;
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const double gX = grad[qy][qx][0];
+          const double gY = grad[qy][qx][1];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double wx  = quadToDof(dx, qx);
+            const double wDx = quadToDofD(dx, qx);
+            gradX[dx][0] += gX * wDx;
+            gradX[dx][1] += gY * wx;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double wy  = quadToDof(dy, qy);
+          const double wDy = quadToDofD(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(dx, dy, e) += ((gradX[dx][0] * wy) +
+                                  (gradX[dx][1] * wDy));
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator3D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D][4];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qz][qy][qx][0] = 0;
+            grad[qz][qy][qx][1] = 0;
+            grad[qz][qy][qx][2] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double gradXY[NUM_QUAD_1D][NUM_QUAD_1D][4];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradXY[qy][qx][0] = 0;
+            gradXY[qy][qx][1] = 0;
+            gradXY[qy][qx][2] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double gradX[NUM_QUAD_1D][2];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] = 0;
+            gradX[qx][1] = 0;
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, dy, dz, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              gradX[qx][0] += s * dofToQuad(qx, dx);
+              gradX[qx][1] += s * dofToQuadD(qx, dx);
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy  = dofToQuad(qy, dy);
+            const double wDy = dofToQuadD(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              const double wx  = gradX[qx][0];
+              const double wDx = gradX[qx][1];
+              gradXY[qy][qx][0] += wDx * wy;
+              gradXY[qy][qx][1] += wx  * wDy;
+              gradXY[qy][qx][2] += wx  * wy;
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz  = dofToQuad(qz, dz);
+          const double wDz = dofToQuadD(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+              grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+              grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+            }
+          }
+        }
+      }
+
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double gradX = grad[qz][qy][qx][0];
+            const double gradY = grad[qz][qy][qx][1];
+            const double gradZ = grad[qz][qy][qx][2];
+
+            grad[qz][qy][qx][0] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+            grad[qz][qy][qx][1] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+            grad[qz][qy][qx][2] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double gradXY[NUM_DOFS_1D][NUM_DOFS_1D][4];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradXY[dy][dx][0] = 0;
+            gradXY[dy][dx][1] = 0;
+            gradXY[dy][dx][2] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double gradX[NUM_DOFS_1D][4];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradX[dx][0] = 0;
+            gradX[dx][1] = 0;
+            gradX[dx][2] = 0;
+          }
+
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const double gX = grad[qz][qy][qx][0];
+            const double gY = grad[qz][qy][qx][1];
+            const double gZ = grad[qz][qy][qx][2];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              const double wx  = quadToDof(dx, qx);
+              const double wDx = quadToDofD(dx, qx);
+              gradX[dx][0] += gX * wDx;
+              gradX[dx][1] += gY * wx;
+              gradX[dx][2] += gZ * wx;
+            }
+          }
+
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy  = quadToDof(dy, qy);
+            const double wDy = quadToDofD(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              gradXY[dy][dx][0] += gradX[dx][0] * wy;
+              gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+              gradXY[dy][dx][2] += gradX[dx][2] * wy;
+            }
+          }
+        }
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz  = quadToDof(dz, qz);
+          const double wDz = quadToDofD(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(dx, dy, dz, e) += ((gradXY[dy][dx][0] * wz) +
+                                        (gradXY[dy][dx][1] * wz) +
+                                        (gradXY[dy][dx][2] * wDz));
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -0,0 +1,435 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian1D_t J,
+                        COEFF_ARGS
+                        SymmOperator1D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+          oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    @exclusive double grad[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuadD[i] = dofToQuadD[i];
+        s_quadToDofD[i] = quadToDofD[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qx] += s * s_dofToQuadD(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] *= oper(qx, e);
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            double s = 0;
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              s += grad[qx] * s_quadToDofD(dx, qx);
+            }
+            solOut(dx, e) += s;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_xDy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_grad[2 * NUM_QUAD_2D] @dim(2, NUM_QUAD_1D, NUM_QUAD_1D);
+
+    @exclusive double r_x[NUM_MAX_1D];
+    @exclusive double r_y[NUM_QUAD_1D];
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id]  = dofToQuad[id];
+        s_dofToQuadD[id] = dofToQuadD[id];
+        s_quadToDof[id]  = quadToDof[id];
+        s_quadToDofD[id] = quadToDofD[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              s_xy(dx, qy) = 0;
+              s_xDy(dx, qy) = 0;
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = solIn(dx, dy, e);
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              double xDy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy  += r_x[dy] * s_dofToQuad(qy, dy);
+                xDy += r_x[dy] * s_dofToQuadD(qy, dy);
+              }
+              s_xy(dx, qy)  = xy;
+              s_xDy(dx, qy) = xDy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double gradX = 0, gradY = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                gradX += s_xy(dx, qy)  * s_dofToQuadD(qx, dx);
+                gradY += s_xDy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+
+              const int q = QUAD_2D_ID(qx, qy);
+              const double O11 = oper(0, q, e);
+              const double O12 = oper(1, q, e);
+              const double O22 = oper(2, q, e);
+
+              s_grad(0, qx, qy) = (O11 * gradX) + (O12 * gradY);
+              s_grad(1, qx, qy) = (O12 * gradX) + (O22 * gradY);
+            }
+          }
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx; @inner) {
+          if (qx < NUM_QUAD_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              s_xy(dy, qx) = 0;
+              s_xDy(dy, qx) = 0;
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              r_x[qy] = s_grad(0, qx, qy);
+              r_y[qy] = s_grad(1, qx, qy);
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double xy  = 0;
+              double xDy = 0;
+              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+                xy  += r_x[qy] * s_quadToDof(dy, qy);
+                xDy += r_y[qy] * s_quadToDofD(dy, qy);
+              }
+              s_xy(dy, qx) = xy;
+              s_xDy(dy, qx) = xDy;
+            }
+          }
+        }
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                s += ((s_xy(dy, qx) * s_quadToDofD(dx, qx)) +
+                      (s_xDy(dy, qx) * s_quadToDof(dx, qx)));
+              }
+              solOut(dx, dy, e) += s;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_Dz[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_xyDz[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+    @exclusive double r_qDz[NUM_QUAD_1D];
+    @exclusive double r_dDxyz[NUM_DOFS_1D];
+    @exclusive double r_dxDyz[NUM_DOFS_1D];
+    @exclusive double r_dxyDz[NUM_DOFS_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id]  = dofToQuad[id];
+          s_dofToQuadD[id] = dofToQuadD[id];
+          s_quadToDof[id]  = quadToDof[id];
+          s_quadToDofD[id] = quadToDofD[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+          r_qDz[qz] = 0;
+        }
+        // Initialize our solution updates in the Z axis
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          r_dDxyz[dz] = 0;
+          r_dxDyz[dz] = 0;
+          r_dxyDz[dz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double s = solIn(dx, dy, dz, e);
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz]  += s * s_dofToQuad(qz, dz);
+              r_qDz[qz] += s * s_dofToQuadD(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy)  = r_qz[qz];
+            s_Dz(dx, dy) = r_qDz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double Dxyz = 0;
+            double xDyz = 0;
+            double xyDz = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy  = s_dofToQuad(qy, dy);
+              const double wDy = s_dofToQuadD(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx  = s_dofToQuad(qx, dx);
+                const double wDx = s_dofToQuadD(qx, dx);
+                const double z  = s_z(dx, dy);
+                const double Dz = s_Dz(dx, dy);
+                Dxyz += wDx * wy  * z;
+                xDyz += wx  * wDy * z;
+                xyDz += wx  * wy  * Dz;
+              }
+            }
+
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double qDxyz = (O11 * Dxyz) + (O12 * xDyz) + (O13 * xyDz);
+            const double qxDyz = (O12 * Dxyz) + (O22 * xDyz) + (O23 * xyDz);
+            const double qxyDz = (O13 * Dxyz) + (O23 * xDyz) + (O33 * xyDz);
+
+            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+              const double wz  = s_quadToDof(dz, qz);
+              const double wDz = s_quadToDofD(dz, qz);
+              r_dDxyz[dz] += wz  * qDxyz;
+              r_dxDyz[dz] += wz  * qxDyz;
+              r_dxyDz[dz] += wDz * qxyDz;
+            }
+          }
+        }
+      }
+      @barrier("s_z_s_Dz_sync_1");
+    }
+    // Iterate over xy planes to compute solution
+    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+      // Place xy plane in shared memory
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            s_z(qx, qy)    = r_dDxyz[dz];
+            s_Dz(qx, qy)   = r_dxDyz[dz];
+            s_xyDz(qx, qy) = r_dxyDz[dz];
+          }
+        }
+      }
+      // Finalize solution in xy plane
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            double solZ = 0;
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              const double wy  = s_quadToDof(dy, qy);
+              const double wDy = s_quadToDofD(dy, qy);
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                const double wx  = s_quadToDof(dx, qx);
+                const double wDx = s_quadToDofD(dx, qx);
+                const double Dxyz = s_z(qx, qy);
+                const double xDyz = s_Dz(qx, qy);
+                const double xyDz = s_xyDz(qx, qy);
+                solZ += ((wDx * wy  * Dxyz) +
+                         (wx  * wDy * xDyz) +
+                         (wx  * wy  * xyDz));
+              }
+            }
+            solOut(dx, dy, dz, e) += solZ;
+          }
+        }
+      }
+      @barrier("s_z_s_Dz_s_xyDz_sync_1");
+    }
+  }
+}
+//======================================

backends/occa/engine.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "url_handler.hpp"
+#include "bilinearform.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Engine::fileOpenerRegistered = false;
+
+void Engine::Init(const std::string &engine_spec)
+{
+   //
+   // Initialize inherited fields
+   //
+   memory_resources[0] = NULL;
+   workers_weights[0]= 1.0;
+   workers_mem_res[0] = 0;
+
+   //
+   // Initialize the OCCA engine
+   //
+   ::occa::properties props(engine_spec);
+   device = new ::occa::device[1];
+   device[0].setup(props);
+
+   okl_path = "mfem-occa://";
+   if (!fileOpenerRegistered)
+   {
+      // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
+      // priority.
+      FileOpener *fo = new FileOpener("mfem-occa://", "MFEM_OCCA_OKL_PATH");
+      // Next in priority is the source path, if it exists.
+      std::string mfem_src_prefix = mfem::GetSourcePath();
+      fo->AddDir(mfem_src_prefix + "/backends/occa");
+      // And last in priority is the install path, if it exists.
+      std::string mfem_install_prefix = mfem::GetInstallPath();
+      fo->AddDir(mfem_install_prefix + "/lib/mfem/occa");
+      ::occa::io::fileOpener::add(fo);
+      fileOpenerRegistered = true;
+   }
+   // std::cout << "OCCA device properties:\n" << device[0].properties();
+
+   force_cuda_aware_mpi = false;
+}
+
+Engine::Engine(const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   comm = _comm;
+   Init(engine_spec);
+}
+#endif
+
+bool Engine::CheckEngine(const mfem::Engine *engine) const
+{
+   return (engine != NULL && util::Is<const Engine>(engine) != NULL &&
+           *util::As<const Engine>(engine) == *this);
+}
+
+bool Engine::CheckLayout(const PLayout *layout) const
+{
+   return (layout != NULL && util::Is<const Layout>(layout) != NULL &&
+           layout->As<Layout>().OccaEngine() == *this);
+}
+
+bool Engine::CheckArray(const PArray *array) const
+{
+   return (array != NULL && util::Is<const Array>(array) != NULL &&
+           array->As<Array>().OccaEngine() == *this);
+}
+
+bool Engine::CheckVector(const PVector *vector) const
+{
+   return (vector != NULL && util::Is<const Vector>(vector) != NULL &&
+           vector->As<Vector>().OccaEngine() == *this);
+}
+
+bool Engine::CheckFESpace(const PFiniteElementSpace *fes) const
+{
+   return (fes != NULL && util::Is<const FiniteElementSpace>(fes) != NULL &&
+           fes->As<FiniteElementSpace>().OccaEngine() == *this);
+}
+
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   return DLayout(new Layout(*this, size));
+}
+
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   return DLayout(new Layout(*this, offsets.Last()));
+}
+
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   return DArray(new Array(layout.As<Layout>(), item_size));
+}
+
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "type_id " << type_id
+               << " is not supported");
+   return DVector(new Vector(layout.As<Layout>()));
+}
+
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
+{
+   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
+}
+
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   return DBilinearForm(new BilinearForm(*this, bf));
+}
+
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/engine.hpp

@@ -0,0 +1,144 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ENGINE_HPP
+#define MFEM_BACKENDS_OCCA_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "../base/backend.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Engine : public mfem::Engine
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // mfem::Backend *backend;
+#ifdef MFEM_USE_MPI
+   // MPI_Comm comm;
+#endif
+   // int num_mem_res;
+   // int num_workers;
+   // MemoryResource **memory_resources;
+   // double *workers_weights;
+   // int *workers_mem_res;
+
+   static bool fileOpenerRegistered;
+   /// An array of OCCA devices. Currently only a single device is supported.
+   ::occa::device *device;
+   std::string okl_path;
+   bool force_cuda_aware_mpi;
+
+   void Init(const std::string &engine_spec);
+
+public:
+   /// TODO: doxygen
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   /// TODO: doxygen
+   Engine(MPI_Comm comm, const std::string &engine_spec);
+#endif
+
+   /// TODO: doxygen
+   virtual ~Engine() { delete [] device; }
+
+   /**
+       @name OCCA specific interface, used by other objects in the OCCA backend
+    */
+   ///@{
+
+   /// Get the associated OCCA device.
+   ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
+
+   /// TODO: doxygen
+   const std::string &GetOklPath() const { return okl_path; }
+
+   /// OCCA device memory allocation.
+   ::occa::memory Alloc(std::size_t bytes) const
+   { return GetDevice().malloc(bytes); }
+
+   /// Two mfem::occa::Engine%s are equal if they use the same OCCA device.
+   bool operator==(const Engine &other) const
+   { return GetDevice() == other.GetDevice(); }
+
+   /// TODO: doxygen
+   bool CheckEngine(const mfem::Engine *e) const;
+
+   /// TODO: doxygen
+   bool CheckLayout(const PLayout *layout) const;
+
+   /// TODO: doxygen
+   bool CheckArray(const PArray *array) const;
+
+   /// TODO: doxygen
+   bool CheckVector(const PVector *vector) const;
+
+   /// TODO: doxygen
+   bool CheckFESpace(const PFiniteElementSpace *fes) const;
+
+#ifdef MFEM_USE_MPI
+   void SetForceCudaAwareMPI(bool force = true)
+   { force_cuda_aware_mpi = force; }
+
+   bool GetForceCudaAwareMPI() const { return force_cuda_aware_mpi; }
+#endif
+
+   ///@}
+   // End: OCCA specific interface
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t size) const;
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
+
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
+
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const;
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
+                                           fespace) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ENGINE_HPP

backends/occa/fespace.cpp

@@ -0,0 +1,532 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "fespace.hpp"
+#include "interpolation.hpp"
+
+#ifdef MFEM_USE_MPI
+#ifdef OMPI_RELEASE_VERSION
+#include <mpi-ext.h> // Check for cuda support
+#endif
+#endif
+
+namespace mfem
+{
+
+namespace occa
+{
+
+FiniteElementSpace::FiniteElementSpace(const Engine &e,
+                                       mfem::FiniteElementSpace &fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(new Layout(e, 0)) // resized in SetupLocalGlobalMaps()
+{
+   vdim     = fespace.GetVDim();
+   ordering = fespace.GetOrdering();
+
+   SetupLocalGlobalMaps();
+   SetupOperators(); // calls virtual methods of 'fes'
+   SetupKernels();
+}
+
+FiniteElementSpace::~FiniteElementSpace()
+{
+   delete restrictionOp;
+   delete prolongationOp;
+}
+
+void FiniteElementSpace::SetupLocalGlobalMaps()
+{
+   const int elements = fes->GetNE();
+
+   if (elements == 0) { return; }
+
+   // Assuming of finite elements are the same.
+   const mfem::FiniteElement &fe = *fes->GetFE(0);
+   const mfem::TensorBasisElement *el =
+      dynamic_cast<const mfem::TensorBasisElement*>(&fe);
+
+   const mfem::Table &e2dTable = fes->GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+
+   globalDofs = fes->GetNDofs();
+   localDofs  = fe.GetDof();
+
+   e_layout->OccaResize(e2dTable.Size_of_connections());
+
+   int *elementDofMap = new int[localDofs];
+   if (el)
+   {
+      ::memcpy(elementDofMap,
+               el->GetDofMap().GetData(),
+               localDofs * sizeof(int));
+   }
+   else
+   {
+      for (int i = 0; i < localDofs; ++i)
+      {
+         elementDofMap[i] = i;
+      }
+   }
+
+   // Allocate device offsets and indices
+   globalToLocalOffsets.allocate(GetDevice(),
+                                 globalDofs + 1);
+   globalToLocalIndices.allocate(GetDevice(),
+                                 localDofs, elements);
+   localToGlobalMap.allocate(GetDevice(),
+                             localDofs, elements);
+
+   int *offsets = globalToLocalOffsets.ptr();
+   int *indices = globalToLocalIndices.ptr();
+   int *l2gMap  = localToGlobalMap.ptr();
+
+   // We'll be keeping a count of how many local nodes point
+   //   to its global dof
+   for (int i = 0; i <= globalDofs; ++i)
+   {
+      offsets[i] = 0;
+   }
+
+   for (int e = 0; e < elements; ++e)
+   {
+      MFEM_ASSERT(e2dTable.RowSize(e) == localDofs, "");
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + d];
+         ++offsets[gid + 1];
+      }
+   }
+   // Aggregate to find offsets for each global dof
+   for (int i = 1; i <= globalDofs; ++i)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+   // For each global dof, fill in all local nodes that point
+   //   to it
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + elementDofMap[d]];
+         const int lid = localDofs*e + d;
+         indices[offsets[gid]++] = lid;
+         l2gMap[lid] = gid;
+      }
+   }
+   // We shifted the offsets vector by 1 by using it
+   //   as a counter. Now we shift it back.
+   for (int i = globalDofs; i > 0; --i)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+
+   delete [] elementDofMap;
+
+   globalToLocalOffsets.keepInDevice();
+   globalToLocalIndices.keepInDevice();
+   localToGlobalMap.keepInDevice();
+}
+
+void FiniteElementSpace::SetupOperators() const
+{
+   // Construct 'restrictionOp' and 'prolongationOp'.
+
+   prolongationOp = restrictionOp = NULL;
+
+   const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
+   const mfem::Operator *P = fes->GetProlongationMatrix();
+
+   if (!P) { return; }
+
+   Layout &v_layout = OccaVLayout();
+   Layout &t_layout = OccaTrueVLayout();
+
+   // Assuming R has one entry per row equal to 1.
+   MFEM_ASSERT(R->Finalized(), "");
+   const int tdofs = R->Height();
+   MFEM_ASSERT(tdofs == (int)t_layout.Size(), "");
+   MFEM_ASSERT(tdofs == R->GetI()[tdofs], "");
+   ::occa::array<int> ltdof_ldof(GetDevice(), tdofs, R->GetJ());
+   ltdof_ldof.keepInDevice();
+
+   restrictionOp = new RestrictionOperator(v_layout, t_layout, ltdof_ldof);
+
+   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
+   if (pmat)
+   {
+      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
+
+      OccaSparseMatrix *occaP  =
+         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
+      OccaSparseMatrix *occaPT =
+         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
+
+      prolongationOp = new ProlongationOperator(*occaP, *occaPT);
+
+      delete occaPT;
+      delete occaP;
+   }
+#ifdef MFEM_USE_MPI
+   else if (fes->Conforming() && dynamic_cast<ParFiniteElementSpace*>(fes))
+   {
+      ParFiniteElementSpace *pfes = static_cast<ParFiniteElementSpace*>(fes);
+      prolongationOp = new OccaConformingProlongation(*this, *pfes,
+                                                      ltdof_ldof.memory());
+   }
+#endif
+   else
+   {
+      prolongationOp = new ProlongationOperator(t_layout, v_layout, P);
+   }
+}
+
+void FiniteElementSpace::SetupKernels()
+{
+   ::occa::properties props("defines: {"
+                            "  TILESIZE: 256,"
+                            "}");
+   props["defines/NUM_VDIM"] = vdim;
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"] = (int) (ordering == Ordering::byVDIM);
+
+   ::occa::device device = GetDevice();
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "GlobalToLocal",
+                                            props);
+   localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "LocalToGlobal",
+                                            props);
+}
+
+
+#ifdef MFEM_USE_MPI
+
+OccaConformingProlongation::OccaConformingProlongation(
+   const FiniteElementSpace &ofes, const mfem::ParFiniteElementSpace &pfes,
+   ::occa::memory ltdof_ldof_)
+
+   : Operator(ofes.OccaTrueVLayout(), ofes.OccaVLayout()),
+     shr_ltdof(ofes.OccaEngine()),
+     ext_ldof(ofes.OccaEngine()),
+     shr_buf(shr_ltdof.OccaLayout(), sizeof(double)),
+     ext_buf(ext_ldof.OccaLayout(), sizeof(double)),
+     shr_buf_offsets(NULL), ext_buf_offsets(NULL),
+     ltdof_ldof(ltdof_ldof_),
+     gc(pfes.GroupComm())
+{
+   MFEM_ASSERT(pfes.Conforming(), "internal error");
+
+   const Engine &engine = ofes.OccaEngine();
+   const std::string &okl_path = engine.GetOklPath();
+   ::occa::device device = engine.GetDevice();
+
+   {
+      Table nbr_ltdof;
+      gc.GetNeighborLTDofTable(nbr_ltdof);
+      shr_ltdof.OccaResize(nbr_ltdof.Size_of_connections(), sizeof(int));
+      shr_ltdof.OccaPush(nbr_ltdof.GetJ());
+      shr_buf.OccaResize(&shr_ltdof.OccaLayout(), sizeof(double));
+      shr_buf_offsets = nbr_ltdof.GetI();
+      {
+         mfem::Array<int> shr_ltdof(nbr_ltdof.GetJ(),
+                                    nbr_ltdof.Size_of_connections());
+         mfem::Array<int> unique_ltdof(shr_ltdof);
+         unique_ltdof.Sort();
+         unique_ltdof.Unique();
+         // Note: the next loop modifies the J array of nbr_ltdof
+         for (int i = 0; i < shr_ltdof.Size(); i++)
+         {
+            shr_ltdof[i] = unique_ltdof.FindSorted(shr_ltdof[i]);
+            MFEM_ASSERT(shr_ltdof[i] != -1, "internal error");
+         }
+         Table unique_shr;
+         Transpose(shr_ltdof, unique_shr, unique_ltdof.Size());
+
+         unq_ltdof = device.malloc(unique_ltdof.Size()*sizeof(int),
+                                   unique_ltdof.GetData());
+         unq_shr_i = device.malloc((unique_shr.Size()+1)*sizeof(int),
+                                   unique_shr.GetI());
+         unq_shr_j = device.malloc(unique_shr.Size_of_connections()*sizeof(int),
+                                   unique_shr.GetJ());
+      }
+      delete [] nbr_ltdof.GetJ();
+      nbr_ltdof.LoseData();
+   }
+   {
+      Table nbr_ldof;
+      gc.GetNeighborLDofTable(nbr_ldof);
+      ext_ldof.OccaResize(nbr_ldof.Size_of_connections(), sizeof(int));
+      ext_ldof.OccaPush(nbr_ldof.GetJ());
+      ext_buf.OccaResize(&ext_ldof.OccaLayout(), sizeof(double));
+      ext_buf_offsets = nbr_ldof.GetI();
+      delete [] nbr_ldof.GetJ();
+      nbr_ldof.LoseData();
+   }
+   host_shr_buf = NULL;
+   host_ext_buf = NULL;
+   // If the device has a separate memory space (e.g. CUDA device) and the MPI
+   // library does not support buffers in that separate memory space, we
+   // allocate separate host buffers to use for MPI communication.
+   if (device.hasSeparateMemorySpace())
+   {
+      bool need_host_buf = true;
+      if (device.mode() == "CUDA")
+      {
+#ifdef MPIX_CUDA_AWARE_SUPPORT
+         need_host_buf = !MPIX_Query_cuda_support();
+#endif
+         if (engine.GetForceCudaAwareMPI()) { need_host_buf = false; }
+         if (gc.GetGroupTopology().MyRank() == 0)
+         {
+            mfem::out << "\nOccaConformingProlongation: CUDA-aware MPI: "
+                      << (need_host_buf ? "NO" : "YES") << "\n\n";
+         }
+      }
+      if (need_host_buf)
+      {
+         host_shr_buf = new char[shr_buf.OccaMem().size()];
+         host_ext_buf = new char[ext_buf.OccaMem().size()];
+      }
+   }
+
+   ExtractSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                         "ExtractSubVector",
+                                         "defines: { TILESIZE: 256 }");
+   SetSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                     "SetSubVector",
+                                     "defines: { TILESIZE: 256 }");
+   AddSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                     "AddSubVector",
+                                     "defines: { TILESIZE: 256 }");
+
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = shr_buf_offsets[nbr];
+      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0) { req_counter++; }
+
+      const int recv_offset = ext_buf_offsets[nbr];
+      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0) { req_counter++; }
+   }
+   requests = new MPI_Request[req_counter];
+}
+
+OccaConformingProlongation::~OccaConformingProlongation()
+{
+   delete [] requests;
+   delete [] host_ext_buf;
+   delete [] host_shr_buf;
+   delete [] ext_buf_offsets;
+   delete [] shr_buf_offsets;
+}
+
+void OccaConformingProlongation::BcastBeginCopy(const ::occa::memory &src,
+                                                std::size_t item_size) const
+{
+   // shr_buf[i] = src[shr_ltdof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (shr_ltdof.Size() == 0) { return; }
+   ExtractSubVector((int)shr_ltdof.Size(), shr_ltdof.OccaMem(), src,
+                    shr_buf.OccaMem());
+   // If the above kernel is executed asynchronously, wait for it to complete:
+   shr_buf.OccaMem().getDevice().finish();
+   if (host_shr_buf)
+   {
+      shr_buf.OccaMem().copyTo(host_shr_buf);
+   }
+}
+
+void OccaConformingProlongation::BcastLocalCopy(const ::occa::memory &src,
+                                                ::occa::memory &dst,
+                                                std::size_t item_size) const
+{
+   // dst[ltdof_ldof[i]] = src[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ltdof_ldof.size<int>() == 0) { return; }
+   SetSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
+}
+
+void OccaConformingProlongation::BcastEndCopy(::occa::memory &dst,
+                                              std::size_t item_size) const
+{
+   // dst[ext_ldof[i]] = ext_buf[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ext_ldof.Size() == 0) { return; }
+   if (host_ext_buf)
+   {
+      ext_buf.OccaMem().copyFrom(host_ext_buf);
+   }
+   SetSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(),
+                ext_buf.OccaMem(), dst);
+}
+
+void OccaConformingProlongation::ReduceBeginCopy(const ::occa::memory &src,
+                                                 std::size_t item_size) const
+{
+   // ext_buf[i] = src[ext_ldof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ext_ldof.Size() == 0) { return; }
+   ExtractSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(), src,
+                    ext_buf.OccaMem());
+   // If the above kernel is executed asynchronously, wait for it to complete:
+   ext_buf.OccaMem().getDevice().finish();
+   if (host_ext_buf)
+   {
+      ext_buf.OccaMem().copyTo(host_ext_buf);
+   }
+}
+
+void OccaConformingProlongation::ReduceLocalCopy(const ::occa::memory &src,
+                                                 ::occa::memory &dst,
+                                                 std::size_t item_size) const
+{
+   // dst[i] = src[ltdof_ldof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ltdof_ldof.size<int>() == 0) { return; }
+   ExtractSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
+}
+
+void OccaConformingProlongation::ReduceEndAssemble(::occa::memory &dst,
+                                                   std::size_t item_size) const
+{
+   // dst[shr_ltdof[i]] += shr_buf[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (unq_ltdof.size<int>() == 0) { return; }
+   if (host_shr_buf)
+   {
+      shr_buf.OccaMem().copyFrom(host_shr_buf);
+   }
+   AddSubVector((int)unq_ltdof.size<int>(), unq_ltdof, unq_shr_i, unq_shr_j,
+                shr_buf.OccaMem(), dst);
+}
+
+void OccaConformingProlongation::Mult_(const Vector &x, Vector &y) const
+{
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+
+   BcastBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'shr_buf'
+
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = shr_buf_offsets[nbr];
+      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0)
+      {
+         void *send_buf;
+         if (host_shr_buf)
+         {
+            send_buf = host_shr_buf + send_offset*sizeof(double);
+         }
+         else
+         {
+            send_buf = (shr_buf.OccaMem() + send_offset*sizeof(double)).ptr();
+         }
+         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41822, gtopo.GetComm(), &requests[req_counter++]);
+      }
+
+      const int recv_offset = ext_buf_offsets[nbr];
+      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0)
+      {
+         void *recv_buf;
+         if (host_ext_buf)
+         {
+            recv_buf = host_ext_buf + recv_offset*sizeof(double);
+         }
+         else
+         {
+            recv_buf = (ext_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
+         }
+         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41822, gtopo.GetComm(), &requests[req_counter++]);
+      }
+   }
+
+   BcastLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
+
+   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+
+   BcastEndCopy(y.OccaMem(), sizeof(double)); // copy from 'ext_buf'
+}
+
+void OccaConformingProlongation::MultTranspose_(const Vector &x,
+                                                Vector &y) const
+{
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+
+   ReduceBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'ext_buf'
+
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = ext_buf_offsets[nbr];
+      const int send_size = ext_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0)
+      {
+         void *send_buf;
+         if (host_ext_buf)
+         {
+            send_buf = host_ext_buf + send_offset*sizeof(double);
+         }
+         else
+         {
+            send_buf = (ext_buf.OccaMem() + send_offset*sizeof(double)).ptr();
+         }
+         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41823, gtopo.GetComm(), &requests[req_counter++]);
+      }
+
+      const int recv_offset = shr_buf_offsets[nbr];
+      const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0)
+      {
+         void *recv_buf;
+         if (host_shr_buf)
+         {
+            recv_buf = host_shr_buf + recv_offset*sizeof(double);
+         }
+         else
+         {
+            recv_buf = (shr_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
+         }
+         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41823, gtopo.GetComm(), &requests[req_counter++]);
+      }
+   }
+
+   ReduceLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
+
+   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+
+   ReduceEndAssemble(y.OccaMem(), sizeof(double)); // assemble from 'shr_buf'
+}
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/fespace.hpp

@@ -0,0 +1,210 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+#define MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "engine.hpp"
+#include "operator.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+/// TODO: doxygen
+class FiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::FiniteElementSpace *fes;
+
+   SharedPtr<Layout> e_layout;
+
+   ::occa::array<int> globalToLocalOffsets;
+   ::occa::array<int> globalToLocalIndices;
+   ::occa::array<int> localToGlobalMap;
+   ::occa::kernel globalToLocalKernel, localToGlobalKernel;
+
+   mfem::Ordering::Type ordering;
+
+   int globalDofs, localDofs;
+   int vdim;
+
+   mutable Operator *prolongationOp, *restrictionOp;
+
+   void SetupLocalGlobalMaps();
+   void SetupOperators() const; // calls virtual methods of 'fes' !!!
+   void SetupKernels();
+
+public:
+   /// TODO: doxygen
+   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
+
+   /// Virtual destructor
+   virtual ~FiniteElementSpace();
+
+   /// TODO: doxygen
+   const Engine &OccaEngine() const { return engine->As<Engine>(); }
+
+   /// TODO: doxygen
+   ::occa::device GetDevice(int idx = 0) const
+   { return OccaEngine().GetDevice(idx); }
+
+   mfem::Mesh* GetMesh() const { return fes->GetMesh(); }
+
+   Layout &OccaVLayout() const
+   { return *fes->GetVLayout().As<Layout>(); }
+
+   Layout &OccaTrueVLayout() const
+   { return *fes->GetTrueVLayout().As<Layout>(); }
+
+   Layout &OccaEVLayout() { return *e_layout; }
+
+   bool hasTensorBasis() const
+   { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
+
+   mfem::Ordering::Type GetOrdering() const { return ordering; }
+
+   int GetGlobalDofs() const { return globalDofs; }
+   int GetLocalDofs() const { return localDofs; }
+
+   int GetDim() const { return fes->GetMesh()->Dimension(); }
+   int GetVDim() const { return vdim; }
+
+   int GetVSize() const { return globalDofs * vdim; }
+   int GetTrueVSize() const { return fes->GetTrueVSize(); }
+   int GetGlobalVSize() const { return globalDofs*vdim; /* FIXME: MPI */ }
+   int GetGlobalTrueVSize() const { return fes->GetTrueVSize(); }
+
+   int GetNE() const { return fes->GetNE(); }
+
+   const mfem::FiniteElementCollection *FEColl() const
+   { return fes->FEColl(); }
+   const mfem::FiniteElement *GetFE(const int idx) const
+   { return fes->GetFE(idx); }
+
+   virtual const mfem::Operator *GetProlongationOperator() const
+   { return prolongationOp; }
+
+   virtual const mfem::Operator *GetRestrictionOperator() const
+   { return restrictionOp; }
+
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
+
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
+
+   const ::occa::array<int> GetLocalToGlobalMap() const
+   { return localToGlobalMap; }
+
+   /// L-vector to E-vector
+   void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
+   {
+      globalToLocalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          globalVec.OccaMem(), localVec.OccaMem());
+   }
+
+   /// E-vector to L-vector, transpose of GlobalToLocal
+   void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
+   {
+      localToGlobalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          localVec.OccaMem(), globalVec.OccaMem());
+   }
+};
+
+
+#ifdef MFEM_USE_MPI
+
+/// OCCA version of mfem::ConformingProlongationOperator
+class OccaConformingProlongation : public Operator
+{
+protected:
+   // size(shr_buf)=size(shr_ltdof)
+   // size(ext_buf)=size(ext_ldof)
+   Array shr_ltdof, ext_ldof;
+   mutable Array shr_buf, ext_buf;
+   mutable char *host_shr_buf, *host_ext_buf;
+   // Offsets into {shr,ext}_buf; size is num. neighbors, i.e.
+   // gc.GetGroupTopology().GetNumNeighbors():
+   int *shr_buf_offsets, *ext_buf_offsets;
+
+   ::occa::memory ltdof_ldof; // shared with the restriction operator
+
+   ::occa::memory unq_ltdof; // enumeration of the unique ltdofs in shr_ltdof
+   ::occa::memory unq_shr_i, unq_shr_j;
+
+   ::occa::kernel ExtractSubVector, SetSubVector, AddSubVector;
+
+   MPI_Request *requests;
+
+   const GroupCommunicator &gc;
+
+   // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
+   //         shr_buf[i] = src[shr_ltdof[i]]
+   void BcastBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
+   // Kernel: copy ltdofs from 'src' to ldofs in 'dst'.
+   //         dst[ltdof_ldof[i]] = src[i]
+   void BcastLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
+                       std::size_t item_size) const;
+   // Kernel: copy ext. dofs from 'ext_buf' to 'dst' - after recv.
+   //         dst[ext_ldof[i]] = ext_buf[i]
+   void BcastEndCopy(::occa::memory &dst, std::size_t item_size) const;
+
+   // Kernel: copy ext. dofs from 'src' to 'ext_buf' - prepare for send.
+   //         ext_buf[i] = src[ext_ldof[i]]
+   void ReduceBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
+   // Kernel: copy owned ldofs from 'src' to ltdofs in 'dst'.
+   //         dst[i] = src[ltdof_ldof[i]]
+   void ReduceLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
+                       std::size_t item_size) const;
+   // Kernel: assemble dofs from 'shr_buf' into to 'dst' - after recv.
+   //         dst[shr_ltdof[i]] += shr_buf[i]
+   void ReduceEndAssemble(::occa::memory &dst, std::size_t item_size) const;
+
+public:
+   OccaConformingProlongation(const FiniteElementSpace &ofes,
+                              const mfem::ParFiniteElementSpace &pfes,
+                              ::occa::memory ltdof_ldof_);
+
+   virtual ~OccaConformingProlongation();
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+};
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_FE_SPACE_HPP

backends/occa/fespace.okl

@@ -0,0 +1,67 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over entries
+  ================================================
+*/
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double *Global_t @dim(NUM_VDIM, globalEntries);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries);
+#else
+typedef double *Global_t @dim(NUM_VDIM, globalEntries) @dimOrder(1, 0);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
+#endif
+
+@kernel void GlobalToLocal(const int globalEntries,
+                           const int localEntries,
+                           @restrict const int * offsets,
+                           @restrict const int * indices,
+                           @restrict const Global_t globalX,
+                           @restrict Local_t localX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        const double dofValue = globalX(v, i);
+        for (int j = offset; j < nextOffset; ++j) {
+          localX(v, indices[j]) = dofValue;
+        }
+      }
+    }
+  }
+}
+
+@kernel void LocalToGlobal(const int globalEntries,
+                           const int localEntries,
+                           @restrict const int * offsets,
+                           @restrict const int * indices,
+                           @restrict const Local_t localX,
+                           @restrict Global_t globalX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        double dofValue = 0;
+        for (int j = offset; j < nextOffset; ++j) {
+          dofValue += localX(v, indices[j]);
+        }
+        globalX(v, i) = dofValue;
+      }
+    }
+  }
+}

backends/occa/geometry.okl

@@ -0,0 +1,181 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef STORE_JACOBIAN
+#  define STORE_JACOBIAN 1
+#endif
+
+#ifndef STORE_JACOBIAN_INV
+#  define STORE_JACOBIAN_INV 1
+#endif
+
+#ifndef STORE_JACOBIAN_DET
+#  define STORE_JACOBIAN_DET 1
+#endif
+
+typedef double* Local1D_t @dim(1, NUM_DOFS, numElements);
+typedef double* Local2D_t @dim(2, NUM_DOFS, numElements);
+typedef double* Local3D_t @dim(3, NUM_DOFS, numElements);
+
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+typedef double* DofToQuadD1D_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* Jacobian1D_t @dim(NUM_QUAD, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+@kernel void InitGeometryInfo1D(const int numElements,
+                                @restrict const DofToQuadD1D_t dofToQuadD,
+                                @restrict const Local1D_t nodes,
+                                @restrict Jacobian1D_t J,
+                                @restrict Jacobian1D_t invJ,
+                                @restrict QLocal_t detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[NUM_DOFS];
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes[d] = nodes(0, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(q, d);
+        J11 += wx * s_nodes[d];
+      }
+#if STORE_JACOBIAN
+      J(q, e) = J11;
+#endif
+#if STORE_JACOBIAN_INV
+      invJ(q, e) = 1.0 / J11;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = J11;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo2D(const int numElements,
+                                @restrict const DofToQuadD2D_t dofToQuadD,
+                                @restrict const Local2D_t nodes,
+                                @restrict Jacobian2D_t J,
+                                @restrict Jacobian2D_t invJ,
+                                @restrict QLocal_t detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0;
+      double J21 = 0, J22 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        J11 += (wx * x); J12 += (wx * y);
+        J21 += (wy * x); J22 += (wy * y);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = (J11 * J22) - (J12 * J21);
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) =  J22 * r_idetJ;
+      invJ(1, 0, q, e) = -J12 * r_idetJ;
+
+      invJ(0, 1, q, e) = -J21 * r_idetJ;
+      invJ(1, 1, q, e) =  J11 * r_idetJ;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo3D(const int numElements,
+                                @restrict const DofToQuadD3D_t dofToQuadD,
+                                @restrict const Local3D_t nodes,
+                                @restrict Jacobian3D_t J,
+                                @restrict Jacobian3D_t invJ,
+                                @restrict QLocal_t detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+        s_nodes(2, d) = nodes(2, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0, J13 = 0;
+      double J21 = 0, J22 = 0, J23 = 0;
+      double J31 = 0, J32 = 0, J33 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double wz = dofToQuadD(2, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        const double z = s_nodes(2, d);
+        J11 += (wx * x); J12 += (wx * y); J13 += (wx * z);
+        J21 += (wy * x); J22 += (wy * y); J23 += (wy * z);
+        J31 += (wz * x); J32 += (wz * y); J33 += (wz * z);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                             (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12; J(2, 0, q, e) = J13;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22; J(2, 1, q, e) = J23;
+      J(0, 2, q, e) = J31; J(1, 2, q, e) = J32; J(2, 2, q, e) = J33;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) = r_idetJ * ((J22 * J33) - (J23 * J32));
+      invJ(1, 0, q, e) = r_idetJ * ((J32 * J13) - (J33 * J12));
+      invJ(2, 0, q, e) = r_idetJ * ((J12 * J23) - (J13 * J22));
+
+      invJ(0, 1, q, e) = r_idetJ * ((J23 * J31) - (J21 * J33));
+      invJ(1, 1, q, e) = r_idetJ * ((J33 * J11) - (J31 * J13));
+      invJ(2, 1, q, e) = r_idetJ * ((J13 * J21) - (J11 * J23));
+
+      invJ(0, 2, q, e) = r_idetJ * ((J21 * J32) - (J22 * J31));
+      invJ(1, 2, q, e) = r_idetJ * ((J31 * J12) - (J32 * J11));
+      invJ(2, 2, q, e) = r_idetJ * ((J11 * J22) - (J12 * J21));
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}

backends/occa/gridfunc.cpp

@@ -0,0 +1,86 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "gridfunc.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir)
+{
+   const int numQuad = ir.GetNPoints();
+
+   const FiniteElement &fe = *(fespace.GetFE(0));
+   const int dim  = fe.GetDim();
+   const int vdim = fespace.GetVDim();
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "FEColl : " << fespace.FEColl()->Name()
+      << "Quad: "    << numQuad
+      << "Dim: "     << dim
+      << "VDim: "    << vdim;
+   std::string hash = ss.str();
+
+   // Kernel defines
+   ::occa::properties props;
+   props["defines/NUM_VDIM"] = vdim;
+
+   SetProperties(fespace, ir, props);
+
+   ::occa::kernel kernel = gridFunctionKernels[hash];
+   if (!kernel.isInitialized())
+   {
+      const std::string &okl_path = fespace.OccaEngine().GetOklPath();
+      kernel = device.buildKernel(okl_path + "gridfunc.okl",
+                                  stringWithDim("GridFuncToQuad", dim),
+                                  props);
+   }
+   return kernel;
+}
+
+void ToQuad(const IntegrationRule &ir, FiniteElementSpace &fespace, Vector &gf,
+            Vector &quadValues)
+{
+   const Engine &engine = fespace.OccaEngine();
+   ::occa::device device = engine.GetDevice();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, fespace, ir);
+
+   const int elements = fespace.GetNE();
+   const int numQuad  = ir.GetNPoints();
+   quadValues.OccaResize(numQuad * elements, sizeof(double));
+
+   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, fespace, ir);
+   g2qKernel(elements,
+             maps.dofToQuad,
+             fespace.GetLocalToGlobalMap(),
+             gf.OccaMem(),
+             quadValues.OccaMem());
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/gridfunc.hpp

@@ -0,0 +1,55 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+#define MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+class IntegrationRule;
+
+namespace occa
+{
+
+// TODO: make this object part of the backend or the engine.
+extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+// TODO: make this a method of the backend or the engine.
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir);
+
+// ToQuad version without the deprecated class.
+//
+// FIXME: This is the action of a global B matrix, mapping L-vector to Q-vector,
+//        so it should be made into an operator that can be constructed by the
+//        FE space class. A batched version, where only a subset of the elements
+//        are processed should be defined as well.
+//
+//        The abstract operator construction method in the FE space class is:
+//           PFiniteElementSpace::GetInterpolationOperator(...)
+void ToQuad(const IntegrationRule &ir, FiniteElementSpace &ofespace, Vector &gf,
+            Vector &quadValues);
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_GRID_FUNC_HPP

backends/occa/gridfunc.okl

@@ -0,0 +1,26 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+#ifdef USING_TENSOR_OPS
+#  ifdef OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/tensor/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/simplex/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"
+#  endif
+#endif

backends/occa/gridfunc/simplex/cpu.okl

@@ -0,0 +1,63 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -0,0 +1,79 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/cpu.okl

@@ -0,0 +1,188 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap1D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal1D_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_out[NUM_VDIM][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          r_out[v][qx] = 0;
+        }
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const int gid = l2gMap(dx, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[v][qx] += r_gf * dofToQuad(qx, dx);
+          }
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          out(v, qx, e) = r_out[v][qx];
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap2D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal2D_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out_xy[v][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double out_x[NUM_VDIM][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            out_x[v][qy] = 0;
+          }
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const int gid = l2gMap(dx, dy, e);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            const double r_gf = gf[v + gid*NUM_VDIM];
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              out_x[v][qy] += r_gf * dofToQuad(qy, dx);
+            }
+          }
+        }
+
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double d2q = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] += d2q * out_x[v][qx];
+            }
+          }
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            out(v, qx, qy, e) = out_xy[v][qy][qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap3D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal3D_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xyz[v][qz][qy][qx] = 0;
+            }
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] = 0;
+            }
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double out_x[NUM_VDIM][NUM_QUAD_1D];
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_x[v][qx] = 0;
+            }
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const int gid = l2gMap(dx, dy, dz, e);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              const double r_gf = gf[v + gid*NUM_VDIM];
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_x[v][qx] += r_gf * dofToQuad(qx, dx);
+              }
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xy[v][qy][qx] += wy * out_x[v][qx];
+              }
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xyz[v][qz][qy][qx] += wz * out_xy[v][qy][qx];
+              }
+            }
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              out(v, qx, qy, qz, e) = out_xyz[v][qz][qy][qx];
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -0,0 +1,183 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap1D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal1D_t out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @exclusive double r_out[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuad[i] = dofToQuad[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double r_gf = gf[l2gMap(dx, e)];
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              r_out[qx] += r_gf * s_dofToQuad(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out(qx, e) = r_out[qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap2D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal2D_t out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id] = dofToQuad[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            double r_x[NUM_DOFS_1D];
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = gf[l2gMap(dx, dy, e)];
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy += r_x[dy] * s_dofToQuad(qy, dy);
+              }
+              s_xy(dx, qy) = xy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double val = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                val += s_xy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+              out(qx, qy, e) = val;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap3D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal3D_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id] = dofToQuad[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double val = gf[l2gMap(dx, dy, dz, e)];
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz] += val * s_dofToQuad(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy) = r_qz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double val = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy = s_dofToQuad(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx = s_dofToQuad(qx, dx);
+                val += wx * wy * s_z(dx, dy);
+              }
+            }
+            out(qx, qy, qz, e) = val;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================