Adding cusparse ILU and incomplete Cholesky solvers on GPU.

Mixed meshes in TMOP

.gitignore

@@ -253,15 +253,6 @@ tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/tmop_tests_*
-tests/unit/ptmop_tests_*
-tests/unit/cube.mesh
-tests/unit/star.mesh
-tests/unit/blade.mesh
-tests/unit/square01.mesh
-tests/unit/toroid-hex.mesh
-tests/unit/beam-hex-nurbs.mesh
-tests/unit/square-disc-nurbs.mesh
 
 # Test script output
 tests/scripts/*.err

CHANGELOG

@@ -16,7 +16,12 @@ Meshing improvements
 - The graph linear ordering library Gecko, previously an external dependency, is
   now included directly in MFEM. As a result, Mesh::GetGeckoElementOrdering is
   always available. The interface has also been improved, see for example the
-  mesh-explorer miniapp.
+  Mesh Explorer miniapp.
+
+- Improved Gmsh reader (version 2.2), which now supports both high-order and
+  periodic meshes. Segments, triangles, quadrilaterals, and tetrahedra are
+  supported up to order 10. Wedges and hexahedra are supported up to order 9.
+  For sample periodic meshes, see the periodic*.msh files in the data directory.
 
 - Added support for finite difference-based gradient and Hessian approximation
   in the TMOP mesh optimization algorithms. This improves the accuracy of the
@@ -27,11 +32,8 @@ Meshing improvements
   the user to specify different discrete functions for controlling the
   size, aspect-ratio, orientation, and skew of elements in the mesh.
 
-- Added TMOP capability for approximate tangential mesh relaxation.
-
-- Added support for reading periodic meshes in Gmsh format (version 2.2). See
-  for example the periodic-annulus-sector and periodic-torus-sector files in
-  the data directory.
+- Added TMOP capability for approximate tangential mesh relaxation. Added
+  support and examples for using TMOP on mixed meshes.
 
 - Added complete action of the TMOP Integrator to account for the spatial
   derivatives of discrete and analytic targets.
@@ -50,11 +52,6 @@ Performance improvements
 Improved GPU capabilities
 -------------------------
 - Added support for Chebyshev accelerated polynomial smoother on GPU.
-- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2 and #7 are available in 2D, #302, #303 and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also support AssemblyLevel::PARTIAL
 
 - Optimized AMD/HIP kernel support.
 
@@ -63,6 +60,8 @@ Improved GPU capabilities
   compute a global sparse matrix. All integrators supported by element assembly
   are also supported by full assembly. See the '-fa' option in Example 9.
 
+- Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.
+
 - Added support for BlockOperator on GPU. See the updated Example 5.
 
 Discretization improvements
@@ -170,6 +169,8 @@ New and updated examples and miniapps
 
 - Added device support in Example 5/5p.
 
+- Added the option to plot a function in Mesh Explorer.
+
 Improved testing
 ----------------
 - Added a GitLab pipeline that automates PR testing on supercomputing systems

CMakeLists.txt

@@ -89,8 +89,38 @@ enable_language(CXX)
 if (MFEM_USE_CUDA)
    # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
    cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
+   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
+   if (NOT CMAKE_CUDA_HOST_COMPILER)
+      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+   endif()
    enable_language(CUDA)
+   set(CMAKE_CUDA_STANDARD 11)
+   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+   set(CMAKE_CUDA_EXTENSIONS OFF)
+   set(CUDA_FLAGS "--expt-extended-lambda")
+   if (CMAKE_VERSION VERSION_LESS 3.18.0)
+      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
+   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
+      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
+      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
+         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
+      else()
+         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
+      endif()
+   else()
+      set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
+   endif()
    message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
+   if (CMAKE_VERSION VERSION_LESS 3.12.0)
+      # CMake versions 3.8 and 3.9 require this to work; 3.10 and 3.11 are not
+      # tested and may not actually need this (but should be ok to keep).
+      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
+      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
+   endif()
+   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
+       "CUDA flags set for MFEM" FORCE)
+   set(CUSPARSE_FOUND TRUE)
+   set(CUSPARSE_LIBRARIES "cusparse")
 endif()
 
 if (XSDK_ENABLE_C)
@@ -296,22 +326,6 @@ if (MFEM_USE_HIOP)
   # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()
 
-# CUDA
-if (MFEM_USE_CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-   set(CMAKE_CUDA_EXTENSIONS OFF)
-   set(CMAKE_CUDA_FLAGS "-arch=${CUDA_ARCH} --expt-extended-lambda"
-       CACHE STRING "CUDA flags set for MFEM" FORCE)
-   if (MFEM_USE_MPI)
-      set(CUDA_CCBIN_COMPILER ${MPI_CXX_COMPILER})
-   else()
-      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
-   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
-   set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CUDA_CCBIN_COMPILER})
-endif()
-
 # OCCA
 if (MFEM_USE_OCCA)
    find_package(OCCA REQUIRED)
@@ -357,7 +371,8 @@ endif()
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
     SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
+    CUSPARSE)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

INSTALL

@@ -663,7 +663,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.6, git-hash a970f63.
+  Versions: libCEED > 0.6, git-hash fe5822c.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -128,7 +128,15 @@ function(add_mfem_miniapp MFEM_EXE_NAME)
   if (MFEM_USE_CUDA)
     set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
       PROPERTY LANGUAGE CUDA)
-    list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
+    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
+      list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
+    else()
+      set(LIST_)
+      foreach(item IN LISTS EXTRA_OPTIONS_LIST)
+        list(APPEND LIST_ "-Xcompiler=${item}")
+      endforeach()
+      set(EXTRA_OPTIONS_LIST ${LIST_})
+    endif()
   endif()
 
   # Actually add the executable

config/defaults.mk

@@ -138,7 +138,6 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
-MFEM_USE_CAMP          = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 
@@ -373,11 +372,6 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
-# CAMP library configuration
-CAMP_DIR = @MFEM_DIR@/../camp
-CAMP_OPT = -I$(CAMP_DIR)/include
-CAMP_LIB = -L$(CAMP_DIR)/lib
-
 # If YES, enable some informational messages
 VERBOSE = NO
 

data/periodic-annulus-sector.geo

@@ -1,13 +1,38 @@
 SetFactory("OpenCASCADE");
 
+// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
+periodic = 1;
+
+// Set the geometry order (1, 2, ..., 9)
+order = 3;
+
+// Set the element type (3 - triangles, 4 - quadrilaterals)
+type = 3;
+
+// Number of radial elements
+nrad = 2;
+
+// Number of azimuthal elements on inner arc
+nazm1 = 3;
+
+// Number of azimuthal elements on outer arc
+nazm2 = 5;
+
+// Note: Using type = 4 with nazm1 != nazm2 can lead to mixed meshes
+//       containing both triangles and quadrilaterals.
+
+// Inner and outer radii
 R1 = 1.0;
 R2 = 2.0;
 
+// Angular size of the sector
+Phi = Pi/3.0;
+
 Point(1) = {0.0, 0, 0, 1.0};
 Point(2) = {R1, 0, 0, 1.0};
 Point(3) = {R2, 0, 0, 1.0};
-Point(4) = {R1*Cos(Pi/3), R1*Sin(Pi/3), 0, 1.0};
-Point(5) = {R2*Cos(Pi/3), R2*Sin(Pi/3), 0, 1.0};
+Point(4) = {R1*Cos(Phi), R1*Sin(Phi), 0, 1.0};
+Point(5) = {R2*Cos(Phi), R2*Sin(Phi), 0, 1.0};
 Line(1) = {2, 3};
 Line(2) = {4, 5};
 Circle(3) = {2, 1, 4};
@@ -15,13 +40,23 @@ Circle(4) = {3, 1, 5};
 Curve Loop(5) = {1, 4, -2, -3};
 Plane Surface(1) = {5};
 
-Transfinite Curve{1} = 7;
-Transfinite Curve{2} = 7;
-Transfinite Curve{3} = 4;
-Transfinite Curve{4} = 10;
+Transfinite Curve{1} = nrad+1;
+Transfinite Curve{2} = nrad+1;
+Transfinite Curve{3} = nazm1+1;
+Transfinite Curve{4} = nazm2+1;
+
+If (nazm1 == nazm2)
+   Transfinite Surface{1};
+EndIf
+
+If (type == 4)
+   Recombine Surface {1};
+EndIf
 
 // Set a rotation periodicity constraint:
-Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Pi/3};
+If (periodic)
+   Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Phi};
+EndIf
 
 // Tag surfaces and volumes with positive integers
 Physical Curve(1) = {3};
@@ -30,8 +65,22 @@ Physical Curve(3) = {1};
 Physical Curve(4) = {2};
 Physical Surface(1) = {1};
 
+// Optimize the high-order mesh
+// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
+// Mesh.ElementOrder = order;
+// Mesh.HighOrderOptimize = 1;
+
 // Generate 2D mesh
 Mesh 2;
+SetOrder order;
 Mesh.MshFileVersion = 2.2;
 
-Save "periodic-annulus-sector.msh";
+// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
+// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
+// Plugin(AnalyseMeshQuality).Run;
+
+If (periodic)
+   Save Sprintf("periodic-annulus-sector-t%01g-o%01g.msh", type, order);
+Else
+   Save Sprintf("annulus-sector-t%01g-o%01g.msh", type, order);
+EndIf

data/periodic-annulus-sector.msh

@@ -2,184 +2,191 @@ $MeshFormat
 2.2 0 8
 $EndMeshFormat
 $Nodes
-55
+136
 1 1 0 0
 2 2 0 0
 3 0.5000000000000001 0.8660254037844386 0
 4 1 1.732050807568877 0
-5 1.166666666666667 0 0
-6 1.333333333333333 0 0
-7 1.5 0 0
+5 1.5 0 0
+6 1.166666666666667 0 0
+7 1.333333333333333 0 0
 8 1.666666666666667 0 0
 9 1.833333333333333 0 0
-10 0.5833333333333335 1.010362971081845 0
-11 0.6666666666666667 1.154700538379251 0
-12 0.7500000000000002 1.299038105676658 0
+10 0.7500000000000002 1.299038105676658 0
+11 0.5833333333333335 1.010362971081845 0
+12 0.6666666666666667 1.154700538379251 0
 13 0.8333333333333335 1.443375672974064 0
 14 0.9166666666666669 1.587713240271471 0
 15 0.9396926207859085 0.3420201433256683 0
 16 0.7660444431189786 0.6427876096865386 0
-17 1.986476715483886 0.2321858282504602 0
-18 1.946089741159648 0.4612317414848793 0
-19 1.879385241571817 0.6840402866513365 0
-20 1.787265280646825 0.8975983604009234 0
-21 1.670975622825874 1.09901795614161 0
-22 1.532088886237958 1.285575219373077 0
-23 1.372483275737469 1.454747283146095 0
-24 1.194317183405575 1.604246385510085 0
-25 1.425989114816062 0.1915326920916892 0
-26 0.8788667344146573 1.13917645290495 0
-27 1.630372059110754 0.7154531062316609 0
-28 1.436395769298814 1.053728612482506 0
-29 1.081023776188756 0.6241293681829633 0
-30 1.168737372335971 1.428012728596308 0
-31 1.821063986059922 0.298149890497067 0
-32 1.234707097211386 0.3469796339295647 0
-33 1.377747393186519 0.6200150626754309 0
-34 1.457047681210906 0.3890895843559762 0
-35 0.917846726184522 0.8957978954532204 0
-36 1.218335619030348 0.9017812086952638 0
-37 1.066623110765233 1.061857005744772 0
-38 1.587029716281926 0.1355955181472859 0
-39 1.744445799211916 0.1441515753740107 0
-40 1.25 0.1443375672974065 0
-41 1.453660070628011 0.8435769396609902 0
-42 1.741367044061892 0.499612708014486 0
-43 1.30550638526547 1.257610469847477 0
-44 1.118213276932792 0.1666674689105279 0
-45 0.9109440214958271 1.306610291787315 0
-46 0.9970618258753989 1.438658589955562 0
-47 0.7499999999999998 1.010362971081845 0
-48 0.7034449005273667 0.8850673702175776 0
-49 1.605449512513618 0.9269067082200894 0
-50 1.561654019115059 0.5298592532912715 0
-51 1.229782222487711 1.096820457143683 0
-52 1.617066998712459 0.3090202662210922 0
-53 1.079645953234324 1.246963713711438 0
-54 1.877063966817811 0.1348974588243076 0
-55 1.055356609656722 1.558136350380461 0
+17 0.993238357741943 0.1160929141252301 0
+18 0.9730448705798238 0.2306158707424401 0
+19 0.8936326403234125 0.4487991802004617 0
+20 0.8354878114129367 0.5495089780708056 0
+21 0.6862416378687343 0.7273736415730481 0
+22 0.597158591702787 0.8021231927550432 0
+23 1.956295201467611 0.4158233816355181 0
+24 1.827090915285202 0.8134732861515996 0
+25 1.618033988749896 1.175570504584944 0
+26 1.338261212717719 1.486289650954786 0
+27 1.995128100519648 0.1395129474882505 0
+28 1.980536137483141 0.278346201920131 0
+29 1.922523391876638 0.551274711633998 0
+30 1.879385241571817 0.6840402866513373 0
+31 1.765895185717855 0.9389431255717802 0
+32 1.696096192312853 1.059838528466408 0
+33 1.532088886237958 1.285575219373077 0
+34 1.438679600677305 1.389316740917992 0
+35 1.231322950651319 1.576021507213442 0
+36 1.118385806941496 1.658075145110082 0
+37 1.162276263405681 0.6710405135499813 0
+38 1.248615852873337 1.079531485311822 0
+39 1.559209616901855 0.5415673055003691 0
+40 1.478306597054007 0.8535007117539289 0
+41 0.9210953433941653 0.9653302893212266 0
+42 1.296548225291847 0.3150268220262836 0
+43 1.055002035226811 1.358510675893086 0
+44 1.704005774249187 0.2344032256041583 0
+45 0.6403651144647218 0.8991270322967013 0
+46 0.7807302289294435 0.9322286608089638 0
+47 0.864063562262777 1.07656622810637 0
+48 0.8070317811313885 1.187802166891514 0
+49 0.7236984477980553 1.043464599594108 0
+50 1.432182741763949 0.1050089406754279 0
+51 1.364365483527898 0.2100178813508558 0
+52 1.197698816861231 0.2100178813508558 0
+53 1.098849408430616 0.1050089406754279 0
+54 1.265516075097282 0.1050089406754278 0
+55 0.8177280765440409 0.7503018362314346 0
+56 0.869411709969103 0.8578160627763305 0
+57 0.7348318576552288 0.8316090412164392 0
+58 1.177596357123201 0.3240245957927452 0
+59 1.058644488954555 0.3330223695592067 0
+60 1.087610484537871 0.2205785356313179 0
+61 1.267619707955123 0.7318605796179638 0
+62 1.372963152504565 0.7926806456859463 0
+63 1.40174301566045 0.9288443029398934 0
+64 1.325179434266893 1.004187894125858 0
+65 1.219835989717452 0.9433678280578752 0
+66 1.191056126561566 0.8072041708039283 0
+67 1.296399571111008 0.8680242368719107 0
+68 1.532241943619239 0.645545107584889 0
+69 1.505274270336623 0.7495229096694089 0
+70 1.294587381237739 0.6278827775334439 0
+71 1.426898499069797 0.5847250415169065 0
+72 1.399930825787181 0.6887028436014264 0
+73 1.139442349713613 1.041464419981624 0
+74 1.030268846553889 1.003397354651425 0
+75 1.001488983398004 0.8672336973974781 0
+76 1.081882623401843 0.7691371054737297 0
+77 1.110662486557728 0.9053007627276766 0
+78 1.207033584034403 0.5523692830420821 0
+79 1.251790904663125 0.4336980525341829 0
+80 1.384102022495183 0.3905403165176455 0
+81 1.471655819698519 0.4660538110090073 0
+82 1.339344701866461 0.5092115470255447 0
+83 1.73779714915742 0.7228379592678562 0
+84 1.648503383029637 0.6322026323841127 0
+85 1.691571478423774 0.4996526642120855 0
+86 1.823933339945692 0.4577380229238018 0
+87 1.787039416783436 0.5922941012619014 0
+88 1.308379426102925 1.350703595740465 0
+89 1.278497639488131 1.215117540526144 0
+90 1.371755231498856 1.111544491736196 0
+91 1.494894610124376 1.14355749816057 0
+92 1.4064614465962 1.25147448186763 0
+93 1.013887168325833 0.451693600067106 0
+94 1.088081715865757 0.5613670568085436 0
+95 1.03019898997678 0.6616228789288338 0
+96 0.8981217165478794 0.6522052443076862 0
+97 0.9637989050473432 0.5564495572737495 0
+98 1.432367408277627 0.2881522898855752 0
+99 1.568186591263407 0.2612777577448668 0
+100 1.655740388466743 0.3367912522362286 0
+101 1.607475002684299 0.4391792788682989 0
+102 1.519921205480963 0.3636657843769371 0
+103 1.184077913657828 1.17252454883891 0
+104 1.119539974442319 1.265517612365998 0
+105 1.010366471282595 1.2274505470358 0
+106 0.9657309073383804 1.096390418178513 0
+107 1.074904410498104 1.134457483508712 0
+108 1.901335258083062 0.07813440853471942 0
+109 1.802670516166124 0.1562688170694388 0
+110 1.636003849499458 0.156268817069439 0
+111 1.568001924749729 0.07813440853471942 0
+112 1.734668591416396 0.07813440853471944 0
+113 0.8516673450756037 1.318862295748801 0
+114 0.9533346901512071 1.338686485820944 0
+115 1.03666802348454 1.48302405311835 0
+116 1.01833401174227 1.607537430343614 0
+117 0.9350006784089369 1.463199863046207 0
+118 1.710829475874804 0.8268157613523761 0
+119 1.594568036464405 0.8401582365531526 0
+120 1.621535709747021 0.7361804344686326 0
+121 1.52488239428597 0.9608573093642675 0
+122 1.571458191517933 1.068213906974606 0
+123 1.448318812892413 1.036200900550232 0
+124 0.908699126206992 1.207626356963657 0
+125 1.500184666513678 0.1831433492101474 0
+126 1.646765991694905 0.9507607885973723 0
+127 0.9498053499729417 0.7597194708525821 0
+128 1.132839036494479 0.4426958263006443 0
+129 1.149421761057113 1.40110366758032 0
+130 1.243841486887416 1.443696659267553 0
+131 1.134903597606542 1.530869109405537 0
+132 1.872198725728137 0.3553499962917315 0
+133 1.788102249988661 0.2948766109479449 0
+134 1.893223337417325 0.2174207916708467 0
+135 1.213959700272622 1.308110604053232 0
+136 1.739836864206217 0.3972646375800152 0
 $EndNodes
 $Elements
-108
-1 1 2 3 1 1 5
-2 1 2 3 1 5 6
-3 1 2 3 1 6 7
-4 1 2 3 1 7 8
-5 1 2 3 1 8 9
-6 1 2 3 1 9 2
-7 1 2 4 2 3 10
-8 1 2 4 2 10 11
-9 1 2 4 2 11 12
-10 1 2 4 2 12 13
-11 1 2 4 2 13 14
-12 1 2 4 2 14 4
-13 1 2 1 3 1 15
-14 1 2 1 3 15 16
-15 1 2 1 3 16 3
-16 1 2 2 4 2 17
-17 1 2 2 4 17 18
-18 1 2 2 4 18 19
-19 1 2 2 4 19 20
-20 1 2 2 4 20 21
-21 1 2 2 4 21 22
-22 1 2 2 4 22 23
-23 1 2 2 4 23 24
-24 1 2 2 4 24 4
-25 2 2 1 1 32 40 25
-26 2 2 1 1 25 34 32
-27 2 2 1 1 33 41 36
-28 2 2 1 1 38 52 25
-29 2 2 1 1 33 36 29
-30 2 2 1 1 26 47 35
-31 2 2 1 1 35 37 26
-32 2 2 1 1 25 52 34
-33 2 2 1 1 32 44 40
-34 2 2 1 1 15 32 29
-35 2 2 1 1 15 29 16
-36 2 2 1 1 36 41 28
-37 2 2 1 1 32 33 29
-38 2 2 1 1 50 52 42
-39 2 2 1 1 32 34 33
-40 2 2 1 1 42 52 31
-41 2 2 1 1 43 53 51
-42 2 2 1 1 27 41 33
-43 2 2 1 1 26 53 45
-44 2 2 1 1 18 31 17
-45 2 2 1 1 29 35 16
-46 2 2 1 1 29 36 35
-47 2 2 1 1 24 30 23
-48 2 2 1 1 30 53 43
-49 2 2 1 1 17 54 2
-50 2 2 1 1 4 55 24
-51 2 2 1 1 28 51 36
-52 2 2 1 1 47 48 35
-53 2 2 1 1 36 37 35
-54 2 2 1 1 37 53 26
-55 2 2 1 1 22 28 21
-56 2 2 1 1 20 27 19
-57 2 2 1 1 33 50 27
-58 2 2 1 1 15 44 32
-59 2 2 1 1 18 42 31
-60 2 2 1 1 30 43 23
-61 2 2 1 1 35 48 16
-62 2 2 1 1 31 54 17
-63 2 2 1 1 9 39 8
-64 2 2 1 1 8 38 7
-65 2 2 1 1 7 25 6
-66 2 2 1 1 22 43 28
-67 2 2 1 1 23 43 22
-68 2 2 1 1 39 54 31
-69 2 2 1 1 19 42 18
-70 2 2 1 1 24 55 30
-71 2 2 1 1 27 42 19
-72 2 2 1 1 13 46 14
-73 2 2 1 1 51 53 37
-74 2 2 1 1 39 52 38
-75 2 2 1 1 6 40 5
-76 2 2 1 1 34 52 50
-77 2 2 1 1 12 45 13
-78 2 2 1 1 30 55 46
-79 2 2 1 1 10 47 11
-80 2 2 1 1 8 39 38
-81 2 2 1 1 28 49 21
-82 2 2 1 1 7 38 25
-83 2 2 1 1 41 49 28
-84 2 2 1 1 20 49 27
-85 2 2 1 1 11 26 12
-86 2 2 1 1 27 49 41
-87 2 2 1 1 31 52 39
-88 2 2 1 1 25 40 6
-89 2 2 1 1 2 54 9
-90 2 2 1 1 14 55 4
-91 2 2 1 1 45 53 46
-92 2 2 1 1 45 46 13
-93 2 2 1 1 5 44 1
-94 2 2 1 1 21 49 20
-95 2 2 1 1 46 53 30
-96 2 2 1 1 3 48 10
-97 2 2 1 1 34 50 33
-98 2 2 1 1 36 51 37
-99 2 2 1 1 26 45 12
-100 2 2 1 1 11 47 26
-101 2 2 1 1 27 50 42
-102 2 2 1 1 40 44 5
-103 2 2 1 1 43 51 28
-104 2 2 1 1 10 48 47
-105 2 2 1 1 9 54 39
-106 2 2 1 1 46 55 14
-107 2 2 1 1 1 44 15
-108 2 2 1 1 16 48 3
+38
+1 26 2 3 1 1 5 6 7
+2 26 2 3 1 5 2 8 9
+3 26 2 4 2 3 10 11 12
+4 26 2 4 2 10 4 13 14
+5 26 2 1 3 1 15 17 18
+6 26 2 1 3 15 16 19 20
+7 26 2 1 3 16 3 21 22
+8 26 2 2 4 2 23 27 28
+9 26 2 2 4 23 24 29 30
+10 26 2 2 4 24 25 31 32
+11 26 2 2 4 25 26 33 34
+12 26 2 2 4 26 4 35 36
+13 21 2 1 1 3 41 10 45 46 47 48 12 11 49
+14 21 2 1 1 5 42 1 50 51 52 53 6 7 54
+15 21 2 1 1 16 41 3 55 56 46 45 22 21 57
+16 21 2 1 1 1 42 15 53 52 58 59 18 17 60
+17 21 2 1 1 37 40 38 61 62 63 64 65 66 67
+18 21 2 1 1 39 40 37 68 69 62 61 70 71 72
+19 21 2 1 1 38 41 37 73 74 75 76 66 65 77
+20 21 2 1 1 37 42 39 78 79 80 81 71 70 82
+21 21 2 1 1 24 39 23 83 84 85 86 29 30 87
+22 21 2 1 1 26 38 25 88 89 90 91 33 34 92
+23 21 2 1 1 15 37 16 93 94 95 96 20 19 97
+24 21 2 1 1 42 44 39 98 99 100 101 81 80 102
+25 21 2 1 1 38 43 41 103 104 105 106 74 73 107
+26 21 2 1 1 2 44 5 108 109 110 111 8 9 112
+27 21 2 1 1 10 43 4 113 114 115 116 14 13 117
+28 21 2 1 1 24 40 39 118 119 69 68 84 83 120
+29 21 2 1 1 38 40 25 64 63 121 122 91 90 123
+30 21 2 1 1 41 43 10 106 105 114 113 48 47 124
+31 21 2 1 1 5 44 42 111 110 99 98 51 50 125
+32 21 2 1 1 25 40 24 122 121 119 118 31 32 126
+33 21 2 1 1 37 41 16 76 75 56 55 96 95 127
+34 21 2 1 1 15 42 37 59 58 79 78 94 93 128
+35 21 2 1 1 4 43 26 116 115 129 130 35 36 131
+36 21 2 1 1 23 44 2 132 133 109 108 27 28 134
+37 21 2 1 1 26 43 38 130 129 104 103 89 88 135
+38 21 2 1 1 39 44 23 101 100 133 132 86 85 136
 $EndElements
 $Periodic
 1
 1 1 2
 Affine 0.5000000000000001 0.8660254037844386 0 0 -0.8660254037844386 0.5000000000000001 0 0 0 0 1 0 0 0 0 1
-7
-9 14
-6 11
-8 13
+3
 5 10
-7 12
-2 4
 1 3
+2 4
 $EndPeriodic

data/periodic-torus-sector.geo

@@ -1,25 +1,141 @@
-SetFactory("OpenCASCADE");
+// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
+periodic = 1;
 
-R = 1.5;
-r = 0.5;
+// Set the geometry order (1, 2, ..., 10 for tetrahedra or 9 for other types)
+order = 3;
 
-Torus(1) = {0,0,0, R, r, Pi/3};
+// Set the element type (4 - tetrahedra, 6 - wedges, 8 - hexahedra)
+type = 8;
 
-pts() = PointsOf{ Volume{1}; };
+// Minor and major radii
+R1 = 1.0;
+R2 = 2.0;
 
-Characteristic Length{ pts() } = 0.25;
+// Side length of interior square
+A1 = 0.8;
+
+// Angular size of the sector
+Phi = Pi/3.0;
+
+// Number of azimuthal elements
+nazm = 3;
+
+// Number of elements around a quarter of the circle
+narc = 2;
+
+// Number of elements between surface and interior square
+nshl = 1;
+
+lc = 0.5;
+a1 = A1 / Sqrt(2.0);
+
+Point(1) = {R2+R1, 0, 0, lc};
+Point(2) = {R2, 0, R1, lc};
+Point(3) = {R2-R1, 0, 0, lc};
+Point(4) = {R2, 0, -R1, lc};
+Point(5) = {R2, 0, 0, lc};
+Point(6) = {R2+a1, 0, 0, lc};
+Point(7) = {R2, 0, a1, lc};
+Point(8) = {R2-a1, 0, 0, lc};
+Point(9) = {R2, 0, -a1, lc};
+
+Circle(1) = {1,5,2};
+Circle(2) = {2,5,3};
+Circle(3) = {3,5,4};
+Circle(4) = {4,5,1};
+
+Line(5) = {6,1};
+Line(6) = {7,2};
+Line(7) = {8,3};
+Line(8) = {9,4};
+
+Line(9) = {6, 7};
+Line(10) = {7, 8};
+Line(11) = {8, 9};
+Line(12) = {9, 6};
+
+Line Loop(101) = {1, -6, -9, 5};
+Line Loop(102) = {2, -7, -10, 6};
+Line Loop(103) = {3, -8, -11, 7};
+Line Loop(104) = {4, -5, -12, 8};
+Line Loop(105) = {9, 10, 11, 12};
+
+Plane Surface(201) = {101};
+Plane Surface(202) = {102};
+Plane Surface(203) = {103};
+Plane Surface(204) = {104};
+Plane Surface(205) = {105};
+
+Transfinite Curve{1} = narc+1;
+Transfinite Curve{2} = narc+1;
+Transfinite Curve{3} = narc+1;
+Transfinite Curve{4} = narc+1;
+
+Transfinite Curve{5} = nshl+1;
+Transfinite Curve{6} = nshl+1;
+Transfinite Curve{7} = nshl+1;
+Transfinite Curve{8} = nshl+1;
+
+Transfinite Curve{9} = narc+1;
+Transfinite Curve{10} = narc+1;
+Transfinite Curve{11} = narc+1;
+Transfinite Curve{12} = narc+1;
+
+If (type == 8)
+   Recombine Surface {201};
+   Recombine Surface {202};
+   Recombine Surface {203};
+   Recombine Surface {204};
+   Recombine Surface {205};
+
+   Transfinite Surface {201} = {1,2,7,6};
+   Transfinite Surface {202} = {2,3,8,7};
+   Transfinite Surface {203} = {3,4,9,8};
+   Transfinite Surface {204} = {4,1,6,9};
+   Transfinite Surface {205} = {6,7,8,9};
+EndIf
+
+If (type == 4)
+   Extrude { {0,0,1} , {0,0,0} , Phi} {
+      Surface{201,202,203,204,205}; Layers{nazm};
+}
+Else
+   Extrude { {0,0,1} , {0,0,0} , Phi} {
+      Surface{201,202,203,204,205}; Layers{nazm}; Recombine;
+}
+EndIf
 
 // Set a rotation periodicity constraint:
-Periodic Surface{3} = {2} Rotate{{0,0,1}, {0,0,0}, Pi/3};
+If (periodic)
+   Periodic Surface{227} = {201} Rotate{{0,0,1}, {0,0,0}, Phi};
+   Periodic Surface{249} = {202} Rotate{{0,0,1}, {0,0,0}, Phi};
+   Periodic Surface{271} = {203} Rotate{{0,0,1}, {0,0,0}, Phi};
+   Periodic Surface{293} = {204} Rotate{{0,0,1}, {0,0,0}, Phi};
+   Periodic Surface{315} = {205} Rotate{{0,0,1}, {0,0,0}, Phi};
+EndIf
 
 // Tag surfaces and volumes with positive integers
-Physical Surface(1) = {1};
-Physical Surface(2) = {2};
-Physical Surface(3) = {3};
-Physical Volume(1) = {1};
+Physical Surface(1) = {201,202,203,204,205};
+Physical Surface(2) = {227,249,271,293,315};
+Physical Surface(3) = {214,236,258,280};
+Physical Volume(1) = {1,2,3,4,5};
+
+// Optimize the high-order mesh
+// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
+// Mesh.ElementOrder = order;
+// Mesh.HighOrderOptimize = 1;
 
 // Generate 3D mesh
 Mesh 3;
-
+SetOrder order;
 Mesh.MshFileVersion = 2.2;
-Save "periodic-torus-sector.msh";
+
+// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
+// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
+// Plugin(AnalyseMeshQuality).Run;
+
+If (periodic)
+   Save Sprintf("periodic-torus-sector-t%01g-o%01g.msh", type, order);
+Else
+   Save Sprintf("torus-sector-t%01g-o%01g.msh", type, order);
+EndIf

data/square-mixed.mesh

@@ -0,0 +1,118 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+20
+1 3 0 1 6 5
+1 3 1 2 7 6
+1 3 2 3 8 7
+1 3 3 4 9 8
+1 3 5 6 11 10
+1 2 6 7 11
+1 2 7 12 11
+1 2 7 8 13
+1 2 7 13 12
+1 3 8 9 14 13
+1 3 10 11 16 15
+1 2 11 12 17
+1 2 11 17 16
+1 2 12 13 17
+1 2 13 18 17
+1 3 13 14 19 18
+1 3 15 16 21 20
+1 3 16 17 22 21
+1 3 17 18 23 22
+1 3 18 19 24 23
+
+boundary
+16
+2 1 0 1
+2 1 1 2
+2 1 2 3
+2 1 3 4
+2 1 21 20
+2 1 22 21
+2 1 23 22
+2 1 24 23
+1 1 5 0
+1 1 10 5
+1 1 15 10
+1 1 20 15
+1 1 4 9
+1 1 9 14
+1 1 14 19
+1 1 19 24
+
+vertices
+25
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P1
+VDim: 2
+Ordering: 0
+
+0
+0.25
+0.5
+0.75
+1
+0
+0.25
+0.5
+0.75
+1
+0
+0.25
+0.5
+0.75
+1
+0
+0.25
+0.5
+0.75
+1
+0
+0.25
+0.5
+0.75
+1
+0
+0
+0
+0
+0
+0.25
+0.25
+0.25
+0.25
+0.25
+0.5
+0.5
+0.5
+0.5
+0.5
+0.75
+0.75
+0.75
+0.75
+0.75
+1
+1
+1
+1
+1

doc/CodeDocumentation.dox

@@ -144,8 +144,7 @@ namespace mfem {
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
  * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
-
-* - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
+ * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
  * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
  * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes

doc/makefile

@@ -19,7 +19,7 @@ html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
 	@doxygen $(DOXYGEN_CONF)
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
-	@cat warnings.log
+	@cat warnings.log 1>&2
 	@# Generate the log of undocumented methods
 	@( cat $(DOXYGEN_CONF) ; echo "GENERATE_HTML=NO" ; echo "EXTRACT_ALL=NO" ; echo "WARN_LOGFILE=undoc.log" ; echo "QUIET=YES" ) | doxygen - &> /dev/null
 

examples/ex1.cpp

@@ -175,7 +175,8 @@ int main(int argc, char *argv[])
    //    domain integrator.
    BilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddDomainIntegrator(new MassIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: eliminating boundary
@@ -184,19 +185,40 @@ int main(int argc, char *argv[])
    if (static_cond) { a.EnableStaticCondensation(); }
    a.Assemble();
 
-   OperatorPtr A;
+   OperatorPtr A, As;
    Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Array<int> empty_list;
+   a.FormSystemMatrix(empty_list, As);
+   //a.FormLinearSystem(empty_list, x, b, A, X, B);
+   //a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   cout << "Size of linear system: " << A->Height() << endl;
+   //cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
    if (!pa)
    {
 #ifndef MFEM_USE_SUITESPARSE
       // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+      //GSSmoother M((SparseMatrix&)(*A));
+
+      //SparseMatrix &Asp = *As.As<SparseMatrix>();
+      SparseMatrix &Asp = a.SpMat();
+
+      Asp.Finalize();
+      Asp.SortColumnIndices();
+
+      Vector tmpx(B.Size());
+      Vector tmpy(B.Size());
+      tmpx = 1.0;
+      tmpy = 0.0;
+
+      //As.As<SparseMatrix>()->Mult(tmpx, tmpy);
+      Asp.Mult(tmpx, tmpy);
+
+      //IncompleteCholesky M(*As.As<SparseMatrix>());
+      IncompleteCholesky M(Asp);
+      //ILUcusparse M(*A.As<SparseMatrix>());
+      PCG(*As, M, B, X, 1, 200, 1e-12, 0.0);
 #else
       // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
       UMFPackSolver umf_solver;

examples/ex1p.cpp

@@ -122,7 +122,7 @@ int main(int argc, char *argv[])
    {
       int ref_levels =
          (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
+      for (int l = 0; l < ref_levels-1; l++)
       {
          mesh.UniformRefinement();
       }
@@ -134,7 +134,7 @@ int main(int argc, char *argv[])
    ParMesh pmesh(MPI_COMM_WORLD, mesh);
    mesh.Clear();
    {
-      int par_ref_levels = 2;
+      int par_ref_levels = 1;
       for (int l = 0; l < par_ref_levels; l++)
       {
          pmesh.UniformRefinement();
@@ -216,6 +216,13 @@ int main(int argc, char *argv[])
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
+   SparseMatrix Asp;
+   A.As<HypreParMatrix>()->GetDiag(Asp);
+   Vector diag;
+
+   StopWatch sw;
+   sw.Start();
+
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
    //     * With partial assembly, use Jacobi smoothing, for now.
@@ -229,7 +236,14 @@ int main(int argc, char *argv[])
    }
    else
    {
-      prec = new HypreBoomerAMG;
+      //prec = new HypreBoomerAMG;
+      Asp.Finalize();
+      Asp.SortColumnIndices();
+
+      Asp.GetDiag(diag);
+      prec = new OperatorJacobiSmoother(diag, ess_tdof_list);
+      //prec = new IncompleteCholesky(Asp);
+      //prec = new ILUcusparse(Asp);
    }
    CGSolver cg(MPI_COMM_WORLD);
    cg.SetRelTol(1e-12);
@@ -240,6 +254,9 @@ int main(int argc, char *argv[])
    cg.Mult(B, X);
    delete prec;
 
+   sw.Stop();
+   cout << "Step 13 solve time " << sw.RealTime() << endl;
+
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a.RecoverFEMSolution(X, b, x);

examples/ex24.cpp

@@ -70,7 +70,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
    args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H(Curl) or 1: H(Div)");
+                  "Choose between 0: grad, 1: curl, 2: div");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",

examples/ex24p.cpp

@@ -76,7 +76,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
    args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H(Curl) or 1: H(Div)");
+                  "Choose between 0: grad, 1: curl, 2: div");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",

fem/CMakeLists.txt

@@ -50,43 +50,10 @@ set(SRCS
   fespacehierarchy.cpp
   nonlininteg_vectorconvection.cpp
   quadinterpolator.cpp
-  quadinterpolator_det.cpp
-  quadinterpolator_eval_by_nodes.cpp
-  quadinterpolator_eval_by_vdim.cpp
-  quadinterpolator_grad_by_nodes.cpp
-  quadinterpolator_grad_by_vdim.cpp
-  quadinterpolator_grad_phys_by_nodes.cpp
-  quadinterpolator_grad_phys_by_vdim.cpp
   quadinterpolator_face.cpp
   restriction.cpp
   staticcond.cpp
   tmop.cpp
-  tmop_pa.cpp
-  tmop_pa_h2d.cpp
-  tmop_pa_h2d_c0.cpp
-  tmop_pa_h2m.cpp
-  tmop_pa_h2m_c0.cpp
-  tmop_pa_h2s.cpp
-  tmop_pa_h2s_c0.cpp
-  tmop_pa_h3d.cpp
-  tmop_pa_h3d_c0.cpp
-  tmop_pa_h3m.cpp
-  tmop_pa_h3m_c0.cpp
-  tmop_pa_h3s.cpp
-  tmop_pa_h3s_c0.cpp
-  tmop_pa_jp2.cpp
-  tmop_pa_jp3.cpp
-  tmop_pa_jt2_tc.cpp
-  tmop_pa_jt3_datc.cpp
-  tmop_pa_jt3_tc.cpp
-  tmop_pa_p2.cpp
-  tmop_pa_p2_c0.cpp
-  tmop_pa_p3.cpp
-  tmop_pa_p3_c0.cpp
-  tmop_pa_w2.cpp
-  tmop_pa_w2_c0.cpp
-  tmop_pa_w3.cpp
-  tmop_pa_w3_c0.cpp
   tmop_tools.cpp
   gslib.cpp
   transfer.cpp
@@ -116,10 +83,7 @@ set(HDRS
   nonlinearform_ext.hpp
   nonlininteg.hpp
   quadinterpolator.hpp
-  quadinterpolator_eval.hpp
   quadinterpolator_face.hpp
-  quadinterpolator_grad.hpp
-  quadinterpolator_grad_phys.hpp
   restriction.hpp
   fespacehierarchy.hpp
   staticcond.hpp
@@ -132,7 +96,6 @@ set(HDRS
   tfespace.hpp
   tintrules.hpp
   tmop.hpp
-  tmop_pa.hpp
   tmop_tools.hpp
   gslib.hpp
   transfer.hpp

fem/bilinearform.cpp

@@ -627,6 +627,33 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
       MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
                   "Vector for holding diagonal has wrong size!");
       const Operator *P = fes->GetProlongationMatrix();
+      // For an AMR mesh, a convergent diagonal is assembled with |P^T| d_e,
+      // where |P^T| has the entry-wise absolute values of the conforming
+      // prolongation transpose operator.
+      if (P && !fes->Conforming())
+      {
+         Vector local_diag(P->Height());
+         ext->AssembleDiagonal(local_diag);
+         const SparseMatrix *SP = dynamic_cast<const SparseMatrix*>(P);
+#ifdef MFEM_USE_MPI
+         const HypreParMatrix *HP = dynamic_cast<const HypreParMatrix*>(P);
+#endif
+         if (SP)
+         {
+            SP->AbsMultTranspose(local_diag, diag);
+         }
+#ifdef MFEM_USE_MPI
+         else if (HP)
+         {
+            HP->AbsMultTranspose(1.0, local_diag, 0.0, diag);
+         }
+#endif
+         else
+         {
+            MFEM_ABORT("Prolongation matrix has unexpected type.");
+         }
+         return;
+      }
       if (!IsIdentityProlongation(P))
       {
          Vector local_diag(P->Height());

fem/bilinearform_ext.cpp

@@ -96,6 +96,9 @@ void PABilinearFormExtension::Assemble()
       integrators[i]->AssemblePA(*a->FESpace());
    }
 
+   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
+               "Partial assembly does not support AddBoundaryIntegrator yet.");
+
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int intFaceIntegratorCount = intFaceIntegrators.Size();
    for (int i = 0; i < intFaceIntegratorCount; ++i)
@@ -320,6 +323,9 @@ void EABilinearFormExtension::Assemble()
               GetTraceElement(0, trialFes->GetMesh()->GetFaceBaseGeometry(0)) ->
               GetDof();
 
+   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
+               "Element assembly does not support AddBoundaryIntegrator yet.");
+
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int intFaceIntegratorCount = intFaceIntegrators.Size();
    if (intFaceIntegratorCount>0)
@@ -794,6 +800,12 @@ void PAMixedBilinearFormExtension::Assemble()
    {
       integrators[i]->AssemblePA(*trialFes, *testFes);
    }
+   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
+               "Partial assembly does not support AddBoundaryIntegrator yet.");
+   MFEM_VERIFY(a->GetTFBFI()->Size() == 0,
+               "Partial assembly does not support AddTraceFaceIntegrator yet.");
+   MFEM_VERIFY(a->GetBTFBFI()->Size() == 0,
+               "Partial assembly does not support AddBdrTraceFaceIntegrator yet.");
 }
 
 void PAMixedBilinearFormExtension::Update()

fem/bilininteg.hpp

@@ -20,6 +20,13 @@
 namespace mfem
 {
 
+// Local maximum size of dofs and quads in 1D
+constexpr int HCURL_MAX_D1D = 5;
+constexpr int HCURL_MAX_Q1D = 6;
+
+constexpr int HDIV_MAX_D1D = 5;
+constexpr int HDIV_MAX_Q1D = 6;
+
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
@@ -2390,8 +2397,11 @@ protected:
    Vector pa_data;
    const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
    const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
+   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
+   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D, fetype;
+   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
+   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
    VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
@@ -2412,6 +2422,8 @@ public:
 
    using BilinearFormIntegrator::AssemblePA;
    virtual void AssemblePA(const FiniteElementSpace &fes);
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
    virtual void AddMultPA(const Vector &x, Vector &y) const;
    virtual void AssembleDiagonalPA(Vector& diag);
 };

fem/bilininteg_convection_pa.cpp

@@ -13,10 +13,6 @@
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
 
-#include "restriction.hpp"
-#include "tmop_pa.hpp"
-#include "../linalg/kernels.hpp"
-
 using namespace std;
 
 namespace mfem
@@ -72,53 +68,47 @@ static void PAConvectionSetup3D(const int Q1D,
                                 const double alpha,
                                 Vector &op)
 {
-   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
+   const int NQ = Q1D*Q1D*Q1D;
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
    const bool const_v = vel.Size() == 3;
-   const auto V = const_v ?
-                  Reshape(vel.Read(), 3,1,1,1,1) :
-                  Reshape(vel.Read(), 3,Q1D,Q1D,Q1D,NE);
-   auto y = Reshape(op.Write(), Q1D,Q1D,Q1D,3,NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   auto V =
+      const_v ? Reshape(vel.Read(), 3,1,1) : Reshape(vel.Read(), 3,NQ,NE);
+   auto y = Reshape(op.Write(), NQ, 3, NE);
+   MFEM_FORALL(e, NE,
    {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               const double J11 = J(qx,qy,qz,0,0,e);
-               const double J12 = J(qx,qy,qz,0,1,e);
-               const double J13 = J(qx,qy,qz,0,2,e);
-               const double J21 = J(qx,qy,qz,1,0,e);
-               const double J22 = J(qx,qy,qz,1,1,e);
-               const double J23 = J(qx,qy,qz,1,2,e);
-               const double J31 = J(qx,qy,qz,2,0,e);
-               const double J32 = J(qx,qy,qz,2,1,e);
-               const double J33 = J(qx,qy,qz,2,2,e);
-               const double w = alpha * W(qx,qy,qz);
-               const double v0 = const_v ? V(0,0,0,0,0) : V(0,qx,qy,qz,e);
-               const double v1 = const_v ? V(1,0,0,0,0) : V(1,qx,qy,qz,e);
-               const double v2 = const_v ? V(2,0,0,0,0) : V(2,qx,qy,qz,e);
-               const double wx = w * v0;
-               const double wy = w * v1;
-               const double wz = w * v2;
-               // adj(J)
-               const double A11 = (J22 * J33) - (J23 * J32);
-               const double A12 = (J32 * J13) - (J12 * J33);
-               const double A13 = (J12 * J23) - (J22 * J13);
-               const double A21 = (J31 * J23) - (J21 * J33);
-               const double A22 = (J11 * J33) - (J13 * J31);
-               const double A23 = (J21 * J13) - (J11 * J23);
-               const double A31 = (J21 * J32) - (J31 * J22);
-               const double A32 = (J31 * J12) - (J11 * J32);
-               const double A33 = (J11 * J22) - (J12 * J21);
-               // q . J^{-1} = q . adj(J)
-               y(qx,qy,qz,0,e) = wx * A11 + wy * A12 + wz * A13;
-               y(qx,qy,qz,1,e) = wx * A21 + wy * A22 + wz * A23;
-               y(qx,qy,qz,2,e) = wx * A31 + wy * A32 + wz * A33;
-            }
-         }
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double w = alpha * W[q];
+         const double v0 = const_v ? V(0,0,0) : V(0,q,e);
+         const double v1 = const_v ? V(1,0,0) : V(1,q,e);
+         const double v2 = const_v ? V(2,0,0) : V(2,q,e);
+         const double wx = w * v0;
+         const double wy = w * v1;
+         const double wz = w * v2;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+         // q . J^{-1} = q . adj(J)
+         y(q,0,e) =  wx * A11 + wy * A12 + wz * A13;
+         y(q,1,e) =  wx * A21 + wy * A22 + wz * A23;
+         y(q,2,e) =  wx * A31 + wy * A32 + wz * A33;
       }
    });
 }
@@ -194,8 +184,8 @@ void PAConvectionApply2D(const int ne,
             Gu[dy][qx] = 0.0;
             for (int dx = 0; dx < D1D; ++dx)
             {
-               const double bx = B(qx,dx);
-               const double gx = G(qx,dx);
+               const double bx  = B(qx,dx);
+               const double gx  = G(qx,dx);
                const double x = u[dy][dx];
                Bu[dy][qx] += bx * x;
                Gu[dy][qx] += gx * x;
@@ -212,8 +202,8 @@ void PAConvectionApply2D(const int ne,
             BGu[qy][qx] = 0.0;
             for (int dy = 0; dy < D1D; ++dy)
             {
-               const double bx = B(qy,dy);
-               const double gx = G(qy,dy);
+               const double bx  = B(qy,dy);
+               const double gx  = G(qy,dy);
                GBu[qy][qx] += gx * Bu[dy][qx];
                BGu[qy][qx] += bx * Gu[dy][qx];
             }
@@ -242,7 +232,7 @@ void PAConvectionApply2D(const int ne,
             BDGu[dy][qx] = 0.0;
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               const double w = Bt(dy,qy);
+               const double w  = Bt(dy,qy);
                BDGu[dy][qx] += w * DGu[qy][qx];
             }
          }
@@ -254,7 +244,7 @@ void PAConvectionApply2D(const int ne,
             double BBDGu = 0.0;
             for (int qx = 0; qx < Q1D; ++qx)
             {
-               const double w = Bt(dx,qx);
+               const double w  = Bt(dx,qx);
                BBDGu += w * BDGu[dy][qx];
             }
             y(dx,dy,e) += BBDGu;
@@ -320,7 +310,7 @@ void SmemPAConvectionApply2D(const int ne,
             {
                const double bx = B(qx,dx);
                const double gx = G(qx,dx);
-               const double x = u[tidz][dy][dx];
+               const double x  = u[tidz][dy][dx];
                Bu[tidz][dy][qx] += bx * x;
                Gu[tidz][dy][qx] += gx * x;
             }
@@ -337,8 +327,8 @@ void SmemPAConvectionApply2D(const int ne,
             BGu[tidz][qy][qx] = 0.0;
             for (int dy = 0; dy < D1D; ++dy)
             {
-               const double bx = B(qy,dy);
-               const double gx = G(qy,dy);
+               const double bx  = B(qy,dy);
+               const double gx  = G(qy,dy);
                GBu[tidz][qy][qx] += gx * Bu[tidz][dy][qx];
                BGu[tidz][qy][qx] += bx * Gu[tidz][dy][qx];
             }
@@ -369,7 +359,7 @@ void SmemPAConvectionApply2D(const int ne,
             BDGu[tidz][dy][qx] = 0.0;
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               const double w = Bt(dy,qy);
+               const double w  = Bt(dy,qy);
                BDGu[tidz][dy][qx] += w * DGu[tidz][qy][qx];
             }
          }
@@ -382,7 +372,7 @@ void SmemPAConvectionApply2D(const int ne,
             double BBDGu = 0.0;
             for (int qx = 0; qx < Q1D; ++qx)
             {
-               const double w = Bt(dx,qx);
+               const double w  = Bt(dx,qx);
                BBDGu += w * BDGu[tidz][dy][qx];
             }
             y(dx,dy,e) += BBDGu;
@@ -446,8 +436,8 @@ void PAConvectionApply3D(const int ne,
                Gu[dz][dy][qx] = 0.0;
                for (int dx = 0; dx < D1D; ++dx)
                {
-                  const double bx = B(qx,dx);
-                  const double gx = G(qx,dx);
+                  const double bx  = B(qx,dx);
+                  const double gx  = G(qx,dx);
                   const double x = u[dz][dy][dx];
                   Bu[dz][dy][qx] += bx * x;
                   Gu[dz][dy][qx] += gx * x;
@@ -469,8 +459,8 @@ void PAConvectionApply3D(const int ne,
                BGu[dz][qy][qx] = 0.0;
                for (int dy = 0; dy < D1D; ++dy)
                {
-                  const double bx = B(qy,dy);
-                  const double gx = G(qy,dy);
+                  const double bx  = B(qy,dy);
+                  const double gx  = G(qy,dy);
                   BBu[dz][qy][qx] += bx * Bu[dz][dy][qx];
                   GBu[dz][qy][qx] += gx * Bu[dz][dy][qx];
                   BGu[dz][qy][qx] += bx * Gu[dz][dy][qx];
@@ -492,8 +482,8 @@ void PAConvectionApply3D(const int ne,
                BBGu[qz][qy][qx] = 0.0;
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  const double bx = B(qz,dz);
-                  const double gx = G(qz,dz);
+                  const double bx  = B(qz,dz);
+                  const double gx  = G(qz,dz);
                   GBBu[qz][qy][qx] += gx * BBu[dz][qy][qx];
                   BGBu[qz][qy][qx] += bx * GBu[dz][qy][qx];
                   BBGu[qz][qy][qx] += bx * BGu[dz][qy][qx];
@@ -531,7 +521,7 @@ void PAConvectionApply3D(const int ne,
                BDGu[dz][qy][qx] = 0.0;
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  const double w = Bt(dz,qz);
+                  const double w  = Bt(dz,qz);
                   BDGu[dz][qy][qx] += w * DGu[qz][qy][qx];
                }
             }
@@ -547,7 +537,7 @@ void PAConvectionApply3D(const int ne,
                BBDGu[dz][dy][qx] = 0.0;
                for (int qy = 0; qy < Q1D; ++qy)
                {
-                  const double w = Bt(dy,qy);
+                  const double w  = Bt(dy,qy);
                   BBDGu[dz][dy][qx] += w * BDGu[dz][qy][qx];
                }
             }
@@ -562,7 +552,7 @@ void PAConvectionApply3D(const int ne,
                double BBBDGu = 0.0;
                for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const double w = Bt(dx,qx);
+                  const double w  = Bt(dx,qx);
                   BBBDGu += w * BBDGu[dz][dy][qx];
                }
                y(dx,dy,dz,e) += BBBDGu;
@@ -635,8 +625,8 @@ void SmemPAConvectionApply3D(const int ne,
                double Gu_ = 0.0;
                for (int dx = 0; dx < D1D; ++dx)
                {
-                  const double bx = B(qx,dx);
-                  const double gx = G(qx,dx);
+                  const double bx  = B(qx,dx);
+                  const double gx  = G(qx,dx);
                   const double x = u[dz][dy][dx];
                   Bu_ += bx * x;
                   Gu_ += gx * x;
@@ -661,8 +651,8 @@ void SmemPAConvectionApply3D(const int ne,
                double BGu_ = 0.0;
                for (int dy = 0; dy < D1D; ++dy)
                {
-                  const double bx = B(qy,dy);
-                  const double gx = G(qy,dy);
+                  const double bx  = B(qy,dy);
+                  const double gx  = G(qy,dy);
                   BBu_ += bx * Bu[dz][dy][qx];
                   GBu_ += gx * Bu[dz][dy][qx];
                   BGu_ += bx * Gu[dz][dy][qx];
@@ -688,8 +678,8 @@ void SmemPAConvectionApply3D(const int ne,
                double BBGu_ = 0.0;
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  const double bx = B(qz,dz);
-                  const double gx = G(qz,dz);
+                  const double bx  = B(qz,dz);
+                  const double gx  = G(qz,dz);
                   GBBu_ += gx * BBu[dz][qy][qx];
                   BGBu_ += bx * GBu[dz][qy][qx];
                   BBGu_ += bx * BGu[dz][qy][qx];
@@ -731,7 +721,7 @@ void SmemPAConvectionApply3D(const int ne,
                double BDGu_ = 0.0;
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  const double w = Bt(dz,qz);
+                  const double w  = Bt(dz,qz);
                   BDGu_ += w * DGu[qz][qy][qx];
                }
                BDGu[dz][qy][qx] = BDGu_;
@@ -749,7 +739,7 @@ void SmemPAConvectionApply3D(const int ne,
                double BBDGu_ = 0.0;
                for (int qy = 0; qy < Q1D; ++qy)
                {
-                  const double w = Bt(dy,qy);
+                  const double w  = Bt(dy,qy);
                   BBDGu_ += w * BDGu[dz][qy][qx];
                }
                BBDGu[dz][dy][qx] = BBDGu_;
@@ -766,7 +756,7 @@ void SmemPAConvectionApply3D(const int ne,
                double BBBDGu = 0.0;
                for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const double w = Bt(dx,qx);
+                  const double w  = Bt(dx,qx);
                   BBBDGu += w * BBDGu[dz][dy][qx];
                }
                y(dx,dy,dz,e) = BBBDGu;
@@ -776,117 +766,6 @@ void SmemPAConvectionApply3D(const int ne,
    });
 }
 
-template<int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0, int T_MAX = 0>
-static void QEvalVGF2D(const int NE,
-                       const double *b_,
-                       const double *x_,
-                       double *y_,
-                       const int vdim = 1,
-                       const int d1d = 0,
-                       const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto X = Reshape(x_, D1D, D1D, VDIM, NE);
-   auto C = Reshape(y_, VDIM, Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
-   {
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      mfem::kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-
-      MFEM_SHARED double DD[NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         mfem::kernels::LoadX<MD1,NBZ>(e,D1D,c,X,DD);
-         mfem::kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,DD,DQ);
-         mfem::kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ,QQ);
-
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               double G;
-               mfem::kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,G);
-               C(c,qx,qy,e) = G;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0, int T_MAX = 0>
-static void QEvalVGF3D(const int NE,
-                       const double *b_,
-                       const double *x_,
-                       double *y_,
-                       const int vdim = 1,
-                       const int d1d = 0,
-                       const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto X = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-   auto C = Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      mfem::kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-
-      MFEM_SHARED double DDD[MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         mfem::kernels::LoadX<MD1>(e,D1D,c,X,DDD);
-         mfem::kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD,DDQ);
-         mfem::kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ,DQQ);
-         mfem::kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ,QQQ);
-
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qz,z,Q1D)
-               {
-                  double G;
-                  mfem::kernels::PullEval<MQ1>(qx,qy,qz,QQQ,G);
-                  C(c,qx,qy,qz,e) = G;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
 void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    // Assumes tensor-product elements
@@ -899,90 +778,16 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    const int nq = ir->GetNPoints();
    dim = mesh->Dimension();
    ne = fes.GetNE();
-   const DofToQuad::Mode mode = DofToQuad::TENSOR;
-#ifdef MFEM_USE_UMPIRE
-   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
-      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
-#else
-   const MemoryType temp_type = Device::GetDeviceMemoryType();
-#endif
-
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode, temp_type);
-   maps = &el.GetDofToQuad(*ir, mode);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, temp_type);
+   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
    Vector vel;
-   if (VectorConstantCoefficient *cQ =
-          dynamic_cast<VectorConstantCoefficient*>(Q))
+   if (VectorConstantCoefficient *cQ = dynamic_cast<VectorConstantCoefficient*>(Q))
    {
       vel = cQ->GetVec();
    }
-   else if (VectorGridFunctionCoefficient *vgfQ =
-               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
-   {
-      Vector xe;
-      vel.SetSize(dim * nq * ne, temp_type);
-
-      const GridFunction *gf = vgfQ->GetGridFunction();
-      const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-      const FiniteElementSpace &gf_fes = *gf->FESpace();
-
-      const int vdim = gf_fes.GetVDim();
-      const Operator *R = gf_fes.GetElementRestriction(ordering);
-      const FiniteElement &el_gf = *gf_fes.GetFE(0);
-      const DofToQuad *maps_gf = &el_gf.GetDofToQuad(*ir, mode);
-      const int D1D = maps_gf->ndof;
-      const int Q1D = maps_gf->nqpt;
-
-      MFEM_VERIFY(R,"");
-      MFEM_VERIFY(vdim == dim, "");
-      MFEM_VERIFY(dim==2 || dim==3,"");
-
-      xe.SetSize(R->Height(), Device::GetMemoryType());
-      xe.UseDevice(true);
-      R->Mult(*gf, xe);
-
-      const auto B = maps_gf->B.Read();
-      const auto x = xe.Read();
-      auto y = vel.Write();
-
-      const int id = (D1D << 4 ) | Q1D;
-      if (dim == 2)
-      {
-         switch (id)
-         {
-            case 0x22: QEvalVGF2D<2,2,2>(ne,B,x,y); break;
-            case 0x33: QEvalVGF2D<2,3,3>(ne,B,x,y); break;
-            case 0x34: QEvalVGF2D<2,3,4>(ne,B,x,y); break;
-            default:
-            {
-               constexpr int MAX_DQ = 8;
-               MFEM_VERIFY(D1D <= MAX_DQ, "");
-               MFEM_VERIFY(Q1D <= MAX_DQ, "");
-               QEvalVGF2D<0,0,0,MAX_DQ>(ne,B,x,y,vdim,D1D,Q1D);
-            }
-         }
-      }
-      if (dim == 3)
-      {
-         switch (id)
-         {
-            case 0x23: QEvalVGF3D<3,2,3>(ne,B,x,y); break;
-            case 0x34: QEvalVGF3D<3,3,4>(ne,B,x,y); break;
-            case 0x35: QEvalVGF3D<3,3,5>(ne,B,x,y); break;
-            case 0x46: QEvalVGF3D<3,4,6>(ne,B,x,y); break;
-            case 0x48: QEvalVGF3D<3,4,8>(ne,B,x,y); break;
-            default:
-            {
-               constexpr int MAX_DQ = 6;
-               MFEM_VERIFY(D1D <= MAX_DQ, "");
-               MFEM_VERIFY(Q1D <= MAX_DQ, "");
-               QEvalVGF3D<0,0,0,MAX_DQ>(ne,B,x,y,vdim,D1D,Q1D);
-            }
-         }
-      }
-   }
    else if (VectorQuadratureFunctionCoefficient* cQ =
                dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
    {
@@ -1036,12 +841,9 @@ static void PAConvectionApply(const int dim,
       switch ((D1D << 4 ) | Q1D)
       {
          case 0x22: return SmemPAConvectionApply2D<2,2,8>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x33: return SmemPAConvectionApply2D<3,3,4>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x34: return SmemPAConvectionApply2D<3,4,4>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x44: return SmemPAConvectionApply2D<4,4,4>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x46: return SmemPAConvectionApply2D<4,6,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x33: return SmemPAConvectionApply2D<3,3,3>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x44: return SmemPAConvectionApply2D<4,4,2>(NE,B,G,Bt,Gt,op,x,y);
          case 0x55: return SmemPAConvectionApply2D<5,5,2>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x58: return SmemPAConvectionApply2D<5,8,2>(NE,B,G,Bt,Gt,op,x,y);
          case 0x66: return SmemPAConvectionApply2D<6,6,1>(NE,B,G,Bt,Gt,op,x,y);
          case 0x77: return SmemPAConvectionApply2D<7,7,1>(NE,B,G,Bt,Gt,op,x,y);
          case 0x88: return SmemPAConvectionApply2D<8,8,1>(NE,B,G,Bt,Gt,op,x,y);
@@ -1054,12 +856,8 @@ static void PAConvectionApply(const int dim,
       switch ((D1D << 4 ) | Q1D)
       {
          case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x24: return SmemPAConvectionApply3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x26: return SmemPAConvectionApply3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
          case 0x34: return SmemPAConvectionApply3D<3,4>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x35: return SmemPAConvectionApply3D<3,5>(NE,B,G,Bt,Gt,op,x,y);
          case 0x45: return SmemPAConvectionApply3D<4,5>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x48: return SmemPAConvectionApply3D<4,8>(NE,B,G,Bt,Gt,op,x,y);
          case 0x56: return SmemPAConvectionApply3D<5,6>(NE,B,G,Bt,Gt,op,x,y);
          case 0x67: return SmemPAConvectionApply3D<6,7>(NE,B,G,Bt,Gt,op,x,y);
          case 0x78: return SmemPAConvectionApply3D<7,8>(NE,B,G,Bt,Gt,op,x,y);

fem/bilininteg_diffusion_ea.cpp

@@ -130,8 +130,8 @@ static void EADiffusionAssemble2D(const int NE,
 
 template<int T_D1D = 0, int T_Q1D = 0>
 static void EADiffusionAssemble3D(const int NE,
-                                  const Array<double> &g,
                                   const Array<double> &b,
+                                  const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
                                   const int d1d = 0,

fem/bilininteg_diffusion_pa.cpp

@@ -96,26 +96,28 @@ void PADiffusionSetup2D<2>(const int Q1D,
                            const Vector &c,
                            Vector &d)
 {
-   const int NQ = Q1D*Q1D;
    const bool const_c = c.Size() == 1;
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
-   auto D = Reshape(d.Write(), NQ, 3, NE);
-
-   MFEM_FORALL(e, NE,
+   const auto W = Reshape(w.Read(), Q1D,Q1D);
+   const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
+   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
    {
-      for (int q = 0; q < NQ; ++q)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double c_detJ = W[q] * coeff / ((J11*J22)-(J21*J12));
-         D(q,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
-         D(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
-         D(q,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            const double J11 = J(qx,qy,0,0,e);
+            const double J21 = J(qx,qy,1,0,e);
+            const double J12 = J(qx,qy,0,1,e);
+            const double J22 = J(qx,qy,1,1,e);
+            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+            const double c_detJ = W(qx,qy) * coeff / ((J11*J22)-(J21*J12));
+            D(qx,qy,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
+            D(qx,qy,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
+            D(qx,qy,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
+         }
       }
    });
 }
@@ -131,33 +133,35 @@ void PADiffusionSetup2D<3>(const int Q1D,
 {
    constexpr int DIM = 2;
    constexpr int SDIM = 3;
-   const int NQ = Q1D*Q1D;
    const bool const_c = c.Size() == 1;
-
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, SDIM, DIM, NE);
-   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
-   auto D = Reshape(d.Write(), NQ, 3, NE);
-   MFEM_FORALL(e, NE,
+   const auto W = Reshape(w.Read(), Q1D,Q1D);
+   const auto J = Reshape(j.Read(), Q1D,Q1D,SDIM,DIM,NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
+   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
    {
-      for (int q = 0; q < NQ; ++q)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         const double wq = W[q];
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double E = J11*J11 + J21*J21 + J31*J31;
-         const double G = J12*J12 + J22*J22 + J32*J32;
-         const double F = J11*J12 + J21*J22 + J31*J32;
-         const double iw = 1.0 / sqrt(E*G - F*F);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double alpha = wq * coeff * iw;
-         D(q,0,e) =  alpha * G; // 1,1
-         D(q,1,e) = -alpha * F; // 1,2
-         D(q,2,e) =  alpha * E; // 2,2
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            const double wq = W(qx,qy);
+            const double J11 = J(qx,qy,0,0,e);
+            const double J21 = J(qx,qy,1,0,e);
+            const double J31 = J(qx,qy,2,0,e);
+            const double J12 = J(qx,qy,0,1,e);
+            const double J22 = J(qx,qy,1,1,e);
+            const double J32 = J(qx,qy,2,1,e);
+            const double E = J11*J11 + J21*J21 + J31*J31;
+            const double G = J12*J12 + J22*J22 + J32*J32;
+            const double F = J11*J12 + J21*J22 + J31*J32;
+            const double iw = 1.0 / sqrt(E*G - F*F);
+            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+            const double alpha = wq * coeff * iw;
+            D(qx,qy,0,e) =  alpha * G; // 1,1
+            D(qx,qy,1,e) = -alpha * F; // 1,2
+            D(qx,qy,2,e) =  alpha * E; // 2,2
+         }
       }
    });
 }
@@ -282,10 +286,9 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
    const int nq = ir->GetNPoints();
    dim = mesh->Dimension();
    ne = fes.GetNE();
-   const DofToQuad::Mode mode = DofToQuad::TENSOR;
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    const int sdim = mesh->SpaceDimension();
-   maps = &el.GetDofToQuad(*ir, mode);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * ne, Device::GetDeviceMemoryType());
@@ -740,7 +743,6 @@ static void PADiffusionAssembleDiagonal(const int dim,
          case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,B,G,D,Y);
          case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,B,G,D,Y);
          case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,B,G,D,Y);
-         case 0x46: return SmemPADiffusionDiagonal3D<4,6>(NE,B,G,D,Y);
          case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,B,G,D,Y);
          case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,B,G,D,Y);
          case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,B,G,D,Y);
@@ -1701,7 +1703,6 @@ static void PADiffusionApply(const int dim,
       switch (ID)
       {
          case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
-         case 0x24: return SmemPADiffusionApply3D<2,4>(NE,B,G,D,X,Y);
          case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
          case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,D,X,Y);
          case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,D,X,Y);

fem/bilininteg_hcurl.cpp

@@ -19,10 +19,6 @@ using namespace std;
 namespace mfem
 {
 
-// Local maximum size of dofs and quads in 1D
-constexpr int HCURL_MAX_D1D = 5;
-constexpr int HCURL_MAX_Q1D = 6;
-
 // PA H(curl) Mass Assemble 2D kernel
 void PAHcurlSetup2D(const int Q1D,
                     const int coeffDim,
@@ -33,11 +29,11 @@ void PAHcurlSetup2D(const int Q1D,
                     Vector &op)
 {
    const int NQ = Q1D*Q1D;
+   const bool symmetric = (coeffDim != 4);
    auto W = w.Read();
-
    auto J = Reshape(j.Read(), NQ, 2, 2, NE);
    auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), NQ, 3, NE);
+   auto y = Reshape(op.Write(), NQ, symmetric ? 3 : 4, NE);
 
    MFEM_FORALL(e, NE,
    {
@@ -47,12 +43,39 @@ void PAHcurlSetup2D(const int Q1D,
          const double J21 = J(q,1,0,e);
          const double J12 = J(q,0,1,e);
          const double J22 = J(q,1,1,e);
-         const double c_detJ1 = W[q] * coeff(0, q, e) / ((J11*J22)-(J21*J12));
-         const double c_detJ2 = coeffDim == 2 ? W[q] * coeff(1, q, e)
-         / ((J11*J22)-(J21*J12)) : c_detJ1;
-         y(q,0,e) =  (c_detJ2*J12*J12 + c_detJ1*J22*J22); // 1,1
-         y(q,1,e) = -(c_detJ2*J12*J11 + c_detJ1*J22*J21); // 1,2
-         y(q,2,e) =  (c_detJ2*J11*J11 + c_detJ1*J21*J21); // 2,2
+
+         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
+         {
+            // First compute entries of R = MJ^{-T}, without det J factor.
+            const double M11 = coeff(0, q, e);
+            const double M12 = coeff(1, q, e);
+            const double M21 = symmetric ? M12 : coeff(2, q, e);
+            const double M22 = symmetric ? coeff(2, q, e) : coeff(3, q, e);
+            const double R11 = M11*J22 - M12*J12;
+            const double R21 = M21*J22 - M22*J12;
+            const double R12 = -M11*J21 + M12*J11;
+            const double R22 = -M21*J21 + M22*J11;
+
+            // Now set y to J^{-1}R.
+            const double w_detJ = W[q] / ((J11*J22)-(J21*J12));
+            y(q,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(q,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(q,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
+            (J22*R12 - J12*R22)); // 2,2 or 1,2
+            if (!symmetric)
+            {
+               y(q,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+            }
+         }
+         else  // Vector or scalar coefficient version
+         {
+            const double c_detJ1 = W[q] * coeff(0, q, e) / ((J11*J22)-(J21*J12));
+            const double c_detJ2 = (coeffDim == 2) ? W[q] * coeff(1, q, e)
+                                   / ((J11*J22)-(J21*J12)) : c_detJ1;
+            y(q,0,e) =  (c_detJ2*J12*J12 + c_detJ1*J22*J22); // 1,1
+            y(q,1,e) = -(c_detJ2*J12*J11 + c_detJ1*J22*J21); // 1,2
+            y(q,2,e) =  (c_detJ2*J11*J11 + c_detJ1*J21*J21); // 2,2
+         }
       }
    });
 }
@@ -67,10 +90,11 @@ void PAHcurlSetup3D(const int Q1D,
                     Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
+   const bool symmetric = (coeffDim != 9);
    auto W = w.Read();
    auto J = Reshape(j.Read(), NQ, 3, 3, NE);
    auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), NQ, 6, NE);
+   auto y = Reshape(op.Write(), NQ, symmetric ? 6 : 9, NE);
 
    MFEM_FORALL(e, NE,
    {
@@ -89,9 +113,6 @@ void PAHcurlSetup3D(const int Q1D,
          /* */               J21 * (J12 * J33 - J32 * J13) +
          /* */               J31 * (J12 * J23 - J22 * J13);
          const double w_detJ = W[q] / detJ;
-         const double D1 = coeff(0, q, e);
-         const double D2 = coeffDim == 3 ? coeff(1, q, e) : D1;
-         const double D3 = coeffDim == 3 ? coeff(2, q, e) : D1;
          // adj(J)
          const double A11 = (J22 * J33) - (J23 * J32);
          const double A12 = (J32 * J13) - (J12 * J33);
@@ -102,13 +123,66 @@ void PAHcurlSetup3D(const int Q1D,
          const double A31 = (J21 * J32) - (J31 * J22);
          const double A32 = (J31 * J12) - (J11 * J32);
          const double A33 = (J11 * J22) - (J12 * J21);
-         // detJ J^{-1} J^{-T} = (1/detJ) adj(J) D adj(J)^T
-         y(q,0,e) = w_detJ * (D1*A11*A11 + D2*A12*A12 + D3*A13*A13); // 1,1
-         y(q,1,e) = w_detJ * (D1*A11*A21 + D2*A12*A22 + D3*A13*A23); // 2,1
-         y(q,2,e) = w_detJ * (D1*A11*A31 + D2*A12*A32 + D3*A13*A33); // 3,1
-         y(q,3,e) = w_detJ * (D1*A21*A21 + D2*A22*A22 + D3*A23*A23); // 2,2
-         y(q,4,e) = w_detJ * (D1*A21*A31 + D2*A22*A32 + D3*A23*A33); // 3,2
-         y(q,5,e) = w_detJ * (D1*A31*A31 + D2*A32*A32 + D3*A33*A33); // 3,3
+
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+         {
+            // First compute entries of R = MJ^{-T} = M adj(J)^T, without det J factor.
+            const double M11 = coeff(0, q, e);
+            const double M12 = coeff(1, q, e);
+            const double M13 = coeff(2, q, e);
+            const double M21 = (!symmetric) ? coeff(3, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(4, q, e) : coeff(3, q, e);
+            const double M23 = (!symmetric) ? coeff(5, q, e) : coeff(4, q, e);
+            const double M31 = (!symmetric) ? coeff(6, q, e) : M13;
+            const double M32 = (!symmetric) ? coeff(7, q, e) : M23;
+            const double M33 = (!symmetric) ? coeff(8, q, e) : coeff(5, q, e);
+
+            const double R11 = M11*A11 + M12*A12 + M13*A13;
+            const double R12 = M11*A21 + M12*A22 + M13*A23;
+            const double R13 = M11*A31 + M12*A32 + M13*A33;
+            const double R21 = M21*A11 + M22*A12 + M23*A13;
+            const double R22 = M21*A21 + M22*A22 + M23*A23;
+            const double R23 = M21*A31 + M22*A32 + M23*A33;
+            const double R31 = M31*A11 + M32*A12 + M33*A13;
+            const double R32 = M31*A21 + M32*A22 + M33*A23;
+            const double R33 = M31*A31 + M32*A32 + M33*A33;
+
+            // Now set y to J^{-1} R = adj(J) R
+            y(q,0,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+            const double Y12 = w_detJ * (A11*R12 + A12*R22 + A13*R32);
+            y(q,1,e) = Y12; // 1,2
+            y(q,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+
+            const double Y21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
+            const double Y22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
+            const double Y23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
+
+            const double Y33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
+
+            y(q,3,e) = symmetric ? Y22 : Y21; // 2,2 or 2,1
+            y(q,4,e) = symmetric ? Y23 : Y22; // 2,3 or 2,2
+            y(q,5,e) = symmetric ? Y33 : Y23; // 3,3 or 2,3
+
+            if (!symmetric)
+            {
+               y(q,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+               y(q,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+               y(q,8,e) = Y33; // 3,3
+            }
+         }
+         else  // Vector or scalar coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeffDim == 3 ? coeff(1, q, e) : D1;
+            const double D3 = coeffDim == 3 ? coeff(2, q, e) : D1;
+            // detJ J^{-1} D J^{-T} = (1/detJ) adj(J) D adj(J)^T
+            y(q,0,e) = w_detJ * (D1*A11*A11 + D2*A12*A12 + D3*A13*A13); // 1,1
+            y(q,1,e) = w_detJ * (D1*A11*A21 + D2*A12*A22 + D3*A13*A23); // 2,1
+            y(q,2,e) = w_detJ * (D1*A11*A31 + D2*A12*A32 + D3*A13*A33); // 3,1
+            y(q,3,e) = w_detJ * (D1*A21*A21 + D2*A22*A22 + D3*A23*A23); // 2,2
+            y(q,4,e) = w_detJ * (D1*A21*A31 + D2*A22*A32 + D3*A23*A33); // 3,2
+            y(q,5,e) = w_detJ * (D1*A31*A31 + D2*A32*A32 + D3*A33*A33); // 3,3
+         }
       }
    });
 }
@@ -116,6 +190,7 @@ void PAHcurlSetup3D(const int Q1D,
 void PAHcurlMassApply2D(const int D1D,
                         const int Q1D,
                         const int NE,
+                        const bool symmetric,
                         const Array<double> &_Bo,
                         const Array<double> &_Bc,
                         const Array<double> &_Bot,
@@ -132,7 +207,7 @@ void PAHcurlMassApply2D(const int D1D,
    auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
    auto Bot = Reshape(_Bot.Read(), D1D-1, Q1D);
    auto Bct = Reshape(_Bct.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, 3, NE);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
    auto x = Reshape(_x.Read(), 2*(D1D-1)*D1D, NE);
    auto y = Reshape(_y.ReadWrite(), 2*(D1D-1)*D1D, NE);
 
@@ -194,12 +269,13 @@ void PAHcurlMassApply2D(const int D1D,
          for (int qx = 0; qx < Q1D; ++qx)
          {
             const double O11 = op(qx,qy,0,e);
-            const double O12 = op(qx,qy,1,e);
-            const double O22 = op(qx,qy,2,e);
+            const double O21 = op(qx,qy,1,e);
+            const double O12 = symmetric ? O21 : op(qx,qy,2,e);
+            const double O22 = symmetric ? op(qx,qy,2,e) : op(qx,qy,3,e);
             const double massX = mass[qy][qx][0];
             const double massY = mass[qy][qx][1];
             mass[qy][qx][0] = (O11*massX)+(O12*massY);
-            mass[qy][qx][1] = (O12*massX)+(O22*massY);
+            mass[qy][qx][1] = (O21*massX)+(O22*massY);
          }
       }
 
@@ -215,7 +291,7 @@ void PAHcurlMassApply2D(const int D1D,
             double massX[MAX_D1D];
             for (int dx = 0; dx < D1Dx; ++dx)
             {
-               massX[dx] = 0;
+               massX[dx] = 0.0;
             }
             for (int qx = 0; qx < Q1D; ++qx)
             {
@@ -244,6 +320,7 @@ void PAHcurlMassApply2D(const int D1D,
 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                    const int Q1D,
                                    const int NE,
+                                   const bool symmetric,
                                    const Array<double> &_Bo,
                                    const Array<double> &_Bc,
                                    const Vector &_op,
@@ -254,7 +331,7 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
 
    auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, 3, NE);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
    auto diag = Reshape(_diag.ReadWrite(), 2*(D1D-1)*D1D, NE);
 
    MFEM_FORALL(e, NE,
@@ -277,7 +354,8 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
                {
                   const double wy = (c == 1) ? Bo(qy,dy) : Bc(qy,dy);
 
-                  mass[qx] += wy * wy * ((c == 0) ? op(qx,qy,0,e) : op(qx,qy,2,e));
+                  mass[qx] += wy * wy * ((c == 0) ? op(qx,qy,0,e) :
+                  op(qx,qy,symmetric ? 2 : 3, e));
                }
             }
 
@@ -299,6 +377,7 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
 void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const int Q1D,
                                    const int NE,
+                                   const bool symmetric,
                                    const Array<double> &_Bo,
                                    const Array<double> &_Bc,
                                    const Vector &_op,
@@ -313,7 +392,7 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
 
    auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
    auto diag = Reshape(_diag.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
    MFEM_FORALL(e, NE,
@@ -326,7 +405,8 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
          const int D1Dy = (c == 1) ? D1D - 1 : D1D;
          const int D1Dx = (c == 0) ? D1D - 1 : D1D;
 
-         const int opc = (c == 0) ? 0 : ((c == 1) ? 3 : 5);
+         const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
+         (symmetric ? 5 : 8));
 
          double mass[MAX_Q1D];
 
@@ -369,6 +449,7 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
 void PAHcurlMassApply3D(const int D1D,
                         const int Q1D,
                         const int NE,
+                        const bool symmetric,
                         const Array<double> &_Bo,
                         const Array<double> &_Bc,
                         const Array<double> &_Bot,
@@ -388,7 +469,7 @@ void PAHcurlMassApply3D(const int D1D,
    auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
    auto Bot = Reshape(_Bot.Read(), D1D-1, Q1D);
    auto Bct = Reshape(_Bct.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
    auto x = Reshape(_x.Read(), 3*(D1D-1)*D1D*D1D, NE);
    auto y = Reshape(_y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
@@ -483,15 +564,18 @@ void PAHcurlMassApply3D(const int D1D,
                const double O11 = op(qx,qy,qz,0,e);
                const double O12 = op(qx,qy,qz,1,e);
                const double O13 = op(qx,qy,qz,2,e);
-               const double O22 = op(qx,qy,qz,3,e);
-               const double O23 = op(qx,qy,qz,4,e);
-               const double O33 = op(qx,qy,qz,5,e);
+               const double O21 = symmetric ? O12 : op(qx,qy,qz,3,e);
+               const double O22 = symmetric ? op(qx,qy,qz,3,e) : op(qx,qy,qz,4,e);
+               const double O23 = symmetric ? op(qx,qy,qz,4,e) : op(qx,qy,qz,5,e);
+               const double O31 = symmetric ? O13 : op(qx,qy,qz,6,e);
+               const double O32 = symmetric ? O23 : op(qx,qy,qz,7,e);
+               const double O33 = symmetric ? op(qx,qy,qz,5,e) : op(qx,qy,qz,8,e);
                const double massX = mass[qz][qy][qx][0];
                const double massY = mass[qz][qy][qx][1];
                const double massZ = mass[qz][qy][qx][2];
                mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
-               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
-               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
+               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
             }
          }
       }
@@ -512,7 +596,7 @@ void PAHcurlMassApply3D(const int D1D,
             {
                for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  massXY[dy][dx] = 0;
+                  massXY[dy][dx] = 0.0;
                }
             }
             for (int qy = 0; qy < Q1D; ++qy)
@@ -727,7 +811,7 @@ static void PACurlCurlApply2D(const int D1D,
       {
          for (int qx = 0; qx < Q1D; ++qx)
          {
-            curl[qy][qx] = 0;
+            curl[qy][qx] = 0.0;
          }
       }
 
@@ -789,7 +873,7 @@ static void PACurlCurlApply2D(const int D1D,
             double gradX[MAX_D1D];
             for (int dx = 0; dx < D1Dx; ++dx)
             {
-               gradX[dx] = 0;
+               gradX[dx] = 0.0;
             }
             for (int qx = 0; qx < Q1D; ++qx)
             {
@@ -1757,7 +1841,7 @@ void PAHcurlH1Apply3D(const int D1D,
             {
                for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  massXY[dy][dx] = 0;
+                  massXY[dy][dx] = 0.0;
                }
             }
             for (int qy = 0; qy < Q1D; ++qy)
@@ -1900,7 +1984,7 @@ void PAHcurlH1Apply2D(const int D1D,
             double massX[MAX_D1D];
             for (int dx = 0; dx < D1Dx; ++dx)
             {
-               massX[dx] = 0;
+               massX[dx] = 0.0;
             }
             for (int qx = 0; qx < Q1D; ++qx)
             {
@@ -1926,15 +2010,14 @@ void PAHcurlH1Apply2D(const int D1D,
    }); // end of element loop
 }
 
-// PA H(curl) Mass Assemble 3D kernel
-static void PAHcurlL2Setup3D(const int Q1D,
-                             const int coeffDim,
-                             const int NE,
-                             const Array<double> &w,
-                             Vector &_coeff,
-                             Vector &op)
+// PA H(curl) assemble kernel
+void PAHcurlL2Setup(const int NQ,
+                    const int coeffDim,
+                    const int NE,
+                    const Array<double> &w,
+                    Vector &_coeff,
+                    Vector &op)
 {
-   const int NQ = Q1D*Q1D*Q1D;
    auto W = w.Read();
    auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
    auto y = Reshape(op.Write(), coeffDim, NQ, NE);
@@ -2035,7 +2118,7 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
    if (testType == mfem::FiniteElement::CURL &&
        trialType == mfem::FiniteElement::CURL && dim == 3)
    {
-      PAHcurlL2Setup3D(quad1D, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+      PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
    }
    else if (testType == mfem::FiniteElement::DIV &&
             trialType == mfem::FiniteElement::CURL && dim == 3 &&
@@ -2346,7 +2429,7 @@ static void PAHcurlL2Apply3D(const int D1D,
             {
                for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  massXY[dy][dx] = 0;
+                  massXY[dy][dx] = 0.0;
                }
             }
             for (int qy = 0; qy < Q1D; ++qy)
@@ -2354,7 +2437,7 @@ static void PAHcurlL2Apply3D(const int D1D,
                double massX[MAX_D1D];
                for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  massX[dx] = 0;
+                  massX[dx] = 0.0;
                }
                for (int qx = 0; qx < Q1D; ++qx)
                {
@@ -2425,7 +2508,7 @@ static void PAHcurlHdivApply3D(const int D1D,
    auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
    auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, 6, NE);
    auto x = Reshape(_x.Read(), 3*(D1D-1)*D1D*D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), 3*(D1Dtest-1)*(D1Dtest-1)*D1D, NE);
+   auto y = Reshape(_y.ReadWrite(), 3*(D1Dtest-1)*(D1Dtest-1)*D1Dtest, NE);
 
    MFEM_FORALL(e, NE,
    {
@@ -2700,7 +2783,7 @@ static void PAHcurlHdivApply3D(const int D1D,
             {
                for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  massXY[dy][dx] = 0;
+                  massXY[dy][dx] = 0.0;
                }
             }
             for (int qy = 0; qy < Q1D; ++qy)
@@ -2708,7 +2791,7 @@ static void PAHcurlHdivApply3D(const int D1D,
                double massX[HCURL_MAX_D1D];
                for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  massX[dx] = 0;
+                  massX[dx] = 0.0;
                }
                for (int qx = 0; qx < Q1D; ++qx)
                {
@@ -2844,7 +2927,7 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
 
    if (trialType == mfem::FiniteElement::CURL && dim == 3)
    {
-      PAHcurlL2Setup3D(quad1D, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+      PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
    }
    else
    {

fem/bilininteg_hdiv.cpp

@@ -23,11 +23,6 @@ using namespace std;
 namespace mfem
 {
 
-// Local maximum size of dofs and quads in 1D
-constexpr int HDIV_MAX_D1D = 5;
-constexpr int HDIV_MAX_Q1D = 6;
-
-
 // PA H(div) Mass Assemble 2D kernel
 void PAHdivSetup2D(const int Q1D,
                    const int NE,

fem/bilininteg_mass_pa.cpp

@@ -25,7 +25,6 @@ namespace mfem
 
 void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
 {
-
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
@@ -46,38 +45,22 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
-   const DofToQuad::Mode mode = DofToQuad::TENSOR;
-   const int flags = GeometricFactors::JACOBIANS |
-                     GeometricFactors::COORDINATES;
-#ifdef MFEM_USE_UMPIRE
-   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
-      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
-#else
-   const MemoryType temp_type = Device::GetDeviceMemoryType();
-#endif
-   geom = mesh->GetGeometricFactors(*ir, flags, mode, temp_type);
-   maps = &el.GetDofToQuad(*ir, mode);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
+                                    GeometricFactors::JACOBIANS);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
-   Vector *coeff{nullptr};
-   bool own_coeff{true};
+   Vector coeff;
    if (Q == nullptr)
    {
-      coeff = new Vector;
-      coeff->SetSize(1);
-      (*coeff)(0) = 1.0;
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
    }
    else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
    {
-      coeff = new Vector;
-      coeff->SetSize(1);
-      (*coeff)(0) = cQ->constant;
-   }
-   else if (QuadratureCoefficient* cQ = dynamic_cast<QuadratureCoefficient*>(Q))
-   {
-      coeff = cQ->Data();
-      own_coeff = false;
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
    }
    else if (QuadratureFunctionCoefficient* cQ =
                dynamic_cast<QuadratureFunctionCoefficient*>(Q))
@@ -90,13 +73,12 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
                   "IntegrationRule used within integrator and in"
                   " QuadratureFunction appear to be different");
       qFun.Read();
-      coeff->MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
    }
    else
    {
-      coeff = new Vector;
-      coeff->SetSize(nq * ne);
-      auto C = Reshape(coeff->HostWrite(), nq, ne);
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
       for (int e = 0; e < ne; ++e)
       {
          ElementTransformation& T = *fes.GetElementTransformation(e);
@@ -110,24 +92,27 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    if (dim==2)
    {
       const int NE = ne;
-      const int NQ = nq;
-      const bool const_c = coeff->Size() == 1;
-      auto w = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
-      auto C =
-         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      MFEM_FORALL(e, NE,
+      const int Q1D = quad1D;
+      const bool const_c = coeff.Size() == 1;
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
+      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
+                     Reshape(coeff.Read(), Q1D,Q1D,NE);
+      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
+      MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
       {
-         for (int q = 0; q < NQ; ++q)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            const double J11 = J(q,0,0,e);
-            const double J12 = J(q,1,0,e);
-            const double J21 = J(q,0,1,e);
-            const double J22 = J(q,1,1,e);
-            const double detJ = (J11*J22)-(J21*J12);
-            const double coeff = const_c ? C(0,0) : C(q,e);
-            v(q,e) =  w[q] * coeff * detJ;
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               const double J11 = J(qx,qy,0,0,e);
+               const double J12 = J(qx,qy,1,0,e);
+               const double J21 = J(qx,qy,0,1,e);
+               const double J22 = J(qx,qy,1,1,e);
+               const double detJ = (J11*J22)-(J21*J12);
+               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+               v(qx,qy,e) =  W(qx,qy) * coeff * detJ;
+            }
          }
       });
    }
@@ -135,13 +120,12 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    {
       const int NE = ne;
       const int Q1D = quad1D;
-      const bool const_c = coeff->Size() == 1;
-      const auto W = Reshape(ir->GetWeights().Read(),Q1D,Q1D,Q1D);
+      const bool const_c = coeff.Size() == 1;
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
       const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
-      const auto C = const_c ?
-                     Reshape(coeff->Read(), 1,1,1,1) :
-                     Reshape(coeff->Read(), Q1D,Q1D,Q1D,NE);
-      auto V = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
+                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
+      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
       MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
       {
          MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -151,26 +135,24 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
                MFEM_FOREACH_THREAD(qz,z,Q1D)
                {
                   const double J11 = J(qx,qy,qz,0,0,e);
-                  const double J12 = J(qx,qy,qz,0,1,e);
-                  const double J13 = J(qx,qy,qz,0,2,e);
                   const double J21 = J(qx,qy,qz,1,0,e);
-                  const double J22 = J(qx,qy,qz,1,1,e);
-                  const double J23 = J(qx,qy,qz,1,2,e);
                   const double J31 = J(qx,qy,qz,2,0,e);
+                  const double J12 = J(qx,qy,qz,0,1,e);
+                  const double J22 = J(qx,qy,qz,1,1,e);
                   const double J32 = J(qx,qy,qz,2,1,e);
+                  const double J13 = J(qx,qy,qz,0,2,e);
+                  const double J23 = J(qx,qy,qz,1,2,e);
                   const double J33 = J(qx,qy,qz,2,2,e);
                   const double detJ = J11 * (J22 * J33 - J32 * J23) -
                   /* */               J21 * (J12 * J33 - J32 * J13) +
                   /* */               J31 * (J12 * J23 - J22 * J13);
                   const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-                  V(qx,qy,qz,e) = W(qx,qy,qz) * coeff * detJ;
+                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * detJ;
                }
             }
          }
       });
    }
-
-   if (own_coeff) { delete coeff; }
 }
 
 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
@@ -472,12 +454,8 @@ static void PAMassAssembleDiagonal(const int dim, const int D1D,
       switch ((D1D << 4 ) | Q1D)
       {
          case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
-         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
-         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
          case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
-         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
          case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
-         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
          case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
          case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
          case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
@@ -1211,13 +1189,10 @@ static void PAMassApply(const int dim,
          case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
          case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply2D<3,5,16>(NE,B,Bt,D,X,Y);
          case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
          case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply2D<4,6,8>(NE,B,Bt,D,X,Y);
          case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
          case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
-         case 0x57: return SmemPAMassApply2D<5,7,8>(NE,B,Bt,D,X,Y);
          case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
          case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
          case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
@@ -1225,7 +1200,6 @@ static void PAMassApply(const int dim,
          case 0x99: return SmemPAMassApply2D<9,9,2>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply2D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
-      mfem::out << "Unknown 2D kernel 0x" << std::hex << id << std::endl;
    }
    else if (dim == 3)
    {
@@ -1234,9 +1208,7 @@ static void PAMassApply(const int dim,
          case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
          case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
          case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
-         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
          case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
          case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
          case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
@@ -1248,8 +1220,8 @@ static void PAMassApply(const int dim,
          case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
-      mfem::out << "Unknown 3D kernel 0x" << std::hex << id << std::endl;
    }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
    MFEM_ABORT("Unknown kernel.");
 }
 

fem/bilininteg_vectorfe.cpp

@@ -34,6 +34,7 @@ void PAHcurlSetup3D(const int Q1D,
 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                    const int Q1D,
                                    const int NE,
+                                   const bool symmetric,
                                    const Array<double> &_Bo,
                                    const Array<double> &_Bc,
                                    const Vector &_op,
@@ -42,6 +43,7 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
 void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const int Q1D,
                                    const int NE,
+                                   const bool symmetric,
                                    const Array<double> &_Bo,
                                    const Array<double> &_Bc,
                                    const Vector &_op,
@@ -50,6 +52,7 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
 void PAHcurlMassApply2D(const int D1D,
                         const int Q1D,
                         const int NE,
+                        const bool symmetric,
                         const Array<double> &_Bo,
                         const Array<double> &_Bc,
                         const Array<double> &_Bot,
@@ -61,6 +64,7 @@ void PAHcurlMassApply2D(const int D1D,
 void PAHcurlMassApply3D(const int D1D,
                         const int Q1D,
                         const int NE,
+                        const bool symmetric,
                         const Array<double> &_Bo,
                         const Array<double> &_Bc,
                         const Array<double> &_Bot,
@@ -143,20 +147,573 @@ void PAHdivMassApply3D(const int D1D,
                        const Vector &_x,
                        Vector &_y);
 
+void PAHcurlL2Setup(const int NQ,
+                    const int coeffDim,
+                    const int NE,
+                    const Array<double> &w,
+                    Vector &_coeff,
+                    Vector &op);
+
+// PA H(curl) x H(div) mass assemble 3D kernel, with factor
+// dF^{-1} C dF for a vector or matrix coefficient C.
+// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
+void PAHcurlHdivSetup3D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const bool transpose,
+                        const Array<double> &_w,
+                        const Vector &j,
+                        Vector &_coeff,
+                        Vector &op)
+{
+   const int NQ = Q1D*Q1D*Q1D;
+   const bool symmetric = (coeffDim != 9);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 9, NQ, NE);
+
+   const int i11 = 0;
+   const int i12 = transpose ? 3 : 1;
+   const int i13 = transpose ? 6 : 2;
+   const int i21 = transpose ? 1 : 3;
+   const int i22 = 4;
+   const int i23 = transpose ? 7 : 5;
+   const int i31 = transpose ? 2 : 6;
+   const int i32 = transpose ? 5 : 7;
+   const int i33 = 8;
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double detJ = J11 * (J22 * J33 - J32 * J23) -
+         /* */               J21 * (J12 * J33 - J32 * J13) +
+         /* */               J31 * (J12 * J23 - J22 * J13);
+         const double w_detJ = W[q] / detJ;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+         {
+            // First compute entries of R = MJ
+            const double M11 = (!symmetric) ? coeff(i11, q, e) : coeff(0, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M13 = (!symmetric) ? coeff(i13, q, e) : coeff(2, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(3, q, e);
+            const double M23 = (!symmetric) ? coeff(i23, q, e) : coeff(4, q, e);
+            const double M31 = (!symmetric) ? coeff(i31, q, e) : M13;
+            const double M32 = (!symmetric) ? coeff(i32, q, e) : M23;
+            const double M33 = (!symmetric) ? coeff(i33, q, e) : coeff(5, q, e);
+
+            const double R11 = M11*J11 + M12*J12 + M13*J13;
+            const double R12 = M11*J21 + M12*J22 + M13*J23;
+            const double R13 = M11*J31 + M12*J32 + M13*J33;
+            const double R21 = M21*J11 + M22*J12 + M23*J13;
+            const double R22 = M21*J21 + M22*J22 + M23*J23;
+            const double R23 = M21*J31 + M22*J32 + M23*J33;
+            const double R31 = M31*J11 + M32*J12 + M33*J13;
+            const double R32 = M31*J21 + M32*J22 + M33*J23;
+            const double R33 = M31*J31 + M32*J32 + M33*J33;
+
+            // Now set y to detJ J^{-1} R = adj(J) R
+            y(i11,q,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+            y(i12,q,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
+            y(i13,q,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+            y(i21,q,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
+            y(i22,q,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
+            y(i23,q,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
+            y(i31,q,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+            y(i32,q,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+            y(i33,q,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
+         }
+         else if (coeffDim == 3)  // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double D3 = coeff(2, q, e);
+            // detJ J^{-1} DJ = adj(J) DJ
+            y(i11,q,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
+            y(i12,q,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
+            y(i13,q,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
+            y(i21,q,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
+            y(i22,q,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
+            y(i23,q,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
+            y(i31,q,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
+            y(i32,q,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
+            y(i33,q,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
+         }
+      }
+   });
+}
+
+// PA H(curl) x H(div) mass assemble 2D kernel, with factor
+// dF^{-1} C dF for a vector or matrix coefficient C.
+// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
+void PAHcurlHdivSetup2D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const bool transpose,
+                        const Array<double> &_w,
+                        const Vector &j,
+                        Vector &_coeff,
+                        Vector &op)
+{
+   const int NQ = Q1D*Q1D;
+   const bool symmetric = (coeffDim != 4);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 4, NQ, NE);
+
+   const int i11 = 0;
+   const int i12 = transpose ? 2 : 1;
+   const int i21 = transpose ? 1 : 2;
+   const int i22 = 3;
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double w_detJ = W[q] / (J11*J22) - (J21*J12);
+
+         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
+         {
+            // First compute entries of R = MJ
+            const double M11 = coeff(i11, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(2, q, e);
+
+            const double R11 = M11*J11 + M12*J21;
+            const double R12 = M11*J12 + M12*J22;
+            const double R21 = M21*J11 + M22*J21;
+            const double R22 = M21*J12 + M22*J22;
+
+            // Now set y to J^{-1} R
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+         }
+         else if (coeffDim == 2) // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double R11 = D1*J11;
+            const double R12 = D1*J12;
+            const double R21 = D2*J21;
+            const double R22 = D2*J22;
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+         }
+      }
+   });
+}
+
+// Mass operator for H(curl) and H(div) functions, using Piola transformations
+// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
+void PAHcurlHdivMassApply3D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const Array<double> &_Bo,
+                            const Array<double> &_Bc,
+                            const Array<double> &_Bot,
+                            const Array<double> &_Bct,
+                            const Vector &_op,
+                            const Vector &_x,
+                            Vector &_y)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+   constexpr static int VDIM = 3;
+
+   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
+   auto Bot = Reshape(_Bot.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(_Bct.Read(), D1Dtest, Q1D);
+   auto op = Reshape(_op.Read(), scalarCoeff ? 1 : 9, Q1D, Q1D, Q1D, NE);
+   auto x = Reshape(_x.Read(), 3*(D1D-1)*D1D*(trialHcurl ? D1D : D1D-1), NE);
+   auto y = Reshape(_y.ReadWrite(), 3*(D1Dtest-1)*D1Dtest*
+                    (trialHcurl ? D1Dtest-1 : D1Dtest), NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  mass[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z trial components
+      {
+         const int D1Dz = trialHcurl ? ((c == 2) ? D1D - 1 : D1D) :
+                          ((c == 2) ? D1D : D1D - 1);
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
+                          ((c == 1) ? D1D : D1D - 1);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
+                          ((c == 0) ? D1D : D1D - 1);
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double massXY[MAX_Q1D][MAX_Q1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massXY[qy][qx] = 0.0;
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massX[MAX_Q1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] = 0.0;
+               }
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double t = x(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
+                                       ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
+                                    ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx = massX[qx];
+                     massXY[qy][qx] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = trialHcurl ? ((c == 2) ? Bo(qz,dz) : Bc(qz,dz)) :
+                                 ((c == 2) ? Bc(qz,dz) : Bo(qz,dz));
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(0,qx,qy,qz,e);
+               const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,qz,e);
+               const double O13 = scalarCoeff ? 0.0 : op(2,qx,qy,qz,e);
+               const double O21 = scalarCoeff ? 0.0 : op(3,qx,qy,qz,e);
+               const double O22 = scalarCoeff ? O11 : op(4,qx,qy,qz,e);
+               const double O23 = scalarCoeff ? 0.0 : op(5,qx,qy,qz,e);
+               const double O31 = scalarCoeff ? 0.0 : op(6,qx,qy,qz,e);
+               const double O32 = scalarCoeff ? 0.0 : op(7,qx,qy,qz,e);
+               const double O33 = scalarCoeff ? O11 : op(8,qx,qy,qz,e);
+               const double massX = mass[qz][qy][qx][0];
+               const double massY = mass[qz][qy][qx][1];
+               const double massZ = mass[qz][qy][qx][2];
+               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
+               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
+            }
+         }
+      }
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double massXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z test components
+         {
+            const int D1Dz = trialHcurl ? ((c == 2) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 2) ? D1Dtest - 1 : D1Dtest);
+            const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 1) ? D1Dtest - 1 : D1Dtest);
+            const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 0) ? D1Dtest - 1 : D1Dtest);
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massXY[dy][dx] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massX[HDIV_MAX_D1D];
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] = 0.0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massX[dx] += mass[qz][qy][qx][c] * (trialHcurl ?
+                                                         ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
+                                                         ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
+                                    ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massXY[dy][dx] += massX[dx] * wy;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = trialHcurl ? ((c == 2) ? Bct(dz,qz) : Bot(dz,qz)) :
+                                 ((c == 2) ? Bot(dz,qz) : Bct(dz,qz));
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) +=
+                        massXY[dy][dx] * wz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         }  // loop c
+      }  // loop qz
+   }); // end of element loop
+}
+
+// Mass operator for H(curl) and H(div) functions, using Piola transformations
+// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
+void PAHcurlHdivMassApply2D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const Array<double> &_Bo,
+                            const Array<double> &_Bc,
+                            const Array<double> &_Bot,
+                            const Array<double> &_Bct,
+                            const Vector &_op,
+                            const Vector &_x,
+                            Vector &_y)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+   constexpr static int VDIM = 2;
+
+   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
+   auto Bot = Reshape(_Bot.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(_Bct.Read(), D1Dtest, Q1D);
+   auto op = Reshape(_op.Read(), scalarCoeff ? 1 : 4, Q1D, Q1D, NE);
+   auto x = Reshape(_x.Read(), 2*(D1D-1)*D1D, NE);
+   auto y = Reshape(_y.ReadWrite(), 2*(D1Dtest-1)*D1Dtest, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      double mass[MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               mass[qy][qx][c] = 0.0;
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y trial components
+      {
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
+                          ((c == 1) ? D1D : D1D - 1);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
+                          ((c == 0) ? D1D : D1D - 1);
+
+         for (int dy = 0; dy < D1Dy; ++dy)
+         {
+            double massX[MAX_Q1D];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               massX[qx] = 0.0;
+            }
+
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               const double t = x(dx + (dy * D1Dx) + osc, e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
+                                    ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
+               }
+            }
+
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
+                                 ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  mass[qy][qx][c] += massX[qx] * wy;
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double O11 = op(0,qx,qy,e);
+            const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,e);
+            const double O21 = scalarCoeff ? 0.0 : op(2,qx,qy,e);
+            const double O22 = scalarCoeff ? O11 : op(3,qx,qy,e);
+            const double massX = mass[qy][qx][0];
+            const double massY = mass[qy][qx][1];
+            mass[qy][qx][0] = (O11*massX)+(O12*massY);
+            mass[qy][qx][1] = (O21*massX)+(O22*massY);
+         }
+      }
+
+      osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y test components
+      {
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
+                          ((c == 1) ? D1Dtest - 1 : D1Dtest);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
+                          ((c == 0) ? D1Dtest - 1 : D1Dtest);
+
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double massX[HDIV_MAX_D1D];
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               massX[dx] = 0.0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] += mass[qy][qx][c] * (trialHcurl ?
+                                                  ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
+                                                  ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
+               }
+            }
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
+                                 ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  y(dx + (dy * D1Dx) + osc, e) += massX[dx] * wy;
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy;
+      }  // loop c
+   }); // end of element loop
+}
+
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   // Assumes tensor-product elements
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement *fel = fes.GetFE(0);
+   AssemblePA(fes, fes);
+}
 
-   const VectorTensorFiniteElement *el =
-      dynamic_cast<const VectorTensorFiniteElement*>(fel);
-   MFEM_VERIFY(el != NULL, "Only VectorTensorFiniteElement is supported!");
+void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
+                                        const FiniteElementSpace &test_fes)
+{
+   // Assumes tensor-product elements
+   Mesh *mesh = trial_fes.GetMesh();
+
+   const FiniteElement *trial_fel = trial_fes.GetFE(0);
+   const VectorTensorFiniteElement *trial_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
+   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const FiniteElement *test_fel = test_fes.GetFE(0);
+   const VectorTensorFiniteElement *test_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
+   MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
 
    const IntegrationRule *ir
-      = IntRule ? IntRule : &MassIntegrator::GetRule(*el, *el,
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
                                                      *mesh->GetElementTransformation(0));
-   const int dims = el->GetDim();
+   const int dims = trial_el->GetDim();
    MFEM_VERIFY(dims == 2 || dims == 3, "");
 
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
@@ -164,36 +721,101 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
-   ne = fes.GetNE();
+   ne = trial_fes.GetNE();
+   MFEM_VERIFY(ne == test_fes.GetNE(),
+               "Different meshes for test and trial spaces");
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   mapsO = &el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
    dofs1D = mapsC->ndof;
    quad1D = mapsC->nqpt;
 
+   mapsCtest = &test_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsOtest = &test_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1Dtest = mapsCtest->ndof;
+
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
+   trial_fetype = trial_el->GetDerivType();
+   test_fetype = test_el->GetDerivType();
 
-   const int coeffDim = VQ ? VQ->GetVDim() : 1;
+   const int MQsymmDim = MQ ? (MQ->GetWidth() * (MQ->GetWidth() + 1)) / 2 : 0;
+   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
+   const int MQdim = MQ ? (MQ->IsSymmetric() ? MQsymmDim : MQfullDim) : 0;
+   const int coeffDim = MQ ? MQdim : (VQ ? VQ->GetVDim() : 1);
+
+   symmetric = MQ ? MQ->IsSymmetric() : true;
+
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+
+   if ((trial_curl && test_div) || (trial_div && test_curl))
+      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
+                      Device::GetMemoryType());
+   else
+      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
+                      Device::GetMemoryType());
 
    Vector coeff(coeffDim * ne * nq);
    coeff = 1.0;
    auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || VQ)
+   if (Q || VQ || MQ)
    {
       Vector D(VQ ? coeffDim : 0);
+      DenseMatrix M;
+      Vector Msymm;
+      if (MQ)
+      {
+         if (symmetric)
+         {
+            Msymm.SetSize(MQsymmDim);
+         }
+         else
+         {
+            M.SetSize(dim);
+         }
+      }
+
       if (VQ)
       {
          MFEM_VERIFY(coeffDim == dim, "");
       }
+      if (MQ)
+      {
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+      }
 
       for (int e=0; e<ne; ++e)
       {
          ElementTransformation *tr = mesh->GetElementTransformation(e);
          for (int p=0; p<nq; ++p)
          {
-            if (VQ)
+            if (MQ)
+            {
+               if (MQ->IsSymmetric())
+               {
+                  MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<MQsymmDim; ++i)
+                  {
+                     coeffh(i, p, e) = Msymm[i];
+                  }
+               }
+               else
+               {
+                  MQ->Eval(M, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<dim; ++i)
+                     for (int j=0; j<dim; ++j)
+                     {
+                        coeffh(j+(i*dim), p, e) = M(i,j);
+                     }
+               }
+            }
+            else if (VQ)
             {
                VQ->Eval(D, *tr, ir->IntPoint(p));
                for (int i=0; i<coeffDim; ++i)
@@ -209,28 +831,44 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       }
    }
 
-   fetype = el->GetDerivType();
-
-   if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
+   if (trial_curl && test_curl && dim == 3)
    {
       PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
-   else if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
+   else if (trial_curl && test_curl && dim == 2)
    {
       PAHcurlSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
-   else if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
+   else if (trial_div && test_div && dim == 3)
    {
       PAHdivSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
-   else if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 2)
+   else if (trial_div && test_div && dim == 2)
    {
       PAHdivSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
+   else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
+            test_fel->GetOrder() == trial_fel->GetOrder())
+   {
+      if (coeffDim == 1)
+      {
+         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+      }
+      else
+      {
+         const bool tr = (trial_div && test_curl);
+         if (dim == 3)
+            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+                               geom->J, coeff, pa_data);
+         else
+            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+                               geom->J, coeff, pa_data);
+      }
+   }
    else
    {
       MFEM_ABORT("Unknown kernel.");
@@ -241,12 +879,13 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 {
    if (dim == 3)
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne,
+         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
                                        mapsO->B, mapsC->B, pa_data, diag);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
@@ -258,12 +897,13 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
    }
    else
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         PAHcurlMassAssembleDiagonal2D(dofs1D, quad1D, ne,
+         PAHcurlMassAssembleDiagonal2D(dofs1D, quad1D, ne, symmetric,
                                        mapsO->B, mapsC->B, pa_data, diag);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
@@ -277,18 +917,37 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+
    if (dim == 3)
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_curl && test_curl)
       {
-         PAHcurlMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_div && test_div)
       {
          PAHdivMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                            mapsC->Bt, pa_data, x, y);
       }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(VQ || MQ);
+         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                true, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                mapsCtest->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(VQ || MQ);
+         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                mapsCtest->Bt, pa_data, x, y);
+      }
       else
       {
          MFEM_ABORT("Unknown kernel.");
@@ -296,16 +955,23 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_curl && test_curl)
       {
-         PAHcurlMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_div && test_div)
       {
          PAHdivMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                            mapsC->Bt, pa_data, x, y);
       }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(VQ || MQ);
+         PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                trial_curl, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                mapsCtest->Bt, pa_data, x, y);
+      }
       else
       {
          MFEM_ABORT("Unknown kernel.");

fem/coefficient.cpp

@@ -12,7 +12,6 @@
 // Implementation of Coefficient class
 
 #include "fem.hpp"
-#include "../linalg/dtensor.hpp"
 
 #include <cmath>
 #include <limits>
@@ -22,13 +21,6 @@ namespace mfem
 
 using namespace std;
 
-double QuadratureCoefficient::Eval(ElementTransformation & T,
-                                   const IntegrationPoint & ip)
-{
-   auto coeff = mfem::Reshape(qData->HostRead(), nip, NE);
-   return coeff(ip.index, T.ElementNo);
-}
-
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -327,6 +319,31 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
    }
 }
 
+void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
+                                              ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
+               "MatrixFunctionCoefficient is not symmetric");
+
+   double x[3];
+   Vector transip(x, 3);
+
+   T.Transform(ip, transip);
+
+   K.SetSize((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
+
+   if (SymmFunction)
+   {
+      (*SymmFunction)(transip, K);
+   }
+
+   if (Q)
+   {
+      K *= Q->Eval(T, ip, GetTime());
+   }
+}
+
 MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
    : MatrixCoefficient (dim)
 {

fem/coefficient.hpp

@@ -87,33 +87,6 @@ public:
    { return (constant); }
 };
 
-
-/// class for quadrature coefficient
-class QuadratureCoefficient : public Coefficient
-{
-
-private:
-   const int nip;
-   const int NE;
-public:
-   Vector *qData{nullptr};
-
-   //Set external data
-   QuadratureCoefficient(Vector *Data, int in_nip, int in_NE)
-      : qData(Data), nip(in_nip), NE(in_NE)
-   { }
-
-   virtual double Eval(ElementTransformation &T,
-                       const IntegrationPoint &ip);
-
-   Vector *Data()
-   {
-      return qData;
-   }
-
-};
-
-/// class for piecewise constant coefficient
 /** @brief A piecewise constant coefficient with the constants keyed
     off the element attribute numbers. */
 class PWConstCoefficient : public Coefficient
@@ -722,13 +695,16 @@ class MatrixCoefficient
 protected:
    int height, width;
    double time;
+   bool symmetric;
 
 public:
    /// Construct a dim x dim matrix coefficient.
-   explicit MatrixCoefficient(int dim) { height = width = dim; time = 0.; }
+   explicit MatrixCoefficient(int dim, bool symm=false)
+   { height = width = dim; time = 0.; symmetric = symm; }
 
    /// Construct a h x w matrix coefficient.
-   MatrixCoefficient(int h, int w) : height(h), width(w), time(0.) { }
+   MatrixCoefficient(int h, int w, bool symm=false) :
+      height(h), width(w), time(0.), symmetric(symm) { }
 
    /// Set the time for time dependent coefficients
    void SetTime(double t) { time = t; }
@@ -745,6 +721,9 @@ public:
    /// For backward compatibility get the width of the matrix.
    int GetVDim() const { return width; }
 
+   void SetSymmetric(bool s) { symmetric = s; }
+   bool IsSymmetric() const { return symmetric; }
+
    /** @brief Evaluate the matrix coefficient in the element described by @a T
        at the point @a ip, storing the result in @a K. */
    /** @note When this method is called, the caller must make sure that the
@@ -753,6 +732,15 @@ public:
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip) = 0;
 
+   /** @brief Evaluate the upper triangular entries of the matrix coefficient
+       in the symmetric case, similarly to Eval. Matrix entry (i,j) is stored
+       in K[j - i + os_i] for 0 <= i <= j < width, os_0 = 0,
+       os_{i+1} = os_i + width - i. That is, K = {M(0,0), ..., M(0,w-1),
+       M(1,1), ..., M(1,w-1), ..., M(w-1,w-1) with w = width. */
+   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
+                              const IntegrationPoint &ip)
+   { mfem_error("MatrixCoefficient::EvalSymmetric"); }
+
    virtual ~MatrixCoefficient() { }
 };
 
@@ -780,6 +768,7 @@ class MatrixFunctionCoefficient : public MatrixCoefficient
 {
 private:
    void (*Function)(const Vector &, DenseMatrix &);
+   void (*SymmFunction)(const Vector &, Vector &);
    void (*TDFunction)(const Vector &, double, DenseMatrix &);
    Coefficient *Q;
    DenseMatrix mat;
@@ -817,10 +806,26 @@ public:
       mat.SetSize(0);
    }
 
+   /// Construct a symmetric square matrix coefficient from a C-function
+   /// defining a vector function used by EvalSymmetric
+   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
+                             Coefficient *q = NULL)
+      : MatrixCoefficient(dim, true), Q(q)
+   {
+      SymmFunction = F;
+      Function = NULL;
+      TDFunction = NULL;
+      mat.SetSize(0);
+   }
+
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip);
 
+   /// Evaluate the symmetric matrix coefficient at @a ip.
+   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
+                              const IntegrationPoint &ip);
+
    virtual ~MatrixFunctionCoefficient() { }
 };
 

fem/eltrans.hpp

@@ -77,6 +77,9 @@ public:
 
    ElementTransformation();
 
+   /** @brief Force the reevaluation of the Jacobian in the next call. */
+   void Reset() { EvalState = 0; }
+
    /** @brief Set the integration point @a ip that weights and Jacobians will
        be evaluated at. */
    void SetIntPoint(const IntegrationPoint *ip)
@@ -357,9 +360,17 @@ private:
    // Evaluate the Hessian of the transformation at the IntPoint and store it
    // in d2Fdx2.
    virtual const DenseMatrix &EvalHessian();
+
 public:
+   IsoparametricTransformation() : FElem(NULL) {}
+
    /// Set the element that will be used to compute the transformations
-   void SetFE(const FiniteElement *FE) { FElem = FE; geom = FE->GetGeomType(); }
+   void SetFE(const FiniteElement *FE)
+   {
+      MFEM_ASSERT(FE != NULL, "Must provide a valid FiniteElement object!");
+      EvalState = (FE != FElem) ? 0 : EvalState;
+      FElem = FE; geom = FE->GetGeomType();
+   }
 
    /// Get the current element used to compute the transformations
    const FiniteElement* GetFE() const { return FElem; }
@@ -374,12 +385,15 @@ public:
        the column-vector of all basis functions evaluated at \f$ \hat x \f$ .
        The columns of @a P represent the control points in physical space
        defining the transformation. */
-   void SetPointMat(const DenseMatrix &pm) { PointMat = pm; }
+   void SetPointMat(const DenseMatrix &pm) { PointMat = pm; EvalState = 0; }
 
    /// Return the stored point matrix.
    const DenseMatrix &GetPointMat() const { return PointMat; }
 
-   /// Write access to the stored point matrix. Use with caution.
+   /// @brief Write access to the stored point matrix. Use with caution.
+   /** If the point matrix is altered using this member function the Reset
+       function should also be called to force the reevaluation of the
+       Jacobian, etc.. */
    DenseMatrix &GetPointMat() { return PointMat; }
 
    /// Set the FiniteElement Geometry for the reference elements being used.

fem/fe.cpp

@@ -7034,6 +7034,95 @@ void Poly_1D::Basis::Eval(const double y, Vector &u, Vector &d) const
    }
 }
 
+void Poly_1D::Basis::Eval(const double y, Vector &u, Vector &d,
+                          Vector &d2) const
+{
+   MFEM_VERIFY(etype == Barycentric,
+               "Basis::Eval with second order derivatives not implemented for"
+               " etype = " << etype);
+   switch (etype)
+   {
+      case ChangeOfBasis:
+      {
+         CalcBasis(Ai.Width() - 1, y, x, w);
+         Ai.Mult(x, u);
+         Ai.Mult(w, d);
+         // set d2 (not implemented yet)
+         break;
+      }
+      case Barycentric:
+      {
+         int i, k, p = x.Size() - 1;
+         double l, lp, lp2, lk, sk, si, sk2;
+
+         if (p == 0)
+         {
+            u(0) = 1.0;
+            d(0) = 0.0;
+            d2(0) = 0.0;
+            return;
+         }
+
+         lk = 1.0;
+         for (k = 0; k < p; k++)
+         {
+            if (y >= (x(k) + x(k+1))/2)
+            {
+               lk *= y - x(k);
+            }
+            else
+            {
+               for (i = k+1; i <= p; i++)
+               {
+                  lk *= y - x(i);
+               }
+               break;
+            }
+         }
+         l = lk * (y - x(k));
+
+         sk = 0.0;
+         sk2 = 0.0;
+         for (i = 0; i < k; i++)
+         {
+            si = 1.0/(y - x(i));
+            sk += si;
+            sk2 -= si * si;
+            u(i) = l * si * w(i);
+         }
+         u(k) = lk * w(k);
+         for (i++; i <= p; i++)
+         {
+            si = 1.0/(y - x(i));
+            sk += si;
+            sk2 -= si * si;
+            u(i) = l * si * w(i);
+         }
+         lp = l * sk + lk;
+         lp2 = lp * sk + l * sk2 + sk * lk;
+
+         for (i = 0; i < k; i++)
+         {
+            d(i) = (lp * w(i) - u(i))/(y - x(i));
+            d2(i) = (lp2 * w(i) - 2 * d(i))/(y - x(i));
+         }
+         d(k) = sk * u(k);
+         d2(k) = sk2 * u(k) + sk * d(k);
+         for (i++; i <= p; i++)
+         {
+            d(i) = (lp * w(i) - u(i))/(y - x(i));
+            d2(i) = (lp2 * w(i) - 2 * d(i))/(y - x(i));
+         }
+         break;
+      }
+      case Positive:
+         CalcBernstein(x.Size() - 1, y, u, d);
+         break;
+
+      default: break;
+   }
+}
+
 const int *Poly_1D::Binom(const int p)
 {
    if (binom.NumCols() <= p)
@@ -7589,6 +7678,7 @@ H1_SegmentElement::H1_SegmentElement(const int p, const int btype)
 #ifndef MFEM_THREAD_SAFE
    shape_x.SetSize(p+1);
    dshape_x.SetSize(p+1);
+   d2shape_x.SetSize(p+1);
 #endif
 
    Nodes.IntPoint(0).x = cp[0];
@@ -7637,6 +7727,25 @@ void H1_SegmentElement::CalcDShape(const IntegrationPoint &ip,
    }
 }
 
+void H1_SegmentElement::CalcHessian(const IntegrationPoint &ip,
+                                    DenseMatrix &Hessian) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1), dshape_x(p+1), d2shape_x(p+1);
+#endif
+
+   basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);
+
+   Hessian(0,0) = d2shape_x(0);
+   Hessian(1,0) = d2shape_x(p);
+   for (int i = 1; i < p; i++)
+   {
+      Hessian(i+1,0) = d2shape_x(i);
+   }
+}
+
 void H1_SegmentElement::ProjectDelta(int vertex, Vector &dofs) const
 {
    const int p = order;
@@ -7677,6 +7786,8 @@ H1_QuadrilateralElement::H1_QuadrilateralElement(const int p, const int btype)
    shape_y.SetSize(p1);
    dshape_x.SetSize(p1);
    dshape_y.SetSize(p1);
+   d2shape_x.SetSize(p1);
+   d2shape_y.SetSize(p1);
 #endif
 
    int o = 0;
@@ -7730,6 +7841,30 @@ void H1_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
    }
 }
 
+void H1_QuadrilateralElement::CalcHessian(const IntegrationPoint &ip,
+                                          DenseMatrix &Hessian) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1), shape_y(p+1), dshape_x(p+1), dshape_y(p+1),
+          d2shape_x(p+1), d2shape_y(p+1);
+#endif
+
+   basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);
+   basis1d.Eval(ip.y, shape_y, dshape_y, d2shape_y);
+
+   for (int o = 0, j = 0; j <= p; j++)
+   {
+      for (int i = 0; i <= p; i++)
+      {
+         Hessian(dof_map[o],0) = d2shape_x(i)*  shape_y(j);
+         Hessian(dof_map[o],1) =  dshape_x(i)* dshape_y(j);
+         Hessian(dof_map[o],2) =   shape_x(i)*d2shape_y(j);  o++;
+      }
+   }
+}
+
 void H1_QuadrilateralElement::ProjectDelta(int vertex, Vector &dofs) const
 {
    const int p = order;
@@ -7793,6 +7928,9 @@ H1_HexahedronElement::H1_HexahedronElement(const int p, const int btype)
    dshape_x.SetSize(p1);
    dshape_y.SetSize(p1);
    dshape_z.SetSize(p1);
+   d2shape_x.SetSize(p1);
+   d2shape_y.SetSize(p1);
+   d2shape_z.SetSize(p1);
 #endif
 
    int o = 0;
@@ -7849,6 +7987,35 @@ void H1_HexahedronElement::CalcDShape(const IntegrationPoint &ip,
          }
 }
 
+void H1_HexahedronElement::CalcHessian(const IntegrationPoint &ip,
+                                       DenseMatrix &Hessian) const
+{
+   const int p = order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape_x(p+1),  shape_y(p+1),  shape_z(p+1);
+   Vector dshape_x(p+1), dshape_y(p+1), dshape_z(p+1);
+   Vector d2shape_x(p+1), d2shape_y(p+1), ds2hape_z(p+1);
+#endif
+
+   basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);
+   basis1d.Eval(ip.y, shape_y, dshape_y, d2shape_y);
+   basis1d.Eval(ip.z, shape_z, dshape_z, d2shape_z);
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j <= p; j++)
+         for (int i = 0; i <= p; i++)
+         {
+            Hessian(dof_map[o],0) = d2shape_x(i)*  shape_y(j)*  shape_z(k);
+            Hessian(dof_map[o],1) =  dshape_x(i)* dshape_y(j)*  shape_z(k);
+            Hessian(dof_map[o],2) =  dshape_x(i)*  shape_y(j)* dshape_z(k);
+            Hessian(dof_map[o],3) =   shape_x(i)*d2shape_y(j)*  shape_z(k);
+            Hessian(dof_map[o],4) =   shape_x(i)* dshape_y(j)* dshape_z(k);
+            Hessian(dof_map[o],5) =   shape_x(i)*  shape_y(j)*d2shape_z(k);
+            o++;
+         }
+}
+
 void H1_HexahedronElement::ProjectDelta(int vertex, Vector &dofs) const
 {
    const int p = order;

fem/fe.hpp

@@ -446,7 +446,7 @@ public:
    /** Each row of the result DenseMatrix @a Hessian contains upper triangular
        part of the Hessian of one shape function.
        The order in 2D is {u_xx, u_xy, u_yy}.
-       The size (#dof x (#dim (#dim-1)/2) of @a Hessian must be set in advance.*/
+       The size (#dof x (#dim (#dim+1)/2) of @a Hessian must be set in advance.*/
    virtual void CalcHessian (const IntegrationPoint &ip,
                              DenseMatrix &Hessian) const;
 
@@ -1850,6 +1850,7 @@ public:
       Basis(const int p, const double *nodes, EvalType etype = Barycentric);
       void Eval(const double x, Vector &u) const;
       void Eval(const double x, Vector &u, Vector &d) const;
+      void Eval(const double x, Vector &u, Vector &d, Vector &d2) const;
    };
 
 private:
@@ -2100,7 +2101,7 @@ class H1_SegmentElement : public NodalTensorFiniteElement
 {
 private:
 #ifndef MFEM_THREAD_SAFE
-   mutable Vector shape_x, dshape_x;
+   mutable Vector shape_x, dshape_x, d2shape_x;
 #endif
 
 public:
@@ -2109,6 +2110,8 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;
    virtual void ProjectDelta(int vertex, Vector &dofs) const;
 };
 
@@ -2118,7 +2121,7 @@ class H1_QuadrilateralElement : public NodalTensorFiniteElement
 {
 private:
 #ifndef MFEM_THREAD_SAFE
-   mutable Vector shape_x, shape_y, dshape_x, dshape_y;
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
 #endif
 
 public:
@@ -2128,6 +2131,8 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;
    virtual void ProjectDelta(int vertex, Vector &dofs) const;
 };
 
@@ -2137,7 +2142,8 @@ class H1_HexahedronElement : public NodalTensorFiniteElement
 {
 private:
 #ifndef MFEM_THREAD_SAFE
-   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z;
+   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z,
+           d2shape_x, d2shape_y, d2shape_z;
 #endif
 
 public:
@@ -2146,6 +2152,8 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;
    virtual void ProjectDelta(int vertex, Vector &dofs) const;
 };
 

fem/fespace.cpp

@@ -944,7 +944,7 @@ const Operator *FiniteElementSpace::GetFaceRestriction(
 }
 
 const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
-   const IntegrationRule &ir, const DofToQuad::Mode mode) const
+   const IntegrationRule &ir) const
 {
    for (int i = 0; i < E2Q_array.Size(); i++)
    {
@@ -952,13 +952,13 @@ const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
       if (qi->IntRule == &ir) { return qi; }
    }
 
-   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, ir, mode);
+   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, ir);
    E2Q_array.Append(qi);
    return qi;
 }
 
 const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
-   const QuadratureSpace &qs, const DofToQuad::Mode mode) const
+   const QuadratureSpace &qs) const
 {
    for (int i = 0; i < E2Q_array.Size(); i++)
    {
@@ -966,7 +966,7 @@ const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
       if (qi->qspace == &qs) { return qi; }
    }
 
-   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, qs, mode);
+   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, qs);
    E2Q_array.Append(qi);
    return qi;
 }
@@ -983,8 +983,8 @@ const FaceQuadratureInterpolator
          if (qi->IntRule == &ir) { return qi; }
       }
 
-      FaceQuadratureInterpolator *qi =
-         new FaceQuadratureInterpolator(*this, ir, type);
+      FaceQuadratureInterpolator *qi = new FaceQuadratureInterpolator(*this, ir,
+                                                                      type);
       E2IFQ_array.Append(qi);
       return qi;
    }

fem/fespace.hpp

@@ -367,7 +367,7 @@ public:
        All elements will use the same IntegrationRule, @a ir as the target
        quadrature points. */
    const QuadratureInterpolator *GetQuadratureInterpolator(
-      const IntegrationRule &ir, const DofToQuad::Mode = DofToQuad::FULL) const;
+      const IntegrationRule &ir) const;
 
    /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
        quadrature point values and/or derivatives (Q-vectors). */
@@ -378,7 +378,7 @@ public:
        The target quadrature points in the elements are described by the given
        QuadratureSpace, @a qs. */
    const QuadratureInterpolator *GetQuadratureInterpolator(
-      const QuadratureSpace &qs, const DofToQuad::Mode = DofToQuad::FULL) const;
+      const QuadratureSpace &qs) const;
 
    /** @brief Return a FaceQuadratureInterpolator that interpolates E-vectors to
        quadrature point values and/or derivatives (Q-vectors). */

fem/nonlinearform.cpp

@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.
 
 #include "fem.hpp"
-#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -28,7 +27,7 @@ void NonlinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
          // This is the default behavior.
          break;
       case AssemblyLevel::PARTIAL:
-         ext = new PANonlinearForm(this);
+         ext = new PANonlinearFormExtension(this);
          break;
       default:
          mfem_error("Unknown assembly level for this form.");
@@ -81,13 +80,6 @@ void NonlinearForm::SetEssentialVDofs(const Array<int> &ess_vdofs_list)
 
 double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
 {
-   if (ext)
-   {
-      MFEM_VERIFY(!fnfi.Size(), "Interior faces terms not yet implemented!");
-      MFEM_VERIFY(!bfnfi.Size(), "Boundary face terms not yet implemented!");
-      return ext->GetGridFunctionEnergy(x);
-   }
-
    Array<int> vdofs;
    Vector el_x;
    const FiniteElement *fe;
@@ -146,14 +138,6 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
    if (ext)
    {
       ext->Mult(px, py);
-      if (Serial())
-      {
-         if (cP) { cP->MultTranspose(py, y); }
-         const int N = ess_tdof_list.Size();
-         const auto tdof = ess_tdof_list.Read();
-         auto Y = y.ReadWrite();
-         MFEM_FORALL(i, N, Y[tdof[i]] = 0.0; );
-      }
       return;
    }
 
@@ -280,16 +264,7 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const
 {
    if (ext)
    {
-      Operator &grad = ext->GetGradient(Prolongate(x));
-      hGrad.Reset(&grad, false);
-      if (Serial())
-      {
-         Operator *Gop;
-         if (cP) { hGrad.Reset(new RAPOperator(*cP, grad, *cP)); }
-         hGrad.Ptr()->Operator::FormSystemOperator(ess_tdof_list, Gop);
-         hGrad.Reset(Gop);
-      }
-      return *hGrad.Ptr();
+      MFEM_ABORT("Not yet implemented!");
    }
 
    const int skip_zeros = 0;
@@ -451,31 +426,7 @@ void NonlinearForm::Update()
 
 void NonlinearForm::Setup()
 {
-   if (ext) { return ext->Setup(); }
-}
-
-void NonlinearForm::AssembleGradientDiagonal(Vector &diag) const
-{
-   if (ext)
-   {
-      MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
-                  "Vector for holding diagonal has wrong size!");
-      const Operator *P = fes->GetProlongationMatrix();
-      if (!IsIdentityProlongation(P))
-      {
-         Vector local_diag(P->Height());
-         ext->AssembleGradientDiagonal(local_diag);
-         P->MultTranspose(local_diag, diag);
-      }
-      else
-      {
-         ext->AssembleGradientDiagonal(diag);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Not implemented. Can be obtained through GetGradient().");
-   }
+   if (ext) { return ext->AssemblePA(); }
 }
 
 NonlinearForm::~NonlinearForm()

fem/nonlinearform.hpp

@@ -45,7 +45,6 @@ protected:
    Array<Array<int>*>              bfnfi_marker; // not owned
 
    mutable SparseMatrix *Grad, *cGrad; // owned
-   mutable OperatorHandle hGrad;
 
    /// A list of all essential true dofs
    Array<int> ess_tdof_list;
@@ -166,15 +165,6 @@ public:
    /// Setup the NonlinearForm
    virtual void Setup();
 
-   /** @brief Assemble the diagonal of the gradient into diag
-
-       For adaptively refined meshes, this returns P^T d_e, where d_e is the
-       locally assembled diagonal on each element and P^T is the transpose of
-       the conforming prolongation. In general this is not the correct diagonal
-       for an AMR mesh. */
-   void AssembleGradientDiagonal(Vector &diag) const;
-
-
    /// Get the finite element space prolongation matrix
    virtual const Operator *GetProlongation() const { return P; }
    /// Get the finite element space restriction matrix

fem/nonlinearform_ext.cpp

@@ -13,101 +13,62 @@
 // PABilinearFormExtension and MFBilinearFormExtension.
 
 #include "nonlinearform.hpp"
-#include "../general/forall.hpp"
 
 namespace mfem
 {
 
-NonlinearFormExtension::NonlinearFormExtension(const NonlinearForm *nlf)
-   : Operator(nlf->FESpace()->GetTrueVSize()), nlf(nlf) { }
-
-PANonlinearForm::PANonlinearForm(NonlinearForm *nlf):
-   NonlinearFormExtension(nlf),
-   x_grad(NULL),
-   fes(*nlf->FESpace()),
-   dnfi(*nlf->GetDNFI()),
-   R(fes.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC))
+NonlinearFormExtension::NonlinearFormExtension(NonlinearForm *form)
+   : Operator(form->FESpace()->GetTrueVSize()), n(form)
 {
-   MFEM_VERIFY(R, "Not yet implemented!");
-   xe.SetSize(R->Height(), Device::GetMemoryType());
-   ye.SetSize(R->Height(), Device::GetMemoryType());
-   ye.UseDevice(true);
+   // empty
 }
 
-double PANonlinearForm::GetGridFunctionEnergy(const Vector &x) const
+PANonlinearFormExtension::PANonlinearFormExtension(NonlinearForm *form):
+   NonlinearFormExtension(form), fes(*form->FESpace())
 {
-   double energy = 0.0;
-
-   R->Mult(x, xe);
-   for (int i = 0; i < dnfi.Size(); i++)
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   elem_restrict_lex = fes.GetElementRestriction(ordering);
+   if (elem_restrict_lex)
    {
-      energy += dnfi[i]->GetGridFunctionEnergyPA(xe);
+      localX.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
+      localY.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
+      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
    }
-   return energy;
 }
 
-void PANonlinearForm::Setup()
+void PANonlinearFormExtension::AssemblePA()
 {
-   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AssemblePA(fes); }
-}
-
-void PANonlinearForm::Mult(const Vector &x, Vector &y) const
-{
-   ye = 0.0;
-   R->Mult(x, xe);
-   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AddMultPA(xe, ye); }
-   R->MultTranspose(ye, y);
-}
-
-void PANonlinearForm::AssembleGradientDiagonal(Vector &diag) const
-{
-   MFEM_VERIFY(x_grad, "GetGradient() has not been called");
-   R->Mult(*x_grad, xe);
-
-   ye = 0.0;
-   for (int i = 0; i < dnfi.Size(); ++i)
+   Array<NonlinearFormIntegrator*> &integrators = *n->GetDNFI();
+   const int Ni = integrators.Size();
+   for (int i = 0; i < Ni; ++i)
    {
-      dnfi[i]->AssembleGradientDiagonalPA(xe, ye);
+      integrators[i]->AssemblePA(*n->FESpace());
    }
-   R->MultTranspose(ye, diag);
 }
 
-Operator &PANonlinearForm::GetGradient(const Vector &x) const
+void PANonlinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
-   // Store the last x that was used to compute the gradient.
-   x_grad = &x;
-
-   Grad.Reset(new PANonlinearForm::Gradient(x, *this));
-   return *Grad.Ptr();
+   Array<NonlinearFormIntegrator*> &integrators = *n->GetDNFI();
+   const int iSz = integrators.Size();
+   if (elem_restrict_lex)
+   {
+      elem_restrict_lex->Mult(x, localX);
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AddMultPA(localX, localY);
+      }
+      elem_restrict_lex->MultTranspose(localY, y);
+   }
+   else
+   {
+      y.UseDevice(true); // typically this is a large vector, so store on device
+      y = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AddMultPA(x, y);
+      }
+   }
 }
 
-PANonlinearForm::Gradient::Gradient(const Vector &x, const PANonlinearForm &e):
-   Operator(e.fes.GetVSize()), R(e.R), dnfi(e.dnfi)
-{
-   ge.UseDevice(true);
-   ge.SetSize(R->Height(), Device::GetMemoryType());
-   R->Mult(x, ge);
-
-   xe.UseDevice(true);
-   xe.SetSize(R->Height(), Device::GetMemoryType());
-
-   ye.UseDevice(true);
-   ye.SetSize(R->Height(), Device::GetMemoryType());
-
-   ze.UseDevice(true);
-   ze.SetSize(R->Height(), Device::GetMemoryType());
-
-   // Do we still need to do this?
-   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AssemblePA(e.fes); }
 }
-
-void PANonlinearForm::Gradient::Mult(const Vector &x, Vector &y) const
-{
-   ze = x;
-   ye = 0.0;
-   R->Mult(ze, xe);
-   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AddMultGradPA(ge, xe, ye); }
-   R->MultTranspose(ye, y);
-}
-
-} // namespace mfem

fem/nonlinearform_ext.hpp

@@ -17,60 +17,28 @@
 
 namespace mfem
 {
-
 class NonlinearForm;
-class NonlinearFormIntegrator;
 
-/** @brief Class extending the NonlinearForm class to support the different
-    AssemblyLevel%s. */
 class NonlinearFormExtension : public Operator
 {
 protected:
-   const NonlinearForm *nlf;
+   NonlinearForm *n; ///< Not owned
 public:
-   NonlinearFormExtension(const NonlinearForm*);
-   virtual void Setup() = 0;
-   virtual Operator &GetGradient(const Vector&) const = 0;
-   virtual double GetGridFunctionEnergy(const Vector &x) const = 0;
-   virtual void AssembleGradientDiagonal(Vector &diag) const
-   {
-      MFEM_ABORT("Not implemented for this assembly level!");
-   }
+   NonlinearFormExtension(NonlinearForm *form);
+   virtual void AssemblePA() = 0;
 };
 
-class PANonlinearForm;
-
-
 /// Data and methods for partially-assembled nonlinear forms
-class PANonlinearForm : public NonlinearFormExtension
+class PANonlinearFormExtension : public NonlinearFormExtension
 {
-private:
-   class Gradient : public Operator
-   {
-   protected:
-      const Operator *R;
-      mutable Vector ge, xe, ye, ze;
-      const Array<NonlinearFormIntegrator*> &dnfi;
-   public:
-      Gradient(const Vector &x, const PANonlinearForm &ext);
-      virtual void Mult(const Vector &x, Vector &y) const;
-   };
-
 protected:
-   mutable Vector xe, ye;
-   mutable const Vector *x_grad;
-   mutable OperatorHandle Grad;
-   const FiniteElementSpace &fes;
-   const Array<NonlinearFormIntegrator*> &dnfi;
-   const Operator *R;
-
+   const FiniteElementSpace &fes; // Not owned
+   mutable Vector localX, localY;
+   const Operator *elem_restrict_lex; // Not owned
 public:
-   PANonlinearForm(NonlinearForm *nlf);
-   void Setup();
+   PANonlinearFormExtension(NonlinearForm*);
+   void AssemblePA();
    void Mult(const Vector &x, Vector &y) const;
-   Operator &GetGradient(const Vector &x) const;
-   double GetGridFunctionEnergy(const Vector &x) const;
-   void AssembleGradientDiagonal(Vector &diag) const;
 };
 }
 #endif // NONLINEARFORM_EXT_HPP

fem/nonlininteg.cpp

@@ -15,13 +15,6 @@
 namespace mfem
 {
 
-double NonlinearFormIntegrator::GetGridFunctionEnergyPA(const Vector &x) const
-{
-   mfem_error ("NonlinearFormIntegrator::GetGridFunctionEnergyPA(...)\n"
-               "   is not implemented for this class.");
-   return 0.0;
-}
-
 void NonlinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
    mfem_error ("NonlinearFormIntegrator::AssemblePA(...)\n"
@@ -41,20 +34,6 @@ void NonlinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
                "   is not implemented for this class.");
 }
 
-void NonlinearFormIntegrator::AddMultGradPA(const Vector&,
-                                            const Vector&, Vector&) const
-{
-   mfem_error ("NonlinearFormIntegrator::AddMultGradPA(...)\n"
-               "   is not implemented for this class.");
-}
-
-void NonlinearFormIntegrator::AssembleGradientDiagonalPA(const mfem::Vector &x,
-                                                         mfem::Vector &diag) const
-{
-   mfem_error ("NonlinearFormIntegrator::AssembleDiagonalPA(...)\n"
-               "   is not implemented for this class.");
-}
-
 void NonlinearFormIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr,
    const Vector &elfun, Vector &elvect)

fem/nonlininteg.hpp

@@ -68,9 +68,6 @@ public:
                                    ElementTransformation &Tr,
                                    const Vector &elfun);
 
-   /// Compute the local energy with partial assembly.
-   virtual double GetGridFunctionEnergyPA(const Vector &x) const;
-
    /// Method defining partial assembly.
    /** The result of the partial assembly is stored internally so that it can be
        used later in the methods AddMultPA(). */
@@ -91,12 +88,6 @@ public:
        called. */
    virtual void AddMultPA(const Vector &x, Vector &y) const;
 
-   /// Method for partially assembled gradient action.
-   virtual void AddMultGradPA(const Vector &g,
-                              const Vector &x, Vector &y) const;
-
-   virtual void AssembleGradientDiagonalPA(const Vector &x, Vector &diag) const;
-
    virtual ~NonlinearFormIntegrator() { }
 };
 

fem/pfespace.cpp

@@ -3147,7 +3147,7 @@ static void SetSubVector(const int N,
                          const Array<int> &indices,
                          const Vector &in, Vector &out)
 {
-   auto y = out.ReadWrite();
+   auto y = out.Write();
    const auto x = in.Read();
    const auto I = indices.Read();
    MFEM_FORALL(i, N, y[I[i]] = x[i];);
@@ -3234,7 +3234,7 @@ static void AddSubVector(const int num_unique_dst_indices,
                          const Vector &src,
                          Vector &dst)
 {
-   auto y = dst.ReadWrite();
+   auto y = dst.Write();
    const auto x = src.Read();
    const auto DST_I = unique_dst_indices.Read();
    const auto SRC_O = unique_to_src_offsets.Read();

fem/pgridfunc.cpp

@@ -711,9 +711,9 @@ void ParGridFunction::SaveAsOne(std::ostream &out)
    int *nfdofs = new int[NRanks];
    int *nrdofs = new int[NRanks];
 
-   HostReadWrite();
-   values[0] = data;
+   double * h_data = const_cast<double *>(this->HostRead());
 
+   values[0] = h_data;
    nv[0]     = pfes -> GetVSize();
    nvdofs[0] = pfes -> GetNVDofs();
    nedofs[0] = pfes -> GetNEDofs();
@@ -814,7 +814,7 @@ void ParGridFunction::SaveAsOne(std::ostream &out)
       MPI_Send(&nvdofs[0], 1, MPI_INT, 0, 456, MyComm);
       MPI_Send(&nedofs[0], 1, MPI_INT, 0, 457, MyComm);
       MPI_Send(&nfdofs[0], 1, MPI_INT, 0, 458, MyComm);
-      MPI_Send(data, nv[0], MPI_DOUBLE, 0, 460, MyComm);
+      MPI_Send(h_data, nv[0], MPI_DOUBLE, 0, 460, MyComm);
    }
 
    delete [] values;

fem/pnonlinearform.cpp

@@ -14,7 +14,6 @@
 #ifdef MFEM_USE_MPI
 
 #include "fem.hpp"
-#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -50,7 +49,6 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 
    if (fnfi.Size())
    {
-      MFEM_VERIFY(!NonlinearForm::ext,"");
       // Terms over shared interior faces in parallel.
       ParFiniteElementSpace *pfes = ParFESpace();
       ParMesh *pmesh = pfes->GetParMesh();
@@ -88,16 +86,15 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 
    P->MultTranspose(aux2, y);
 
-   const int N = ess_tdof_list.Size();
-   const auto idx = ess_tdof_list.Read();
-   auto Y = y.ReadWrite();
-   MFEM_FORALL(i, N, Y[idx[i]] = 0.0; );
+   y.HostReadWrite();
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      y(ess_tdof_list[i]) = 0.0;
+   }
 }
 
 const SparseMatrix &ParNonlinearForm::GetLocalGradient(const Vector &x) const
 {
-   if (NonlinearForm::ext) { MFEM_ABORT("Not yet implemented!"); }
-
    NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
 
    return *Grad;
@@ -107,20 +104,16 @@ Operator &ParNonlinearForm::GetGradient(const Vector &x) const
 {
    ParFiniteElementSpace *pfes = ParFESpace();
 
-   Operator &grad = NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
-
    pGrad.Clear();
 
+   NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
+
    OperatorHandle dA(pGrad.Type()), Ph(pGrad.Type());
 
    if (fnfi.Size() == 0)
    {
-      if (NonlinearForm::ext) { dA.Reset(&grad, false); }
-      else
-      {
-         dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
-                                pfes->GetDofOffsets(), Grad);
-      }
+      dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
+                             pfes->GetDofOffsets(), Grad);
    }
    else
    {

fem/quadinterpolator.hpp

@@ -41,11 +41,10 @@ protected:
    const FiniteElementSpace *fespace;  ///< Not owned
    const QuadratureSpace *qspace;      ///< Not owned
    const IntegrationRule *IntRule;     ///< Not owned
-
    mutable QVectorLayout q_layout;     ///< Output Q-vector layout
-   mutable bool use_tensor_products;   ///< Tensor product evaluation mmode
 
-public:
+   mutable bool use_tensor_products;
+
    static const int MAX_NQ2D = 100;
    static const int MAX_ND2D = 100;
    static const int MAX_VDIM2D = 3;
@@ -54,6 +53,7 @@ public:
    static const int MAX_ND3D = 1000;
    static const int MAX_VDIM3D = 3;
 
+public:
    enum EvalFlags
    {
       VALUES       = 1 << 0,  ///< Evaluate the values at quadrature points
@@ -61,28 +61,21 @@ public:
       /** @brief Assuming the derivative at quadrature points form a matrix,
           this flag can be used to compute and store their determinants. This
           flag can only be used in Mult(). */
-      DETERMINANTS = 1 << 2,
-      PHYSICAL_DERIVATIVES = 1 << 3 ///< Evaluate the physical derivatives
+      DETERMINANTS = 1 << 2
    };
 
    QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const IntegrationRule &ir,
-                          const bool use_tensor_products = false);
+                          const IntegrationRule &ir);
 
    QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const QuadratureSpace &qs,
-                          const bool use_tensor_products = false);
+                          const QuadratureSpace &qs);
 
    /** @brief Disable the use of tensor product evaluations, for tensor-product
        elements, e.g. quads and hexes. */
-   void DisableTensorProducts() const { use_tensor_products = false; }
-
-   /** @brief Enable the use of tensor product evaluations, for tensor-product
-       elements, e.g. quads and hexes. */
-   void EnableTensorProducts() const { use_tensor_products = true; }
-
-   /** @brief Query the current evaluation mode. */
-   bool UseTensorProducts() const { return use_tensor_products; }
+   /** Currently, tensor product evaluations are not implemented and this method
+       has no effect. */
+   void DisableTensorProducts(bool disable = true) const
+   { use_tensor_products = !disable; }
 
    /** @brief Query the current output Q-vector layout. The default value is
        QVectorLayout::byNODES. */
@@ -90,7 +83,8 @@ public:
 
    /** @brief Set the desired output Q-vector layout. The default value is
        QVectorLayout::byNODES. */
-   void SetOutputLayout(QVectorLayout layout) const { q_layout = layout; }
+   void SetOutputLayout(QVectorLayout out_layout) const
+   { q_layout = out_layout; }
 
    /// Interpolate the E-vector @a e_vec to quadrature points.
    /** The @a eval_flags are a bitwise mask of constants from the EvalFlags
@@ -105,36 +99,26 @@ public:
              Vector &q_val, Vector &q_der, Vector &q_det) const;
 
    /// Interpolate the values of the E-vector @a e_vec at quadrature points.
-   template <QVectorLayout>
-   void Values(const Vector &e_vec, Vector &q_val) const;
    void Values(const Vector &e_vec, Vector &q_val) const;
 
    /** @brief Interpolate the derivatives of the E-vector @a e_vec at quadrature
        points. */
-   template <QVectorLayout>
-   void Derivatives(const Vector &e_vec, Vector &q_der) const;
    void Derivatives(const Vector &e_vec, Vector &q_der) const;
 
    /** @brief Interpolate the derivatives in physical space of the E-vector
        @a e_vec at quadrature points. */
-   template <QVectorLayout>
    void PhysDerivatives(const Vector &e_vec, Vector &q_der) const;
-   void PhysDerivatives(const Vector &e_vec, Vector &q_der) const;
-
-   /// Compute the determinant of the E-vector @a e_vec at quadrature points.
-   void Determinants(const Vector &e_vec, Vector &q_det) const;
 
    /// Perform the transpose operation of Mult(). (TODO)
    void MultTranspose(unsigned eval_flags, const Vector &q_val,
                       const Vector &q_der, Vector &e_vec) const;
 
    // Compute kernels follow (cannot be private or protected with nvcc)
+
    /// Template compute kernel for 2D.
    template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Mult2D(const int NE,
+   static void Eval2D(const int NE,
                       const int vdim,
-                      const QVectorLayout q_layout,
-                      const GeometricFactors *geom,
                       const DofToQuad &maps,
                       const Vector &e_vec,
                       Vector &q_val,
@@ -144,10 +128,8 @@ public:
 
    /// Template compute kernel for 3D.
    template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Mult3D(const int NE,
+   static void Eval3D(const int NE,
                       const int vdim,
-                      const QVectorLayout q_layout,
-                      const GeometricFactors *geom,
                       const DofToQuad &maps,
                       const Vector &e_vec,
                       Vector &q_val,

fem/quadinterpolator_det.cpp

@@ -1,208 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop_pa.hpp"
-#include "quadinterpolator.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-#include "../fem/kernels.hpp"
-#include "../linalg/kernels.hpp"
-
-using namespace mfem;
-
-namespace mfem
-{
-
-template<int T_D1D = 0, int T_Q1D = 0, int MAX_D1D = 0, int MAX_Q1D = 0>
-static void Det2D(const int NE,
-                  const double *b,
-                  const double *g,
-                  const double *x,
-                  double *y,
-                  const int vdim = 1,
-                  const int d1d = 0,
-                  const int q1d = 0)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x,  D1D, D1D, DIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double J[4];
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,J);
-            Y(qx,qy,e) = kernels::Det<2>(J);
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int MAX_D1D = 0, int MAX_Q1D = 0>
-static void Det3D(const int NE,
-                  const double *b,
-                  const double *g,
-                  const double *x,
-                  double *y,
-                  const int vdim = 1,
-                  const int d1d = 0,
-                  const int q1d = 0)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double sm0[9][MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[9][MDQ*MDQ*MDQ];
-
-      double (*DDD)[MD1*MD1*MD1] = (double (*)[MD1*MD1*MD1]) (sm0);
-      double (*DDQ)[MD1*MD1*MQ1] = (double (*)[MD1*MD1*MQ1]) (sm1);
-      double (*DQQ)[MD1*MQ1*MQ1] = (double (*)[MD1*MQ1*MQ1]) (sm0);
-      double (*QQQ)[MQ1*MQ1*MQ1] = (double (*)[MQ1*MQ1*MQ1]) (sm1);
-
-      kernels::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double J[9];
-               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, J);
-               Y(qx,qy,qz,e) = kernels::Det<3>(J);
-            }
-         }
-      }
-   });
-}
-
-void QuadratureInterpolator::Determinants(const Vector &e_vec,
-                                          Vector &q_det) const
-{
-   if (use_tensor_products)
-   {
-      const int NE = fespace->GetNE();
-      if (NE == 0) { return; }
-      const int vdim = fespace->GetVDim();
-      const int dim = fespace->GetMesh()->Dimension();
-      const FiniteElement *fe = fespace->GetFE(0);
-      const IntegrationRule *ir =
-         IntRule ? IntRule : &qspace->GetElementIntRule(0);
-      const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
-      const int D1D = maps.ndof;
-      const int Q1D = maps.nqpt;
-      const double *B = maps.B.Read();
-      const double *G = maps.G.Read();
-      const double *X = e_vec.Read();
-      double *Y = q_det.Write();
-
-      const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
-
-      switch (id)
-      {
-         case 0x2222: return Det2D<2,2>(NE,B,G,X,Y);
-         case 0x2223: return Det2D<2,3>(NE,B,G,X,Y);
-         case 0x2224: return Det2D<2,4>(NE,B,G,X,Y);
-         case 0x2226: return Det2D<2,6>(NE,B,G,X,Y);
-         case 0x2234: return Det2D<3,4>(NE,B,G,X,Y);
-         case 0x2236: return Det2D<3,6>(NE,B,G,X,Y);
-         case 0x2244: return Det2D<4,4>(NE,B,G,X,Y);
-         case 0x2246: return Det2D<4,6>(NE,B,G,X,Y);
-         case 0x2256: return Det2D<5,6>(NE,B,G,X,Y);
-
-         case 0x3324: return Det3D<2,4>(NE,B,G,X,Y);
-         case 0x3333: return Det3D<3,3>(NE,B,G,X,Y);
-         case 0x3335: return Det3D<3,5>(NE,B,G,X,Y);
-         case 0x3336: return Det3D<3,6>(NE,B,G,X,Y);
-         //case 0x3348: return Det3D<4,8>(NE,B,G,X,Y);
-
-         default:
-         {
-            if (dim == 2)
-            {
-               constexpr int MD1 = 8;
-               constexpr int MQ1 = 8;
-               MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
-                           << " are not supported!");
-               MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
-                           << MQ1 << " 1D points are not supported!");
-               return Det2D<0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
-            }
-            if (dim == 3)
-            {
-               constexpr int MD1 = 6;
-               constexpr int MQ1 = 6;
-               MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
-                           << " are not supported!");
-               MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
-                           << MQ1 << " 1D points are not supported!");
-               return Det3D<0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
-            }
-         }
-
-      }
-      MFEM_ABORT("Kernel " << std::hex << id << std::dec << " not supported yet");
-   }
-   else
-   {
-      Vector empty;
-      Mult(e_vec, DETERMINANTS, empty, empty, q_det);
-   }
-}
-
-} // namespace mfem

fem/quadinterpolator_eval.hpp

@@ -1,233 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1, int MAX_D1D = 0, int MAX_Q1D = 0>
-static void Eval2D(const int NE,
-                   const double *b_,
-                   const double *x_,
-                   double *y_,
-                   const int vdim = 0,
-                   const int d1d = 0,
-                   const int q1d = 0)
-{
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, VDIM, NE):
-            Reshape(y_, VDIM, Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED double s_B[MQ1*MD1];
-      DeviceTensor<2,double> B(s_B, Q1D, D1D);
-
-      MFEM_SHARED double s_DD[NBZ][MD1*MD1];
-      DeviceTensor<2,double> DD((double*)(s_DD+tidz), MD1, MD1);
-
-      MFEM_SHARED double s_DQ[NBZ][MD1*MQ1];
-      DeviceTensor<2,double> DQ((double*)(s_DQ+tidz), MD1, MQ1);
-
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B(q,d) = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               DD(dx,dy) = x(dx,dy,c,e);
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  u += B(qx,dx) *  DD(dx,dy);
-               }
-               DQ(dy,qx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DQ(dy,qx) * B(qy,dy);
-               }
-               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
-               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int MAX_D1D = 0, int MAX_Q1D = 0>
-static void Eval3D(const int NE,
-                   const double *b_,
-                   const double *x_,
-                   double *y_,
-                   const int vdim = 0,
-                   const int d1d = 0,
-                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
-            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED double s_B[MQ1*MD1];
-      DeviceTensor<2,double> B(s_B, Q1D, D1D);
-
-      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-      DeviceTensor<3,double> DDD(sm0, MD1, MD1, MD1);
-      DeviceTensor<3,double> DDQ(sm1, MD1, MD1, MQ1);
-      DeviceTensor<3,double> DQQ(sm0, MD1, MQ1, MQ1);
-
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B(q,d) = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  DDD(dx,dy,dz) = x(dx,dy,dz,c,e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     u += B(qx,dx) * DDD(dx,dy,dz);
-                  }
-                  DDQ(dz,dy,qx) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  for (int dy = 0; dy < D1D; ++dy)
-                  {
-                     u += DDQ(dz,dy,qx) * B(qy,dy);
-                  }
-                  DQQ(dz,qy,qx) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  for (int dz = 0; dz < D1D; ++dz)
-                  {
-                     u +=  DQQ(dz,qy,qx) * B(qz,dz);
-                  }
-                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
-                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-} // namespace mfem

fem/quadinterpolator_eval_by_nodes.cpp

@@ -1,110 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "quadinterpolator_eval.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-template<>
-void QuadratureInterpolator::Values<QVectorLayout::byNODES>(
-   const Vector &e_vec, Vector &q_val) const
-{
-   const int NE = fespace->GetNE();
-   if (NE == 0) { return; }
-   const int vdim = fespace->GetVDim();
-   const int dim = fespace->GetMesh()->Dimension();
-   const FiniteElement *fe = fespace->GetFE(0);
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const double *B = maps.B.Read();
-   const double *X = e_vec.Read();
-   double *Y = q_val.Write();
-
-   constexpr QVectorLayout L = QVectorLayout::byNODES;
-
-   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
-
-   switch (id)
-   {
-      case 0x2133: return Eval2D<L,1,3,3>(NE,B,X,Y);
-      case 0x2124: return Eval2D<L,1,2,4>(NE,B,X,Y);
-      case 0x2132: return Eval2D<L,1,3,2>(NE,B,X,Y);
-      case 0x2134: return Eval2D<L,1,3,4>(NE,B,X,Y);
-      case 0x2143: return Eval2D<L,1,4,3>(NE,B,X,Y);
-      case 0x2144: return Eval2D<L,1,4,4>(NE,B,X,Y);
-
-      case 0x2222: return Eval2D<L,2,2,2>(NE,B,X,Y);
-      case 0x2223: return Eval2D<L,2,2,3>(NE,B,X,Y);
-      case 0x2224: return Eval2D<L,2,2,4>(NE,B,X,Y);
-      case 0x2225: return Eval2D<L,2,2,5>(NE,B,X,Y);
-      case 0x2226: return Eval2D<L,2,2,6>(NE,B,X,Y);
-      case 0x2233: return Eval2D<L,2,3,3>(NE,B,X,Y);
-      case 0x2234: return Eval2D<L,2,3,4>(NE,B,X,Y);
-      case 0x2236: return Eval2D<L,2,3,6>(NE,B,X,Y);
-      case 0x2243: return Eval2D<L,2,4,3>(NE,B,X,Y);
-      case 0x2244: return Eval2D<L,2,4,4>(NE,B,X,Y);
-      case 0x2245: return Eval2D<L,2,4,5>(NE,B,X,Y);
-      case 0x2246: return Eval2D<L,2,4,6>(NE,B,X,Y);
-      case 0x2247: return Eval2D<L,2,4,7>(NE,B,X,Y);
-      case 0x2256: return Eval2D<L,2,5,6>(NE,B,X,Y);
-
-      case 0x3124: return Eval3D<L,1,2,4>(NE,B,X,Y);
-      case 0x3133: return Eval3D<L,1,3,3>(NE,B,X,Y);
-      case 0x3134: return Eval3D<L,1,3,4>(NE,B,X,Y);
-      case 0x3136: return Eval3D<L,1,3,6>(NE,B,X,Y);
-      case 0x3143: return Eval3D<L,1,4,3>(NE,B,X,Y);
-      case 0x3144: return Eval3D<L,1,4,4>(NE,B,X,Y);
-      case 0x3148: return Eval3D<L,1,4,8>(NE,B,X,Y);
-
-      case 0x3222: return Eval3D<L,2,2,2>(NE,B,X,Y);
-      case 0x3223: return Eval3D<L,2,2,3>(NE,B,X,Y);
-      case 0x3234: return Eval3D<L,2,3,4>(NE,B,X,Y);
-
-      case 0x3323: return Eval3D<L,3,2,3>(NE,B,X,Y);
-      case 0x3324: return Eval3D<L,3,2,4>(NE,B,X,Y);
-      case 0x3325: return Eval3D<L,3,2,5>(NE,B,X,Y);
-      case 0x3326: return Eval3D<L,3,2,6>(NE,B,X,Y);
-      case 0x3333: return Eval3D<L,3,3,3>(NE,B,X,Y);
-      case 0x3334: return Eval3D<L,3,3,4>(NE,B,X,Y);
-      case 0x3335: return Eval3D<L,3,3,5>(NE,B,X,Y);
-      case 0x3336: return Eval3D<L,3,3,6>(NE,B,X,Y);
-      case 0x3343: return Eval3D<L,3,4,3>(NE,B,X,Y);
-      case 0x3344: return Eval3D<L,3,4,4>(NE,B,X,Y);
-      case 0x3346: return Eval3D<L,3,4,6>(NE,B,X,Y);
-      case 0x3347: return Eval3D<L,3,4,7>(NE,B,X,Y);
-      case 0x3348: return Eval3D<L,3,4,8>(NE,B,X,Y);
-
-      default:
-      {
-         constexpr int MD1 = 8;
-         constexpr int MQ1 = 8;
-         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
-                     << " are not supported!");
-         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
-                     << MQ1 << " 1D points are not supported!");
-         if (dim == 2) { Eval2D<L,0,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
-         if (dim == 3) { Eval3D<L,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
-         return;
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Kernel not supported yet");
-}
-
-} // namespace mfem

fem/quadinterpolator_eval_by_vdim.cpp

@@ -1,79 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "quadinterpolator_eval.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-template<>
-void QuadratureInterpolator::Values<QVectorLayout::byVDIM>(
-   const Vector &e_vec, Vector &q_val) const
-{
-   const int NE = fespace->GetNE();
-   if (NE == 0) { return; }
-   const int vdim = fespace->GetVDim();
-   const int dim = fespace->GetMesh()->Dimension();
-   const FiniteElement *fe = fespace->GetFE(0);
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const double *B = maps.B.Read();
-   const double *X = e_vec.Read();
-   double *Y = q_val.Write();
-
-   constexpr QVectorLayout L = QVectorLayout::byVDIM;
-
-   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
-
-   switch (id)
-   {
-      case 0x2124: return Eval2D<L,1,2,4,8>(NE,B,X,Y);
-      case 0x2136: return Eval2D<L,1,3,6,4>(NE,B,X,Y);
-      case 0x2148: return Eval2D<L,1,4,8,2>(NE,B,X,Y);
-
-      case 0x2224: return Eval2D<L,2,2,4,8>(NE,B,X,Y);
-      case 0x2234: return Eval2D<L,2,3,4,8>(NE,B,X,Y);
-      case 0x2236: return Eval2D<L,2,3,6,4>(NE,B,X,Y);
-      case 0x2248: return Eval2D<L,2,4,8,2>(NE,B,X,Y);
-
-      case 0x3124: return Eval3D<L,1,2,4>(NE,B,X,Y);
-      case 0x3136: return Eval3D<L,1,3,6>(NE,B,X,Y);
-      case 0x3148: return Eval3D<L,1,4,8>(NE,B,X,Y);
-
-      case 0x3324: return Eval3D<L,3,2,4>(NE,B,X,Y);
-      case 0x3336: return Eval3D<L,3,3,6>(NE,B,X,Y);
-      case 0x3348: return Eval3D<L,3,4,8>(NE,B,X,Y);
-
-      default:
-      {
-         constexpr int MD1 = 8;
-         constexpr int MQ1 = 8;
-         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
-                     << " are not supported!");
-         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
-                     << MQ1 << " 1D points are not supported!");
-         if (dim == 2) { Eval2D<L,0,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
-         if (dim == 3) { Eval3D<L,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
-         return;
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Kernel not supported yet");
-}
-
-} // namespace mfem

fem/quadinterpolator_face.cpp

@@ -495,9 +495,8 @@ void FaceQuadratureInterpolator::Mult(
    }
 }
 
-
-void FaceQuadratureInterpolator::Values(const Vector &e_vec,
-                                        Vector &q_val) const
+void FaceQuadratureInterpolator::Values(
+   const Vector &e_vec, Vector &q_val) const
 {
    Vector q_der, q_det, q_nor;
    Mult(e_vec, VALUES, q_val, q_der, q_det, q_nor);

fem/quadinterpolator_grad.hpp

@@ -1,282 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1, int MAX_D1D = 0, int MAX_Q1D = 0>
-static void Grad2D(const int NE,
-                   const double *b_,
-                   const double *g_,
-                   const double *x_,
-                   double *y_,
-                   const int vdim = 0,
-                   const int d1d = 0,
-                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, VDIM, 2, NE):
-            Reshape(y_, VDIM, 2, Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      const int tidz = MFEM_THREAD_ID(z);
-      MFEM_SHARED double s_B[MQ1*MD1];
-      MFEM_SHARED double s_G[MQ1*MD1];
-      DeviceTensor<2,double> B(s_B, Q1D, D1D);
-      DeviceTensor<2,double> G(s_G, Q1D, D1D);
-
-      MFEM_SHARED double s_X[NBZ][MD1*MD1];
-      DeviceTensor<2,double> X((double*)(s_X+tidz), MD1, MD1);
-
-      MFEM_SHARED double s_DQ[2][NBZ][MD1*MQ1];
-      DeviceTensor<2,double> DQ0((double*)(s_DQ[0]+tidz), MD1, MQ1);
-      DeviceTensor<2,double> DQ1((double*)(s_DQ[1]+tidz), MD1, MQ1);
-
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B(q,d) = b(q,d);
-               G(q,d) = g(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int c = 0; c < VDIM; ++c)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               X(dx,dy) = x(dx,dy,c,e);
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double input = X(dx,dy);
-                  u += input * B(qx,dx);
-                  v += input * G(qx,dx);
-               }
-               DQ0(dy,qx) = u;
-               DQ1(dy,qx) = v;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DQ1(dy,qx) * B(qy,dy);
-                  v += DQ0(dy,qx) * G(qy,dy);
-               }
-               if (Q_LAYOUT == QVectorLayout::byNODES)
-               {
-                  y(qx,qy,c,0,e) = u;
-                  y(qx,qy,c,1,e) = v;
-               }
-               if (Q_LAYOUT == QVectorLayout::byVDIM)
-               {
-                  y(c,0,qx,qy,e) = u;
-                  y(c,1,qx,qy,e) = v;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int MAX_D1D = 0, int MAX_Q1D = 0>
-static void Grad3D(const int NE,
-                   const double *b_,
-                   const double *g_,
-                   const double *x_,
-                   double *y_,
-                   const int vdim = 0,
-                   const int d1d = 0,
-                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, Q1D, VDIM, 3, NE):
-            Reshape(y_, VDIM, 3, Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-
-      const int tidz = MFEM_THREAD_ID(z);
-      MFEM_SHARED double s_B[MQ1*MD1];
-      MFEM_SHARED double s_G[MQ1*MD1];
-      DeviceTensor<2,double> B(s_B, Q1D, D1D);
-      DeviceTensor<2,double> G(s_G, Q1D, D1D);
-
-      MFEM_SHARED double sm0[3][MQ1*MQ1*MQ1];
-      MFEM_SHARED double sm1[3][MQ1*MQ1*MQ1];
-      DeviceTensor<3,double> X((double*)(sm0+2), MD1, MD1, MD1);
-      DeviceTensor<3,double> DDQ0((double*)(sm0+0), MD1, MD1, MQ1);
-      DeviceTensor<3,double> DDQ1((double*)(sm0+1), MD1, MD1, MQ1);
-      DeviceTensor<3,double> DQQ0((double*)(sm1+0), MD1, MQ1, MQ1);
-      DeviceTensor<3,double> DQQ1((double*)(sm1+1), MD1, MQ1, MQ1);
-      DeviceTensor<3,double> DQQ2((double*)(sm1+2), MD1, MQ1, MQ1);
-
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B(q,d) = b(q,d);
-               G(q,d) = g(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int c = 0; c < VDIM; ++c)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dz,z,D1D)
-               {
-                  X(dx,dy,dz) = x(dx,dy,dz,c,e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  double v = 0.0;
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     const double input = X(dx,dy,dz);
-                     u += input * B(qx,dx);
-                     v += input * G(qx,dx);
-                  }
-                  DDQ0(dz,dy,qx) = u;
-                  DDQ1(dz,dy,qx) = v;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  double v = 0.0;
-                  double w = 0.0;
-                  for (int dy = 0; dy < D1D; ++dy)
-                  {
-                     u += DDQ1(dz,dy,qx) * B(qy,dy);
-                     v += DDQ0(dz,dy,qx) * G(qy,dy);
-                     w += DDQ0(dz,dy,qx) * B(qy,dy);
-                  }
-                  DQQ0(dz,qy,qx) = u;
-                  DQQ1(dz,qy,qx) = v;
-                  DQQ2(dz,qy,qx) = w;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  double v = 0.0;
-                  double w = 0.0;
-                  for (int dz = 0; dz < D1D; ++dz)
-                  {
-                     u += DQQ0(dz,qy,qx) * B(qz,dz);
-                     v += DQQ1(dz,qy,qx) * B(qz,dz);
-                     w += DQQ2(dz,qy,qx) * G(qz,dz);
-                  }
-                  if (Q_LAYOUT == QVectorLayout::byNODES)
-                  {
-                     y(qx,qy,qz,c,0,e) = u;
-                     y(qx,qy,qz,c,1,e) = v;
-                     y(qx,qy,qz,c,2,e) = w;
-                  }
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     y(c,0,qx,qy,qz,e) = u;
-                     y(c,1,qx,qy,qz,e) = v;
-                     y(c,2,qx,qy,qz,e) = w;
-                  }
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-} // namespace mfem

fem/quadinterpolator_grad_by_nodes.cpp

@@ -1,109 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "quadinterpolator_grad.hpp"
-
-namespace mfem
-{
-
-template<>
-void QuadratureInterpolator::Derivatives<QVectorLayout::byNODES>(
-   const Vector &e_vec, Vector &q_der) const
-{
-   const int NE = fespace->GetNE();
-   if (NE == 0) { return; }
-
-   const int vdim = fespace->GetVDim();
-   const int dim = fespace->GetMesh()->Dimension();
-   const FiniteElement *fe = fespace->GetFE(0);
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const double *B = maps.B.Read();
-   const double *G = maps.G.Read();
-   const double *X = e_vec.Read();
-   double *Y = q_der.Write();
-
-   constexpr QVectorLayout L = QVectorLayout::byNODES;
-
-   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
-
-   switch (id)
-   {
-      case 0x2133: return Grad2D<L,1,3,3,16>(NE,B,G,X,Y);
-      case 0x2134: return Grad2D<L,1,3,4,16>(NE,B,G,X,Y);
-      case 0x2143: return Grad2D<L,1,4,3,16>(NE,B,G,X,Y);
-      case 0x2144: return Grad2D<L,1,4,4,16>(NE,B,G,X,Y);
-
-      case 0x2222: return Grad2D<L,2,2,2,16>(NE,B,G,X,Y);
-      case 0x2223: return Grad2D<L,2,2,3,8>(NE,B,G,X,Y);
-      case 0x2224: return Grad2D<L,2,2,4,4>(NE,B,G,X,Y);
-      case 0x2225: return Grad2D<L,2,2,5,4>(NE,B,G,X,Y);
-      case 0x2226: return Grad2D<L,2,2,6,2>(NE,B,G,X,Y);
-
-      case 0x2233: return Grad2D<L,2,3,3,2>(NE,B,G,X,Y);
-      case 0x2234: return Grad2D<L,2,3,4,4>(NE,B,G,X,Y);
-      case 0x2243: return Grad2D<L,2,4,3,4>(NE,B,G,X,Y);
-      case 0x2236: return Grad2D<L,2,3,6,2>(NE,B,G,X,Y);
-
-      case 0x2244: return Grad2D<L,2,4,4,2>(NE,B,G,X,Y);
-      case 0x2245: return Grad2D<L,2,4,5,2>(NE,B,G,X,Y);
-      case 0x2246: return Grad2D<L,2,4,6,2>(NE,B,G,X,Y);
-      case 0x2247: return Grad2D<L,2,4,7,2>(NE,B,G,X,Y);
-
-      case 0x2256: return Grad2D<L,2,5,6,2>(NE,B,G,X,Y);
-
-      case 0x3124: return Grad3D<L,1,2,4>(NE,B,G,X,Y);
-      case 0x3133: return Grad3D<L,1,3,3>(NE,B,G,X,Y);
-      case 0x3134: return Grad3D<L,1,3,4>(NE,B,G,X,Y);
-      case 0x3136: return Grad3D<L,1,3,6>(NE,B,G,X,Y);
-      case 0x3144: return Grad3D<L,1,4,4>(NE,B,G,X,Y);
-      case 0x3148: return Grad3D<L,1,4,8>(NE,B,G,X,Y);
-
-      case 0x3323: return Grad3D<L,3,2,3>(NE,B,G,X,Y);
-      case 0x3324: return Grad3D<L,3,2,4>(NE,B,G,X,Y);
-      case 0x3325: return Grad3D<L,3,2,5>(NE,B,G,X,Y);
-      case 0x3326: return Grad3D<L,3,2,6>(NE,B,G,X,Y);
-
-      case 0x3333: return Grad3D<L,3,3,3>(NE,B,G,X,Y);
-      case 0x3334: return Grad3D<L,3,3,4>(NE,B,G,X,Y);
-      case 0x3335: return Grad3D<L,3,3,5>(NE,B,G,X,Y);
-      case 0x3336: return Grad3D<L,3,3,6>(NE,B,G,X,Y);
-      case 0x3344: return Grad3D<L,3,4,4>(NE,B,G,X,Y);
-      case 0x3346: return Grad3D<L,3,4,6>(NE,B,G,X,Y);
-      case 0x3347: return Grad3D<L,3,4,7>(NE,B,G,X,Y);
-      case 0x3348: return Grad3D<L,3,4,8>(NE,B,G,X,Y);
-      default:
-      {
-         constexpr int MD1 = 8;
-         constexpr int MQ1 = 8;
-         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
-                     << " are not supported!");
-         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
-                     << MQ1 << " 1D points are not supported!");
-         if (dim == 2)
-         {
-            return Grad2D<L,0,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
-         }
-         if (dim == 3)
-         {
-            return Grad3D<L,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
-         }
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Kernel not supported yet");
-}
-
-} // namespace mfem

fem/quadinterpolator_grad_by_vdim.cpp

@@ -1,76 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "quadinterpolator_grad.hpp"
-
-namespace mfem
-{
-
-template<>
-void QuadratureInterpolator::Derivatives<QVectorLayout::byVDIM>(
-   const Vector &e_vec, Vector &q_der) const
-{
-   const int NE = fespace->GetNE();
-   if (NE == 0) { return; }
-
-   const int vdim = fespace->GetVDim();
-   const int dim = fespace->GetMesh()->Dimension();
-   const FiniteElement *fe = fespace->GetFE(0);
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const double *B = maps.B.Read();
-   const double *G = maps.G.Read();
-   const double *X = e_vec.Read();
-   double *Y = q_der.Write();
-
-   constexpr QVectorLayout L = QVectorLayout::byVDIM;
-
-   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
-
-   switch (id)
-   {
-      case 0x2134: return Grad2D<L,1,3,4,8>(NE,B,G,X,Y);
-      case 0x2146: return Grad2D<L,1,4,6,4>(NE,B,G,X,Y);
-      case 0x2158: return Grad2D<L,1,5,8,2>(NE,B,G,X,Y);
-
-      case 0x2234: return Grad2D<L,2,3,4,8>(NE,B,G,X,Y);
-      case 0x2246: return Grad2D<L,2,4,6,4>(NE,B,G,X,Y);
-      case 0x2258: return Grad2D<L,2,5,8,2>(NE,B,G,X,Y);
-
-      case 0x3134: return Grad3D<L,1,3,4>(NE,B,G,X,Y);
-      case 0x3146: return Grad3D<L,1,4,6>(NE,B,G,X,Y);
-      case 0x3158: return Grad3D<L,1,5,8>(NE,B,G,X,Y);
-
-      case 0x3334: return Grad3D<L,3,3,4>(NE,B,G,X,Y);
-      case 0x3346: return Grad3D<L,3,4,6>(NE,B,G,X,Y);
-      case 0x3358: return Grad3D<L,3,5,8>(NE,B,G,X,Y);
-      default:
-      {
-         constexpr int MD1 = 8;
-         constexpr int MQ1 = 8;
-         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
-                     << " are not supported!");
-         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
-                     << MQ1 << " 1D points are not supported!");
-         if (dim == 2) { Grad2D<L,0,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D); }
-         if (dim == 3) { Grad3D<L,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D); }
-         return;
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Kernel not supported yet");
-}
-
-} // namespace mfem

fem/quadinterpolator_grad_phys.hpp

@@ -1,303 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1, int MAX_D1D = 0, int MAX_Q1D = 0>
-static void PhysGrad2D(const int NE,
-                       const double *b_,
-                       const double *g_,
-                       const double *j_,
-                       const double *x_,
-                       double *y_,
-                       const int vdim = 0,
-                       const int d1d = 0,
-                       const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto j = Reshape(j_, Q1D, Q1D, 2, 2, NE);
-   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, VDIM, 2, NE):
-            Reshape(y_, VDIM, 2, Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      const int tidz = MFEM_THREAD_ID(z);
-      MFEM_SHARED double s_B[MQ1][MD1];
-      MFEM_SHARED double s_G[MQ1][MD1];
-      DeviceTensor<2,double> B((double*)(s_B), Q1D, D1D);
-      DeviceTensor<2,double> G((double*)(s_G), Q1D, D1D);
-
-      MFEM_SHARED double s_X[NBZ][MD1*MD1];
-      DeviceTensor<2,double> X((double*)(s_X+tidz), MD1, MD1);
-
-      MFEM_SHARED double sm[2][NBZ][MD1*MQ1];
-      DeviceTensor<2,double> DQ0((double*)(sm[0]+tidz), MD1, MQ1);
-      DeviceTensor<2,double> DQ1((double*)(sm[1]+tidz), MD1, MQ1);
-
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B(q,d) = b(q,d);
-               G(q,d) = g(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int c = 0; c < VDIM; ++c)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               X(dx,dy) = x(dx,dy,c,e);
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double input = X(dx,dy);
-                  u += input * B(qx,dx);
-                  v += input * G(qx,dx);
-               }
-               DQ0(dy,qx) = u;
-               DQ1(dy,qx) = v;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DQ1(dy,qx) * B(qy,dy);
-                  v += DQ0(dy,qx) * G(qy,dy);
-               }
-               double Jloc[4], Jinv[4];
-               Jloc[0] = j(qx,qy,0,0,e);
-               Jloc[1] = j(qx,qy,1,0,e);
-               Jloc[2] = j(qx,qy,0,1,e);
-               Jloc[3] = j(qx,qy,1,1,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               if (Q_LAYOUT == QVectorLayout::byVDIM)
-               {
-                  y(c,0,qx,qy,e) = Jinv[0]*u + Jinv[1]*v;
-                  y(c,1,qx,qy,e) = Jinv[2]*u + Jinv[3]*v;
-               }
-               if (Q_LAYOUT == QVectorLayout::byNODES)
-               {
-                  y(qx,qy,c,0,e) = Jinv[0]*u + Jinv[1]*v;
-                  y(qx,qy,c,1,e) = Jinv[2]*u + Jinv[3]*v;
-
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int MAX_D = 0, int MAX_Q = 0>
-static  void PhysGrad3D(const int NE,
-                        const double *b_,
-                        const double *g_,
-                        const double *j_,
-                        const double *x_,
-                        double *y_,
-                        const int vdim = 1,
-                        const int d1d = 0,
-                        const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto j = Reshape(j_, Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, Q1D, VDIM, 3, NE):
-            Reshape(y_, VDIM, 3, Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D;
-      const int tidz = MFEM_THREAD_ID(z);
-      MFEM_SHARED double s_B[MQ1][MD1];
-      MFEM_SHARED double s_G[MQ1][MD1];
-      DeviceTensor<2,double> B((double*)(s_B), Q1D, D1D);
-      DeviceTensor<2,double> G((double*)(s_G), Q1D, D1D);
-
-      MFEM_SHARED double sm0[3][MQ1*MQ1*MQ1];
-      MFEM_SHARED double sm1[3][MQ1*MQ1*MQ1];
-      DeviceTensor<3,double> X((double*)(sm0+2), MD1, MD1, MD1);
-      DeviceTensor<3,double> DDQ0((double*)(sm0+0), MD1, MD1, MQ1);
-      DeviceTensor<3,double> DDQ1((double*)(sm0+1), MD1, MD1, MQ1);
-      DeviceTensor<3,double> DQQ0((double*)(sm1+0), MD1, MQ1, MQ1);
-      DeviceTensor<3,double> DQQ1((double*)(sm1+1), MD1, MQ1, MQ1);
-      DeviceTensor<3,double> DQQ2((double*)(sm1+2), MD1, MQ1, MQ1);
-
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B(q,d) = b(q,d);
-               G(q,d) = g(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int c = 0; c < VDIM; ++c)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dz,z,D1D)
-               {
-                  X(dx,dy,dz) = x(dx,dy,dz,c,e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  double v = 0.0;
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     const double coords = X(dx,dy,dz);
-                     u += coords * B(qx,dx);
-                     v += coords * G(qx,dx);
-                  }
-                  DDQ0(dz,dy,qx) = u;
-                  DDQ1(dz,dy,qx) = v;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  double v = 0.0;
-                  double w = 0.0;
-                  for (int dy = 0; dy < D1D; ++dy)
-                  {
-                     u += DDQ1(dz,dy,qx) * B(qy,dy);
-                     v += DDQ0(dz,dy,qx) * G(qy,dy);
-                     w += DDQ0(dz,dy,qx) * B(qy,dy);
-                  }
-                  DQQ0(dz,qy,qx) = u;
-                  DQQ1(dz,qy,qx) = v;
-                  DQQ2(dz,qy,qx) = w;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  double u = 0.0;
-                  double v = 0.0;
-                  double w = 0.0;
-                  for (int dz = 0; dz < D1D; ++dz)
-                  {
-                     u += DQQ0(dz,qy,qx) * B(qz,dz);
-                     v += DQQ1(dz,qy,qx) * B(qz,dz);
-                     w += DQQ2(dz,qy,qx) * G(qz,dz);
-                  }
-                  double Jloc[9], Jinv[9];
-                  for (int col = 0; col < 3; col++)
-                  {
-                     for (int row = 0; row < 3; row++)
-                     {
-                        Jloc[row+3*col] = j(qx,qy,qz,row,col,e);
-                     }
-                  }
-                  kernels::CalcInverse<3>(Jloc, Jinv);
-                  if (Q_LAYOUT == QVectorLayout::byNODES)
-                  {
-                     y(qx,qy,qz,c,0,e) = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                     y(qx,qy,qz,c,1,e) = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                     y(qx,qy,qz,c,2,e) = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  }
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     y(c,0,qx,qy,qz,e) = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                     y(c,1,qx,qy,qz,e) = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                     y(c,2,qx,qy,qz,e) = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  }
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-} // namespace mfem

fem/quadinterpolator_grad_phys_by_nodes.cpp

@@ -1,110 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "quadinterpolator_grad_phys.hpp"
-
-namespace mfem
-{
-
-template<>
-void QuadratureInterpolator::PhysDerivatives<QVectorLayout::byNODES>(
-   const Vector &e_vec, Vector &q_der) const
-{
-   const int NE = fespace->GetNE();
-   if (NE == 0) { return; }
-   Mesh *mesh = fespace->GetMesh();
-   const int vdim = fespace->GetVDim();
-   const int dim = fespace->GetMesh()->Dimension();
-   const FiniteElement *fe = fespace->GetFE(0);
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   constexpr DofToQuad::Mode mode = DofToQuad::TENSOR;
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
-   const GeometricFactors *geom =
-      mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
-
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-
-   const double *B = maps.B.Read();
-   const double *G = maps.G.Read();
-   const double *J = geom->J.Read();
-   const double *X = e_vec.Read();
-   double *Y = q_der.Write();
-
-   constexpr QVectorLayout L = QVectorLayout::byNODES;
-
-   const int id = (vdim<<8) | (D1D<<4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x133: return PhysGrad2D<L,1,3,3,8>(NE,B,G,J,X,Y);
-         case 0x134: return PhysGrad2D<L,1,3,4,8>(NE,B,G,J,X,Y);
-         case 0x143: return PhysGrad2D<L,1,4,3,4>(NE,B,G,J,X,Y);
-         case 0x144: return PhysGrad2D<L,1,4,4,4>(NE,B,G,J,X,Y);
-         case 0x146: return PhysGrad2D<L,1,4,6,4>(NE,B,G,J,X,Y);
-         case 0x158: return PhysGrad2D<L,1,5,8,2>(NE,B,G,J,X,Y);
-
-         case 0x233: return PhysGrad2D<L,2,3,3,8>(NE,B,G,J,X,Y);
-         case 0x234: return PhysGrad2D<L,2,3,4,8>(NE,B,G,J,X,Y);
-         case 0x243: return PhysGrad2D<L,2,4,3,4>(NE,B,G,J,X,Y);
-         case 0x244: return PhysGrad2D<L,2,4,4,4>(NE,B,G,J,X,Y);
-         case 0x246: return PhysGrad2D<L,2,4,6,4>(NE,B,G,J,X,Y);
-         case 0x258: return PhysGrad2D<L,2,5,8,2>(NE,B,G,J,X,Y);
-         default:
-         {
-            constexpr int MD = MAX_D1D;
-            constexpr int MQ = MAX_Q1D;
-            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
-                        << " are not supported!");
-            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
-                        << " 1D points are not supported!");
-            PhysGrad2D<L,0,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
-            return;
-         }
-      }
-   }
-   if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x133: return PhysGrad3D<L,1,3,3>(NE,B,G,J,X,Y);
-         case 0x134: return PhysGrad3D<L,1,3,4>(NE,B,G,J,X,Y);
-         case 0x144: return PhysGrad3D<L,1,4,4>(NE,B,G,J,X,Y);
-         case 0x146: return PhysGrad3D<L,1,4,6>(NE,B,G,J,X,Y);
-         case 0x158: return PhysGrad3D<L,1,5,8>(NE,B,G,J,X,Y);
-
-         case 0x333: return PhysGrad3D<L,3,3,3>(NE,B,G,J,X,Y);
-         case 0x334: return PhysGrad3D<L,3,3,4>(NE,B,G,J,X,Y);
-         case 0x344: return PhysGrad3D<L,3,4,4>(NE,B,G,J,X,Y);
-         case 0x346: return PhysGrad3D<L,3,4,6>(NE,B,G,J,X,Y);
-         case 0x358: return PhysGrad3D<L,3,5,8>(NE,B,G,J,X,Y);
-         default:
-         {
-            constexpr int MD = 8;
-            constexpr int MQ = 8;
-            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
-                        << " are not supported!");
-            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
-                        << " 1D points are not supported!");
-            PhysGrad3D<L,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
-            return;
-         }
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Unknown kernel");
-}
-
-} // namespace mfem

fem/quadinterpolator_grad_phys_by_vdim.cpp

@@ -1,101 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "quadinterpolator.hpp"
-#include "quadinterpolator_grad_phys.hpp"
-
-namespace mfem
-{
-
-template<>
-void QuadratureInterpolator::PhysDerivatives<QVectorLayout::byVDIM>(
-   const Vector &e_vec, Vector &q_der) const
-{
-   const int NE = fespace->GetNE();
-   if (NE == 0) { return; }
-   Mesh *mesh = fespace->GetMesh();
-   const int vdim = fespace->GetVDim();
-   const int dim = fespace->GetMesh()->Dimension();
-   const FiniteElement *fe = fespace->GetFE(0);
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   constexpr DofToQuad::Mode mode = DofToQuad::TENSOR;
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
-   const GeometricFactors *geom =
-      mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
-
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-
-   const double *B = maps.B.Read();
-   const double *G = maps.G.Read();
-   const double *J = geom->J.Read();
-   const double *X = e_vec.Read();
-   double *Y = q_der.Write();
-
-   constexpr QVectorLayout L = QVectorLayout::byVDIM;
-
-   const int id = (vdim<<8) | (D1D<<4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x134: return PhysGrad2D<L,1,3,4,8>(NE, B, G, J, X, Y);
-         case 0x146: return PhysGrad2D<L,1,4,6,4>(NE, B, G, J, X, Y);
-         case 0x158: return PhysGrad2D<L,1,5,8,2>(NE, B, G, J, X, Y);
-
-         case 0x233: return PhysGrad2D<L,2,3,3,8>(NE, B, G, J, X, Y);
-         case 0x234: return PhysGrad2D<L,2,3,4,8>(NE, B, G, J, X, Y);
-         case 0x246: return PhysGrad2D<L,2,4,6,4>(NE, B, G, J, X, Y);
-         case 0x258: return PhysGrad2D<L,2,5,8,2>(NE, B, G, J, X, Y);
-         default:
-         {
-            constexpr int MD = MAX_D1D;
-            constexpr int MQ = MAX_Q1D;
-            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
-                        << " are not supported!");
-            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
-                        << " 1D points are not supported!");
-            PhysGrad2D<L,0,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
-            return;
-         }
-      }
-   }
-   if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x134: return PhysGrad3D<L,1,3,4>(NE, B, G, J, X, Y);
-         case 0x146: return PhysGrad3D<L,1,4,6>(NE, B, G, J, X, Y);
-         case 0x158: return PhysGrad3D<L,1,5,8>(NE, B, G, J, X, Y);
-
-         case 0x334: return PhysGrad3D<L,3,3,4>(NE, B, G, J, X, Y);
-         case 0x346: return PhysGrad3D<L,3,4,6>(NE, B, G, J, X, Y);
-         case 0x358: return PhysGrad3D<L,3,5,8>(NE, B, G, J, X, Y);
-         default:
-         {
-            constexpr int MD = 8;
-            constexpr int MQ = 8;
-            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
-                        << " are not supported!");
-            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
-                        << " 1D points are not supported!");
-            PhysGrad3D<L,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
-            return;
-         }
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Unknown kernel");
-}
-
-} // namespace mfem

fem/tmop.cpp

@@ -13,7 +13,6 @@
 #include "linearform.hpp"
 #include "pgridfunc.hpp"
 #include "tmop_tools.hpp"
-#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -442,8 +441,8 @@ void TMOP_Metric_058::AssembleH(const DenseMatrix &Jpt,
 double TMOP_Metric_077::EvalW(const DenseMatrix &Jpt) const
 {
    ie.SetJacobian(Jpt.GetData());
-   const double I2b = ie.Get_I2b();
-   return  0.5*(I2b*I2b + 1./(I2b*I2b) - 2.);
+   const double I2 = ie.Get_I2b();
+   return  0.5*(I2*I2 + 1./(I2*I2) - 2.);
 }
 
 void TMOP_Metric_077::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
@@ -1039,10 +1038,11 @@ void DiscreteAdaptTC::SetTspecAtIndex(int idx, const ParGridFunction &tspec_)
 {
    const int vdim     = tspec_.FESpace()->GetVDim(),
              dof_cnt  = tspec_.Size()/vdim;
-   const auto tspec__d = tspec_.Read();
-   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   MFEM_FORALL(i, dof_cnt*vdim, tspec_d[i+offset] = tspec__d[i];);
+   for (int i = 0; i < dof_cnt*vdim; i++)
+   {
+      tspec(i+idx*dof_cnt) = tspec_(i);
+   }
+
    FinalizeParDiscreteTargetSpec(tspec_);
 }
 
@@ -1102,33 +1102,34 @@ void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
    // make a copy of tspec->tspec_temp, increase its size, and
    // copy data from tspec_temp -> tspec, then add new entries
    Vector tspec_temp = tspec;
-   tspec.UseDevice(true);
-   tspec_sav.UseDevice(true);
    tspec.SetSize(ncomp*dof_cnt);
 
-   const auto tspec_temp_d = tspec_temp.Read();
-   auto tspec_d = tspec.ReadWrite();
-   MFEM_FORALL(i, tspec_temp.Size(), tspec_d[i] = tspec_temp_d[i];);
+   for (int i = 0; i < tspec_temp.Size(); i++)
+   {
+      tspec(i) = tspec_temp(i);
+   }
 
-   const auto tspec__d = tspec_.Read();
-   const int offset = (ncomp-vdim)*dof_cnt;
-   MFEM_FORALL(i, dof_cnt*vdim, tspec_d[i+offset] = tspec__d[i];);
+   for (int i = 0; i < dof_cnt*vdim; i++)
+   {
+      tspec(i+(ncomp-vdim)*dof_cnt) = tspec_(i);
+   }
 }
 
 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const GridFunction &tspec_)
 {
    const int vdim     = tspec_.FESpace()->GetVDim(),
              dof_cnt  = tspec_.Size()/vdim;
+   for (int i = 0; i < dof_cnt*vdim; i++)
+   {
+      tspec(i+idx*dof_cnt) = tspec_(i);
+   }
 
-   const auto tspec__d = tspec_.Read();
-   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   MFEM_FORALL(i, dof_cnt*vdim, tspec_d[i+offset] = tspec__d[i];);
    FinalizeSerialDiscreteTargetSpec();
 }
 
 void DiscreteAdaptTC::SetSerialDiscreteTargetSize(const GridFunction &tspec_)
 {
+
    if (sizeidx > -1) { SetTspecAtIndex(sizeidx, tspec_); return; }
    sizeidx = ncomp;
    SetDiscreteTargetBase(tspec_);
@@ -1256,17 +1257,16 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
 
          Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
                 par_vals_c1, par_vals_c2, par_vals_c3;
+
          Array<int> dofs;
          DenseMatrix D_rho(dim), Q_phi(dim), R_theta(dim);
          tspec_fesv->GetElementVDofs(e_id, dofs);
-         tspec.UseDevice(true);
          tspec.GetSubVector(dofs, tspec_vals);
 
          for (int q = 0; q < nqp; q++)
          {
             const IntegrationPoint &ip = ir.IntPoint(q);
             tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-
             Jtr(q) = Wideal; // Initialize to identity
             for (int d = 0; d < 4; d++)
             {
@@ -1872,17 +1872,6 @@ void AdaptivityEvaluator::SetParMetaInfo(const ParMesh &m,
 }
 #endif
 
-void AdaptivityEvaluator::ClearGeometricFactors()
-{
-#ifdef MFEM_USE_MPI
-   if (pmesh) pmesh->DeleteGeometricFactors();
-   if (pfes) pfes->GetParMesh()->DeleteGeometricFactors();
-#else
-   if (mesh) mesh->DeleteGeometricFactors();
-   if (fes) fes->GetMesh()->DeleteGeometricFactors();
-#endif
-}
-
 AdaptivityEvaluator::~AdaptivityEvaluator()
 {
    delete fes;
@@ -1910,7 +1899,6 @@ void TMOP_Integrator::EnableLimiting(const GridFunction &n0,
 {
    EnableLimiting(n0, w0, lfunc);
    lim_dist = &dist;
-   if (PA.enabled) { EnableLimitingPA(n0); }
 }
 void TMOP_Integrator::EnableLimiting(const GridFunction &n0, Coefficient &w0,
                                      TMOP_LimiterFunction *lfunc)
@@ -1980,11 +1968,11 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
    Jpt.SetSize(dim);
    PMatI.UseExternalData(elfun.GetData(), dof, dim);
 
-   const IntegrationRule *ir = EnergyIntegrationRule(el);
+   const IntegrationRule &ir = EnergyIntegrationRule(el);
 
    energy = 0.0;
-   DenseTensor Jtr(dim, dim, ir->GetNPoints());
-   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+   DenseTensor Jtr(dim, dim, ir.GetNPoints());
+   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
 
    // Limited case.
    Vector shape, p, p0, d_vals;
@@ -2001,11 +1989,11 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
       nodes0->GetSubVector(pos_dofs, pos0V);
       if (lim_dist)
       {
-         lim_dist->GetValues(T.ElementNo, *ir, d_vals);
+         lim_dist->GetValues(T.ElementNo, ir, d_vals);
       }
       else
       {
-         d_vals.SetSize(ir->GetNPoints()); d_vals = 1.0;
+         d_vals.SetSize(ir.GetNPoints()); d_vals = 1.0;
       }
    }
 
@@ -2031,13 +2019,13 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
    Vector zeta_q, zeta0_q;
    if (adaptive_limiting)
    {
-      zeta->GetValues(T.ElementNo, *ir, zeta_q);
-      zeta_0->GetValues(T.ElementNo, *ir, zeta0_q);
+      zeta->GetValues(T.ElementNo, ir, zeta_q);
+      zeta_0->GetValues(T.ElementNo, ir, zeta0_q);
    }
 
-   for (int i = 0; i < ir->GetNPoints(); i++)
+   for (int i = 0; i < ir.GetNPoints(); i++)
    {
-      const IntegrationPoint &ip = ir->IntPoint(i);
+      const IntegrationPoint &ip = ir.IntPoint(i);
       const DenseMatrix &Jtr_i = Jtr(i);
       metric->SetTargetJacobian(Jtr_i);
       CalcInverse(Jtr_i, Jrt);
@@ -2056,8 +2044,7 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
          PMatI.MultTranspose(shape, p);
          pos0.MultTranspose(shape, p0);
          val += lim_normal *
-                lim_func->Eval(p, p0, d_vals(i)) *
-                coeff0->Eval(*Tpr, ip);
+                lim_func->Eval(p, p0, d_vals(i)) * coeff0->Eval(*Tpr, ip);
       }
 
       if (adaptive_limiting)
@@ -2118,14 +2105,14 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
    elvect.SetSize(dof*dim);
    PMatO.UseExternalData(elvect.GetData(), dof, dim);
 
-   const IntegrationRule *ir = ActionIntegrationRule(el);
-   const int nqp = ir->GetNPoints();
+   const IntegrationRule &ir = ActionIntegrationRule(el);
+   const int nqp = ir.GetNPoints();
 
    elvect = 0.0;
    Vector weights(nqp);
    DenseTensor Jtr(dim, dim, nqp);
    DenseTensor dJtr(dim, dim, dim*nqp);
-   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
 
    // Limited case.
    DenseMatrix pos0;
@@ -2142,7 +2129,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       nodes0->GetSubVector(pos_dofs, pos0V);
       if (lim_dist)
       {
-         lim_dist->GetValues(T.ElementNo, *ir, d_vals);
+         lim_dist->GetValues(T.ElementNo, ir, d_vals);
       }
       else
       {
@@ -2162,7 +2149,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
       if (exact_action)
       {
-         targetC->ComputeElementTargetsGradient(*ir, elfun, *Tpr, dJtr);
+         targetC->ComputeElementTargetsGradient(ir, elfun, *Tpr, dJtr);
       }
    }
 
@@ -2172,7 +2159,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
 
    for (int q = 0; q < nqp; q++)
    {
-      const IntegrationPoint &ip = ir->IntPoint(q);
+      const IntegrationPoint &ip = ir.IntPoint(q);
       const DenseMatrix &Jtr_q = Jtr(q);
       metric->SetTargetJacobian(Jtr_q);
       CalcInverse(Jtr_q, Jrt);
@@ -2199,7 +2186,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
          DenseMatrix dwdx(dim);
          for (int d = 0; d < dim; d++)
          {
-            const DenseMatrix &dJtr_q = dJtr(q + d*ir->GetNPoints());
+            const DenseMatrix &dJtr_q = dJtr(q + d * nqp);
             Mult(Jrt, dJtr_q, dwdx );
             d_detW_dx(d) = dwdx.Trace();
          }
@@ -2234,7 +2221,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       }
    }
 
-   if (zeta) { AssembleElemVecAdaptLim(el, weights, *Tpr, *ir, PMatO); }
+   if (zeta) { AssembleElemVecAdaptLim(el, weights, *Tpr, ir, PMatO); }
 
    delete Tpr;
 }
@@ -2253,13 +2240,13 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
    PMatI.UseExternalData(elfun.GetData(), dof, dim);
    elmat.SetSize(dof*dim);
 
-   const IntegrationRule *ir = GradientIntegrationRule(el);
-   const int nqp = ir->GetNPoints();
+   const IntegrationRule &ir = GradientIntegrationRule(el);
+   const int nqp = ir.GetNPoints();
 
    elmat = 0.0;
    Vector weights(nqp);
    DenseTensor Jtr(dim, dim, nqp);
-   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
 
    // Limited case.
    DenseMatrix pos0, grad_grad;
@@ -2276,7 +2263,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
       nodes0->GetSubVector(pos_dofs, pos0V);
       if (lim_dist)
       {
-         lim_dist->GetValues(T.ElementNo, *ir, d_vals);
+         lim_dist->GetValues(T.ElementNo, ir, d_vals);
       }
       else
       {
@@ -2298,7 +2285,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
 
    for (int q = 0; q < nqp; q++)
    {
-      const IntegrationPoint &ip = ir->IntPoint(q);
+      const IntegrationPoint &ip = ir.IntPoint(q);
       const DenseMatrix &Jtr_q = Jtr(q);
       metric->SetTargetJacobian(Jtr_q);
       CalcInverse(Jtr_q, Jrt);
@@ -2315,7 +2302,6 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
 
       // TODO: derivatives of adaptivity-based targets.
 
-      // TODO optimize by symmetry.
       if (coeff0)
       {
          el.CalcShape(ip, shape);
@@ -2341,7 +2327,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
       }
    }
 
-   if (zeta) { AssembleElemGradAdaptLim(el, weights, *Tpr, *ir, elmat); }
+   if (zeta) { AssembleElemGradAdaptLim(el, weights, *Tpr, ir, elmat); }
 
    delete Tpr;
 }
@@ -2512,10 +2498,10 @@ void TMOP_Integrator::AssembleElementVectorFD(const FiniteElement &el,
    // Contributions from adaptive limiting (exact derivatives).
    if (zeta)
    {
-      const IntegrationRule *ir = ActionIntegrationRule(el);
-      const int nqp = ir->GetNPoints();
+      const IntegrationRule &ir = ActionIntegrationRule(el);
+      const int nqp = ir.GetNPoints();
       DenseTensor Jtr(dim, dim, nqp);
-      targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+      targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
 
       IsoparametricTransformation Tpr;
       Tpr.SetFE(&el);
@@ -2527,11 +2513,11 @@ void TMOP_Integrator::AssembleElementVectorFD(const FiniteElement &el,
       Vector weights(nqp);
       for (int q = 0; q < nqp; q++)
       {
-         weights(q) = ir->IntPoint(q).weight * Jtr(q).Det();
+         weights(q) = ir.IntPoint(q).weight * Jtr(q).Det();
       }
 
       PMatO.UseExternalData(elvect.GetData(), dof, dim);
-      AssembleElemVecAdaptLim(el, weights, Tpr, *ir, PMatO);
+      AssembleElemVecAdaptLim(el, weights, Tpr, ir, PMatO);
    }
 }
 
@@ -2608,10 +2594,10 @@ void TMOP_Integrator::AssembleElementGradFD(const FiniteElement &el,
    // Contributions from adaptive limiting.
    if (zeta)
    {
-      const IntegrationRule *ir = GradientIntegrationRule(el);
-      const int nqp = ir->GetNPoints();
+      const IntegrationRule &ir = GradientIntegrationRule(el);
+      const int nqp = ir.GetNPoints();
       DenseTensor Jtr(dim, dim, nqp);
-      targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
+      targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
 
       IsoparametricTransformation Tpr;
       Tpr.SetFE(&el);
@@ -2623,10 +2609,10 @@ void TMOP_Integrator::AssembleElementGradFD(const FiniteElement &el,
       Vector weights(nqp);
       for (int q = 0; q < nqp; q++)
       {
-         weights(q) = ir->IntPoint(q).weight * Jtr(q).Det();
+         weights(q) = ir.IntPoint(q).weight * Jtr(q).Det();
       }
 
-      AssembleElemGradAdaptLim(el, weights, Tpr, *ir, elmat);
+      AssembleElemGradAdaptLim(el, weights, Tpr, ir, elmat);
    }
 }
 
@@ -2656,33 +2642,32 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
    Array<int> vdofs;
    Vector x_vals;
    const FiniteElementSpace* const fes = x.FESpace();
-   const FiniteElement *fe = fes->GetFE(0);
 
-   const int dof = fes->GetFE(0)->GetDof(), dim = fes->GetFE(0)->GetDim();
-
-   DSh.SetSize(dof, dim);
+   const int dim = fes->GetMesh()->Dimension();
    Jrt.SetSize(dim);
    Jpr.SetSize(dim);
    Jpt.SetSize(dim);
 
-   const IntegrationRule *ir = EnergyIntegrationRule(*fe);
-   const int nqp =  ir->GetNPoints();
-   DenseTensor Jtr(dim, dim, nqp);
-
    metric_energy = 0.0;
    lim_energy = 0.0;
    for (int i = 0; i < fes->GetNE(); i++)
    {
-      fe = fes->GetFE(i);
+      const FiniteElement *fe = fes->GetFE(i);
+      const IntegrationRule &ir = EnergyIntegrationRule(*fe);
+      const int nqp = ir.GetNPoints();
+      DenseTensor Jtr(dim, dim, nqp);
+      const int dof = fe->GetDof();
+      DSh.SetSize(dof, dim);
+
       fes->GetElementVDofs(i, vdofs);
       x.GetSubVector(vdofs, x_vals);
       PMatI.UseExternalData(x_vals.GetData(), dof, dim);
 
-      targetC->ComputeElementTargets(i, *fe, *ir, x_vals, Jtr);
+      targetC->ComputeElementTargets(i, *fe, ir, x_vals, Jtr);
 
       for (int q = 0; q < nqp; q++)
       {
-         const IntegrationPoint &ip = ir->IntPoint(q);
+         const IntegrationPoint &ip = ir.IntPoint(q);
          metric->SetTargetJacobian(Jtr(q));
          CalcInverse(Jtr(q), Jrt);
          const double weight = ip.weight * Jtr(q).Det();
@@ -2706,9 +2691,9 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
                                     const FiniteElementSpace &fes)
 {
    const FiniteElement *fe = fes.GetFE(0);
-   const IntegrationRule *ir = EnergyIntegrationRule(*fe);
+   const IntegrationRule &ir = EnergyIntegrationRule(*fe);
    const int NE = fes.GetMesh()->GetNE(), dim = fe->GetDim(),
-             dof = fe->GetDof(), nsp = ir->GetNPoints();
+             dof = fe->GetDof(), nsp = ir.GetNPoints();
 
    Array<int> xdofs(dof * dim);
    DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
@@ -2725,7 +2710,7 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
       detv_sum = 0.;
       for (int j = 0; j < nsp; j++)
       {
-         fes.GetFE(i)->CalcDShape(ir->IntPoint(j), dshape);
+         fes.GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
          MultAtB(pos, dshape, Jpr);
          detv_sum += std::fabs(Jpr.Det());
       }
@@ -2737,8 +2722,6 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
 
 void TMOP_Integrator::UpdateAfterMeshChange(const Vector &new_x)
 {
-   PA.setup_Jtr = false;
-   PA.setup_Grad = false;
    // Update zeta if adaptive limiting is enabled.
    if (zeta) { adapt_eval->ComputeAtNewPosition(new_x, *zeta); }
 }
@@ -2894,49 +2877,6 @@ void TMOPComboIntegrator::ParEnableNormalization(const ParGridFunction &x)
 }
 #endif
 
-void TMOPComboIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   for (int i = 0; i < tmopi.Size(); i++)
-   {
-      tmopi[i]->AssemblePA(fes);
-   }
-}
-
-void TMOPComboIntegrator::AssembleGradientDiagonalPA(const Vector &xe,
-                                                     Vector &de) const
-{
-   for (int i = 0; i < tmopi.Size(); i++)
-   {
-      tmopi[i]->AssembleGradientDiagonalPA(xe, de);
-   }
-}
-
-void TMOPComboIntegrator::AddMultPA(const Vector &xe, Vector &ye) const
-{
-   for (int i = 0; i < tmopi.Size(); i++)
-   {
-      tmopi[i]->AddMultPA(xe, ye);
-   }
-}
-
-void TMOPComboIntegrator::AddMultGradPA(const Vector &xe, const Vector &re,
-                                        Vector &ce) const
-{
-   for (int i = 0; i < tmopi.Size(); i++)
-   {
-      tmopi[i]->AddMultGradPA(xe, re, ce);
-   }
-}
-
-double TMOPComboIntegrator::GetGridFunctionEnergyPA(const Vector &xe) const
-{
-   double energy = 0.0;
-   for (int i = 0; i < tmopi.Size(); i++)
-   {
-      energy += tmopi[i]->GetGridFunctionEnergyPA(xe);
-   }
-   return energy;
-}
 
 void InterpolateTMOP_QualityMetric(TMOP_QualityMetric &metric,
                                    const TargetConstructor &tc,

fem/tmop.hpp

@@ -68,10 +68,6 @@ public:
    */
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const = 0;
-
-   /** @brief Return the metric ID.
-    */
-   virtual int Id() const { return 0; }
 };
 
 
@@ -89,8 +85,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
-   virtual int Id() const { return 1; }
 };
 
 /// Skew metric, 2D.
@@ -182,8 +176,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
-   virtual int Id() const { return 2; }
 };
 
 /// Shape & area, ideal barrier metric, 2D
@@ -200,8 +192,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
-   virtual int Id() const { return 7; }
 };
 
 /// Shape & area metric, 2D
@@ -288,6 +278,7 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
 };
 
 /// Shape, ideal barrier metric, 2D
@@ -305,6 +296,7 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
 };
 
 /// Area, ideal barrier metric, 2D
@@ -322,7 +314,6 @@ public:
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
 
-   virtual int Id() const { return 77; }
 };
 
 /// Shape & orientation metric, 2D.
@@ -409,8 +400,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
-   virtual int Id() const { return 302; }
 };
 
 /// Shape, ideal barrier metric, 3D
@@ -427,8 +416,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
-   virtual int Id() const { return 303; }
 };
 
 /// Volume metric, 3D
@@ -445,8 +432,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
-   virtual int Id() const { return 315; }
 };
 
 /// Volume, ideal barrier metric, 3D
@@ -481,8 +466,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
-   virtual int Id() const { return 321; }
 };
 
 /// Shifted barrier form of 3D metric 16 (volume, ideal barrier metric), 3D
@@ -606,8 +589,6 @@ public:
 
    virtual void ComputeAtNewPosition(const Vector &new_nodes,
                                      Vector &new_field) = 0;
-
-   void ClearGeometricFactors();
 };
 
 /** @brief Base class representing target-matrix construction algorithms for
@@ -683,14 +664,9 @@ public:
        nodes are used by all target types except IDEAL_SHAPE_UNIT_SIZE. */
    void SetNodes(const GridFunction &n) { nodes = &n; avg_volume = 0.0; }
 
-   /** @brief Get the nodes to be used in the target-matrix construction. */
-   const GridFunction *GetNodes() const { return nodes; }
-
    /// Used by target type IDEAL_SHAPE_EQUAL_SIZE. The default volume scale is 1.
    void SetVolumeScale(double vol_scale) { volume_scale = vol_scale; }
 
-   const TargetType &Type() const { return target_type; }
-
    /// Checks if the target matrices contain non-trivial size specification.
    virtual bool ContainsVolumeInfo() const;
 
@@ -702,16 +678,6 @@ public:
                                       const Vector &elfun,
                                       DenseTensor &Jtr) const;
 
-   template<int DIM>
-   bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
-                                const IntegrationRule *ir,
-                                DenseTensor &Jtr,
-                                const Vector &xe = Vector()) const;
-   virtual bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
-                                        const IntegrationRule *ir,
-                                        DenseTensor &Jtr,
-                                        const Vector &xe = Vector()) const;
-
    virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
                                               const Vector &elfun,
                                               IsoparametricTransformation &Tpr,
@@ -757,11 +723,6 @@ public:
                                       const Vector &elfun,
                                       DenseTensor &Jtr) const;
 
-   virtual bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
-                                        const IntegrationRule *ir,
-                                        DenseTensor &Jtr,
-                                        const Vector &xe = Vector()) const;
-
    virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
                                               const Vector &elfun,
                                               IsoparametricTransformation &Tpr,
@@ -804,19 +765,12 @@ protected:
    // Owned.
    AdaptivityEvaluator *adapt_eval;
 
-   // PA extension
-   struct { mutable Vector tspec_e; } PA;
-
-   void FinalizeSerialDiscreteTargetSpec();
-#ifdef MFEM_USE_MPI
-   void FinalizeParDiscreteTargetSpec(const ParGridFunction &tspec_);
-#endif
-
-public: // MFEM_FORALL nvcc restriction that it must be public
    void SetDiscreteTargetBase(const GridFunction &tspec_);
    void SetTspecAtIndex(int idx, const GridFunction &tspec_);
+   void FinalizeSerialDiscreteTargetSpec();
 #ifdef MFEM_USE_MPI
    void SetTspecAtIndex(int idx, const ParGridFunction &tspec_);
+   void FinalizeParDiscreteTargetSpec(const ParGridFunction &tspec_);
 #endif
 
 public:
@@ -901,7 +855,6 @@ public:
    const Vector &GetTspecPert1H()   { return tspec_pert1h; }
    const Vector &GetTspecPert2H()   { return tspec_pert2h; }
    const Vector &GetTspecPertMixH() { return tspec_pertmix; }
-   const FiniteElementSpace *GetTspecFesv() const { return tspec_fesv; }
 
    /** @brief Given an element and quadrature rule, computes ref->target
        transformation Jacobians for each quadrature point in the element.
@@ -913,11 +866,6 @@ public:
                                       const Vector &elfun,
                                       DenseTensor &Jtr) const;
 
-   virtual bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
-                                        const IntegrationRule *ir,
-                                        DenseTensor &Jtr,
-                                        const Vector &xe = Vector()) const;
-
    virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
                                               const Vector &elfun,
                                               IsoparametricTransformation &Tpr,
@@ -942,6 +890,10 @@ protected:
    TMOP_QualityMetric *metric;        // not owned
    const TargetConstructor *targetC;  // not owned
 
+   // Custom integration rules.
+   IntegrationRules *IntegRules;
+   int integ_order;
+
    // Weight Coefficient multiplying the quality metric term.
    Coefficient *coeff1; // not owned, if NULL -> coeff1 is 1.
    // Normalization factor for the metric term.
@@ -993,25 +945,10 @@ protected:
    //        output - the result of AssembleElementVector() (dof x dim).
    DenseMatrix DSh, DS, Jrt, Jpr, Jpt, P, PMatI, PMatO;
 
-   // PA extension
-   struct
-   {
-      bool enabled;
-      int dim, ne, nq;
-      mutable DenseTensor Jtr;
-      mutable bool setup_Grad, setup_Jtr;
-      mutable Vector E, O, W, X0, H, C0, LD, H0;
-      const DofToQuad *maps;
-      const DofToQuad *maps_lim = nullptr;
-      const GeometricFactors *geom;
-      const FiniteElementSpace *fes;
-      const Operator *R;
-      const IntegrationRule *ir;
-   } PA;
-
    void ComputeNormalizationEnergies(const GridFunction &x,
                                      double &metric_energy, double &lim_energy);
 
+
    void AssembleElementVectorExact(const FiniteElement &el,
                                    ElementTransformation &T,
                                    const Vector &elfun, Vector &elvect);
@@ -1055,17 +992,21 @@ protected:
       nodes0 = NULL; coeff0 = NULL; lim_dist = NULL; lim_func = NULL;
    }
 
-   const IntegrationRule *EnergyIntegrationRule(const FiniteElement &el) const
+   const IntegrationRule &EnergyIntegrationRule(const FiniteElement &el) const
    {
-      return (IntRule) ? IntRule
-             /*     */ : &(IntRules.Get(el.GetGeomType(), 2*el.GetOrder() + 3));
+      if (IntegRules)
+      {
+         return IntegRules->Get(el.GetGeomType(), integ_order);
+      }
+      return (IntRule) ? *IntRule
+             /*     */ : IntRules.Get(el.GetGeomType(), 2*el.GetOrder() + 3);
    }
-   const IntegrationRule *ActionIntegrationRule(const FiniteElement &el) const
+   const IntegrationRule &ActionIntegrationRule(const FiniteElement &el) const
    {
       // TODO the energy most likely needs less integration points.
       return EnergyIntegrationRule(el);
    }
-   const IntegrationRule *GradientIntegrationRule(const FiniteElement &el) const
+   const IntegrationRule &GradientIntegrationRule(const FiniteElement &el) const
    {
       // TODO the action and energy most likely need less integration points.
       return EnergyIntegrationRule(el);
@@ -1075,17 +1016,25 @@ public:
    /** @param[in] m  TMOP_QualityMetric that will be integrated (not owned).
        @param[in] tc Target-matrix construction algorithm to use (not owned). */
    TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
-      : metric(m), targetC(tc),
+      : metric(m), targetC(tc), IntegRules(NULL), integ_order(-1),
         coeff1(NULL), metric_normal(1.0),
         nodes0(NULL), coeff0(NULL),
         lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
         zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
-   { PA.enabled = false; }
+   { }
 
    ~TMOP_Integrator();
 
+   /// Prescribe a set of integration rules; relevant for mixed meshes.
+   /** This function has priority over SetIntRule(), if both are called. */
+   void SetIntegrationRules(IntegrationRules &irules, int order)
+   {
+      IntegRules = &irules;
+      integ_order = order;
+   }
+
    /// Sets a scaling Coefficient for the quality metric term of the integrator.
    /** With this addition, the integrator becomes
           @f$ \int w1 W(Jpt) dx @f$.
@@ -1150,45 +1099,6 @@ public:
    virtual void AssembleElementGrad(const FiniteElement &el,
                                     ElementTransformation &T,
                                     const Vector &elfun, DenseMatrix &elmat);
-   /// PA extension
-   void SetupGradPA(const Vector &xe) const;
-   void EnableLimitingPA(const GridFunction &n0);
-   void ComputeElementTargetsPA(const Vector &xe = Vector()) const;
-
-   using NonlinearFormIntegrator::GetGridFunctionEnergyPA;
-   double GetGridFunctionEnergyPA_2D(const Vector&) const;
-   double GetGridFunctionEnergyPA_C0_2D(const Vector&) const;
-   double GetGridFunctionEnergyPA_3D(const Vector&) const;
-   double GetGridFunctionEnergyPA_C0_3D(const Vector&) const;
-   virtual double GetGridFunctionEnergyPA(const Vector&) const;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace&);
-
-   virtual void AssembleGradientDiagonalPA(const Vector&, Vector&) const;
-   void AssembleDiagonalPA_2D(Vector&) const;
-   void AssembleDiagonalPA_3D(Vector&) const;
-   void AssembleDiagonalPA_C0_2D(Vector&) const;
-   void AssembleDiagonalPA_C0_3D(Vector&) const;
-
-   using NonlinearFormIntegrator::AddMultPA;
-   void AddMultPA_2D(const Vector&, Vector&) const;
-   void AddMultPA_3D(const Vector&, Vector&) const;
-   void AddMultPA_C0_2D(const Vector&, Vector&) const;
-   void AddMultPA_C0_3D(const Vector&, Vector&) const;
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   using NonlinearFormIntegrator::AddMultGradPA;
-   void AddMultGradPA_2D(const Vector&, Vector&) const;
-   void AddMultGradPA_3D(const Vector&, const Vector&, Vector&) const;
-   void AddMultGradPA_C0_2D(const Vector&, const Vector&, Vector&) const;
-   void AddMultGradPA_C0_3D(const Vector&, const Vector&, Vector&) const;
-   virtual void AddMultGradPA(const Vector&, const Vector&, Vector&) const;
-
-   void AssembleGradPA_2D(const Vector&) const;
-   void AssembleGradPA_3D(const Vector&) const;
-   void AssembleGradPA_C0_2D(const Vector&) const;
-   void AssembleGradPA_C0_3D(const Vector&) const;
 
    DiscreteAdaptTC *GetDiscreteAdaptTC() const { return discr_tc; }
 
@@ -1211,17 +1121,6 @@ public:
 
    /** @brief Flag to control if exact action of Integration is effected. */
    void SetExactActionFlag(bool flag_) { exact_action = flag_; }
-
-   void ReleaseTemporaryMemory()
-   {
-      if (PA.enabled)
-      {
-         PA.H.GetMemory().DeleteDevice();
-         PA.H0.GetMemory().DeleteDevice();
-         //PA.Jtr.GetMemory().DeleteDevice();
-         //PA.setup_Jtr = false;
-      }
-   }
 };
 
 class TMOPComboIntegrator : public NonlinearFormIntegrator
@@ -1270,17 +1169,6 @@ public:
 #ifdef MFEM_USE_MPI
    void ParEnableNormalization(const ParGridFunction &x);
 #endif
-
-   /// PA extension
-   using NonlinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace&);
-   virtual void AssembleGradientDiagonalPA(const Vector&, Vector&) const;
-   using NonlinearFormIntegrator::AddMultPA;
-   virtual void AddMultPA(const Vector&, Vector&) const;
-   using NonlinearFormIntegrator::AddMultGradPA;
-   virtual void AddMultGradPA(const Vector&, const Vector&, Vector&) const;
-   using NonlinearFormIntegrator::GetGridFunctionEnergyPA;
-   virtual double GetGridFunctionEnergyPA(const Vector&) const;
 };
 
 /// Interpolates the @a metric's values at the nodes of @a metric_gf.

fem/tmop_pa.cpp

@@ -1,363 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "linearform.hpp"
-#include "pgridfunc.hpp"
-#include "tmop_tools.hpp"
-#include "quadinterpolator.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-
-void TMOP_Integrator::SetupGradPA(const Vector &xe) const
-{
-   MFEM_VERIFY(PA.R, "PA extension setup has not been done!");
-   PA.setup_Grad = true;
-
-   if (PA.dim == 2)
-   {
-      AssembleGradPA_2D(xe);
-      if (coeff0) { AssembleGradPA_C0_2D(xe); }
-   }
-
-   if (PA.dim == 3)
-   {
-      AssembleGradPA_3D(xe);
-      if (coeff0) { AssembleGradPA_C0_3D(xe); }
-   }
-}
-
-// We might come here w/o knowing that PA will be used.
-// It is the case when EnableLimiting is called before the Setup => AssemblePA.
-void TMOP_Integrator::EnableLimitingPA(const GridFunction &n0)
-{
-   MFEM_VERIFY(PA.enabled, "EnableLimitingPA but PA is not enabled!");
-   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-
-   // Nodes0
-   const FiniteElementSpace *n0_fes = n0.FESpace();
-   const Operator *n0_R = n0_fes->GetElementRestriction(ordering);
-   PA.X0.SetSize(n0_R->Height(), Device::GetMemoryType());
-   PA.X0.UseDevice(true);
-   n0_R->Mult(n0, PA.X0);
-
-   // Get the 1D maps for the distance FE space.
-   const IntegrationRule &ir = *EnergyIntegrationRule(*n0.FESpace()->GetFE(0));
-   PA.maps_lim =
-      &lim_dist->FESpace()->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-
-   // lim_dist & lim_func checks
-   MFEM_VERIFY(lim_dist, "No lim_dist!")
-   const FiniteElementSpace *ld_fes = lim_dist->FESpace();
-   const Operator *ld_R = ld_fes->GetElementRestriction(ordering);
-   MFEM_VERIFY(ld_R, "No lim_dist restriction operator found!");
-   PA.LD.SetSize(ld_R->Height(), Device::GetMemoryType());
-   PA.LD.UseDevice(true);
-   ld_R->Mult(*lim_dist, PA.LD);
-
-   // Only TMOP_QuadraticLimiter is supported
-   MFEM_VERIFY(lim_func, "No lim_func!")
-   MFEM_VERIFY(dynamic_cast<TMOP_QuadraticLimiter*>(lim_func),
-               "Only TMOP_QuadraticLimiter is supported");
-}
-
-bool TargetConstructor::ComputeElementTargetsPA(const FiniteElementSpace *fes,
-                                                const IntegrationRule *ir,
-                                                DenseTensor &Jtr,
-                                                const Vector &xe) const
-{
-   MFEM_VERIFY(Jtr.SizeI() == Jtr.SizeJ() && Jtr.SizeI() > 1, "");
-   const int dim = Jtr.SizeI();
-   if (dim == 2) { return ComputeElementTargetsPA<2>(fes, ir, Jtr, xe); }
-   if (dim == 3) { return ComputeElementTargetsPA<3>(fes, ir, Jtr, xe); }
-   return false;
-}
-
-bool AnalyticAdaptTC::ComputeElementTargetsPA(const FiniteElementSpace *fes,
-                                              const IntegrationRule *ir,
-                                              DenseTensor &Jtr,
-                                              const Vector &xe) const
-{
-   return false;
-}
-
-
-// Code paths leading to ComputeElementTargets:
-// - GetElementEnergy(elfun) which is done through GetGridFunctionEnergyPA(x)
-// - AssembleElementVectorExact(elfun)
-// - AssembleElementGradExact(elfun)
-// - EnableNormalization(x) -> ComputeNormalizationEnergies(x)
-// - (AssembleElementVectorFD(elfun))
-// - (AssembleElementGradFD(elfun))
-// ============================================================================
-//      - TargetConstructor():
-//          - IDEAL_SHAPE_UNIT_SIZE: Wideal
-//          - IDEAL_SHAPE_EQUAL_SIZE: α * Wideal
-//          - IDEAL_SHAPE_GIVEN_SIZE: β * Wideal
-//          - GIVEN_SHAPE_AND_SIZE:   β * Wideal
-//      - AnalyticAdaptTC(elfun):
-//          - GIVEN_FULL: matrix_tspec->Eval(Jtr(elfun))
-//      - DiscreteAdaptTC():
-//          - IDEAL_SHAPE_GIVEN_SIZE: size^{1.0/dim} * Jtr(i)  (size)
-//          - GIVEN_SHAPE_AND_SIZE:   Jtr(i) *= D_rho          (ratio)
-//                                    Jtr(i) *= Q_phi          (skew)
-//                                    Jtr(i) *= R_theta        (orientation)
-void TMOP_Integrator::ComputeElementTargetsPA(const Vector &xe) const
-{
-   PA.setup_Jtr = false;
-   const FiniteElementSpace *fes = PA.fes;
-   const IntegrationRule *ir = EnergyIntegrationRule(*fes->GetFE(0));
-   const TargetConstructor::TargetType &target_type = targetC->Type();
-   const DiscreteAdaptTC *discr_tc = GetDiscreteAdaptTC();
-
-   // Skip when TargetConstructor needs the nodes but have not been set
-   const bool use_nodes =
-      target_type == TargetConstructor::IDEAL_SHAPE_EQUAL_SIZE ||
-      target_type == TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE ||
-      target_type == TargetConstructor::GIVEN_SHAPE_AND_SIZE;
-   if (targetC && !discr_tc && use_nodes && !targetC->GetNodes()) { return; }
-
-   // Try to use the TargetConstructor ComputeElementTargetsPA
-   PA.setup_Jtr = targetC->ComputeElementTargetsPA(fes, ir, PA.Jtr);
-   if (PA.setup_Jtr) { return; }
-
-   // Defaulting to host version
-   PA.Jtr.HostWrite();
-
-   const int NE = PA.ne;
-   const int NQ = PA.nq;
-   const int dim = PA.dim;
-   DenseTensor &Jtr = PA.Jtr;
-
-   Vector x;
-   const bool useable_input_vector = xe.Size() > 0;
-   const bool use_input_vector = target_type == TargetConstructor::GIVEN_FULL;
-
-   if (use_input_vector && !useable_input_vector) { return; }
-
-   if (discr_tc && !discr_tc->GetTspecFesv()) { return; }
-
-   if (use_input_vector)
-   {
-      x.SetSize(PA.R->Width(), Device::GetMemoryType());
-      x.UseDevice(true);
-      PA.R->MultTranspose(xe, x);
-      // Scale by weights
-      const int N = PA.W.Size();
-      const auto W = Reshape(PA.W.Read(), N);
-      auto X = Reshape(x.ReadWrite(), N);
-      MFEM_FORALL(i, N, X(i) /= W(i););
-   }
-
-   // Use TargetConstructor::ComputeElementTargets to fill the PA.Jtr
-   Vector elfun;
-   Array<int> vdofs;
-   DenseTensor J;
-   for (int e = 0; e < NE; e++)
-   {
-      const FiniteElement &fe = *fes->GetFE(e);
-      if (use_input_vector)
-      {
-         fes->GetElementVDofs(e, vdofs);
-         x.GetSubVector(vdofs, elfun);
-      }
-      J.UseExternalData(Jtr(e*NQ).Data(), dim, dim, NQ);
-      targetC->ComputeElementTargets(e, fe, *ir, elfun, J);
-   }
-   PA.setup_Jtr = true;
-}
-
-void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   PA.enabled = true;
-   MFEM_ASSERT(fes.GetMesh()->GetNE() > 0, "");
-   PA.ir = EnergyIntegrationRule(*fes.GetFE(0));
-   const IntegrationRule *ir = PA.ir;
-   MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
-               "PA Only supports Ordering::byNODES!");
-
-   PA.fes = &fes;
-   Mesh *mesh = fes.GetMesh();
-   const int nq = PA.nq = ir->GetNPoints();
-   const int ne = PA.ne = fes.GetMesh()->GetNE();
-   const int dim = PA.dim = mesh->Dimension();
-   MFEM_VERIFY(PA.dim == 2 || PA.dim == 3, "Not yet implemented!");
-
-   const DofToQuad::Mode mode = DofToQuad::TENSOR;
-   PA.maps = &fes.GetFE(0)->GetDofToQuad(*ir, mode);
-   PA.geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
-
-   // Energy vector
-   PA.E.UseDevice(true);
-   PA.E.SetSize(ne*nq, Device::GetDeviceMemoryType());
-
-   // Setup initialization
-   PA.setup_Jtr = false;
-   PA.setup_Grad = false;
-
-
-#ifdef MFEM_USE_UMPIRE
-   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
-      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
-#else
-   const MemoryType temp_type = Device::GetDeviceMemoryType();
-#endif
-
-   // H for Grad
-   PA.H.SetSize(dim*dim * dim*dim * nq*ne, temp_type);
-   // H0 for coeff0
-   PA.H0.SetSize(dim * dim * nq*ne, temp_type);
-
-   // Restriction setup
-   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   PA.R = fes.GetElementRestriction(ordering);
-   MFEM_VERIFY(PA.R, "Not yet implemented!");
-
-   // Weight of the R^t
-   PA.W.SetSize(PA.R->Width(), Device::GetDeviceMemoryType());
-   PA.W.UseDevice(true);
-   PA.O.SetSize(dim*ne*nq, Device::GetDeviceMemoryType());
-   PA.O.UseDevice(true);
-   PA.O = 1.0;
-   PA.R->MultTranspose(PA.O, PA.W);
-
-   // Scalar vector of '1'
-   PA.O.SetSize(ne*nq, Device::GetDeviceMemoryType());
-   PA.O = 1.0;
-
-   // TargetConstructor TargetType setup
-   PA.Jtr.SetSize(dim, dim, PA.ne*PA.nq);//, temp_type);
-   ComputeElementTargetsPA();
-
-   // Coeff0 PA.C0
-   PA.C0.UseDevice(true);
-   if (coeff0 == nullptr)
-   {
-      PA.C0.SetSize(1, Device::GetMemoryType());
-      PA.C0.HostWrite();
-      PA.C0(0) = 0.0;
-   }
-   else if (ConstantCoefficient* cQ =
-               dynamic_cast<ConstantCoefficient*>(coeff0))
-   {
-      PA.C0.SetSize(1, Device::GetMemoryType());
-      PA.C0.HostWrite();
-      PA.C0(0) = cQ->constant;
-   }
-   else
-   {
-      PA.C0.SetSize(PA.nq * PA.ne, Device::GetMemoryType());
-      auto C0 = Reshape(PA.C0.HostWrite(), PA.nq, PA.ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C0(q,e) = coeff0->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-
-   if (coeff0)
-   {
-      MFEM_VERIFY(nodes0, "nodes0 has not been set!");
-      EnableLimitingPA(*nodes0);
-   }
-}
-
-void TMOP_Integrator::AssembleGradientDiagonalPA(const Vector &xe,
-                                                 Vector &de) const
-{
-   MFEM_VERIFY(PA.R, "PA extension setup has not been done!");
-
-   if (!PA.setup_Jtr) { ComputeElementTargetsPA(xe); }
-
-   if (!PA.setup_Grad) { SetupGradPA(xe); }
-
-   if (PA.dim == 2)
-   {
-      AssembleDiagonalPA_2D(de);
-      if (coeff0) { AssembleDiagonalPA_C0_2D(de); }
-   }
-   else if (PA.dim == 3)
-   {
-      AssembleDiagonalPA_3D(de);
-      if (coeff0) { AssembleDiagonalPA_C0_3D(de); }
-   }
-   else
-   {
-      MFEM_ABORT("3D diagonal computation is WIP.");
-   }
-}
-
-void TMOP_Integrator::AddMultPA(const Vector &xe, Vector &ye) const
-{
-   if (!PA.setup_Jtr) { ComputeElementTargetsPA(); }
-
-   if (PA.dim == 2)
-   {
-      AddMultPA_2D(xe,ye);
-      if (coeff0) { AddMultPA_C0_2D(xe,ye); }
-   }
-
-   if (PA.dim == 3)
-   {
-      AddMultPA_3D(xe,ye);
-      if (coeff0) { AddMultPA_C0_3D(xe,ye); }
-   }
-}
-
-void TMOP_Integrator::AddMultGradPA(const Vector &xe,
-                                    const Vector &re, Vector &ce) const
-{
-   if (!PA.setup_Jtr) { ComputeElementTargetsPA(xe); }
-
-   if (!PA.setup_Grad) { SetupGradPA(xe); }
-
-   if (PA.dim == 2)
-   {
-      AddMultGradPA_2D(re,ce);
-      if (coeff0) { AddMultGradPA_C0_2D(xe,re,ce); }
-   }
-
-   if (PA.dim == 3)
-   {
-      AddMultGradPA_3D(xe,re,ce);
-      if (coeff0) { AddMultGradPA_C0_3D(xe,re,ce); }
-   }
-}
-
-double TMOP_Integrator::GetGridFunctionEnergyPA(const Vector &xe) const
-{
-   double energy = 0.0;
-
-   ComputeElementTargetsPA(xe);
-
-   if (PA.dim == 2)
-   {
-      energy = GetGridFunctionEnergyPA_2D(xe);
-      if (coeff0) { energy += GetGridFunctionEnergyPA_C0_2D(xe); }
-   }
-
-   if (PA.dim == 3)
-   {
-      energy = GetGridFunctionEnergyPA_3D(xe);
-      if (coeff0) { energy += GetGridFunctionEnergyPA_C0_3D(xe); }
-   }
-
-   return energy;
-}
-
-} // namespace mfem

fem/tmop_pa.hpp

@@ -1,143 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_TMOP_PA_HPP
-#define MFEM_TMOP_PA_HPP
-
-#include "../config/config.hpp"
-#include "../linalg/dtensor.hpp"
-
-#include "../fem/kernels.hpp"
-
-#include <unordered_map>
-
-namespace mfem
-{
-
-namespace kernels
-{
-
-/// Generic emplace
-template<typename K, const int N,
-         typename Key_t = typename K::Key_t,
-         typename Kernel_t = typename K::Kernel_t>
-void emplace(std::unordered_map<Key_t, Kernel_t> &map)
-{
-   constexpr Key_t key = K::template GetKey<N>();
-   constexpr Kernel_t value = K::template GetValue<key>();
-   map.emplace(key, value);
-}
-
-/// Instances
-template<class K, typename T, T... idx>
-struct instances
-{
-   static void Fill(std::unordered_map<typename K::Key_t,
-                    typename K::Kernel_t> &map)
-   {
-      using unused = int[];
-      (void) unused {0, (emplace<K,idx>(map), 0)... };
-   }
-};
-
-/// Cat instances
-template<class K, typename Offset, typename Lhs, typename Rhs> struct cat;
-template<class K, typename T, T Offset, T... Lhs, T... Rhs>
-struct cat<K, std::integral_constant<T, Offset>,
-          instances<K, T, Lhs...>,
-          instances<K, T, Rhs...> >
-{ using type = instances<K, T, Lhs..., (Offset + Rhs)...>; };
-
-/// Sequence, empty and one element terminal cases
-template<class K, typename T, typename N>
-struct sequence
-{
-   using Lhs = std::integral_constant<T, N::value/2>;
-   using Rhs = std::integral_constant<T, N::value-Lhs::value>;
-   using type = typename cat<K, Lhs,
-         typename sequence<K, T, Lhs>::type,
-         typename sequence<K, T, Rhs>::type>::type;
-};
-
-template<class K, typename T>
-struct sequence<K, T, std::integral_constant<T,0> >
-{ using type = instances<K,T>; };
-
-template<class K, typename T>
-struct sequence<K, T, std::integral_constant<T,1> >
-{ using type = instances<K,T,0>; };
-
-/// Make_sequence
-template<class Instance, typename T = typename Instance::Key_t>
-using make_sequence =
-   typename sequence<Instance, T, std::integral_constant<T,Instance::N> >::type;
-
-/// Instantiator class
-template<class Instance,
-         typename Key_t = typename Instance::Key_t,
-         typename Return_t = typename Instance::Return_t,
-         typename Kernel_t = typename Instance::Kernel_t>
-class Instantiator
-{
-private:
-   using map_t = std::unordered_map<Key_t, Kernel_t>;
-   map_t map;
-
-public:
-   Instantiator() { make_sequence<Instance>().Fill(map); }
-
-   bool Find(const Key_t id)
-   {
-      return (map.find(id) != map.end()) ? true : false;
-   }
-
-   Kernel_t At(const Key_t id) { return map.at(id); }
-};
-
-/// MFEM_REGISTER_TMOP_KERNELS macro:
-/// - forward declaration of the kernel
-/// - kernel pointer declaration
-/// - struct K##name##_T definition
-/// - Instantiator definition
-/// - re-use kernel return type and name before its body
-#define MFEM_REGISTER_TMOP_KERNELS(return_t, kernel, ...) \
-template<int T_D1D = 0, int T_Q1D = 0, int T_MAX = 0> \
-    return_t kernel(__VA_ARGS__);\
-typedef return_t (*kernel##_p)(__VA_ARGS__);\
-struct K##kernel##_T {\
-   static const int N = 14;\
-   using Key_t = std::size_t;\
-   using Kernel_t = kernel##_p;\
-   using Return_t = return_t;\
-   template<Key_t I> static constexpr Key_t GetKey() noexcept { return \
-     I==0 ? 0x22 : I==1 ? 0x23 : I==2 ? 0x24 : I==3 ? 0x25 : I==4 ? 0x26 :\
-     I==5 ? 0x33 : I==6 ? 0x34 : I==7 ? 0x35 : I==8 ? 0x36  :\
-     I==9 ? 0x44 : I==10 ? 0x45 : I==11 ? 0x46 :\
-     I==12 ? 0x55 : I==13 ? 0x56 : 0; }\
-   template<Key_t ID> static constexpr Kernel_t GetValue() noexcept\
-   { return &kernel<(ID>>4)&0xF, ID&0xF>; }\
-};\
-static kernels::Instantiator<K##kernel##_T> K##kernel;\
-template<int T_D1D, int T_Q1D, int T_MAX> return_t kernel(__VA_ARGS__)
-
-/// MFEM_LAUNCH_TMOP_KERNEL macro
-#define MFEM_LAUNCH_TMOP_KERNEL(kernel, id, ...)\
-if (K##kernel.Find(id)) { return K##kernel.At(id)(__VA_ARGS__,0,0); }\
-else {\
-   constexpr int T_MAX = 4;\
-   MFEM_VERIFY(D1D <= MAX_D1D && Q1D <= MAX_Q1D, "Max size error!");\
-   return kernel<0,0,T_MAX>(__VA_ARGS__,D1D,Q1D); }
-
-} // namespace kernels
-
-} // namespace mfem
-
-#endif // MFEM_TMOP_PA_HPP

fem/tmop_pa_h2d.cpp

@@ -1,161 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-/* // Original i-j assembly (old invariants code).
-   for (int e = 0; e < NE; e++)
-   {
-      for (int q = 0; q < nqp; q++)
-      {
-         el.CalcDShape(ip, DSh);
-         Mult(DSh, Jrt, DS);
-         for (int i = 0; i < dof; i++)
-         {
-            for (int j = 0; j < dof; j++)
-            {
-               for (int r = 0; r < dim; r++)
-               {
-                  for (int c = 0; c < dim; c++)
-                  {
-                     for (int rr = 0; rr < dim; rr++)
-                     {
-                        for (int cc = 0; cc < dim; cc++)
-                        {
-                           const double H = h(r, c, rr, cc);
-                           A(e, i + r*dof, j + rr*dof) +=
-                                 weight_q * DS(i, c) * DS(j, cc) * H;
-                        }
-                     }
-                  }
-               }
-            }
-         }
-      }
-   }*/
-
-MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_2D,
-                           const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const DenseTensor &j,
-                           const Vector &h,
-                           Vector &diagonal,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b.Read(), Q1D, D1D);
-   const auto G = Reshape(g.Read(), Q1D, D1D);
-   const auto J = Reshape(j.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto H = Reshape(h.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, NE);
-
-   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
-   {
-      constexpr int DIM = 2;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      MFEM_SHARED double qd[DIM*DIM*MQ1*MD1];
-      DeviceTensor<4,double> QD(qd, DIM, DIM, MQ1, MD1);
-
-      for (int v = 0; v < DIM; v++)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QD(0,0,qx,dy) = 0.0;
-               QD(0,1,qx,dy) = 0.0;
-               QD(1,0,qx,dy) = 0.0;
-               QD(1,1,qx,dy) = 0.0;
-
-               MFEM_UNROLL(MQ1);
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double *Jtr = &J(0,0,qx,qy,e);
-
-                  // Jrt = Jtr^{-1}
-                  double j[4];
-                  ConstDeviceMatrix Jrt(j,2,2);
-                  kernels::CalcInverse<2>(Jtr, j);
-
-                  const double gg = G(qy,dy) * G(qy,dy);
-                  const double gb = G(qy,dy) * B(qy,dy);
-                  const double bb = B(qy,dy) * B(qy,dy);
-                  const double bgb[4] = { bb, gb, gb, gg };
-                  ConstDeviceMatrix BG(bgb,2,2);
-
-                  for (int i = 0; i < DIM; i++)
-                  {
-                     for (int j = 0; j < DIM; j++)
-                     {
-                        const double Jij = Jrt(i,i) * Jrt(j,j);
-                        const double alpha = Jij * BG(i,j);
-                        QD(i,j,qx,dy) += alpha * H(v,i,v,j,qx,qy,e);
-                     }
-                  }
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double d = 0.0;
-               MFEM_UNROLL(MQ1);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const double gg = G(qx,dx) * G(qx,dx);
-                  const double gb = G(qx,dx) * B(qx,dx);
-                  const double bb = B(qx,dx) * B(qx,dx);
-                  d += gg * QD(0,0,qx,dy);
-                  d += gb * QD(0,1,qx,dy);
-                  d += gb * QD(1,0,qx,dy);
-                  d += bb * QD(1,1,qx,dy);
-               }
-               D(dx,dy,v,e) += d;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-void TMOP_Integrator::AssembleDiagonalPA_2D(Vector &D) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const Vector &H = PA.H;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_2D,id,N,B,G,J,H,D);
-}
-
-} // namespace mfem

fem/tmop_pa_h2d_c0.cpp

@@ -1,96 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_C0_2D,
-                           const int NE,
-                           const Array<double> &b,
-                           const Vector &h0,
-                           Vector &diagonal,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b.Read(), Q1D, D1D);
-   const auto H0 = Reshape(h0.Read(), DIM, DIM, Q1D, Q1D, NE);
-
-   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
-   {
-      constexpr int DIM = 2;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      MFEM_SHARED double qd[MQ1*MD1];
-      DeviceTensor<2,double> QD(qd, MQ1, MD1);
-
-      for (int v = 0; v < DIM; v++)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QD(qx,dy) = 0.0;
-               MFEM_UNROLL(MQ1);
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double bb = B(qy,dy) * B(qy,dy);
-                  QD(qx,dy) += bb * H0(v,v,qx,qy,e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double d = 0.0;
-               MFEM_UNROLL(MQ1);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const double bb = B(qx,dx) * B(qx,dx);
-                  d += bb * QD(qx,dy);
-               }
-               D(dx,dy,v,e) += d;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-void TMOP_Integrator::AssembleDiagonalPA_C0_2D(Vector &D) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = PA.maps->B;
-   const Vector &H0 = PA.H0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_C0_2D,id,N,B,H0,D);
-}
-
-} // namespace mfem

fem/tmop_pa_h2m.cpp

@@ -1,128 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_2D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const DenseTensor &j_,
-                           const Vector &h_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
-   const auto H = Reshape(h_.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const double *Jtr = &J(0,0,qx,qy,e);
-
-            // Jrt = Jtr^{-1}
-            double Jrt[4];
-            kernels::CalcInverse<2>(Jtr, Jrt);
-
-            // Jpr = X^T.DSh
-            double Jpr[4];
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
-
-            // Jpt = Jpr . Jrt
-            double Jpt[4];
-            kernels::Mult(2,2,2, Jpr, Jrt, Jpt);
-
-            // B = Jpt : H
-            double B[4];
-            DeviceMatrix M(B,2,2);
-            ConstDeviceMatrix J(Jpt,2,2);
-            for (int i = 0; i < DIM; i++)
-            {
-               for (int j = 0; j < DIM; j++)
-               {
-                  M(i,j) = 0.0;
-                  for (int r = 0; r < DIM; r++)
-                  {
-                     for (int c = 0; c < DIM; c++)
-                     {
-                        M(i,j) += H(r,c,i,j,qx,qy,e) * J(r,c);
-                     }
-                  }
-               }
-            }
-            // C = Jrt . B
-            double C[4];
-            kernels::MultABt(2,2,2, Jrt, B, C);
-
-            // Overwrite QQ = Jrt . (Jpt : H)^t
-            kernels::PushGrad<MQ1,NBZ>(qx,qy, C, QQ);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBGt<MD1,MQ1>(D1D,Q1D,b,g,BG);
-      kernels::GradYt<MD1,MQ1,NBZ>(D1D,Q1D,BG,QQ,DQ);
-      kernels::GradXt<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultGradPA_2D(const Vector &R, Vector &C) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const Vector &H = PA.H;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_2D,id,N,B,G,J,H,R,C);
-}
-
-} // namespace mfem

fem/tmop_pa_h2m_c0.cpp

@@ -1,107 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Vector &h0_,
-                           const Vector &r_,
-                           Vector &c_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto H0 = Reshape(h0_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto R = Reshape(r_.Read(), D1D, D1D, DIM, NE);
-
-   auto Y = Reshape(c_.ReadWrite(), D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      constexpr int DIM = 2;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[2][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[2][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,R,XY);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY,DQ);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Xh = X^T . Sh
-            double Xh[2];
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,Xh);
-
-            double B[4];
-            DeviceMatrix H(B,2,2);
-            for (int i = 0; i < DIM; i++)
-            {
-               for (int j = 0; j < DIM; j++)
-               {
-                  H(i,j) = H0(i,j,qx,qy,e);
-               }
-            }
-
-            // p2 = B . Xh
-            double p2[2];
-            kernels::Mult(2,2,B,Xh,p2);
-            kernels::PushEval<MQ1,NBZ>(qx,qy,p2,QQ);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::EvalXt<MD1,MQ1,NBZ>(D1D,Q1D,B,QQ,DQ);
-      kernels::EvalYt<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultGradPA_C0_2D(const Vector &X, const Vector &R,
-                                          Vector &C) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = PA.maps->B;
-   const Vector &H0 = PA.H0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_C0_2D,id,N,B,H0,R,C);
-}
-
-} // namespace mfem

fem/tmop_pa_h2s.cpp

@@ -1,247 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-#include "../linalg/dinvariants.hpp"
-
-namespace mfem
-{
-
-using Args = kernels::InvariantsEvaluator2D::Buffers;
-
-// weight * ddI1
-static MFEM_HOST_DEVICE inline
-void EvalH_001(const int e, const int qx, const int qy,
-               const double weight, const double *Jpt,
-               DeviceTensor<7,double> H)
-{
-   constexpr int DIM = 2;
-   double ddI1[4];
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).ddI1(ddI1));
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi1(ie.Get_ddI1(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               const double h = ddi1(r,c);
-               H(r,c,i,j,qx,qy,e) = weight * h;
-            }
-         }
-      }
-   }
-}
-
-// 0.5 * weight * dI1b
-static MFEM_HOST_DEVICE inline
-void EvalH_002(const int e, const int qx, const int qy,
-               const double weight, const double *Jpt,
-               DeviceTensor<7,double> H)
-{
-   constexpr int DIM = 2;
-   double ddI1[4], ddI1b[4], dI2b[4];
-   kernels::InvariantsEvaluator2D ie(Args()
-                                     .J(Jpt)
-                                     .ddI1(ddI1)
-                                     .ddI1b(ddI1b)
-                                     .dI2b(dI2b));
-   const double w = 0.5 * weight;
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               const double h = ddi1b(r,c);
-               H(r,c,i,j,qx,qy,e) = w * h;
-            }
-         }
-      }
-   }
-}
-
-static MFEM_HOST_DEVICE inline
-void EvalH_007(const int e, const int qx, const int qy,
-               const double weight, const double *Jpt,
-               DeviceTensor<7,double> H)
-{
-   constexpr int DIM = 2;
-   double ddI1[4], ddI2[4], dI1[4], dI2[4], dI2b[4];
-   kernels::InvariantsEvaluator2D ie(Args()
-                                     .J(Jpt)
-                                     .ddI1(ddI1)
-                                     .ddI2(ddI2)
-                                     .dI1(dI1)
-                                     .dI2(dI2)
-                                     .dI2b(dI2b));
-   const double c1 = 1./ie.Get_I2();
-   const double c2 = weight*c1*c1;
-   const double c3 = ie.Get_I1()*c2;
-   ConstDeviceMatrix di1(ie.Get_dI1(),DIM,DIM);
-   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
-
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi1(ie.Get_ddI1(i,j),DIM,DIM);
-         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               H(r,c,i,j,qx,qy,e) =
-                  weight * (1.0 + c1) * ddi1(r,c)
-                  - c3 * ddi2(r,c)
-                  - c2 * ( di1(i,j) * di2(r,c) + di2(i,j) * di1(r,c) )
-                  + 2.0 * c1 * c3 * di2(r,c) * di2(i,j);
-            }
-         }
-      }
-   }
-}
-
-static MFEM_HOST_DEVICE inline
-void EvalH_077(const int e, const int qx, const int qy,
-               const double weight, const double *Jpt,
-               DeviceTensor<7,double> H)
-{
-   constexpr int DIM = 2;
-   double dI2[4], dI2b[4], ddI2[4];
-   kernels::InvariantsEvaluator2D ie(Args()
-                                     .J(Jpt)
-                                     .dI2(dI2)
-                                     .dI2b(dI2b)
-                                     .ddI2(ddI2));
-   const double I2 = ie.Get_I2(), I2inv_sq = 1.0 / (I2 * I2);
-   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               H(r,c,i,j,qx,qy,e) =
-                  weight * 0.5 * (1.0 - I2inv_sq) * ddi2(r,c)
-                  + weight * (I2inv_sq / I2) * di2(r,c) * di2(i,j);
-            }
-         }
-      }
-   }
-}
-
-MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
-                           const Vector &x_,
-                           const double metric_normal,
-                           const int mid,
-                           const int NE,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const DenseTensor &j_,
-                           Vector &h_,
-                           const int d1d,
-                           const int q1d)
-{
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
-               "Metric not yet implemented!");
-
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto W = Reshape(w_.Read(), Q1D, Q1D);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
-   auto H = Reshape(h_.Write(), DIM, DIM, DIM, DIM, Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double s_BG[2][MQ1*MD1];
-      MFEM_SHARED double s_X[2][NBZ][MD1*MD1];
-      MFEM_SHARED double s_DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double s_QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,s_X);
-      kernels::LoadBG<MD1,MQ1>(D1D, Q1D, b, g, s_BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D, Q1D, s_BG, s_X, s_DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D, Q1D, s_BG, s_DQ, s_QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const double *Jtr = &J(0,0,qx,qy,e);
-            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
-
-            // Jrt = Jtr^{-1}
-            double Jrt[4];
-            kernels::CalcInverse<2>(Jtr, Jrt);
-
-            // Jpr = X^t.DSh
-            double Jpr[4];
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,s_QQ,Jpr);
-
-            // Jpt = Jpr.Jrt
-            double Jpt[4];
-            kernels::Mult(2,2,2, Jpr, Jrt, Jpt);
-
-            // metric->AssembleH
-            if (mid ==  1) { EvalH_001(e,qx,qy,weight,Jpt,H); }
-            if (mid ==  2) { EvalH_002(e,qx,qy,weight,Jpt,H); }
-            if (mid ==  7) { EvalH_007(e,qx,qy,weight,Jpt,H); }
-            if (mid == 77) { EvalH_077(e,qx,qy,weight,Jpt,H); }
-         } // qx
-      } // qy
-   });
-}
-
-void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int M = metric->Id();
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double mn = metric_normal;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   Vector &H = PA.H;
-
-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,M,N,W,B,G,J,H);
-}
-
-} // namespace mfem

fem/tmop_pa_h2s_c0.cpp

@@ -1,125 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_C0_2D,
-                           const double lim_normal,
-                           const Vector &lim_dist,
-                           const Vector &c0_,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &bld_,
-                           Vector &h0_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const bool const_c0 = c0_.Size() == 1;
-   const auto C0 = const_c0 ?
-                   Reshape(c0_.Read(), 1, 1, 1) :
-                   Reshape(c0_.Read(), Q1D, Q1D, NE);
-   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, NE);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
-
-   auto H0 = Reshape(h0_.Write(), DIM, DIM, Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      MFEM_SHARED double BLD[MQ1*MD1];
-
-      MFEM_SHARED double XY[NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,LD,XY);
-
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,BLD,XY,DQ);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,BLD,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const double *Jtr = &J(0,0,qx,qy,e);
-            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = W(qx,qy) * detJtr;
-            const double coeff0 = const_c0 ? C0(0,0,0) : C0(qx,qy,e);
-            const double weight_m = weight * lim_normal * coeff0;
-
-            double D;
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,D);
-            const double dist = D; // GetValues, default comp set to 0
-
-            // lim_func->Eval_d2(p1, p0, d_vals(q), grad_grad);
-            // d2.Diag(1.0 / (dist * dist), x.Size());
-            const double c = 1.0 / (dist * dist);
-            double grad_grad[4];
-            kernels::Diag<2>(c, grad_grad);
-            ConstDeviceMatrix gg(grad_grad,DIM,DIM);
-
-            for (int i = 0; i < DIM; i++)
-            {
-               for (int j = 0; j < DIM; j++)
-               {
-                  H0(i,j,qx,qy,e) = weight_m * gg(i,j);
-               }
-            }
-         }
-      }
-   });
-}
-
-void TMOP_Integrator::AssembleGradPA_C0_2D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double ln = lim_normal;
-   const Vector &LD = PA.LD;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W   = PA.ir->GetWeights();
-   const Array<double> &B   = PA.maps->B;
-   const Array<double> &BLD = PA.maps_lim->B;
-   const Vector &C0 = PA.C0;
-   Vector &H0 = PA.H0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_C0_2D,id,ln,LD,C0,N,J,W,B,BLD,H0);
-}
-
-} // namespace mfem

fem/tmop_pa_h3d.cpp

@@ -1,150 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_3D,
-                           const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const DenseTensor &j,
-                           const Vector &h,
-                           Vector &diagonal,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b.Read(), Q1D, D1D);
-   const auto G = Reshape(g.Read(), Q1D, D1D);
-   const auto J = Reshape(j.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto H = Reshape(h.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, Q1D, NE);
-
-   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      constexpr int DIM = 3;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      MFEM_SHARED double qqd[MQ1*MQ1*MD1];
-      MFEM_SHARED double qdd[MQ1*MD1*MD1];
-      DeviceTensor<3,double> QQD(qqd, MQ1, MQ1, MD1);
-      DeviceTensor<3,double> QDD(qdd, MQ1, MD1, MD1);
-
-      for (int v = 0; v < DIM; ++v)
-      {
-         for (int i = 0; i < DIM; i++)
-         {
-            for (int j = 0; j < DIM; j++)
-            {
-               // first tensor contraction, along z direction
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qy,y,Q1D)
-                  {
-                     MFEM_FOREACH_THREAD(dz,z,D1D)
-                     {
-                        QQD(qx,qy,dz) = 0.0;
-                        MFEM_UNROLL(MQ1);
-                        for (int qz = 0; qz < Q1D; ++qz)
-                        {
-                           const double *Jtr = &J(0,0,qx,qy,qz,e);
-                           double jrt[9];
-                           ConstDeviceMatrix Jrt(jrt,3,3);
-                           kernels::CalcInverse<3>(Jtr, jrt);
-                           const double Bz = B(qz,dz);
-                           const double Gz = G(qz,dz);
-                           const double L = i==2 ? Gz : Bz;
-                           const double R = j==2 ? Gz : Bz;
-                           const double Jij = Jrt(i,i) * Jrt(j,j);
-                           const double h = H(v,i,v,j,qx,qy,qz,e);
-                           QQD(qx,qy,dz) += L * Jij * h * R;
-                        }
-                     }
-                  }
-               }
-               MFEM_SYNC_THREAD;
-               // second tensor contraction, along y direction
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  MFEM_FOREACH_THREAD(dz,z,D1D)
-                  {
-                     MFEM_FOREACH_THREAD(dy,y,D1D)
-                     {
-                        QDD(qx,dy,dz) = 0.0;
-                        MFEM_UNROLL(MQ1);
-                        for (int qy = 0; qy < Q1D; ++qy)
-                        {
-                           const double By = B(qy,dy);
-                           const double Gy = G(qy,dy);
-                           const double L = i==1 ? Gy : By;
-                           const double R = j==1 ? Gy : By;
-                           QDD(qx,dy,dz) += L * QQD(qx,qy,dz) * R;
-                        }
-                     }
-                  }
-               }
-               MFEM_SYNC_THREAD;
-               // third tensor contraction, along x direction
-               MFEM_FOREACH_THREAD(dz,z,D1D)
-               {
-                  MFEM_FOREACH_THREAD(dy,y,D1D)
-                  {
-                     MFEM_FOREACH_THREAD(dx,x,D1D)
-                     {
-                        double d = 0.0;
-                        MFEM_UNROLL(MQ1);
-                        for (int qx = 0; qx < Q1D; ++qx)
-                        {
-                           const double Bx = B(qx,dx);
-                           const double Gx = G(qx,dx);
-                           const double L = i==0 ? Gx : Bx;
-                           const double R = j==0 ? Gx : Bx;
-                           d += L * QDD(qx,dy,dz) * R;
-                        }
-                        D(dx,dy,dz,v,e) += d;
-                     }
-                  }
-               }
-               MFEM_SYNC_THREAD;
-            }
-         }
-      }
-   });
-}
-
-void TMOP_Integrator::AssembleDiagonalPA_3D(Vector &D) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const Vector &H = PA.H;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_3D,id,N,B,G,J,H,D);
-}
-
-} // namespace mfem

fem/tmop_pa_h3d_c0.cpp

@@ -1,123 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_C0_3D,
-                           const int NE,
-                           const Array<double> &b,
-                           const Vector &h0,
-                           Vector &diagonal,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b.Read(), Q1D, D1D);
-   const auto H0 = Reshape(h0.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-
-   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      constexpr int DIM = 3;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      MFEM_SHARED double qqd[MQ1*MQ1*MD1];
-      MFEM_SHARED double qdd[MQ1*MD1*MD1];
-      DeviceTensor<3,double> QQD(qqd, MQ1, MQ1, MD1);
-      DeviceTensor<3,double> QDD(qdd, MQ1, MD1, MD1);
-
-      for (int v = 0; v < DIM; ++v)
-      {
-         // first tensor contraction, along z direction
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(dz,z,D1D)
-               {
-                  QQD(qx,qy,dz) = 0.0;
-                  MFEM_UNROLL(MQ1);
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     const double Bz = B(qz,dz);
-                     QQD(qx,qy,dz) += Bz * H0(v,v,qx,qy,qz,e) * Bz;
-                  }
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // second tensor contraction, along y direction
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1D)
-               {
-                  QDD(qx,dy,dz) = 0.0;
-                  MFEM_UNROLL(MQ1);
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     const double By = B(qy,dy);
-                     QDD(qx,dy,dz) += By * QQD(qx,qy,dz) * By;
-                  }
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // third tensor contraction, along x direction
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  double d = 0.0;
-                  MFEM_UNROLL(MQ1);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const double Bx = B(qx,dx);
-                     d += Bx * QDD(qx,dy,dz) * Bx;
-                  }
-                  D(dx,dy,dz, v, e) += d;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-void TMOP_Integrator::AssembleDiagonalPA_C0_3D(Vector &D) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = PA.maps->B;
-   const Vector &H0 = PA.H0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_C0_3D,id,N,B,H0,D);
-}
-
-} // namespace mfem

fem/tmop_pa_h3m.cpp

@@ -1,132 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_3D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const DenseTensor &j_,
-                           const Vector &h_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
-   const auto H = Reshape(h_.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      constexpr int DIM = 3;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[9][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double *Jtr = &J(0,0,qx,qy,qz,e);
-
-               // Jrt = Jtr^{-1}
-               double Jrt[9];
-               kernels::CalcInverse<3>(Jtr, Jrt);
-
-               // Jpr = X^T.DSh
-               double Jpr[9];
-               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, Jpr);
-
-               // Jpt = X^T.DS = (X^T.DSh).Jrt = Jpr.Jrt
-               double Jpt[9];
-               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
-
-               // B = Jpt : H
-               double B[9];
-               DeviceMatrix M(B,3,3);
-               ConstDeviceMatrix J(Jpt,3,3);
-               for (int i = 0; i < DIM; i++)
-               {
-                  for (int j = 0; j < DIM; j++)
-                  {
-                     M(i,j) = 0.0;
-                     for (int r = 0; r < DIM; r++)
-                     {
-                        for (int c = 0; c < DIM; c++)
-                        {
-                           M(i,j) += H(r,c,i,j,qx,qy,qz,e) * J(r,c);
-                        }
-                     }
-                  }
-               }
-
-               // Y +=  DS . M^t += DSh . (Jrt . M^t)
-               double A[9];
-               kernels::MultABt(3,3,3, Jrt, B, A);
-               kernels::PushGrad<MQ1>(qx,qy,qz, A, QQQ);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBGt<MD1,MQ1>(D1D,Q1D,b,g,BG);
-      kernels::GradZt<MD1,MQ1>(D1D,Q1D,BG,QQQ,DQQ);
-      kernels::GradYt<MD1,MQ1>(D1D,Q1D,BG,DQQ,DDQ);
-      kernels::GradXt<MD1,MQ1>(D1D,Q1D,BG,DDQ,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultGradPA_3D(const Vector &X, const Vector &R,
-                                       Vector &C) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const Vector &H = PA.H;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_3D,id,N,B,G,J,H,R,C);
-}
-
-} // namespace mfem

fem/tmop_pa_h3m_c0.cpp

@@ -1,111 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Vector &h0_,
-                           const Vector &r_,
-                           Vector &c_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto H0 = Reshape(h0_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto R = Reshape(r_.Read(), D1D, D1D, D1D, DIM, NE);
-
-   auto Y = Reshape(c_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      constexpr int DIM = 3;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-
-      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[3][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[3][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[3][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,R,DDD);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD,DDQ);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ,DQQ);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               // Xh = X^T . Sh
-               double Xh[3];
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,Xh);
-
-               double B[9];
-               DeviceMatrix H(B,3,3);
-               for (int i = 0; i < DIM; i++)
-               {
-                  for (int j = 0; j < DIM; j++)
-                  {
-                     H(i,j) = H0(i,j,qx,qy,qz,e);
-                  }
-               }
-
-               // p2 = B . Xh
-               double p2[3];
-               kernels::Mult(3,3,B,Xh,p2);
-               kernels::PushEval<MQ1>(qx,qy,qz,p2,QQQ);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::EvalXt<MD1,MQ1>(D1D,Q1D,B,QQQ,DQQ);
-      kernels::EvalYt<MD1,MQ1>(D1D,Q1D,B,DQQ,DDQ);
-      kernels::EvalZt<MD1,MQ1>(D1D,Q1D,B,DDQ,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultGradPA_C0_3D(const Vector &X, const Vector &R,
-                                          Vector &C) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = PA.maps->B;
-   const Vector &H0 = PA.H0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_C0_3D,id,N,B,H0,R,C);
-}
-
-} // namespace mfem

fem/tmop_pa_h3s.cpp

@@ -1,281 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-#include "../linalg/dinvariants.hpp"
-
-namespace mfem
-{
-
-using Args = kernels::InvariantsEvaluator3D::Buffers;
-
-// dP_302 = (dI2b*dI1b + dI1b*dI2b)/9 + (I1b/9)*ddI2b + (I2b/9)*ddI1b
-static MFEM_HOST_DEVICE inline
-void EvalH_302(const int e, const int qx, const int qy, const int qz,
-               const double weight, const double *J, DeviceTensor<8,double> dP)
-{
-   double B[9];
-   double         dI1b[9],          ddI1b[9];
-   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
-   double dI3b[9];
-   constexpr int DIM = 3;
-   kernels::InvariantsEvaluator3D ie(Args()
-                                     .J(J).B(B)
-                                     .dI1b(dI1b).ddI1b(ddI1b)
-                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
-                                     .dI3b(dI3b));
-   const double c1 = weight/9.;
-   const double I1b = ie.Get_I1b();
-   const double I2b = ie.Get_I2b();
-   ConstDeviceMatrix di1b(ie.Get_dI1b(),DIM,DIM);
-   ConstDeviceMatrix di2b(ie.Get_dI2b(),DIM,DIM);
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
-         ConstDeviceMatrix ddi2b(ie.Get_ddI2b(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               const double dp =
-                  (di2b(r,c)*di1b(i,j) + di1b(r,c)*di2b(i,j))
-                  + ddi2b(r,c)*I1b
-                  + ddi1b(r,c)*I2b;
-               dP(r,c,i,j,qx,qy,qz,e) = c1 * dp;
-            }
-         }
-      }
-   }
-}
-
-// dP_303 = ddI1b/3
-static MFEM_HOST_DEVICE inline
-void EvalH_303(const int e, const int qx, const int qy, const int qz,
-               const double weight, const double *J, DeviceTensor<8,double> dP)
-{
-   double B[9];
-   double         dI1b[9], ddI1[9], ddI1b[9];
-   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
-   double dI3b[9], ddI3b[9];
-   constexpr int DIM = 3;
-   kernels::InvariantsEvaluator3D ie(Args()
-                                     .J(J).B(B)
-                                     .dI1b(dI1b).ddI1(ddI1).ddI1b(ddI1b)
-                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
-                                     .dI3b(dI3b).ddI3b(ddI3b));
-   const double c1 = weight/3.;
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               const double dp = ddi1b(r,c);
-               dP(r,c,i,j,qx,qy,qz,e) = c1 * dp;
-            }
-         }
-      }
-   }
-}
-
-// dP_315 = 2*(dI3b x dI3b) + 2*(I3b - 1)*ddI3b
-static MFEM_HOST_DEVICE inline
-void EvalH_315(const int e, const int qx, const int qy, const int qz,
-               const double weight, const double *J, DeviceTensor<8,double> dP)
-{
-   double dI3b[9], ddI3b[9];
-   constexpr int DIM = 3;
-   kernels::InvariantsEvaluator3D ie(Args().
-                                     J(J).
-                                     dI3b(dI3b).ddI3b(ddI3b));
-
-   double sign_detJ;
-   const double I3b = ie.Get_I3b(sign_detJ);
-   ConstDeviceMatrix di3b(ie.Get_dI3b(sign_detJ),DIM,DIM);
-
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi3b(ie.Get_ddI3b(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               const double dp = 2.0 * weight * (I3b - 1.0) * ddi3b(r,c) +
-                                 2.0 * weight * di3b(r,c) * di3b(i,j);
-               dP(r,c,i,j,qx,qy,qz,e) = dp;
-            }
-         }
-      }
-   }
-}
-
-// dP_321 = ddI1 + (-2/I3b^3)*(dI2 x dI3b + dI3b x dI2)
-//               + (1/I3)*ddI2
-//               + (6*I2/I3b^4)*(dI3b x dI3b)
-//               + (-2*I2/I3b^3)*ddI3b
-static MFEM_HOST_DEVICE inline
-void EvalH_321(const int e, const int qx, const int qy, const int qz,
-               const double weight, const double *J, DeviceTensor<8,double> dP)
-{
-   double B[9];
-   double         dI1b[9], ddI1[9], ddI1b[9];
-   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
-   double dI3b[9], ddI3b[9];
-   constexpr int DIM = 3;
-   kernels::InvariantsEvaluator3D ie(Args()
-                                     .J(J).B(B)
-                                     .dI1b(dI1b).ddI1(ddI1).ddI1b(ddI1b)
-                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
-                                     .dI3b(dI3b).ddI3b(ddI3b));
-   double sign_detJ;
-   const double I2 = ie.Get_I2();
-   const double I3b = ie.Get_I3b(sign_detJ);
-
-   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
-   ConstDeviceMatrix di3b(ie.Get_dI3b(sign_detJ),DIM,DIM);
-
-   const double c0 = 1.0/I3b;
-   const double c1 = weight*c0*c0;
-   const double c2 = -2*c0*c1;
-   const double c3 = c2*I2;
-
-   for (int i = 0; i < DIM; i++)
-   {
-      for (int j = 0; j < DIM; j++)
-      {
-         ConstDeviceMatrix ddi1(ie.Get_ddI1(i,j),DIM,DIM);
-         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
-         ConstDeviceMatrix ddi3b(ie.Get_ddI3b(i,j),DIM,DIM);
-         for (int r = 0; r < DIM; r++)
-         {
-            for (int c = 0; c < DIM; c++)
-            {
-               const double dp =
-                  weight * ddi1(r,c)
-                  + c1 * ddi2(r,c)
-                  + c3 * ddi3b(r,c)
-                  + c2 * ((di2(r,c)*di3b(i,j) + di3b(r,c)*di2(i,j)))
-                  -3*c0*c3 * di3b(r,c)*di3b(i,j);
-               dP(r,c,i,j,qx,qy,qz,e) = dp;
-            }
-         }
-      }
-   }
-}
-
-MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
-                           const double metric_normal,
-                           const int mid,
-                           const Vector &x_,
-                           const int NE,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const DenseTensor &j_,
-                           Vector &h_,
-                           const int d1d,
-                           const int q1d)
-{
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
-
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
-   auto H = Reshape(h_.Write(), DIM, DIM, DIM, DIM, Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double s_BG[2][MQ1*MD1];
-      MFEM_SHARED double s_DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double s_DDQ[9][MD1*MD1*MQ1];
-      MFEM_SHARED double s_DQQ[9][MD1*MQ1*MQ1];
-      MFEM_SHARED double s_QQQ[9][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,X,s_DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,s_BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,s_BG,s_DDD,s_DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,s_BG,s_DDQ,s_DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,s_BG,s_DQQ,s_QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double *Jtr = &J(0,0,qx,qy,qz,e);
-               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
-
-               // Jrt = Jtr^{-1}
-               double Jrt[9];
-               kernels::CalcInverse<3>(Jtr, Jrt);
-
-               // Jpr = X^T.DSh
-               double Jpr[9];
-               kernels::PullGrad<MQ1>(qx,qy,qz, s_QQQ, Jpr);
-
-               // Jpt = X^T . DS = (X^T.DSh) . Jrt = Jpr . Jrt
-               double Jpt[9];
-               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
-
-               // metric->AssembleH
-               if (mid == 302) { EvalH_302(e,qx,qy,qz,weight,Jpt,H); }
-               if (mid == 303) { EvalH_303(e,qx,qy,qz,weight,Jpt,H); }
-               if (mid == 315) { EvalH_315(e,qx,qy,qz,weight,Jpt,H); }
-               if (mid == 321) { EvalH_321(e,qx,qy,qz,weight,Jpt,H); }
-            } // qx
-         } // qy
-      } // qz
-   });
-}
-
-void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int M = metric->Id();
-   const int id = (D1D << 4 ) | Q1D;
-   const double mn = metric_normal;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   Vector &H = PA.H;
-
-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,M,X,N,W,B,G,J,H);
-}
-
-} // namespace mfem

fem/tmop_pa_h3s_c0.cpp

@@ -1,129 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
-                           const double lim_normal,
-                           const Vector &lim_dist,
-                           const Vector &c0_,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &bld_,
-                           Vector &h0_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const bool const_c0 = c0_.Size() == 1;
-   const auto C0 = const_c0 ?
-                   Reshape(c0_.Read(), 1, 1, 1, 1) :
-                   Reshape(c0_.Read(), Q1D, Q1D, Q1D, NE);
-   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, D1D, NE);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
-
-   auto H0 = Reshape(h0_.Write(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      constexpr int DIM = 3;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      MFEM_SHARED double BLD[MQ1*MD1];
-
-      MFEM_SHARED double DDD[MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,LD,DDD);
-
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,BLD,DDD,DDQ);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,BLD,DDQ,DQQ);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,BLD,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double *Jtr = &J(0,0,qx,qy,qz,e);
-               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = W(qx,qy,qz) * detJtr;
-               const double coeff0 = const_c0 ? C0(0,0,0,0) : C0(qx,qy,qz,e);
-               const double weight_m = weight * lim_normal * coeff0;
-
-               double D;
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,D);
-               const double dist = D; // GetValues, default comp set to 0
-
-               // lim_func->Eval_d2(p1, p0, d_vals(q), grad_grad);
-               // d2.Diag(1.0 / (dist * dist), x.Size());
-               const double c = 1.0 / (dist * dist);
-               double grad_grad[9];
-               kernels::Diag<3>(c, grad_grad);
-               ConstDeviceMatrix gg(grad_grad,DIM,DIM);
-
-               for (int i = 0; i < DIM; i++)
-               {
-                  for (int j = 0; j < DIM; j++)
-                  {
-                     H0(i,j,qx,qy,qz,e) = weight_m * gg(i,j);
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-void TMOP_Integrator::AssembleGradPA_C0_3D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double ln = lim_normal;
-   const Vector &LD = PA.LD;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &BLD = PA.maps_lim->B;
-   const Vector &C0 = PA.C0;
-   Vector &H0 = PA.H0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_Kernel_C0_3D,id,ln,LD,C0,N,J,W,B,BLD,H0);
-}
-
-} // namespace mfem

fem/tmop_pa_jp2.cpp

@@ -1,178 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "tmop_tools.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(int, CheckDetJpr_Kernel_2D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &x_,
-                           Vector &DetJOk,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
-
-   auto E = Reshape(DetJOk.Write(), Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double J[4];
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,J);
-            const double detJ = kernels::Det<2>(J);
-            E(qx,qy,e) = (detJ <= 0.0) ? 0.0 : 1.0;
-         }
-      }
-   });
-   const double N = DetJOk.Size();
-   const double D = DetJOk * DetJOk;
-   return D < N ? 0 : 1;
-}
-
-int TMOPNewtonSolver::CheckDetJpr_2D(const FiniteElementSpace *fes,
-                                     const Vector &X) const
-{
-   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *R = fes->GetElementRestriction(ordering);
-   Vector XE(R->Height(), Device::GetDeviceMemoryType());
-   XE.UseDevice(true);
-   R->Mult(X, XE);
-
-   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-   const int NE = fes->GetMesh()->GetNE();
-   const int NQ = ir.GetNPoints();
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = maps.B;
-   const Array<double> &G = maps.G;
-
-   Vector E(NE*NQ);
-   E.UseDevice(true);
-
-   MFEM_LAUNCH_TMOP_KERNEL(CheckDetJpr_Kernel_2D,id,NE,B,G,XE,E);
-}
-
-MFEM_REGISTER_TMOP_KERNELS(double, MinDetJpr_Kernel_2D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &x_,
-                           Vector &DetJ,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
-
-   auto D = Reshape(DetJ.Write(), Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double Jpr[4];
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
-            D(qx,qy,e) = kernels::Det<2>(Jpr);
-         }
-      }
-   });
-   return DetJ.Min();
-}
-
-double TMOPNewtonSolver::MinDetJpr_2D(const FiniteElementSpace *fes,
-                                      const Vector &X) const
-{
-   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *R = fes->GetElementRestriction(ordering);
-   Vector XE(R->Height(), Device::GetDeviceMemoryType());
-   XE.UseDevice(true);
-   R->Mult(X, XE);
-
-   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-   const int NE = fes->GetMesh()->GetNE();
-   const int NQ = ir.GetNPoints();
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = maps.B;
-   const Array<double> &G = maps.G;
-
-   Vector E(NE*NQ);
-   E.UseDevice(true);
-
-   MFEM_LAUNCH_TMOP_KERNEL(MinDetJpr_Kernel_2D,id,NE,B,G,XE,E);
-}
-
-} // namespace mfem

fem/tmop_pa_jp3.cpp

@@ -1,181 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "tmop_tools.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(int, CheckDetJpr_Kernel_3D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &x_,
-                           Vector &DetJOk,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
-
-   auto E = Reshape(DetJOk.Write(), Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double J[9];
-               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, J);
-               const double detJ = kernels::Det<3>(J);
-               E(qx,qy,qz,e) = (detJ <= 0.0) ? 0.0 : 1.0;
-            }
-         }
-      }
-   });
-   const double N = DetJOk.Size();
-   const double D = DetJOk * DetJOk;
-   return D < N ? 0 : 1;
-}
-
-int TMOPNewtonSolver::CheckDetJpr_3D(const FiniteElementSpace *fes,
-                                     const Vector &X) const
-{
-   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *R = fes->GetElementRestriction(ordering);
-   Vector XE(R->Height(), Device::GetDeviceMemoryType());
-   XE.UseDevice(true);
-   R->Mult(X, XE);
-
-   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-   const int NE = fes->GetMesh()->GetNE();
-   const int NQ = ir.GetNPoints();
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = maps.B;
-   const Array<double> &G = maps.G;
-
-   Vector E(NE*NQ);
-   E.UseDevice(true);
-
-   MFEM_LAUNCH_TMOP_KERNEL(CheckDetJpr_Kernel_3D,id,NE,B,G,XE,E);
-}
-
-MFEM_REGISTER_TMOP_KERNELS(double, MinDetJpr_Kernel_3D,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &x_,
-                           Vector &DetJ,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
-
-   auto D = Reshape(DetJ.Write(), Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double Jpr[9];
-               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, Jpr);
-               D(qx,qy,qz,e) = kernels::Det<3>(Jpr);
-            }
-         }
-      }
-   });
-   return DetJ.Min();
-}
-
-double TMOPNewtonSolver::MinDetJpr_3D(const FiniteElementSpace *fes,
-                                      const Vector &X) const
-{
-   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *R = fes->GetElementRestriction(ordering);
-   Vector XE(R->Height(), Device::GetDeviceMemoryType());
-   XE.UseDevice(true);
-   R->Mult(X, XE);
-
-   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-   const int NE = fes->GetMesh()->GetNE();
-   const int NQ = ir.GetNPoints();
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const Array<double> &B = maps.B;
-   const Array<double> &G = maps.G;
-
-   Vector E(NE*NQ);
-   E.UseDevice(true);
-
-   MFEM_LAUNCH_TMOP_KERNEL(MinDetJpr_Kernel_3D,id,NE,B,G,XE,E);
-}
-} // namespace mfem

fem/tmop_pa_jt2_tc.cpp

@@ -1,155 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "gridfunc.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_UNIT_SIZE_2D_KERNEL,
-                           const int NE,
-                           const DenseMatrix w_, // copy
-                           DenseTensor &j_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto W = Reshape(w_.Read(), DIM,DIM);
-   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            kernels::Set(DIM,DIM, 1.0, &W(0,0), &J(0,0,qx,qy,e));
-         }
-      }
-   });
-   return true;
-}
-
-MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_GIVEN_SIZE_2D_KERNEL,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const DenseMatrix w_ideal_, // copy
-                           const Vector &x_,
-                           DenseTensor &j_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const double detW = w_ideal_.Det();
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto Wideal = Reshape(w_ideal_.Read(), DIM,DIM);
-   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
-   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double Jtr[4];
-            const double *Wid = &Wideal(0,0);
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jtr);
-            const double detJ = kernels::Det<2>(Jtr);
-            const double alpha = std::pow(detJ/detW,1./2);
-            kernels::Set(DIM,DIM,alpha,Wid,&J(0,0,qx,qy,e));
-         }
-      }
-   });
-   return true;
-}
-
-template<> bool
-TargetConstructor::ComputeElementTargetsPA<2>(const FiniteElementSpace *fes,
-                                              const IntegrationRule *ir,
-                                              DenseTensor &Jtr,
-                                              const Vector&) const
-{
-   MFEM_ASSERT(target_type == IDEAL_SHAPE_UNIT_SIZE || nodes != nullptr, "");
-   MFEM_VERIFY(fes->GetFE(0)->GetGeomType() == Geometry::SQUARE, "");
-   const DenseMatrix &W = Geometries.GetGeomToPerfGeomJac(Geometry::SQUARE);
-   const FiniteElement *fe = fes->GetFE(0);
-   const int NE = fes->GetMesh()->GetNE();
-   const DofToQuad::Mode mode = DofToQuad::TENSOR;
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-
-   const Array<double> &B = maps.B;
-   const Array<double> &G = maps.G;
-
-   switch (target_type)
-   {
-      case IDEAL_SHAPE_UNIT_SIZE: // Jtr(i) = Wideal;
-      {
-         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_UNIT_SIZE_2D_KERNEL,
-                                 id,NE,W,Jtr);
-      }
-      case IDEAL_SHAPE_EQUAL_SIZE: return false;
-      case IDEAL_SHAPE_GIVEN_SIZE:
-      {
-         MFEM_VERIFY(nodes, "");
-         const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-         const Operator *R = fes->GetElementRestriction(ordering);
-         Vector X(R->Height(), Device::GetDeviceMemoryType());
-         X.UseDevice(true);
-         R->Mult(*nodes, X);
-         MFEM_ASSERT(nodes->FESpace()->GetVDim() == 2, "");
-         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_GIVEN_SIZE_2D_KERNEL,
-                                 id,NE,B,G,W,X,Jtr);
-      }
-      case GIVEN_SHAPE_AND_SIZE: return false;
-      default: return false;
-   }
-   return false;
-}
-
-} // namespace mfem

fem/tmop_pa_jt3_datc.cpp

@@ -1,162 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(bool, DatcSize,
-                           const int NE,
-                           const int ncomp,
-                           const int sizeidx,
-                           const DenseMatrix w_, // Copy
-                           const Array<double> &b_,
-                           const Vector &x_,
-                           DenseTensor &j_,
-                           const int d1d,
-                           const int q1d)
-{
-   MFEM_VERIFY(ncomp==1,"");
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= Q1D, "");
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), DIM,DIM);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, ncomp, NE);
-   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D,Q1D, NE);
-
-   const double infinity = std::numeric_limits<double>::infinity();
-   MFEM_VERIFY(sizeidx == 0,"");
-   MFEM_VERIFY(MFEM_CUDA_BLOCKS==256,"");
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      MFEM_SHARED double DDD[MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,sizeidx,X,DDD);
-
-      double min;
-      MFEM_SHARED double min_size[MFEM_CUDA_BLOCKS];
-      DeviceTensor<3,double> M((double*)(min_size),D1D,D1D,D1D);
-      const DeviceTensor<3,const double> D((double*)(DDD+sizeidx),D1D,D1D,D1D);
-      MFEM_FOREACH_THREAD(t,x,MFEM_CUDA_BLOCKS) { min_size[t] = infinity; }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               M(dx,dy,dz) = D(dx,dy,dz);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      for (int wrk = MFEM_CUDA_BLOCKS >> 1; wrk > 0; wrk >>= 1)
-      {
-         MFEM_FOREACH_THREAD(t,x,MFEM_CUDA_BLOCKS)
-         { if (t < wrk) { min_size[t] = fmin(min_size[t], min_size[t+wrk]); } }
-         MFEM_SYNC_THREAD;
-      }
-      min = min_size[0];
-
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD,DDQ);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ,DQQ);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ,QQQ);
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               double T;
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,T);
-               const double shape_par_vals = T;
-               const double size = fmax(shape_par_vals, min);
-               const double alpha = std::pow(size, 1.0/DIM);
-               for (int i = 0; i < DIM; i++)
-               {
-                  for (int j = 0; j < DIM; j++)
-                  {
-                     J(i,j,qx,qy,qz,e) = alpha * W(i,j);
-                  }
-               }
-            }
-         }
-      }
-   });
-   return true;
-}
-
-// PA.Jtr Size = (dim, dim, PA.ne*PA.nq);
-bool DiscreteAdaptTC::ComputeElementTargetsPA(const FiniteElementSpace *pa_fes,
-                                              const IntegrationRule *ir,
-                                              DenseTensor &Jtr,
-                                              const Vector &xe) const
-{
-   MFEM_VERIFY(target_type == IDEAL_SHAPE_GIVEN_SIZE ||
-               target_type == GIVEN_SHAPE_AND_SIZE,"");
-
-   const FiniteElementSpace *fes = tspec_fesv;
-
-   if (!fes) { return false;}
-
-   const FiniteElement &fe = *fes->GetFE(0);
-   const DenseMatrix &W = Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
-   const int DIM = W.Height();
-   const int NE = fes->GetMesh()->GetNE();
-   const DofToQuad::Mode mode = DofToQuad::TENSOR;
-   const DofToQuad &maps = fe.GetDofToQuad(*ir, mode);
-   const Array<double> &B = maps.B;
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-
-   const bool SizeKernel = sizeidx != -1;
-
-   // Until it is not implemented, return on host
-   if (skewidx != -1) { return false; }
-   if (aspectratioidx != -1) { return false; }
-   if (orientationidx != -1) { return false; }
-
-   if (DIM == 3 && SizeKernel)
-   {
-      Vector tspec_e;
-      const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-      const Operator *R = fes->GetElementRestriction(ordering);
-      MFEM_VERIFY(R,"");
-      MFEM_VERIFY(R->Height() == NE*ncomp*D1D*D1D*D1D,"");
-      tspec_e.SetSize(R->Height(), Device::GetDeviceMemoryType());
-      tspec_e.UseDevice(true);
-      tspec.UseDevice(true);
-      R->Mult(tspec, tspec_e);
-      const int id = (D1D << 4 ) | Q1D;
-      MFEM_LAUNCH_TMOP_KERNEL(DatcSize,id,NE,ncomp,sizeidx,W,B,tspec_e,Jtr);
-   }
-   return false;
-}
-
-} // namespace mfem

fem/tmop_pa_jt3_tc.cpp

@@ -1,159 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "gridfunc.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_UNIT_SIZE_3D_KERNEL,
-                           const int NE,
-                           const DenseMatrix w_, // copy
-                           DenseTensor &j_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto W = Reshape(w_.Read(), DIM,DIM);
-   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D,Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               kernels::Set(DIM,DIM, 1.0, &W(0,0), &J(0,0,qx,qy,qz,e));
-            }
-         }
-      }
-   });
-   return true;
-}
-
-MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_GIVEN_SIZE_3D_KERNEL,
-                           const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const DenseMatrix w_ideal_, // copy
-                           const Vector &x_,
-                           DenseTensor &j_,
-                           const int d1d,
-                           const int q1d)
-{
-   constexpr int DIM = 3;
-
-   const double detW = w_ideal_.Det();
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto Wideal = Reshape(w_ideal_.Read(), DIM,DIM);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
-   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D,Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double Jtr[9];
-               const double *Wid = &Wideal(0,0);
-               kernels::PullGrad<MQ1>(qx,qy,qz,QQQ,Jtr);
-               const double detJ = kernels::Det<3>(Jtr);
-               const double alpha = std::pow(detJ/detW,1./3);
-               kernels::Set(DIM,DIM,alpha,Wid,&J(0,0,qx,qy,qz,e));
-            }
-         }
-      }
-   });
-   return true;
-}
-
-template<> bool
-TargetConstructor::ComputeElementTargetsPA<3>(const FiniteElementSpace *fes,
-                                              const IntegrationRule *ir,
-                                              DenseTensor &Jtr,
-                                              const Vector&) const
-{
-   MFEM_ASSERT(target_type == IDEAL_SHAPE_UNIT_SIZE || nodes != nullptr, "");
-   MFEM_VERIFY(fes->GetFE(0)->GetGeomType() == Geometry::CUBE, "");
-   const DenseMatrix &W = Geometries.GetGeomToPerfGeomJac(Geometry::CUBE);
-   const FiniteElement *fe = fes->GetFE(0);
-   const int NE = fes->GetMesh()->GetNE();
-   const DofToQuad::Mode mode = DofToQuad::TENSOR;
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
-   const Array<double> &B = maps.B;
-   const Array<double> &G = maps.G;
-   const int D1D = maps.ndof;
-   const int Q1D = maps.nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-
-   switch (target_type)
-   {
-      case IDEAL_SHAPE_UNIT_SIZE: // Jtr(i) = Wideal;
-      {
-         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_UNIT_SIZE_3D_KERNEL,
-                                 id,NE,W,Jtr);
-      }
-      case IDEAL_SHAPE_EQUAL_SIZE: return false;
-      case IDEAL_SHAPE_GIVEN_SIZE:
-      {
-         MFEM_VERIFY(nodes, "");
-         const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-         const Operator *R = fes->GetElementRestriction(ordering);
-         Vector X(R->Height(), Device::GetDeviceMemoryType());
-         X.UseDevice(true);
-         R->Mult(*nodes, X);
-         MFEM_ASSERT(nodes->FESpace()->GetVDim() == 3, "");
-         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_GIVEN_SIZE_3D_KERNEL,
-                                 id,NE,B,G,W,X,Jtr);
-      }
-      case GIVEN_SHAPE_AND_SIZE: return false;
-      default: return false;
-   }
-   return false;
-}
-
-} // namespace mfem

fem/tmop_pa_p2.cpp

@@ -1,166 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-#include "../linalg/dinvariants.hpp"
-
-namespace mfem
-{
-
-using Args = kernels::InvariantsEvaluator2D::Buffers;
-
-static MFEM_HOST_DEVICE inline
-void EvalP_001(const double *Jpt, double *P)
-{
-   double dI1[4];
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).dI1(dI1));
-   kernels::Set(2,2, 1.0, ie.Get_dI1(), P);
-}
-
-static MFEM_HOST_DEVICE inline
-void EvalP_002(const double *Jpt, double *P)
-{
-   double dI1b[4], dI2b[4];
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).dI1b(dI1b).dI2b(dI2b));
-   kernels::Set(2,2, 1./2., ie.Get_dI1b(), P);
-}
-
-static MFEM_HOST_DEVICE inline
-void EvalP_007(const double *Jpt, double *P)
-{
-   double dI1[4], dI2[4], dI2b[4];
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).dI1(dI1)
-                                     .dI2(dI2).dI2b(dI2b));
-   const double I2 = ie.Get_I2();
-   kernels::Add(2,2, 1.0 + 1.0 / I2,         ie.Get_dI1(),
-                -ie.Get_I1() / (I2*I2), ie.Get_dI2(), P);
-}
-
-static MFEM_HOST_DEVICE inline
-void EvalP_077(const double *Jpt, double *P)
-{
-   double dI2[4], dI2b[4];
-   kernels::InvariantsEvaluator2D ie(Args().
-                                     J(Jpt).
-                                     dI2(dI2).dI2b(dI2b));
-   const double I2 = ie.Get_I2();
-   kernels::Set(2,2, 0.5 * (1.0 - 1.0 / (I2 * I2)), ie.Get_dI2(), P);
-}
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
-                           const double metric_normal,
-                           const int mid,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int d1d,
-                           const int q1d)
-{
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
-               "Metric not yet implemented!");
-
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const double *Jtr = &J(0,0,qx,qy,e);
-            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
-
-            // Jrt = Jtr^{-1}
-            double Jrt[4];
-            kernels::CalcInverse<2>(Jtr, Jrt);
-
-            // Jpr = X{^T}.DSh
-            double Jpr[4];
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
-
-            // Jpt = X{^T}.DS = (X{^T}.DSh).Jrt = Jpr.Jrt
-            double Jpt[4];
-            kernels::Mult(2,2,2, Jpr, Jrt, Jpt);
-
-            // metric->EvalP(Jpt, P);
-            double P[4];
-            if (mid ==  1) { EvalP_001(Jpt, P); }
-            if (mid ==  2) { EvalP_002(Jpt, P); }
-            if (mid ==  7) { EvalP_007(Jpt, P); }
-            if (mid == 77) { EvalP_077(Jpt, P); }
-            for (int i = 0; i < 4; i++) { P[i] *= weight; }
-
-            // PMatO += DS . P^t += DSh . (Jrt . P^t)
-            double A[4];
-            kernels::MultABt(2,2,2, Jrt, P, A);
-            kernels::PushGrad<MQ1,NBZ>(qx,qy,A,QQ);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBGt<MD1,MQ1>(D1D,Q1D,b,g,BG);
-      kernels::GradYt<MD1,MQ1,NBZ>(D1D,Q1D,BG,QQ,DQ);
-      kernels::GradXt<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
-{
-   const int N = PA.ne;
-   const int M = metric->Id();
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const double mn = metric_normal;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,M,N,J,W,B,G,X,Y);
-}
-
-} // namespace mfem

fem/tmop_pa_p2_c0.cpp

@@ -1,148 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_2D,
-                           const double lim_normal,
-                           const Vector &lim_dist,
-                           const Vector &c0_,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &bld_,
-                           const Vector &x0_,
-                           const Vector &x1_,
-                           Vector &y_,
-                           const int d1d,
-                           const int q1d)
-{
-   const bool const_c0 = c0_.Size() == 1;
-
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto C0 = const_c0 ?
-                   Reshape(c0_.Read(), 1, 1, 1) :
-                   Reshape(c0_.Read(), Q1D, Q1D, NE);
-   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, NE);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D);
-   const auto X0 = Reshape(x0_.Read(), D1D, D1D, DIM, NE);
-   const auto X1 = Reshape(x1_.Read(), D1D, D1D, DIM, NE);
-
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      MFEM_SHARED double BLD[MQ1*MD1];
-
-      MFEM_SHARED double XY[NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
-
-      MFEM_SHARED double XY0[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ0[2][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ0[2][NBZ][MQ1*MQ1];
-
-      MFEM_SHARED double XY1[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ1[2][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ1[2][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,LD,XY);
-      kernels::LoadX<MD1,NBZ>(e,D1D,X0,XY0);
-      kernels::LoadX<MD1,NBZ>(e,D1D,X1,XY1);
-
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,BLD,XY,DQ);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,BLD,DQ,QQ);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY0,DQ0);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ0,QQ0);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY1,DQ1);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ1,QQ1);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const double *Jtr = &J(0,0,qx,qy,e);
-            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = W(qx,qy) * detJtr;
-
-            double ld, p0[2], p1[2];
-            const double coeff0 = const_c0 ? C0(0,0,0) : C0(qx,qy,e);
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,ld);
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ0,p0);
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ1,p1);
-
-            const double dist = ld; // GetValues, default comp set to 0
-
-            double d1[2];
-            // Eval_d1
-            // subtract(1.0 / (dist * dist), x, x0, d1);
-            // z = a * (x - y)
-            // grad = a * (x - x0)
-            const double a = 1.0 / (dist * dist);
-            const double w = weight * lim_normal * coeff0;
-            kernels::Subtract<2>(w*a, p1, p0, d1);
-            kernels::PushEval<MQ1,NBZ>(qx,qy,d1,QQ0);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::EvalXt<MD1,MQ1,NBZ>(D1D,Q1D,B,QQ0,DQ0);
-      kernels::EvalYt<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ0,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultPA_C0_2D(const Vector &X, Vector &Y) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double ln = lim_normal;
-   const Vector &LD = PA.LD;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W   = PA.ir->GetWeights();
-   const Array<double> &B   = PA.maps->B;
-   const Array<double> &BLD = PA.maps_lim->B;
-   const Vector &X0 = PA.X0;
-   const Vector &C0 = PA.C0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_C0_2D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,Y);
-}
-
-} // namespace mfem

fem/tmop_pa_p3.cpp

@@ -1,186 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-#include "../linalg/dinvariants.hpp"
-
-namespace mfem
-{
-
-using Args = kernels::InvariantsEvaluator3D::Buffers;
-
-// P_302 = (I1b/9)*dI2b + (I2b/9)*dI1b
-static MFEM_HOST_DEVICE inline
-void EvalP_302(const double *J, double *P)
-{
-   double B[9];
-   double dI1b[9], dI2[9], dI2b[9], dI3b[9];
-   kernels::InvariantsEvaluator3D ie(Args()
-                                     .J(J).B(B)
-                                     .dI1b(dI1b)
-                                     .dI2(dI2).dI2b(dI2b)
-                                     .dI3b(dI3b));
-   const double alpha = ie.Get_I1b()/9.;
-   const double beta = ie.Get_I2b()/9.;
-   kernels::Add(3,3, alpha, ie.Get_dI2b(), beta, ie.Get_dI1b(), P);
-}
-
-// P_303 = dI1b/3
-static MFEM_HOST_DEVICE inline
-void EvalP_303(const double *J, double *P)
-{
-   double B[9];
-   double dI1b[9], dI3b[9];
-   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B).dI1b(dI1b).dI3b(dI3b));
-   kernels::Set(3,3, 1./3., ie.Get_dI1b(), P);
-}
-
-// P_315 = 2*(I3b - 1)*dI3b
-static MFEM_HOST_DEVICE inline
-void EvalP_315(const double *J, double *P)
-{
-   double dI3b[9];
-   kernels::InvariantsEvaluator3D ie(Args().J(J).dI3b(dI3b));
-
-   double sign_detJ;
-   const double I3b = ie.Get_I3b(sign_detJ);
-   kernels::Set(3,3, 2.0 * (I3b - 1.0), ie.Get_dI3b(sign_detJ), P);
-}
-
-// P_321 = dI1 + (1/I3)*dI2 - (2*I2/I3b^3)*dI3b
-static MFEM_HOST_DEVICE inline
-void EvalP_321(const double *J, double *P)
-{
-   double B[9];
-   double dI1[9], dI2[9], dI3b[9];
-   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B)
-                                     .dI1(dI1).dI2(dI2).dI3b(dI3b));
-   double sign_detJ;
-   const double I3 = ie.Get_I3();
-   const double alpha = 1.0/I3;
-   const double beta = -2.*ie.Get_I2()/(I3*ie.Get_I3b(sign_detJ));
-   kernels::Add(3,3, alpha, ie.Get_dI2(), beta, ie.Get_dI3b(sign_detJ), P);
-   kernels::Add(3,3, ie.Get_dI1(), P);
-}
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
-                           const double metric_normal,
-                           const int mid,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int d1d,
-                           const int q1d)
-{
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
-
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double s_BG[2][MQ1*MD1];
-      MFEM_SHARED double s_DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double s_DDQ[9][MD1*MD1*MQ1];
-      MFEM_SHARED double s_DQQ[9][MD1*MQ1*MQ1];
-      MFEM_SHARED double s_QQQ[9][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,X,s_DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,s_BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,s_BG,s_DDD,s_DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,s_BG,s_DDQ,s_DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,s_BG,s_DQQ,s_QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double *Jtr = &J(0,0,qx,qy,qz,e);
-               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
-
-               // Jrt = Jtr^{-1}
-               double Jrt[9];
-               kernels::CalcInverse<3>(Jtr, Jrt);
-
-               // Jpr = X^T.DSh
-               double Jpr[9];
-               kernels::PullGrad<MQ1>(qx,qy,qz, s_QQQ, Jpr);
-
-               // Jpt = X^T.DS = (X^T.DSh).Jrt = Jpr.Jrt
-               double Jpt[9];
-               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
-
-               // metric->EvalP(Jpt, P);
-               double P[9];
-               if (mid == 302) { EvalP_302(Jpt,P); }
-               if (mid == 303) { EvalP_303(Jpt,P); }
-               if (mid == 315) { EvalP_315(Jpt,P); }
-               if (mid == 321) { EvalP_321(Jpt,P); }
-               for (int i = 0; i < 9; i++) { P[i] *= weight; }
-
-               // Y += DS . P^t += DSh . (Jrt . P^t)
-               double A[9];
-               kernels::MultABt(3,3,3, Jrt, P, A);
-               kernels::PushGrad<MQ1>(qx,qy,qz, A, s_QQQ);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBGt<MD1,MQ1>(D1D, Q1D, b, g, s_BG);
-      kernels::GradZt<MD1,MQ1>(D1D,Q1D,s_BG,s_QQQ,s_DQQ);
-      kernels::GradYt<MD1,MQ1>(D1D,Q1D,s_BG,s_DQQ,s_DDQ);
-      kernels::GradXt<MD1,MQ1>(D1D,Q1D,s_BG,s_DDQ,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
-{
-   const int N = PA.ne;
-   const int M = metric->Id();
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const double mn = metric_normal;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,M,N,J,W,B,G,X,Y);
-}
-
-} // namespace mfem

fem/tmop_pa_p3_c0.cpp

@@ -1,154 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
-                           const double lim_normal,
-                           const Vector &lim_dist,
-                           const Vector &c0_,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &bld_,
-                           const Vector &x0_,
-                           const Vector &x1_,
-                           Vector &y_,
-                           const int d1d,
-                           const int q1d)
-{
-   const bool const_c0 = c0_.Size() == 1;
-
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto C0 = const_c0 ?
-                   Reshape(c0_.Read(), 1, 1, 1, 1) :
-                   Reshape(c0_.Read(), Q1D, Q1D, Q1D, NE);
-   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, D1D, NE);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
-   const auto X0 = Reshape(x0_.Read(), D1D, D1D, D1D, DIM, NE);
-   const auto X1 = Reshape(x1_.Read(), D1D, D1D, D1D, DIM, NE);
-
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      MFEM_SHARED double BLD[MQ1*MD1];
-
-      MFEM_SHARED double DDD[MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
-
-      MFEM_SHARED double DDD0[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ0[3][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ0[3][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ0[3][MQ1*MQ1*MQ1];
-
-      MFEM_SHARED double DDD1[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ1[3][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ1[3][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ1[3][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,LD,DDD);
-      kernels::LoadX<MD1>(e,D1D,X0,DDD0);
-      kernels::LoadX<MD1>(e,D1D,X1,DDD1);
-
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,BLD,DDD,DDQ);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,BLD,DDQ,DQQ);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,BLD,DQQ,QQQ);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD0,DDQ0);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ0,DQQ0);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ0,QQQ0);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD1,DDQ1);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ1,DQQ1);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ1,QQQ1);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double *Jtr = &J(0,0,qx,qy,qz,e);
-               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = W(qx,qy,qz) * detJtr;
-
-               double D, p0[3], p1[3];
-               const double coeff0 = const_c0 ? C0(0,0,0,0) : C0(qx,qy,qz,e);
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,D);
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ0,p0);
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ1,p1);
-
-               double d1[3];
-               // Eval_d1
-               // subtract(1.0 / (dist * dist), x, x0, d1);
-               // z = a * (x - y)
-               // grad = a * (x - x0)
-               const double dist = D; // GetValues, default comp set to 0
-               const double a = 1.0 / (dist * dist);
-               const double w = weight * lim_normal * coeff0;
-               kernels::Subtract<3>(w*a, p1, p0, d1);
-               kernels::PushEval<MQ1>(qx,qy,qz,d1,QQQ0);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::EvalXt<MD1,MQ1>(D1D,Q1D,B,QQQ0,DQQ0);
-      kernels::EvalYt<MD1,MQ1>(D1D,Q1D,B,DQQ0,DDQ0);
-      kernels::EvalZt<MD1,MQ1>(D1D,Q1D,B,DDQ0,Y,e);
-   });
-}
-
-void TMOP_Integrator::AddMultPA_C0_3D(const Vector &X, Vector &Y) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double ln = lim_normal;
-   const Vector &LD = PA.LD;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &BLD = PA.maps_lim->B;
-   const Vector &X0 = PA.X0;
-   const Vector &C0 = PA.C0;
-
-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_C0_3D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,Y);
-}
-
-} // namespace mfem

fem/tmop_pa_w2.cpp

@@ -1,155 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-#include "../linalg/dinvariants.hpp"
-
-namespace mfem
-{
-
-using Args = kernels::InvariantsEvaluator2D::Buffers;
-
-static MFEM_HOST_DEVICE inline
-double EvalW_001(const double *Jpt)
-{
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
-   return ie.Get_I1();
-}
-
-static MFEM_HOST_DEVICE inline
-double EvalW_002(const double *Jpt)
-{
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
-   return 0.5 * ie.Get_I1b() - 1.0;
-}
-
-static MFEM_HOST_DEVICE inline
-double EvalW_007(const double *Jpt)
-{
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
-   return ie.Get_I1() * (1.0 + 1.0/ie.Get_I2()) - 4.0;
-}
-
-static MFEM_HOST_DEVICE inline
-double EvalW_077(const double *Jpt)
-{
-   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
-   const double I2 = ie.Get_I2b(); // Get_I2b or Get_I2 ?
-   return 0.5*(I2*I2 + 1./(I2*I2) - 2.);
-}
-
-MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
-                           const double metric_normal,
-                           const int mid,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &x_,
-                           const Vector &ones,
-                           Vector &energy,
-                           const int d1d,
-                           const int q1d)
-{
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
-               "2D metric not yet implemented!");
-
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D);
-   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
-
-   auto E = Reshape(energy.Write(), Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const double *Jtr = &J(0,0,qx,qy,e);
-            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
-
-            // Jrt = Jtr^{-1}
-            double Jrt[4];
-            kernels::CalcInverse<2>(Jtr, Jrt);
-
-            // Jpr = X^t.DSh
-            double Jpr[4];
-            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
-
-            // Jpt = X^T.DS = (X^T.DSh).Jrt = Jpr.Jrt
-            double Jpt[4];
-            kernels::Mult(2,2,2,Jpr,Jrt,Jpt);
-
-            // metric->EvalW(Jpt);
-            const double EvalW =
-            mid ==  1 ? EvalW_001(Jpt) :
-            mid ==  2 ? EvalW_002(Jpt) :
-            mid ==  7 ? EvalW_007(Jpt) :
-            mid == 77 ? EvalW_077(Jpt) : 0.0;
-
-            E(qx,qy,e) = weight * EvalW;
-         }
-      }
-   });
-   return energy * ones;
-}
-
-double TMOP_Integrator::GetGridFunctionEnergyPA_2D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int M = metric->Id();
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double m = metric_normal;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const Vector &O = PA.O;
-   Vector &E = PA.E;
-
-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,m,M,N,J,W,B,G,X,O,E);
-}
-
-} // namespace mfem

fem/tmop_pa_w2_c0.cpp

@@ -1,139 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_2D,
-                           const double lim_normal,
-                           const Vector &lim_dist,
-                           const Vector &c0_,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &bld_,
-                           const Vector &x0_,
-                           const Vector &x1_,
-                           const Vector &ones,
-                           Vector &energy,
-                           const int d1d,
-                           const int q1d)
-{
-   const bool const_c0 = c0_.Size() == 1;
-
-   constexpr int DIM = 2;
-   constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto C0 = const_c0 ?
-                   Reshape(c0_.Read(), 1, 1, 1) :
-                   Reshape(c0_.Read(), Q1D, Q1D, NE);
-   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, NE);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D);
-   const auto X0 = Reshape(x0_.Read(), D1D, D1D, DIM, NE);
-   const auto X1 = Reshape(x1_.Read(), D1D, D1D, DIM, NE);
-
-   auto E = Reshape(energy.Write(), Q1D, Q1D, NE);
-
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      MFEM_SHARED double BLD[MQ1*MD1];
-
-      MFEM_SHARED double XY[NBZ][MD1*MD1];
-      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
-
-      MFEM_SHARED double XY0[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ0[2][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ0[2][NBZ][MQ1*MQ1];
-
-      MFEM_SHARED double XY1[2][NBZ][MD1*MD1];
-      MFEM_SHARED double DQ1[2][NBZ][MD1*MQ1];
-      MFEM_SHARED double QQ1[2][NBZ][MQ1*MQ1];
-
-      kernels::LoadX<MD1,NBZ>(e,D1D,LD,XY);
-      kernels::LoadX<MD1,NBZ>(e,D1D,X0,XY0);
-      kernels::LoadX<MD1,NBZ>(e,D1D,X1,XY1);
-
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,BLD,XY,DQ);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,BLD,DQ,QQ);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY0,DQ0);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ0,QQ0);
-
-      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY1,DQ1);
-      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ1,QQ1);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double ld, p0[2], p1[2];
-            const double *Jtr = &J(0,0,qx,qy,e);
-            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = W(qx,qy) * detJtr;
-            const double coeff0 = const_c0 ? C0(0,0,0) : C0(qx,qy,e);
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,ld);
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ0,p0);
-            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ1,p1);
-            const double dist = ld; // GetValues, default comp set to 0
-            const double id2 = 0.5 / (dist*dist);
-            const double dsq = kernels::DistanceSquared<2>(p1,p0) * id2;
-            E(qx,qy,e) = weight * lim_normal * dsq * coeff0;
-         }
-      }
-   });
-   return energy * ones;
-}
-
-double TMOP_Integrator::GetGridFunctionEnergyPA_C0_2D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double ln = lim_normal;
-   const Vector &LD = PA.LD;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W   = PA.ir->GetWeights();
-   const Array<double> &B   = PA.maps->B;
-   const Array<double> &BLD = PA.maps_lim->B;
-   const Vector &X0 = PA.X0;
-   const Vector &C0 = PA.C0;
-   const Vector &O = PA.O;
-   Vector &E = PA.E;
-
-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_C0_2D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,O,E);
-}
-
-} // namespace mfem

fem/tmop_pa_w3.cpp

@@ -1,165 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-#include "../linalg/dinvariants.hpp"
-
-namespace mfem
-{
-
-using Args = kernels::InvariantsEvaluator3D::Buffers;
-
-// mu_302 = I1b * I2b / 9 - 1
-static MFEM_HOST_DEVICE inline
-double EvalW_302(const double *J)
-{
-   double B[9];
-   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
-   return ie.Get_I1b()*ie.Get_I2b()/9. - 1.;
-}
-
-// mu_303 = I1b/3 - 1
-static MFEM_HOST_DEVICE inline
-double EvalW_303(const double *J)
-{
-   double B[9];
-   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
-   return ie.Get_I1b()/3. - 1.;
-}
-
-// mu_315 = (I3b - 1)^2
-static MFEM_HOST_DEVICE inline
-double EvalW_315(const double *J)
-{
-   double B[9];
-   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
-   const double a = ie.Get_I3b() - 1.0;
-   return a*a;
-}
-
-// mu_321 = I1 + I2/I3 - 6
-static MFEM_HOST_DEVICE inline
-double EvalW_321(const double *J)
-{
-   double B[9];
-   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
-   return ie.Get_I1() + ie.Get_I2()/ie.Get_I3() - 6.0;
-}
-
-MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
-                           const double metric_normal,
-                           const int mid,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Vector &ones,
-                           const Vector &x_,
-                           Vector &energy,
-                           const int d1d,
-                           const int q1d)
-{
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
-
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto g = Reshape(g_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
-   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
-
-   auto E = Reshape(energy.Write(), Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double BG[2][MQ1*MD1];
-      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-
-      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double *Jtr = &J(0,0,qx,qy,qz,e);
-               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
-
-               // Jrt = Jtr^{-1}
-               double Jrt[9];
-               kernels::CalcInverse<3>(Jtr, Jrt);
-
-               // Jpr = X^t.DSh
-               double Jpr[9];
-               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, Jpr);
-
-               // Jpt = X^t.DS = (X^t.DSh).Jrt = Jpr.Jrt
-               double Jpt[9];
-               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
-
-               // metric->EvalW(Jpt);
-               const double EvalW =
-               mid == 302 ? EvalW_302(Jpt) :
-               mid == 303 ? EvalW_303(Jpt) :
-               mid == 315 ? EvalW_315(Jpt) :
-               mid == 321 ? EvalW_321(Jpt) : 0.0;
-
-               E(qx,qy,qz,e) = weight * EvalW;
-            }
-         }
-      }
-   });
-   return energy * ones;
-}
-
-double TMOP_Integrator::GetGridFunctionEnergyPA_3D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int M = metric->Id();
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double mn = metric_normal;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &G = PA.maps->G;
-   const Vector &O = PA.O;
-   Vector &E = PA.E;
-
-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,M,N,J,W,B,G,O,X,E);
-}
-
-} // namespace mfem

fem/tmop_pa_w3_c0.cpp

@@ -1,148 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop.hpp"
-#include "tmop_pa.hpp"
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
-                           const double lim_normal,
-                           const Vector &lim_dist,
-                           const Vector &c0_,
-                           const int NE,
-                           const DenseTensor &j_,
-                           const Array<double> &w_,
-                           const Array<double> &b_,
-                           const Array<double> &bld_,
-                           const Vector &x0_,
-                           const Vector &x1_,
-                           const Vector &ones,
-                           Vector &energy,
-                           const int d1d,
-                           const int q1d)
-{
-   const bool const_c0 = c0_.Size() == 1;
-
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto C0 = const_c0 ?
-                   Reshape(c0_.Read(), 1, 1, 1, 1) :
-                   Reshape(c0_.Read(), Q1D, Q1D, Q1D, NE);
-   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, D1D, NE);
-   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
-   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
-   const auto X0 = Reshape(x0_.Read(), D1D, D1D, D1D, DIM, NE);
-   const auto X1 = Reshape(x1_.Read(), D1D, D1D, D1D, DIM, NE);
-
-   auto E = Reshape(energy.Write(), Q1D, Q1D, Q1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
-
-      MFEM_SHARED double B[MQ1*MD1];
-      MFEM_SHARED double BLD[MQ1*MD1];
-
-      MFEM_SHARED double DDD[MD1*MD1*MD1];
-      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
-
-      MFEM_SHARED double DDD0[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ0[3][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ0[3][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ0[3][MQ1*MQ1*MQ1];
-
-      MFEM_SHARED double DDD1[3][MD1*MD1*MD1];
-      MFEM_SHARED double DDQ1[3][MD1*MD1*MQ1];
-      MFEM_SHARED double DQQ1[3][MD1*MQ1*MQ1];
-      MFEM_SHARED double QQQ1[3][MQ1*MQ1*MQ1];
-
-      kernels::LoadX<MD1>(e,D1D,LD,DDD);
-      kernels::LoadX<MD1>(e,D1D,X0,DDD0);
-      kernels::LoadX<MD1>(e,D1D,X1,DDD1);
-
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,BLD,DDD,DDQ);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,BLD,DDQ,DQQ);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,BLD,DQQ,QQQ);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD0,DDQ0);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ0,DQQ0);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ0,QQQ0);
-
-      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD1,DDQ1);
-      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ1,DQQ1);
-      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ1,QQQ1);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double D, p0[3], p1[3];
-               const double *Jtr = &J(0,0,qx,qy,qz,e);
-               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = W(qx,qy,qz) * detJtr;
-               const double coeff0 = const_c0 ? C0(0,0,0,0) : C0(qx,qy,qz,e);
-
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,D);
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ0,p0);
-               kernels::PullEval<MQ1>(qx,qy,qz,QQQ1,p1);
-
-               const double dist = D; // GetValues, default comp set to 0
-               const double id2 = 0.5 / (dist*dist);
-
-               const double dsq = kernels::DistanceSquared<3>(p1,p0) * id2;
-               E(qx,qy,qz,e) = weight * lim_normal * dsq * coeff0;
-            }
-         }
-      }
-   });
-   return energy * ones;
-}
-
-double TMOP_Integrator::GetGridFunctionEnergyPA_C0_3D(const Vector &X) const
-{
-   const int N = PA.ne;
-   const int D1D = PA.maps->ndof;
-   const int Q1D = PA.maps->nqpt;
-   const int id = (D1D << 4 ) | Q1D;
-   const double ln = lim_normal;
-   const Vector &LD = PA.LD;
-   const DenseTensor &J = PA.Jtr;
-   const Array<double> &W = PA.ir->GetWeights();
-   const Array<double> &B = PA.maps->B;
-   const Array<double> &BLD = PA.maps_lim->B;
-   const Vector &X0 = PA.X0;
-   const Vector &C0 = PA.C0;
-   const Vector &O = PA.O;
-   Vector &E = PA.E;
-
-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_C0_3D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,O,E);
-}
-
-} // namespace mfem

fem/tmop_tools.cpp

@@ -33,11 +33,10 @@ void AdvectorCG::ComputeAtNewPosition(const Vector &new_nodes,
    const int pnt_cnt = new_field.Size()/ncomp;
 
    new_field = field0;
-   new_field.HostReadWrite();
-   Vector new_field_temp;
+
    for (int i = 0; i < ncomp; i++)
    {
-      new_field_temp.MakeRef(new_field, i*pnt_cnt, pnt_cnt);
+      Vector new_field_temp(new_field.GetData()+i*pnt_cnt, pnt_cnt);
       ComputeAtNewPositionScalar(new_nodes, new_field_temp);
    }
 
@@ -73,13 +72,13 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
    if (fes)
    {
       fess = new FiniteElementSpace(fes->GetMesh(), fes->FEColl(), 1);
-      oper = new SerialAdvectorCGOper(nodes0, u, *fess, al);
+      oper = new SerialAdvectorCGOper(nodes0, u, *fess);
    }
 #ifdef MFEM_USE_MPI
    else if (pfes)
    {
       pfess = new ParFiniteElementSpace(pfes->GetParMesh(), pfes->FEColl(), 1);
-      oper  = new ParAdvectorCGOper(nodes0, u, *pfess, al);
+      oper  = new ParAdvectorCGOper(nodes0, u, *pfess);
    }
 #endif
    MFEM_VERIFY(oper != NULL,
@@ -95,7 +94,6 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
    double v_max = 0.0;
    const int s = new_field.Size();
 
-   u.HostReadWrite();
    for (int i = 0; i < s; i++)
    {
       double vel = 0.;
@@ -151,7 +149,6 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
 #endif
 
    // Trim the overshoots and undershoots.
-   new_field.HostReadWrite();
    for (int i = 0; i < s; i++)
    {
       if (new_field(i) < glob_minv) { new_field(i) = glob_minv; }
@@ -167,23 +164,20 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
 
 SerialAdvectorCGOper::SerialAdvectorCGOper(const Vector &x_start,
                                            GridFunction &vel,
-                                           FiniteElementSpace &fes,
-                                           AssemblyLevel al)
+                                           FiniteElementSpace &fes)
    : TimeDependentOperator(fes.GetVSize()),
      x0(x_start), x_now(*fes.GetMesh()->GetNodes()),
-     u(vel), u_coeff(&u), M(&fes), K(&fes), al(al)
+     u(vel), u_coeff(&u), M(&fes), K(&fes)
 {
    ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
    K.AddDomainIntegrator(Kinteg);
-   K.SetAssemblyLevel(al);
    K.Assemble(0);
    K.Finalize(0);
 
    MassIntegrator *Minteg = new MassIntegrator;
    M.AddDomainIntegrator(Minteg);
-   M.SetAssemblyLevel(al);
-   M.Assemble();
-   M.Finalize();
+   M.Assemble(0);
+   M.Finalize(0);
 }
 
 void SerialAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
@@ -192,11 +186,6 @@ void SerialAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    const double t = GetTime();
    add(x0, t, u, x_now);
 
-   if (al == AssemblyLevel::PARTIAL)
-   {
-      K.FESpace()->GetMesh()->DeleteGeometricFactors();
-   }
-
    // Assemble on the new mesh.
    K.BilinearForm::operator=(0.0);
    K.Assemble();
@@ -207,47 +196,32 @@ void SerialAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
 
    di_dt = 0.0;
    CGSolver lin_solver;
-   Solver *prec = nullptr;
-   Array<int> ess_tdof_list;
-   if (al == AssemblyLevel::PARTIAL)
-   {
-      prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-      lin_solver.SetOperator(M);
-   }
-   else
-   {
-      prec = new DSmoother(M.SpMat());
-      lin_solver.SetOperator(M.SpMat());
-   }
-   lin_solver.SetPreconditioner(*prec);
+   DSmoother prec;
+   lin_solver.SetPreconditioner(prec);
+   lin_solver.SetOperator(M.SpMat());
    lin_solver.SetRelTol(1e-12); lin_solver.SetAbsTol(0.0);
    lin_solver.SetMaxIter(100);
    lin_solver.SetPrintLevel(0);
    lin_solver.Mult(rhs, di_dt);
-
-   delete prec;
 }
 
 #ifdef MFEM_USE_MPI
 ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
                                      GridFunction &vel,
-                                     ParFiniteElementSpace &pfes,
-                                     AssemblyLevel al)
+                                     ParFiniteElementSpace &pfes)
    : TimeDependentOperator(pfes.GetVSize()),
      x0(x_start), x_now(*pfes.GetMesh()->GetNodes()),
-     u(vel), u_coeff(&u), M(&pfes), K(&pfes), al(al)
+     u(vel), u_coeff(&u), M(&pfes), K(&pfes)
 {
    ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
    K.AddDomainIntegrator(Kinteg);
-   K.SetAssemblyLevel(al);
    K.Assemble(0);
    K.Finalize(0);
 
    MassIntegrator *Minteg = new MassIntegrator;
    M.AddDomainIntegrator(Minteg);
-   M.SetAssemblyLevel(al);
-   M.Assemble();
-   M.Finalize();
+   M.Assemble(0);
+   M.Finalize(0);
 }
 
 void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
@@ -256,11 +230,6 @@ void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    const double t = GetTime();
    add(x0, t, u, x_now);
 
-   if (al == AssemblyLevel::PARTIAL)
-   {
-      K.ParFESpace()->GetParMesh()->DeleteGeometricFactors();
-   }
-
    // Assemble on the new mesh.
    K.BilinearForm::operator=(0.0);
    K.Assemble();
@@ -272,25 +241,13 @@ void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    HypreParVector *RHS = rhs.ParallelAssemble();
    HypreParVector X(K.ParFESpace());
    X = 0.0;
-
-   OperatorHandle Mop;
-   Solver *prec = nullptr;
-   Array<int> ess_tdof_list;
-   if (al == AssemblyLevel::PARTIAL)
-   {
-      M.FormSystemMatrix(ess_tdof_list, Mop);
-      prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-   }
-   else
-   {
-      Mop.Reset(M.ParallelAssemble());
-      prec = new HypreSmoother;
-      static_cast<HypreSmoother*>(prec)->SetType(HypreSmoother::Jacobi, 1);
-   }
+   HypreParMatrix *Mh  = M.ParallelAssemble();
 
    CGSolver lin_solver(M.ParFESpace()->GetParMesh()->GetComm());
-   lin_solver.SetPreconditioner(*prec);
-   lin_solver.SetOperator(*Mop);
+   HypreSmoother prec;
+   prec.SetType(HypreSmoother::Jacobi, 1);
+   lin_solver.SetPreconditioner(prec);
+   lin_solver.SetOperator(*Mh);
    lin_solver.SetRelTol(1e-8);
    lin_solver.SetAbsTol(0.0);
    lin_solver.SetMaxIter(100);
@@ -298,8 +255,8 @@ void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    lin_solver.Mult(*RHS, X);
    K.ParFESpace()->GetProlongationMatrix()->Mult(X, di_dt);
 
+   delete Mh;
    delete RHS;
-   delete prec;
 }
 #endif
 
@@ -396,13 +353,12 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       energy_in = nlf->GetEnergy(x);
    }
 
-   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
-             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
-   Array<int> xdofs(dof * dim);
-   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
-   Vector posV(pos.Data(), dof * dim);
-   Vector x_out_loc(fes->GetVSize());
+   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetMesh()->Dimension();
+   Array<int> xdofs;
+   DenseMatrix Jpr(dim);
 
+   // Get the local prolongation of the solution vector.
+   Vector x_out_loc(fes->GetVSize());
    if (serial)
    {
       const SparseMatrix *cP = fes->GetConformingProlongation();
@@ -416,27 +372,27 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
    }
 #endif
 
+   // Check if the starting mesh (given by x) is inverted.
+   // Note that x hasn't been modified by the Newton update yet.
    double min_detJ = infinity();
-   if (dim == 1)
+   for (int i = 0; i < NE; i++)
    {
-      for (int i = 0; i < NE; i++)
-      {
-         fes->GetElementVDofs(i, xdofs);
-         x_out_loc.GetSubVector(xdofs, posV);
+      const int dof = fes->GetFE(i)->GetDof();
+      DenseMatrix dshape(dof, dim), pos(dof, dim);
+      Vector posV(pos.Data(), dof * dim);
 
-         for (int j = 0; j < nsp; j++)
-         {
-            fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-            MultAtB(pos, dshape, Jpr);
-            min_detJ = std::min(min_detJ, Jpr.Det());
-         }
+      fes->GetElementVDofs(i, xdofs);
+      x_out_loc.GetSubVector(xdofs, posV);
+
+      const IntegrationRule &irule = GetIntegrationRule(*fes->GetFE(i));
+      const int nsp = irule.GetNPoints();
+      for (int j = 0; j < nsp; j++)
+      {
+         fes->GetFE(i)->CalcDShape(irule.IntPoint(j), dshape);
+         MultAtB(pos, dshape, Jpr);
+         min_detJ = std::min(min_detJ, Jpr.Det());
       }
    }
-   else
-   {
-      min_detJ = dim == 2 ? MinDetJpr_2D(fes, x_out_loc) :
-                 dim == 3 ? MinDetJpr_3D(fes, x_out_loc) : 0.0;
-   }
    double min_detJ_all = min_detJ;
 #ifdef MFEM_USE_MPI
    if (parallel)
@@ -445,18 +401,18 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
                     p_nlf->ParFESpace()->GetComm());
    }
 #endif
-   bool untangling = false;
-   if (min_detJ_all <= 0) { untangling = true; }
+   const bool untangling = (min_detJ_all <= 0) ? true : false;
 
    const bool have_b = (b.Size() == Height());
 
    Vector x_out(x.Size());
    bool x_out_ok = false;
    double scale = 1.0, energy_out = 0.0;
-   double norm0 = Norm(r);
+   const double norm0 = Norm(r);
 
    const double detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
 
+   // Perform the line search.
    for (int i = 0; i < 12; i++)
    {
       add(x, -scale, c, x_out);
@@ -478,26 +434,26 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (!untangling)
       {
          int jac_ok = 1;
-         if (dim == 1)
+         for (int i = 0; i < NE; i++)
          {
-            for (int i = 0; i < NE; i++)
+            const int dof = fes->GetFE(i)->GetDof();
+            DenseMatrix dshape(dof, dim), pos(dof, dim);
+            Vector posV(pos.Data(), dof * dim);
+
+            fes->GetElementVDofs(i, xdofs);
+            x_out_loc.GetSubVector(xdofs, posV);
+
+            const IntegrationRule &irule = GetIntegrationRule(*fes->GetFE(i));
+            const int nsp = irule.GetNPoints();
+            for (int j = 0; j < nsp; j++)
             {
-               fes->GetElementVDofs(i, xdofs);
-               x_out_loc.GetSubVector(xdofs, posV);
-               for (int j = 0; j < nsp; j++)
-               {
-                  fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-                  MultAtB(pos, dshape, Jpr);
-                  if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
-               }
+               fes->GetFE(i)->CalcDShape(irule.IntPoint(j), dshape);
+               MultAtB(pos, dshape, Jpr);
+               if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
             }
-         break2:;
-         }
-         else
-         {
-            jac_ok = dim == 2 ? CheckDetJpr_2D(fes, x_out_loc) :
-                     dim == 3 ? CheckDetJpr_3D(fes, x_out_loc) : 0;
          }
+
+      break2:
          int jac_ok_all = jac_ok;
 #ifdef MFEM_USE_MPI
          if (parallel)
@@ -506,6 +462,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
                           p_nlf->ParFESpace()->GetComm());
          }
 #endif
+
          if (jac_ok_all == 0)
          {
             if (print_level >= 0)
@@ -515,7 +472,6 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       } // endif(!untangling)
 
       ProcessNewState(x_out);
-
       if (serial)
       {
          energy_out = nlf->GetGridFunctionEnergy(x_out_loc);
@@ -622,7 +578,6 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
             Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
             for (int j = 0; j < ati.Size(); j++)
             {
-               ati[j]->UpdateAfterMeshChange(x_loc);
                ati[j]->ComputeFDh(x_loc, *pfesc);
                UpdateDiscreteTC(*ati[j], x_loc);
             }
@@ -659,7 +614,6 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
             Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
             for (int j = 0; j < ati.Size(); j++)
             {
-               ati[j]->UpdateAfterMeshChange(x_loc);
                ati[j]->ComputeFDh(x_loc, *fesc);
                UpdateDiscreteTC(*ati[j], x_loc);
             }

fem/tmop_tools.hpp

@@ -28,14 +28,12 @@ private:
    Vector nodes0;
    Vector field0;
    const double dt_scale;
-   const AssemblyLevel al;
 
    void ComputeAtNewPositionScalar(const Vector &new_nodes, Vector &new_field);
 public:
-   AdvectorCG(AssemblyLevel al = AssemblyLevel::LEGACYFULL,
-              double timestep_scale = 0.5)
+   AdvectorCG(double timestep_scale = 0.5)
       : AdaptivityEvaluator(),
-        ode_solver(), nodes0(), field0(), dt_scale(timestep_scale), al(al) { }
+        ode_solver(), nodes0(), field0(), dt_scale(timestep_scale) { }
 
    virtual void SetInitialField(const Vector &init_nodes,
                                 const Vector &init_field);
@@ -80,14 +78,12 @@ protected:
    GridFunction &u;
    VectorGridFunctionCoefficient u_coeff;
    mutable BilinearForm M, K;
-   const AssemblyLevel al;
 
 public:
    /** Here @a fes is the FESpace of the function that will be moved. Note
        that Mult() moves the nodes of the mesh corresponding to @a fes. */
    SerialAdvectorCGOper(const Vector &x_start, GridFunction &vel,
-                        FiniteElementSpace &fes,
-                        AssemblyLevel al = AssemblyLevel::LEGACYFULL);
+                        FiniteElementSpace &fes);
 
    virtual void Mult(const Vector &ind, Vector &di_dt) const;
 };
@@ -102,14 +98,12 @@ protected:
    GridFunction &u;
    VectorGridFunctionCoefficient u_coeff;
    mutable ParBilinearForm M, K;
-   const AssemblyLevel al;
 
 public:
    /** Here @a pfes is the ParFESpace of the function that will be moved. Note
        that Mult() moves the nodes of the mesh corresponding to @a pfes. */
    ParAdvectorCGOper(const Vector &x_start, GridFunction &vel,
-                     ParFiniteElementSpace &pfes,
-                     AssemblyLevel al = AssemblyLevel::LEGACYFULL);
+                     ParFiniteElementSpace &pfes);
 
    virtual void Mult(const Vector &ind, Vector &di_dt) const;
 };
@@ -124,16 +118,39 @@ protected:
 
    // Quadrature points that are checked for negative Jacobians etc.
    const IntegrationRule &ir;
+   // These fields are relevant for mixed meshes.
+   IntegrationRules *IntegRules;
+   int integ_order;
+
+   const IntegrationRule &GetIntegrationRule(const FiniteElement &el) const
+   {
+      if (IntegRules)
+      {
+         return IntegRules->Get(el.GetGeomType(), integ_order);
+      }
+      return ir;
+   }
 
    void UpdateDiscreteTC(const TMOP_Integrator &ti, const Vector &x_new) const;
 
 public:
 #ifdef MFEM_USE_MPI
    TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
-      : LBFGSSolver(comm), solver_type(type), parallel(true), ir(irule) { }
+      : LBFGSSolver(comm), solver_type(type), parallel(true),
+        ir(irule), IntegRules(NULL), integ_order(-1) { }
 #endif
    TMOPNewtonSolver(const IntegrationRule &irule, int type = 0)
-      : LBFGSSolver(), solver_type(type), parallel(false), ir(irule) { }
+      : LBFGSSolver(), solver_type(type), parallel(false),
+        ir(irule), IntegRules(NULL), integ_order(-1) { }
+
+   /// Prescribe a set of integration rules; relevant for mixed meshes.
+   /** If called, this function has priority over the IntegrationRule given to
+       the constructor of the class. */
+   void SetIntegrationRules(IntegrationRules &irules, int order)
+   {
+      IntegRules = &irules;
+      integ_order = order;
+   }
 
    virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
 
@@ -165,10 +182,6 @@ public:
       else { MFEM_ABORT("Invalid type"); }
    }
    virtual void SetPreconditioner(Solver &pr) { SetSolver(pr); }
-   int CheckDetJpr_2D(const FiniteElementSpace*, const Vector&) const;
-   int CheckDetJpr_3D(const FiniteElementSpace*, const Vector&) const;
-   double MinDetJpr_2D(const FiniteElementSpace*, const Vector&) const;
-   double MinDetJpr_3D(const FiniteElementSpace*, const Vector&) const;
 };
 
 void vis_tmop_metric_s(int order, TMOP_QualityMetric &qm,

general/CMakeLists.txt

@@ -35,7 +35,6 @@ list(APPEND HDRS
   backends.hpp
   binaryio.hpp
   cuda.hpp
-  debug.hpp
   device.hpp
   error.hpp
   gecko.hpp

general/backends.hpp

@@ -51,7 +51,6 @@
       (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
 #define MFEM_SHARED
 #define MFEM_SYNC_THREAD
-#define MFEM_BLOCK_ID(k) 0
 #define MFEM_THREAD_ID(k) 0
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)

general/cuda.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -61,21 +61,6 @@ void* CuMallocManaged(void** dptr, size_t bytes)
    return *dptr;
 }
 
-void* CuMemAllocHostPinned(void** ptr, size_t bytes)
-{
-#ifdef MFEM_USE_CUDA
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "CuMemAllocHostPinned(): allocating " << bytes << " bytes ... "
-             << std::flush;
-#endif
-   MFEM_GPU_CHECK(cudaMallocHost(ptr, bytes));
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "done: " << *ptr << std::endl;
-#endif
-#endif
-   return *ptr;
-}
-
 void* CuMemFree(void *dptr)
 {
 #ifdef MFEM_USE_CUDA
@@ -91,21 +76,6 @@ void* CuMemFree(void *dptr)
    return dptr;
 }
 
-void* CuMemFreeHostPinned(void *ptr)
-{
-#ifdef MFEM_USE_CUDA
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "CuMemFreeHostPinned(): deallocating memory @ " << ptr << " ... "
-             << std::flush;
-#endif
-   MFEM_GPU_CHECK(cudaFreeHost(ptr));
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "done." << std::endl;
-#endif
-#endif
-   return ptr;
-}
-
 void* CuMemcpyHtoD(void* dst, const void* src, size_t bytes)
 {
 #ifdef MFEM_USE_CUDA

general/cuda.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -63,15 +63,9 @@ void* CuMemAlloc(void **d_ptr, size_t bytes);
 /// Allocates managed device memory
 void* CuMallocManaged(void **d_ptr, size_t bytes);
 
-/// Allocates page-locked (pinned) host memory
-void* CuMemAllocHostPinned(void **ptr, size_t bytes);
-
 /// Frees device memory and returns destination ptr.
 void* CuMemFree(void *d_ptr);
 
-/// Frees page-locked (pinned) host memory and returns destination ptr.
-void* CuMemFreeHostPinned(void *ptr);
-
 /// Copies memory from Host to Device and returns destination ptr.
 void* CuMemcpyHtoD(void *d_dst, const void *h_src, size_t bytes);
 

general/debug.hpp

@@ -1,178 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_DEBUG_HPP
-#define MFEM_DEBUG_HPP
-
-#include <string>
-#include <cstring>
-#include <iomanip>
-#include <iostream>
-
-#include "globals.hpp"
-
-#ifdef MFEM_USE_MPI
-#include <mpi.h>
-#endif
-
-namespace mfem
-{
-
-class Debug
-{
-   const bool debug = false;
-public:
-   inline Debug() {}
-
-   inline Debug(const int mpi_rank,
-                const char *FILE, const int LINE,
-                const char *FUNC, int COLOR): debug(true)
-   {
-      if (!debug) { return; }
-      const char *base = Strrnchr(FILE,'/', 2);
-      const char *file = base ? base + 1 : FILE;
-      const uint8_t color = COLOR ? COLOR : 20 + Checksum8(FILE) % 210;
-      mfem::out << "\033[38;5;" << std::to_string(color) << "m";
-      mfem::out << mpi_rank << std::setw(30) << file << ":";
-      mfem::out << "\033[2m" << std::setw(4) << LINE << "\033[22m: ";
-      if (FUNC) { mfem::out << "[" << FUNC << "] "; }
-      mfem::out << "\033[1m";
-   }
-
-   ~Debug()
-   {
-      if (!debug) { return; }
-      mfem::out << "\033[m";
-      mfem::out << std::endl;
-   }
-
-   template <typename T>
-   inline void operator<<(const T &arg) const noexcept { mfem::out << arg; }
-
-   template<typename T, typename... Args>
-   inline void operator()(const char *fmt, const T &arg,
-                          Args... args) const noexcept
-   {
-      if (!debug) { return; }
-      for (; *fmt != '\0'; fmt++ )
-      {
-         if (*fmt == '%')
-         {
-            fmt++;
-            const char c = *fmt;
-            if (c == 'p') { operator<<(arg); }
-            if (c == 's' || c == 'd' || c == 'f') { operator<<(arg); }
-            if (c == 'x' || c == 'X')
-            {
-               mfem::out << std::hex;
-               if (c == 'X') { mfem::out << std::uppercase; }
-               operator<<(arg);
-               mfem::out << std::nouppercase << std::dec;
-            }
-            if (c == '.')
-            {
-               fmt++;
-               const char c = *fmt;
-               char num[8] = { 0 };
-               for (int k = 0; *fmt != '\0'; fmt++, k++)
-               {
-                  if (*fmt == 'e' || *fmt == 'f') { break; }
-                  if (*fmt < 0x30 || *fmt > 0x39) { break; }
-                  num[k] = *fmt;
-               }
-               const int fx = std::atoi(num);
-               if (c == 'e') { mfem::out << std::scientific; }
-               if (c == 'f') { mfem::out << std::fixed; }
-               mfem::out << std::setprecision(fx);
-               operator<<(arg);
-               mfem::out << std::setprecision(6);
-            }
-            return operator()(fmt + 1, args...);
-         }
-         operator<<(*fmt);
-      }
-   }
-
-   template<typename T>
-   inline void operator()(const T &arg) const noexcept
-   {
-      if (!debug) { return; }
-      operator<<(arg);
-   }
-
-   inline void operator()() const noexcept { }
-
-public:
-   static const Debug Set(const char *FILE, const int LINE, const char *FUNC,
-                          int COLOR = 0)
-   {
-      static int mpi_dbg = 0, mpi_rank = 0;
-      static bool env_mpi = false, env_dbg = false;
-      static bool ini_dbg = false;
-      if (!ini_dbg)
-      {
-         const char *DBG = getenv("MFEM_DEBUG");
-         const char *MPI = getenv("MFEM_DEBUG_MPI");
-         env_dbg = DBG != nullptr;
-         env_mpi = MPI != nullptr;
-#ifdef MFEM_USE_MPI
-         int mpi_ini = false;
-         MPI_Initialized(&mpi_ini);
-         if (mpi_ini) { MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); }
-         mpi_dbg = atoi(env_mpi ? MPI : "0");
-#endif
-         ini_dbg = true;
-      }
-      const bool debug = (env_dbg && (!env_mpi || mpi_rank == mpi_dbg));
-      return debug ? Debug(mpi_rank, FILE, LINE, FUNC, COLOR) : Debug();
-   }
-
-private:
-   inline uint8_t Checksum8(const char *bfr)
-   {
-      unsigned int chk = 0;
-      size_t len = strlen(bfr);
-      for (; len; len--,bfr++) { chk += static_cast<unsigned int>(*bfr); }
-      return (uint8_t) chk;
-   }
-
-   inline const char *Strrnchr(const char *s, const unsigned char c, int n)
-   {
-      size_t len = strlen(s);
-      char *p = const_cast<char*>(s) + len - 1;
-      for (; n; n--,p--,len--)
-      {
-         for (; len; p--,len--)
-            if (*p == c) { break; }
-         if (!len) { return nullptr; }
-         if (n == 1) { return p; }
-      }
-      return nullptr;
-   }
-
-};
-
-#ifndef MFEM_DEBUG_COLOR
-#define MFEM_DEBUG_COLOR 0
-#endif
-
-#define dbg(...) \
-    mfem::Debug::Set(__FILE__,__LINE__,__FUNCTION__,MFEM_DEBUG_COLOR).\
-    operator()(__VA_ARGS__)
-
-} // mfem namespace
-
-#define DBG(...) { printf("\033[33m");  \
-                   printf(__VA_ARGS__); \
-                   printf(" \n\033[m"); \
-                   fflush(0); }
-
-#endif // MFEM_DEBUG_HPP

general/device.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -12,10 +12,9 @@
 #include "forall.hpp"
 #include "occa.hpp"
 #ifdef MFEM_USE_CEED
-#include "../fem/ceed/util.hpp"
+#include <ceed.h>
 #endif
 
-#include <unordered_map>
 #include <string>
 #include <map>
 
@@ -34,16 +33,13 @@ occa::device occaDevice;
 
 #ifdef MFEM_USE_CEED
 Ceed ceed = NULL;
-
-ceed::BasisMap ceed_basis_map;
-ceed::RestrMap ceed_restr_map;
 #endif
 
 // Backends listed by priority, high to low:
 static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 {
    Backend::CEED_CUDA, Backend::OCCA_CUDA, Backend::RAJA_CUDA, Backend::CUDA,
-   Backend::CEED_HIP, Backend::RAJA_HIP, Backend::HIP, Backend::DEBUG,
+   Backend::HIP, Backend::DEBUG,
    Backend::OCCA_OMP, Backend::RAJA_OMP, Backend::OMP,
    Backend::CEED_CPU, Backend::OCCA_CPU, Backend::RAJA_CPU, Backend::CPU
 };
@@ -52,7 +48,7 @@ static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 static const char *backend_name[Backend::NUM_BACKENDS] =
 {
    "ceed-cuda", "occa-cuda", "raja-cuda", "cuda",
-   "ceed-hip", "raja-hip", "hip", "debug",
+   "hip", "debug",
    "occa-omp", "raja-omp", "omp",
    "ceed-cpu", "occa-cpu", "raja-cpu", "cpu"
 };
@@ -65,9 +61,15 @@ Device Device::device_singleton;
 bool Device::device_env = false;
 bool Device::mem_host_env = false;
 bool Device::mem_device_env = false;
-bool Device::mem_types_set = false;
 
-Device::Device()
+Device::Device() : mode(Device::SEQUENTIAL),
+   backends(Backend::CPU),
+   destroy_mm(false),
+   mpi_gpu_aware(false),
+   host_mem_type(MemoryType::HOST),
+   host_mem_class(MemoryClass::HOST),
+   device_mem_type(MemoryType::HOST),
+   device_mem_class(MemoryClass::HOST)
 {
    if (getenv("MFEM_MEMORY") && !mem_host_env && !mem_device_env)
    {
@@ -152,18 +154,6 @@ Device::~Device()
    {
       free(device_option);
 #ifdef MFEM_USE_CEED
-      // Destroy FES -> CeedBasis, CeedElemRestriction hash table contents
-      for (auto entry : internal::ceed_basis_map)
-      {
-         CeedBasisDestroy(&entry.second);
-      }
-      internal::ceed_basis_map.clear();
-      for (auto entry : internal::ceed_restr_map)
-      {
-         CeedElemRestrictionDestroy(&entry.second);
-      }
-      internal::ceed_restr_map.clear();
-      // Destroy Ceed context
       CeedDestroy(&internal::ceed);
 #endif
       mm.Destroy();
@@ -220,24 +210,15 @@ void Device::Configure(const std::string &device, const int dev)
       beg = end + 1;
    }
 
-   // OCCA_CUDA and CEED_CUDA need CUDA or RAJA_CUDA:
-   if (Allows(Backend::OCCA_CUDA|Backend::CEED_CUDA) &&
-       !Allows(Backend::RAJA_CUDA))
+   // OCCA_CUDA needs CUDA or RAJA_CUDA:
+   if (Allows(Backend::OCCA_CUDA) && !Allows(Backend::RAJA_CUDA))
    {
       Get().MarkBackend(Backend::CUDA);
    }
-   // CEED_HIP needs HIP:
-   if (Allows(Backend::CEED_HIP))
+   if (Allows(Backend::CEED_CUDA))
    {
-      Get().MarkBackend(Backend::HIP);
+      Get().MarkBackend(Backend::CUDA);
    }
-   // OCCA_OMP will use OMP or RAJA_OMP unless MFEM_USE_OPENMP=NO:
-#ifdef MFEM_USE_OPENMP
-   if (Allows(Backend::OCCA_OMP) && !Allows(Backend::RAJA_OMP))
-   {
-      Get().MarkBackend(Backend::OMP);
-   }
-#endif
 
    // Perform setup.
    Get().Setup(dev);
@@ -252,30 +233,6 @@ void Device::Configure(const std::string &device, const int dev)
    destroy_mm = true;
 }
 
-// static method
-void Device::SetMemoryTypes(MemoryType h_mt, MemoryType d_mt)
-{
-   // If the device and/or the MemoryTypes are configured through the
-   // environment (variables 'MFEM_DEVICE', 'MFEM_MEMORY'), ignore calls to this
-   // method.
-   if (mem_host_env || mem_device_env || device_env) { return; }
-
-   MFEM_VERIFY(!IsConfigured(), "the default MemoryTypes can only be set before"
-               " Device construction and configuration");
-   MFEM_VERIFY(IsHostMemory(h_mt),
-               "invalid host MemoryType, h_mt = " << (int)h_mt);
-   MFEM_VERIFY(IsDeviceMemory(d_mt) || d_mt == h_mt,
-               "invalid device MemoryType, d_mt = " << (int)d_mt
-               << " (h_mt = " << (int)h_mt << ')');
-
-   Get().host_mem_type = h_mt;
-   Get().device_mem_type = d_mt;
-   mem_types_set = true;
-
-   // h_mt and d_mt will be set as dual to each other during configuration by
-   // the call mm.Configure(...) in UpdateMemoryTypeAndClass()
-}
-
 void Device::Print(std::ostream &out)
 {
    out << "Device configuration: ";
@@ -315,14 +272,7 @@ void Device::UpdateMemoryTypeAndClass()
 
 #ifdef MFEM_USE_UMPIRE
    // If MFEM has been compiled with Umpire support, use it as the default
-   if (!mem_host_env && !mem_types_set)
-   {
-      host_mem_type = MemoryType::HOST_UMPIRE;
-      if (!mem_device_env)
-      {
-         device_mem_type = MemoryType::HOST_UMPIRE;
-      }
-   }
+   if (!mem_host_env) { host_mem_type = MemoryType::HOST_UMPIRE; }
 #endif
 
    // Enable the device memory type
@@ -344,7 +294,7 @@ void Device::UpdateMemoryTypeAndClass()
                   device_mem_type = MemoryType::DEVICE;
             }
          }
-         else if (!mem_types_set)
+         else
          {
 #ifndef MFEM_USE_UMPIRE
             device_mem_type = MemoryType::DEVICE;
@@ -370,9 +320,6 @@ void Device::UpdateMemoryTypeAndClass()
       device_mem_type = MemoryType::DEVICE_DEBUG;
    }
 
-   MFEM_VERIFY(!device || IsDeviceMemory(device_mem_type),
-               "invalid device memory configuration!");
-
    // Update the memory manager with the new settings
    mm.Configure(host_mem_type, device_mem_type);
 }
@@ -422,8 +369,6 @@ static void RajaDeviceSetup(const int dev, int &ngpu)
 {
 #ifdef MFEM_USE_CUDA
    if (ngpu <= 0) { DeviceSetup(dev, ngpu); }
-#elif defined(MFEM_USE_HIP)
-   HipDeviceSetup(dev, ngpu);
 #else
    MFEM_CONTRACT_VAR(dev);
    MFEM_CONTRACT_VAR(ngpu);
@@ -490,8 +435,7 @@ static void CeedDeviceSetup(const char* ceed_spec)
    CeedInit(ceed_spec, &internal::ceed);
    const char *ceed_backend;
    CeedGetResource(internal::ceed, &ceed_backend);
-   if (strcmp(ceed_spec, ceed_backend) && strcmp(ceed_spec, "/cpu/self") &&
-       strcmp(ceed_spec, "/gpu/hip"))
+   if (strcmp(ceed_spec, ceed_backend) && strcmp(ceed_spec, "/cpu/self"))
    {
       mfem::out << std::endl << "WARNING!!!\n"
                 "libCEED is not using the requested backend!!!\n"
@@ -529,16 +473,12 @@ void Device::Setup(const int device)
    MFEM_VERIFY(!Allows(Backend::CEED_MASK),
                "the CEED backends require MFEM built with MFEM_USE_CEED=YES");
 #else
-   int ceed_cpu  = Allows(Backend::CEED_CPU);
-   int ceed_cuda = Allows(Backend::CEED_CUDA);
-   int ceed_hip  = Allows(Backend::CEED_HIP);
-   MFEM_VERIFY(ceed_cpu + ceed_cuda + ceed_hip <= 1,
+   MFEM_VERIFY(!Allows(Backend::CEED_CPU) || !Allows(Backend::CEED_CUDA),
                "Only one CEED backend can be enabled at a time!");
 #endif
    if (Allows(Backend::CUDA)) { CudaDeviceSetup(dev, ngpu); }
    if (Allows(Backend::HIP)) { HipDeviceSetup(dev, ngpu); }
-   if (Allows(Backend::RAJA_CUDA) || Allows(Backend::RAJA_HIP))
-   { RajaDeviceSetup(dev, ngpu); }
+   if (Allows(Backend::RAJA_CUDA)) { RajaDeviceSetup(dev, ngpu); }
    // The check for MFEM_USE_OCCA is in the function OccaDeviceSetup().
    if (Allows(Backend::OCCA_MASK)) { OccaDeviceSetup(dev); }
    if (Allows(Backend::CEED_CPU))
@@ -564,17 +504,6 @@ void Device::Setup(const int device)
          CeedDeviceSetup(device_option);
       }
    }
-   if (Allows(Backend::CEED_HIP))
-   {
-      if (!device_option)
-      {
-         CeedDeviceSetup("/gpu/hip");
-      }
-      else
-      {
-         CeedDeviceSetup(device_option);
-      }
-   }
    if (Allows(Backend::DEBUG)) { ngpu = 1; }
 }
 

general/device.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -46,33 +46,26 @@ struct Backend
       /** @brief [device] RAJA CUDA backend. Enabled when MFEM_USE_RAJA = YES
           and MFEM_USE_CUDA = YES. */
       RAJA_CUDA = 1 << 6,
-      /** @brief [device] RAJA HIP backend. Enabled when MFEM_USE_RAJA = YES
-          and MFEM_USE_HIP = YES. */
-      RAJA_HIP = 1 << 7,
       /** @brief [host] OCCA CPU backend: sequential execution on each MPI rank.
           Enabled when MFEM_USE_OCCA = YES. */
-      OCCA_CPU = 1 << 8,
+      OCCA_CPU = 1 << 7,
       /// [host] OCCA OpenMP backend. Enabled when MFEM_USE_OCCA = YES.
-      OCCA_OMP = 1 << 9,
+      OCCA_OMP = 1 << 8,
       /** @brief [device] OCCA CUDA backend. Enabled when MFEM_USE_OCCA = YES
           and MFEM_USE_CUDA = YES. */
-      OCCA_CUDA = 1 << 10,
+      OCCA_CUDA = 1 << 9,
       /** @brief [host] CEED CPU backend. GPU backends can still be used, but
           with expensive memory transfers. Enabled when MFEM_USE_CEED = YES. */
-      CEED_CPU  = 1 << 11,
+      CEED_CPU  = 1 << 10,
       /** @brief [device] CEED CUDA backend working together with the CUDA
           backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_CUDA = YES.
-          NOTE: The current default libCEED CUDA backend is non-deterministic! */
-      CEED_CUDA = 1 << 12,
-      /** @brief [device] CEED HIP backend working together with the HIP
-          backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_HIP = YES. */
-      CEED_HIP = 1 << 13,
+          NOTE: The current default libCEED GPU backend is non-deterministic! */
+      CEED_CUDA = 1 << 11,
       /** @brief [device] Debug backend: host memory is READ/WRITE protected
           while a device is in use. It allows to test the "device" code-path
           (using separate host/device memory pools and host <-> device
-          transfers) without any GPU hardware. As 'DEBUG' is sometimes used
-          as a macro, `_DEVICE` has been added to avoid conflicts. */
-      DEBUG = 1 << 14
+          transfers) without any GPU hardware. */
+      DEBUG = 1 << 12
    };
 
    /** @brief Additional useful constants. For example, the *_MASK constants can
@@ -80,23 +73,23 @@ struct Backend
    enum
    {
       /// Number of backends: from (1 << 0) to (1 << (NUM_BACKENDS-1)).
-      NUM_BACKENDS = 15,
+      NUM_BACKENDS = 13,
 
       /// Biwise-OR of all CPU backends
       CPU_MASK = CPU | RAJA_CPU | OCCA_CPU | CEED_CPU,
       /// Biwise-OR of all CUDA backends
       CUDA_MASK = CUDA | RAJA_CUDA | OCCA_CUDA | CEED_CUDA,
       /// Biwise-OR of all HIP backends
-      HIP_MASK = HIP | RAJA_HIP | CEED_HIP,
+      HIP_MASK = HIP,
       /// Biwise-OR of all OpenMP backends
       OMP_MASK = OMP | RAJA_OMP | OCCA_OMP,
       /// Bitwise-OR of all CEED backends
-      CEED_MASK = CEED_CPU | CEED_CUDA | CEED_HIP,
+      CEED_MASK = CEED_CPU | CEED_CUDA,
       /// Biwise-OR of all device backends
       DEVICE_MASK = CUDA_MASK | HIP_MASK | DEBUG,
 
       /// Biwise-OR of all RAJA backends
-      RAJA_MASK = RAJA_CPU | RAJA_OMP | RAJA_CUDA | RAJA_HIP,
+      RAJA_MASK = RAJA_CPU | RAJA_OMP | RAJA_CUDA,
       /// Biwise-OR of all OCCA backends
       OCCA_MASK = OCCA_CPU | OCCA_OMP | OCCA_CUDA
    };
@@ -124,25 +117,22 @@ private:
    friend class MemoryManager;
    enum MODES {SEQUENTIAL, ACCELERATED};
 
-   static bool device_env, mem_host_env, mem_device_env, mem_types_set;
+   static bool device_env, mem_host_env, mem_device_env;
    static Device device_singleton;
 
-   MODES mode = Device::SEQUENTIAL;
-   int dev = 0;   ///< Device ID of the configured device.
+   MODES mode;
+   int dev = 0; ///< Device ID of the configured device.
    int ngpu = -1; ///< Number of detected devices; -1: not initialized.
-   /// Bitwise-OR of all configured backends.
-   unsigned long backends = Backend::CPU;
+   unsigned long backends; ///< Bitwise-OR of all configured backends.
    /// Set to true during configuration, except in 'device_singleton'.
-   bool destroy_mm = false;
-   bool mpi_gpu_aware = false;
+   bool destroy_mm;
+   bool mpi_gpu_aware;
 
-   MemoryType host_mem_type = MemoryType::HOST;    ///< Current Host MemoryType
-   MemoryClass host_mem_class = MemoryClass::HOST; ///< Current Host MemoryClass
+   MemoryType host_mem_type;      ///< Current Host MemoryType
+   MemoryClass host_mem_class;    ///< Current Host MemoryClass
 
-   /// Current Device MemoryType
-   MemoryType device_mem_type = MemoryType::HOST;
-   /// Current Device MemoryClass
-   MemoryClass device_mem_class = MemoryClass::HOST;
+   MemoryType device_mem_type;    ///< Current Device MemoryType
+   MemoryClass device_mem_class;  ///< Current Device MemoryClass
 
    char *device_option = NULL;
    Device(Device const&);
@@ -183,6 +173,14 @@ public:
        @note This object should be destroyed after all other MFEM objects that
        use the Device are destroyed. */
    Device(const std::string &device, const int dev = 0)
+      : mode(Device::SEQUENTIAL),
+        backends(Backend::CPU),
+        destroy_mm(false),
+        mpi_gpu_aware(false),
+        host_mem_type(MemoryType::HOST),
+        host_mem_class(MemoryClass::HOST),
+        device_mem_type(MemoryType::HOST),
+        device_mem_class(MemoryClass::HOST)
    { Configure(device, dev); }
 
    /// Destructor.
@@ -195,44 +193,25 @@ public:
        * The available backends are described by the Backend class.
        * The string name of a backend is the lowercase version of the
          Backend::Id enumeration constant with '_' replaced by '-', e.g. the
-         string name of 'RAJA_CPU' is 'raja-cpu'. The string name of the debug
-         backend (Backend::Id 'DEBUG_DEVICE') is exceptionally set to 'debug'.
+         string name of 'RAJA_CPU' is 'raja-cpu'.
        * The 'cpu' backend is always enabled with lowest priority.
        * The current backend priority from highest to lowest is:
-         'ceed-cuda', 'occa-cuda', 'raja-cuda', 'cuda',
-         'ceed-hip', 'hip', 'debug',
+         'ceed-cuda', 'occa-cuda', 'raja-cuda', 'cuda', 'hip', 'debug',
          'occa-omp', 'raja-omp', 'omp',
          'ceed-cpu', 'occa-cpu', 'raja-cpu', 'cpu'.
        * Multiple backends can be configured at the same time.
        * Only one 'occa-*' backend can be configured at a time.
        * The backend 'occa-cuda' enables the 'cuda' backend unless 'raja-cuda'
          is already enabled.
-       * The backend 'occa-omp' enables the 'omp' backend (if MFEM was built
-         with MFEM_USE_OPENMP=YES) unless 'raja-omp' is already enabled.
-       * Only one 'ceed-*' backend can be configured at a time.
        * The backend 'ceed-cpu' delegates to a libCEED CPU backend the setup and
          evaluation of the operator.
        * The backend 'ceed-cuda' delegates to a libCEED CUDA backend the setup
-         and evaluation of operators and enables the 'cuda' backend to avoid
-         transfers between host and device.
-       * The backend 'ceed-hip' delegates to a libCEED HIP backend the setup
-         and evaluation of operators and enables the 'hip' backend to avoid
-         transfers between host and device.
+         and evaluation of the operator and enables the 'cuda' backend to avoid
+         transfer between host and device.
        * The 'debug' backend should not be combined with other device backends.
    */
    void Configure(const std::string &device, const int dev = 0);
 
-   /// Set the default host and device MemoryTypes, @a h_mt and @a d_mt.
-   /** The host and device MemoryTypes are also set to be dual to each other.
-
-       These two MemoryType%s are used by most MFEM classes when allocating
-       memory used on host and device, respectively.
-
-       This method can only be called before Device construction and
-       configuration, and the specified memory types must be compatible with
-       the subsequent Device configuration. */
-   static void SetMemoryTypes(MemoryType h_mt, MemoryType d_mt);
-
    /// Print the configuration of the MFEM virtual device object.
    void Print(std::ostream &out = mfem::out);
 

general/forall.hpp

@@ -23,8 +23,8 @@ namespace mfem
 {
 
 // Maximum size of dofs and quads in 1D.
-const int MAX_D1D = 8;
-const int MAX_Q1D = 8;
+const int MAX_D1D = 14;
+const int MAX_Q1D = 14;
 
 // MFEM pragma macros that can be used inside MFEM_FORALL macros.
 #define MFEM_PRAGMA(X) _Pragma(#X)
@@ -89,11 +89,7 @@ void OmpWrap(const int N, HBODY &&h_body)
 /// RAJA Cuda backend
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA)
 
-#if RAJA_VERSION_MAJOR == 0 && RAJA_VERSION_MINOR < 12
 using RAJA::statement::Segs;
-#else
-using RAJA::Segs;
-#endif
 
 template <const int BLOCKS = MFEM_CUDA_BLOCKS, typename DBODY>
 void RajaCudaWrap1D(const int N, DBODY &&d_body)
@@ -151,12 +147,7 @@ void RajaCudaWrap3D(const int N, DBODY &&d_body,
 /// RAJA OpenMP backend
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
 
-#if RAJA_VERSION_MAJOR == 0 && RAJA_VERSION_MINOR < 12
 using RAJA::statement::Segs;
-#else
-using RAJA::Segs;
-#endif
-
 
 template <typename HBODY>
 void RajaOmpWrap(const int N, HBODY &&h_body)

general/hip.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -61,21 +61,6 @@ void* HipMallocManaged(void** dptr, size_t bytes)
    return *dptr;
 }
 
-void* HipMemAllocHostPinned(void** ptr, size_t bytes)
-{
-#ifdef MFEM_USE_HIP
-#ifdef MFEM_TRACK_HIP_MEM
-   mfem::out << "HipMemAllocHostPinned(): allocating " << bytes << " bytes ... "
-             << std::flush;
-#endif
-   MFEM_GPU_CHECK(hipHostMalloc(ptr, bytes, hipHostMallocDefault));
-#ifdef MFEM_TRACK_HIP_MEM
-   mfem::out << "done: " << *ptr << std::endl;
-#endif
-#endif
-   return *ptr;
-}
-
 void* HipMemFree(void *dptr)
 {
 #ifdef MFEM_USE_HIP
@@ -91,21 +76,6 @@ void* HipMemFree(void *dptr)
    return dptr;
 }
 
-void* HipMemFreeHostPinned(void *ptr)
-{
-#ifdef MFEM_USE_HIP
-#ifdef MFEM_TRACK_HIP_MEM
-   mfem::out << "HipMemFreeHostPinned(): deallocating memory @ " << ptr << " ... "
-             << std::flush;
-#endif
-   MFEM_GPU_CHECK(hipHostFree(ptr));
-#ifdef MFEM_TRACK_HIP_MEM
-   mfem::out << "done." << std::endl;
-#endif
-#endif
-   return ptr;
-}
-
 void* HipMemcpyHtoD(void* dst, const void* src, size_t bytes)
 {
 #ifdef MFEM_USE_HIP
@@ -155,7 +125,7 @@ void* HipMemcpyDtoDAsync(void* dst, const void *src, size_t bytes)
 void* HipMemcpyDtoH(void *dst, const void *src, size_t bytes)
 {
 #ifdef MFEM_USE_HIP
-#ifdef MFEM_TRACK_HIP_MEM
+#ifdef MFEM_TRACK_HPI_MEM
    mfem::out << "HipMemcpyDtoH(): copying " << bytes << " bytes from "
              << src << " to " << dst << " ... " << std::flush;
 #endif

general/hip.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -64,15 +64,9 @@ void* HipMemAlloc(void **d_ptr, size_t bytes);
 /// Allocates managed device memory
 void* HipMallocManaged(void **d_ptr, size_t bytes);
 
-/// Allocates page-locked (pinned) host memory
-void* HipMemAllocHostPinned(void **ptr, size_t bytes);
-
 /// Frees device memory
 void* HipMemFree(void *d_ptr);
 
-/// Frees page-locked (pinned) host memory and returns destination ptr.
-void* HipMemFreeHostPinned(void *ptr);
-
 /// Copies memory from Host to Device
 void* HipMemcpyHtoD(void *d_dst, const void *h_src, size_t bytes);
 

general/mem_manager.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -62,39 +62,38 @@ MemoryType GetMemoryType(MemoryClass mc)
    return MemoryType::HOST;
 }
 
+// We want to keep this pairs, as it is checked in MFEM_VERIFY_TYPES
+MemoryType MemoryManager::GetDualMemoryType_(MemoryType mt)
+{
+   switch (mt)
+   {
+      case MemoryType::HOST:           return MemoryType::DEVICE;
+      case MemoryType::HOST_32:        return MemoryType::DEVICE;
+      case MemoryType::HOST_64:        return MemoryType::DEVICE;
+      case MemoryType::HOST_DEBUG:     return MemoryType::DEVICE_DEBUG;
+      case MemoryType::HOST_UMPIRE:    return MemoryType::DEVICE_UMPIRE;
+      case MemoryType::MANAGED:        return MemoryType::MANAGED;
+      case MemoryType::DEVICE:         return MemoryType::HOST;
+      case MemoryType::DEVICE_DEBUG:   return MemoryType::HOST_DEBUG;
+      case MemoryType::DEVICE_UMPIRE:  return MemoryType::HOST_UMPIRE;
+      default: mfem_error("Unknown memory type!");
+   }
+   MFEM_VERIFY(false,"");
+   return MemoryType::HOST;
+}
 
 static void MFEM_VERIFY_TYPES(const MemoryType h_mt, const MemoryType d_mt)
 {
-   MFEM_VERIFY(IsHostMemory(h_mt), "h_mt = " << (int)h_mt);
-   MFEM_VERIFY(IsDeviceMemory(d_mt) || d_mt == MemoryType::DEFAULT,
-               "d_mt = " << (int)d_mt);
-   // If h_mt == MemoryType::HOST_DEBUG, then d_mt == MemoryType::DEVICE_DEBUG
-   //                                      or d_mt == MemoryType::DEFAULT
-   MFEM_VERIFY(h_mt != MemoryType::HOST_DEBUG ||
-               d_mt == MemoryType::DEVICE_DEBUG ||
-               d_mt == MemoryType::DEFAULT,
-               "d_mt = " << MemoryTypeName[(int)d_mt]);
-   // If d_mt == MemoryType::DEVICE_DEBUG, then h_mt == MemoryType::HOST_DEBUG
-   MFEM_VERIFY(d_mt != MemoryType::DEVICE_DEBUG ||
-               h_mt == MemoryType::HOST_DEBUG,
-               "h_mt = " << MemoryTypeName[(int)h_mt]);
-#if 0
+   MFEM_ASSERT(IsHostMemory(h_mt),"");
+   MFEM_ASSERT(IsDeviceMemory(d_mt),"");
    const bool sync =
-      (h_mt == MemoryType::HOST_PINNED && d_mt == MemoryType::DEVICE) ||
-      (h_mt == MemoryType::HOST_PINNED && d_mt == MemoryType::DEVICE_UMPIRE) ||
-      (h_mt == MemoryType::HOST_PINNED && d_mt == MemoryType::DEVICE_UMPIRE_2) ||
-      (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE) ||
       (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE_UMPIRE) ||
-      (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE_UMPIRE_2) ||
       (h_mt == MemoryType::HOST_DEBUG && d_mt == MemoryType::DEVICE_DEBUG) ||
       (h_mt == MemoryType::MANAGED && d_mt == MemoryType::MANAGED) ||
       (h_mt == MemoryType::HOST_64 && d_mt == MemoryType::DEVICE) ||
       (h_mt == MemoryType::HOST_32 && d_mt == MemoryType::DEVICE) ||
-      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE) ||
-      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE_UMPIRE) ||
-      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE_UMPIRE_2);
+      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE);
    MFEM_VERIFY(sync, "");
-#endif
 }
 
 MemoryClass operator*(MemoryClass mc1, MemoryClass mc2)
@@ -136,12 +135,9 @@ struct Memory
    void *const h_ptr;
    void *d_ptr;
    const size_t bytes;
-   const MemoryType h_mt;
-   MemoryType d_mt;
-   mutable bool h_rw, d_rw;
+   const MemoryType h_mt, d_mt;
    Memory(void *p, size_t b, MemoryType h, MemoryType d):
-      h_ptr(p), d_ptr(nullptr), bytes(b), h_mt(h), d_mt(d),
-      h_rw(true), d_rw(true) { }
+      h_ptr(p), d_ptr(nullptr), bytes(b), h_mt(h), d_mt(d) { }
 };
 
 /// Alias class that holds the base memory region and the offset
@@ -177,8 +173,8 @@ public:
    virtual ~HostMemorySpace() { }
    virtual void Alloc(void **ptr, size_t bytes) { *ptr = std::malloc(bytes); }
    virtual void Dealloc(void *ptr) { std::free(ptr); }
-   virtual void Protect(const Memory&, size_t) { }
-   virtual void Unprotect(const Memory&, size_t) { }
+   virtual void Protect(const void*, size_t) { }
+   virtual void Unprotect(const void*, size_t) { }
    virtual void AliasProtect(const void*, size_t) { }
    virtual void AliasUnprotect(const void*, size_t) { }
 };
@@ -356,10 +352,8 @@ public:
    MmuHostMemorySpace(): HostMemorySpace() { MmuInit(); }
    void Alloc(void **ptr, size_t bytes) { MmuAlloc(ptr, bytes); }
    void Dealloc(void *ptr) { MmuDealloc(ptr, maps->memories.at(ptr).bytes); }
-   void Protect(const Memory& mem, size_t bytes)
-   { if (mem.h_rw) { mem.h_rw = false; MmuProtect(mem.h_ptr, bytes); } }
-   void Unprotect(const Memory &mem, size_t bytes)
-   { if (!mem.h_rw) { mem.h_rw = true; MmuAllow(mem.h_ptr, bytes); } }
+   void Protect(const void *ptr, size_t bytes) { MmuProtect(ptr, bytes); }
+   void Unprotect(const void *ptr, size_t bytes) { MmuAllow(ptr, bytes); }
    /// Aliases need to be restricted during protection
    void AliasProtect(const void *ptr, size_t bytes)
    { MmuProtect(MmuAddrR(ptr), MmuLengthR(ptr, bytes)); }
@@ -406,31 +400,6 @@ public:
    { return CuMemcpyDtoH(dst, src, bytes); }
 };
 
-/// The CUDA/HIP page-locked host memory space
-class HostPinnedMemorySpace: public HostMemorySpace
-{
-public:
-   HostPinnedMemorySpace(): HostMemorySpace() { }
-   void Alloc(void ** ptr, size_t bytes) override
-   {
-#ifdef MFEM_USE_CUDA
-      CuMemAllocHostPinned(ptr, bytes);
-#endif
-#ifdef MFEM_USE_HIP
-      HipMemAllocHostPinned(ptr, bytes);
-#endif
-   }
-   void Dealloc(void *ptr) override
-   {
-#ifdef MFEM_USE_CUDA
-      CuMemFreeHostPinned(ptr);
-#endif
-#ifdef MFEM_USE_HIP
-      HipMemFreeHostPinned(ptr);
-#endif
-   }
-};
-
 /// The HIP device memory space
 class HipDeviceMemorySpace: public DeviceMemorySpace
 {
@@ -473,10 +442,8 @@ public:
    MmuDeviceMemorySpace(): DeviceMemorySpace() { }
    void Alloc(Memory &m) { MmuAlloc(&m.d_ptr, m.bytes); }
    void Dealloc(Memory &m) { MmuDealloc(m.d_ptr, m.bytes); }
-   void Protect(const Memory &m)
-   { if (m.d_rw) { m.d_rw = false; MmuProtect(m.d_ptr, m.bytes); } }
-   void Unprotect(const Memory &m)
-   { if (!m.d_rw) { m.d_rw = true; MmuAllow(m.d_ptr, m.bytes); } }
+   void Protect(const Memory &m) { MmuProtect(m.d_ptr, m.bytes); }
+   void Unprotect(const Memory &m) { MmuAllow(m.d_ptr, m.bytes); }
    /// Aliases need to be restricted during protection
    void AliasProtect(const void *ptr, size_t bytes)
    { MmuProtect(MmuAddrR(ptr), MmuLengthR(ptr, bytes)); }
@@ -491,64 +458,54 @@ public:
    { return std::memcpy(dst, src, bytes); }
 };
 
-#ifdef MFEM_USE_UMPIRE
-class UmpireMemorySpace
-{
-protected:
-   umpire::ResourceManager &rm;
-   umpire::Allocator allocator;
-   bool owns_allocator{false};
-
-public:
-   // TODO: this only releases unused memory
-   virtual ~UmpireMemorySpace() { if (owns_allocator) { allocator.release(); } }
-   UmpireMemorySpace(const char * name, const char * space)
-      : rm(umpire::ResourceManager::getInstance())
-   {
-      if (!rm.isAllocator(name))
-      {
-         allocator = rm.makeAllocator<umpire::strategy::DynamicPool>(
-                        name, rm.getAllocator(space));
-         owns_allocator = true;
-      }
-      else
-      {
-         allocator = rm.getAllocator(name);
-         owns_allocator = false;
-      }
-   }
-};
-
+#ifndef MFEM_USE_UMPIRE
+class UmpireHostMemorySpace : public NoHostMemorySpace { };
+class UmpireDeviceMemorySpace : public NoDeviceMemorySpace { };
+#else
 /// The Umpire host memory space
-class UmpireHostMemorySpace : public HostMemorySpace, public UmpireMemorySpace
+class UmpireHostMemorySpace : public HostMemorySpace
 {
 private:
+   const char *name;
+   umpire::ResourceManager &rm;
+   umpire::Allocator h_allocator;
    umpire::strategy::AllocationStrategy *strat;
 public:
-   UmpireHostMemorySpace(const char * name)
-      : HostMemorySpace(),
-        UmpireMemorySpace(name, "HOST"),
-        strat(allocator.getAllocationStrategy()) {}
-   void Alloc(void **ptr, size_t bytes) override
-   { *ptr = allocator.allocate(bytes); }
-   void Dealloc(void *ptr) override { allocator.deallocate(ptr); }
+   ~UmpireHostMemorySpace() { h_allocator.release(); }
+   UmpireHostMemorySpace():
+      HostMemorySpace(),
+      name(mm.GetUmpireAllocatorHostName()),
+      rm(umpire::ResourceManager::getInstance()),
+      h_allocator(rm.isAllocator(name)? rm.getAllocator(name):
+                  rm.makeAllocator<umpire::strategy::DynamicPool>
+                  (name, rm.getAllocator("HOST"))),
+      strat(h_allocator.getAllocationStrategy()) { }
+   void Alloc(void **ptr, size_t bytes) { *ptr = h_allocator.allocate(bytes); }
+   void Dealloc(void *ptr) { h_allocator.deallocate(ptr); }
    void Insert(void *ptr, size_t bytes)
    { rm.registerAllocation(ptr, {ptr, bytes, strat}); }
 };
 
 /// The Umpire device memory space
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-class UmpireDeviceMemorySpace : public DeviceMemorySpace,
-   public UmpireMemorySpace
+#ifdef MFEM_USE_CUDA
+class UmpireDeviceMemorySpace : public DeviceMemorySpace
 {
+private:
+   const char *name;
+   umpire::ResourceManager &rm;
+   umpire::Allocator d_allocator;
 public:
-   UmpireDeviceMemorySpace(const char * name)
-      : DeviceMemorySpace(),
-        UmpireMemorySpace(name, "DEVICE") {}
-   void Alloc(Memory &base) override
-   { base.d_ptr = allocator.allocate(base.bytes); }
-   void Dealloc(Memory &base) override { rm.deallocate(base.d_ptr); }
-   void *HtoD(void *dst, const void *src, size_t bytes) override
+   ~UmpireDeviceMemorySpace() { d_allocator.release(); }
+   UmpireDeviceMemorySpace():
+      DeviceMemorySpace(),
+      name(mm.GetUmpireAllocatorDeviceName()),
+      rm(umpire::ResourceManager::getInstance()),
+      d_allocator(rm.isAllocator(name)? rm.getAllocator(name):
+                  rm.makeAllocator<umpire::strategy::DynamicPool>
+                  (name, rm.getAllocator("DEVICE"))) { }
+   void Alloc(Memory &base) { base.d_ptr = d_allocator.allocate(base.bytes); }
+   void Dealloc(Memory &base) { d_allocator.deallocate(base.d_ptr); }
+   void *HtoD(void *dst, const void *src, size_t bytes)
    {
 #ifdef MFEM_USE_CUDA
       return CuMemcpyHtoD(dst, src, bytes);
@@ -558,7 +515,7 @@ public:
 #endif
       //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
-   void *DtoD(void* dst, const void* src, size_t bytes) override
+   void *DtoD(void* dst, const void* src, size_t bytes)
    {
 #ifdef MFEM_USE_CUDA
       return CuMemcpyDtoD(dst, src, bytes);
@@ -568,7 +525,7 @@ public:
 #endif
       //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
-   void *DtoH(void *dst, const void *src, size_t bytes) override
+   void *DtoH(void *dst, const void *src, size_t bytes)
    {
 #ifdef MFEM_USE_CUDA
       return CuMemcpyDtoH(dst, src, bytes);
@@ -580,12 +537,8 @@ public:
    }
 };
 #else
-class UmpireDeviceMemorySpace : public NoDeviceMemorySpace
-{
-public:
-   UmpireDeviceMemorySpace(const char * /*unused*/) {}
-};
-#endif // MFEM_USE_CUDA || MFEM_USE_HIP
+class UmpireDeviceMemorySpace : public NoDeviceMemorySpace { };
+#endif // MFEM_USE_CUDA
 #endif // MFEM_USE_UMPIRE
 
 /// Memory space controller class
@@ -615,7 +568,7 @@ public:
       host[static_cast<int>(MT::HOST_64)] = new Aligned64HostMemorySpace();
       // HOST_DEBUG is delayed, as it reroutes signals
       host[static_cast<int>(MT::HOST_DEBUG)] = nullptr;
-      host[static_cast<int>(MT::HOST_UMPIRE)] = nullptr;
+      host[static_cast<int>(MT::HOST_UMPIRE)] = new UmpireHostMemorySpace();
       host[static_cast<int>(MT::MANAGED)] = new UvmHostMemorySpace();
 
       // Filling the device memory backends, shifting with the device size
@@ -625,7 +578,6 @@ public:
       device[static_cast<int>(MemoryType::DEVICE)-shift] = nullptr;
       device[static_cast<int>(MT::DEVICE_DEBUG)-shift] = nullptr;
       device[static_cast<int>(MT::DEVICE_UMPIRE)-shift] = nullptr;
-      device[static_cast<int>(MT::DEVICE_UMPIRE_2)-shift] = nullptr;
    }
 
    HostMemorySpace* Host(const MemoryType mt)
@@ -658,19 +610,8 @@ public:
 private:
    HostMemorySpace* NewHostCtrl(const MemoryType mt)
    {
-      switch (mt)
-      {
-         case MT::HOST_DEBUG: return new MmuHostMemorySpace();
-#ifdef MFEM_USE_UMPIRE
-         case MT::HOST_UMPIRE:
-            return new UmpireHostMemorySpace(
-                      MemoryManager::GetUmpireHostAllocatorName());
-#else
-         case MT::HOST_UMPIRE: return new NoHostMemorySpace();
-#endif
-         case MT::HOST_PINNED: return new HostPinnedMemorySpace();
-         default: MFEM_ABORT("Unknown host memory controller!");
-      }
+      if (mt == MT::HOST_DEBUG) { return new MmuHostMemorySpace(); }
+      MFEM_ABORT("Unknown host memory controller!");
       return nullptr;
    }
 
@@ -678,17 +619,7 @@ private:
    {
       switch (mt)
       {
-#ifdef MFEM_USE_UMPIRE
-         case MT::DEVICE_UMPIRE:
-            return new UmpireDeviceMemorySpace(
-                      MemoryManager::GetUmpireDeviceAllocatorName());
-         case MT::DEVICE_UMPIRE_2:
-            return new UmpireDeviceMemorySpace(
-                      MemoryManager::GetUmpireDevice2AllocatorName());
-#else
-         case MT::DEVICE_UMPIRE: return new NoDeviceMemorySpace();
-         case MT::DEVICE_UMPIRE_2: return new NoDeviceMemorySpace();
-#endif
+         case MT::DEVICE_UMPIRE: return new UmpireDeviceMemorySpace();
          case MT::DEVICE_DEBUG: return new MmuDeviceMemorySpace();
          case MT::DEVICE:
          {
@@ -715,50 +646,20 @@ void *MemoryManager::New_(void *h_tmp, size_t bytes, MemoryType mt,
                           unsigned &flags)
 {
    MFEM_ASSERT(exists, "Internal error!");
-   if (IsHostMemory(mt))
-   {
-      MFEM_ASSERT(mt != MemoryType::HOST && h_tmp == nullptr,
-                  "Internal error!");
-      // d_mt = MemoryType::DEFAULT means d_mt = GetDualMemoryType(h_mt),
-      // evaluated at the time when the device pointer is allocated, see
-      // GetDevicePtr() and GetAliasDevicePtr()
-      const MemoryType d_mt = MemoryType::DEFAULT;
-      // We rely on the next call using lazy dev alloc
-      return New_(h_tmp, bytes, mt, d_mt, Mem::VALID_HOST, flags);
-   }
-   else
-   {
-      const MemoryType h_mt = GetDualMemoryType(mt);
-      return New_(h_tmp, bytes, h_mt, mt, Mem::VALID_DEVICE, flags);
-   }
-}
-
-void *MemoryManager::New_(void *h_tmp, size_t bytes, MemoryType h_mt,
-                          MemoryType d_mt, unsigned valid_flags,
-                          unsigned &flags)
-{
-   MFEM_ASSERT(exists, "Internal error!");
-   MFEM_ASSERT(IsHostMemory(h_mt), "h_mt must be host type");
-   MFEM_ASSERT(IsDeviceMemory(d_mt) || d_mt == h_mt ||
-               d_mt == MemoryType::DEFAULT,
-               "d_mt must be device type, the same is h_mt, or DEFAULT");
-   MFEM_ASSERT((h_mt != MemoryType::HOST || h_tmp != nullptr) &&
-               (h_mt == MemoryType::HOST || h_tmp == nullptr),
-               "Internal error");
-   MFEM_ASSERT((valid_flags & ~(Mem::VALID_HOST | Mem::VALID_DEVICE)) == 0,
-               "Internal error");
-   void *h_ptr;
+   MFEM_ASSERT(mt != MemoryType::HOST, "Internal error!");
+   const bool is_host_mem = IsHostMemory(mt);
+   const MemType dual_mt = GetDualMemoryType_(mt);
+   const MemType h_mt = is_host_mem ? mt : dual_mt;
+   const MemType d_mt = is_host_mem ? dual_mt : mt;
+   MFEM_VERIFY_TYPES(h_mt, d_mt);
+   void *h_ptr = h_tmp;
    if (h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
-   else { h_ptr = h_tmp; }
-   flags = Mem::REGISTERED | Mem::OWNS_INTERNAL | Mem::OWNS_HOST |
-           Mem::OWNS_DEVICE | valid_flags;
-   // The other New_() method relies on this lazy allocation behavior.
-   mm.Insert(h_ptr, bytes, h_mt, d_mt); // lazy dev alloc
-   // mm.InsertDevice(nullptr, h_ptr, bytes, h_mt, d_mt); // non-lazy dev alloc
-
-   // MFEM_VERIFY_TYPES(h_mt, mt); // done by mm.Insert() above
+   flags = Mem::REGISTERED;
+   flags |= Mem::OWNS_INTERNAL | Mem::OWNS_HOST | Mem::OWNS_DEVICE;
+   flags |= is_host_mem ? Mem::VALID_HOST : Mem::VALID_DEVICE;
+   if (is_host_mem) { mm.Insert(h_ptr, bytes, h_mt, d_mt); }
+   else { mm.InsertDevice(nullptr, h_ptr, bytes, h_mt, d_mt); }
    CheckHostMemoryType_(h_mt, h_ptr);
-
    return h_ptr;
 }
 
@@ -770,12 +671,9 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
    MFEM_ASSERT(exists, "Internal error!");
    MFEM_ASSERT(!alias, "Cannot register an alias!");
    const bool is_host_mem = IsHostMemory(mt);
-   const MemType h_mt = is_host_mem ? mt : GetDualMemoryType(mt);
-   const MemType d_mt = is_host_mem ? MemoryType::DEFAULT : mt;
-   // d_mt = MemoryType::DEFAULT means d_mt = GetDualMemoryType(h_mt),
-   // evaluated at the time when the device pointer is allocated, see
-   // GetDevicePtr() and GetAliasDevicePtr()
-
+   const MemType dual_mt = GetDualMemoryType_(mt);
+   const MemType h_mt = is_host_mem ? mt : dual_mt;
+   const MemType d_mt = is_host_mem ? dual_mt : mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
 
    if (ptr == nullptr && h_tmp == nullptr)
@@ -796,42 +694,17 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
    }
    else // DEVICE TYPES
    {
-      MFEM_VERIFY(ptr, "cannot register NULL device pointer");
-      if (h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
-      else { h_ptr = h_tmp; }
+      h_ptr = h_tmp;
+      if (own && h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
       mm.InsertDevice(ptr, h_ptr, bytes, h_mt, d_mt);
       flags = own ? flags | Mem::OWNS_DEVICE : flags & ~Mem::OWNS_DEVICE;
-      flags |= (Mem::OWNS_HOST | Mem::VALID_DEVICE);
+      flags = own ? flags | Mem::OWNS_HOST   : flags & ~Mem::OWNS_HOST;
+      flags |= Mem::VALID_DEVICE;
    }
    CheckHostMemoryType_(h_mt, h_ptr);
    return h_ptr;
 }
 
-void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,
-                              MemoryType h_mt, MemoryType d_mt,
-                              bool own, bool alias, unsigned &flags)
-{
-   MFEM_CONTRACT_VAR(alias);
-   MFEM_ASSERT(exists, "Internal error!");
-   MFEM_ASSERT(!alias, "Cannot register an alias!");
-   MFEM_VERIFY_TYPES(h_mt, d_mt);
-
-   if (h_ptr == nullptr && d_ptr == nullptr)
-   {
-      MFEM_VERIFY(bytes == 0, "internal error");
-      return;
-   }
-
-   flags |= Mem::REGISTERED | Mem::OWNS_INTERNAL;
-
-   mm.InsertDevice(d_ptr, h_ptr, bytes, h_mt, d_mt);
-   flags = (own ? flags | (Mem::OWNS_HOST | Mem::OWNS_DEVICE) :
-            flags & ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE)) |
-           Mem::VALID_HOST;
-
-   CheckHostMemoryType_(h_mt, h_ptr);
-}
-
 void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
                            unsigned base_flags, unsigned &flags)
 {
@@ -841,34 +714,6 @@ void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
            ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE);
 }
 
-void MemoryManager::SetDeviceMemoryType_(void *h_ptr, unsigned flags,
-                                         MemoryType d_mt)
-{
-   MFEM_VERIFY(h_ptr, "cannot set the device memory type: Memory is empty!");
-   if (!(flags & Mem::ALIAS))
-   {
-      auto mem_iter = maps->memories.find(h_ptr);
-      MFEM_VERIFY(mem_iter != maps->memories.end(), "internal error");
-      internal::Memory &mem = mem_iter->second;
-      if (mem.d_mt == d_mt) { return; }
-      MFEM_VERIFY(mem.d_ptr == nullptr, "cannot set the device memory type:"
-                  " device memory is allocated!");
-      mem.d_mt = d_mt;
-   }
-   else
-   {
-      auto alias_iter = maps->aliases.find(h_ptr);
-      MFEM_VERIFY(alias_iter != maps->aliases.end(), "internal error");
-      internal::Alias &alias = alias_iter->second;
-      internal::Memory &base_mem = *alias.mem;
-      if (base_mem.d_mt == d_mt) { return; }
-      MFEM_VERIFY(base_mem.d_ptr == nullptr,
-                  "cannot set the device memory type:"
-                  " alias' base device memory is allocated!");
-      base_mem.d_mt = d_mt;
-   }
-}
-
 MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType mt, unsigned flags)
 {
    const bool alias = flags & Mem::ALIAS;
@@ -902,16 +747,6 @@ MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType mt, unsigned flags)
    return mt;
 }
 
-void MemoryManager::DeleteDevice_(void *h_ptr, unsigned & flags)
-{
-   const bool owns_device = flags & Mem::OWNS_DEVICE;
-   if (owns_device)
-   {
-      mm.EraseDevice(h_ptr);
-      flags = (flags | Mem::VALID_HOST) & ~Mem::VALID_DEVICE;
-   }
-}
-
 bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
                                       MemoryType h_mt, size_t bytes,
                                       unsigned flags)
@@ -925,12 +760,11 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
    const bool known = mm.IsKnown(h_ptr);
    const bool alias = mm.IsAlias(h_ptr);
    const bool check = known || ((flags & Mem::ALIAS) && alias);
-   MFEM_VERIFY(check, "Unknown host pointer: " << h_ptr);
+   MFEM_VERIFY(check,"");
    const internal::Memory &mem =
       (flags & Mem::ALIAS) ?
       *maps->aliases.at(h_ptr).mem : maps->memories.at(h_ptr);
-   MemoryType d_mt = mem.d_mt;
-   if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
+   const MemoryType &d_mt = mem.d_mt;
    switch (mc)
    {
       case MemoryClass::HOST_32:
@@ -949,7 +783,6 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
          MFEM_VERIFY(d_mt == MemoryType::DEVICE ||
                      d_mt == MemoryType::DEVICE_DEBUG ||
                      d_mt == MemoryType::DEVICE_UMPIRE ||
-                     d_mt == MemoryType::DEVICE_UMPIRE_2 ||
                      d_mt == MemoryType::MANAGED,"");
          return true;
       }
@@ -1031,6 +864,7 @@ void *MemoryManager::Write_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
       if (flags & Mem::ALIAS)
       { return mm.GetAliasDevicePtr(h_ptr, bytes, false); }
       else { return mm.GetDevicePtr(h_ptr, bytes, false); }
+
    }
 }
 
@@ -1135,8 +969,11 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
       {
          if (dst_h_ptr != src_d_ptr && bytes != 0)
          {
+            internal::Memory &dst_h_base = maps->memories.at(dst_h_ptr);
             internal::Memory &src_d_base = maps->memories.at(src_d_ptr);
+            MemoryType dst_h_mt = dst_h_base.h_mt;
             MemoryType src_d_mt = src_d_base.d_mt;
+            ctrl->Host(dst_h_mt)->Unprotect(dst_h_ptr, bytes);
             ctrl->Device(src_d_mt)->DtoH(dst_h_ptr, src_d_ptr, bytes);
          }
       }
@@ -1261,7 +1098,7 @@ void MemoryManager::Insert(void *h_ptr, size_t bytes,
 void MemoryManager::InsertDevice(void *d_ptr, void *h_ptr, size_t bytes,
                                  MemoryType h_mt, MemoryType d_mt)
 {
-   // MFEM_VERIFY_TYPES(h_mt, d_mt); // done by Insert() below
+   MFEM_VERIFY_TYPES(h_mt, d_mt);
    MFEM_ASSERT(h_ptr != NULL, "internal error");
    Insert(h_ptr, bytes, h_mt, d_mt);
    internal::Memory &mem = maps->memories.at(h_ptr);
@@ -1314,22 +1151,6 @@ void MemoryManager::Erase(void *h_ptr, bool free_dev_ptr)
    maps->memories.erase(mem_map_iter);
 }
 
-void MemoryManager::EraseDevice(void *h_ptr)
-{
-   if (!h_ptr) { return; }
-   auto mem_map_iter = maps->memories.find(h_ptr);
-   if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
-   auto it = maps->aliases.find(h_ptr);
-   if (it != maps->aliases.end())
-   {
-      fprintf(stderr, "count = %lu\n", it->second.counter);
-      mfem_error("cannot delete aliased obj!");
-   }
-   internal::Memory &mem = mem_map_iter->second;
-   if (mem.d_ptr) { ctrl->Device(mem.d_mt)->Dealloc(mem);}
-   mem.d_ptr = nullptr;
-}
-
 void MemoryManager::EraseAlias(void *alias_ptr)
 {
    if (!alias_ptr) { return; }
@@ -1350,21 +1171,16 @@ void *MemoryManager::GetDevicePtr(const void *h_ptr, size_t bytes,
    }
    internal::Memory &mem = maps->memories.at(h_ptr);
    const MemoryType &h_mt = mem.h_mt;
-   MemoryType &d_mt = mem.d_mt;
+   const MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
-   if (!mem.d_ptr)
-   {
-      if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
-   }
-   // Aliases might have done some protections
+   if (!mem.d_ptr) { ctrl->Device(d_mt)->Alloc(mem); }
    ctrl->Device(d_mt)->Unprotect(mem);
    if (copy_data)
    {
       MFEM_ASSERT(bytes <= mem.bytes, "invalid copy size");
       ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes);
    }
-   ctrl->Host(h_mt)->Protect(mem, bytes);
+   ctrl->Host(h_mt)->Protect(h_ptr, bytes);
    return mem.d_ptr;
 }
 
@@ -1383,18 +1199,13 @@ void *MemoryManager::GetAliasDevicePtr(const void *alias_ptr, size_t bytes,
    const size_t offset = alias.offset;
    internal::Memory &mem = *alias.mem;
    const MemoryType &h_mt = mem.h_mt;
-   MemoryType &d_mt = mem.d_mt;
+   const MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
-   if (!mem.d_ptr)
-   {
-      if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
-   }
+   if (!mem.d_ptr) { ctrl->Device(d_mt)->Alloc(mem); }
    void *alias_h_ptr = static_cast<char*>(mem.h_ptr) + offset;
    void *alias_d_ptr = static_cast<char*>(mem.d_ptr) + offset;
    MFEM_ASSERT(alias_h_ptr == alias_ptr, "internal error");
    MFEM_ASSERT(bytes <= alias.bytes, "internal error");
-   mem.d_rw = false;
    ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes);
    ctrl->Host(h_mt)->AliasUnprotect(alias_ptr, bytes);
    if (copy) { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
@@ -1410,8 +1221,8 @@ void *MemoryManager::GetHostPtr(const void *ptr, size_t bytes, bool copy)
    const MemoryType &h_mt = mem.h_mt;
    const MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
+   ctrl->Host(h_mt)->Unprotect(mem.h_ptr, bytes);
    // Aliases might have done some protections
-   ctrl->Host(h_mt)->Unprotect(mem, bytes);
    if (mem.d_ptr) { ctrl->Device(d_mt)->Unprotect(mem); }
    if (copy && mem.d_ptr) { ctrl->Device(d_mt)->DtoH(mem.h_ptr, mem.d_ptr, bytes); }
    if (mem.d_ptr) { ctrl->Device(d_mt)->Protect(mem); }
@@ -1429,7 +1240,6 @@ void *MemoryManager::GetAliasHostPtr(const void *ptr, size_t bytes,
    void *alias_h_ptr = static_cast<char*>(mem->h_ptr) + alias.offset;
    void *alias_d_ptr = static_cast<char*>(mem->d_ptr) + alias.offset;
    MFEM_ASSERT(alias_h_ptr == ptr,  "internal error");
-   mem->h_rw = false;
    ctrl->Host(h_mt)->AliasUnprotect(alias_h_ptr, bytes);
    if (mem->d_ptr) { ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes); }
    if (copy_data && mem->d_ptr)
@@ -1451,48 +1261,23 @@ MemoryManager::MemoryManager() { Init(); }
 
 MemoryManager::~MemoryManager() { if (exists) { Destroy(); } }
 
-void MemoryManager::SetDualMemoryType(MemoryType mt, MemoryType dual_mt)
-{
-   MFEM_VERIFY(!configured, "changing the dual MemoryTypes is not allowed after"
-               " MemoryManager configuration!");
-   UpdateDualMemoryType(mt, dual_mt);
-}
-
-void MemoryManager::UpdateDualMemoryType(MemoryType mt, MemoryType dual_mt)
-{
-   MFEM_VERIFY((int)mt < MemoryTypeSize,
-               "invalid MemoryType, mt = " << (int)mt);
-   MFEM_VERIFY((int)dual_mt < MemoryTypeSize,
-               "invalid dual MemoryType, dual_mt = " << (int)dual_mt);
-
-   if ((IsHostMemory(mt) && IsDeviceMemory(dual_mt)) ||
-       (IsDeviceMemory(mt) && IsHostMemory(dual_mt)))
-   {
-      dual_map[(int)mt] = dual_mt;
-   }
-   else
-   {
-      // mt + dual_mt is not a pair of host + device types: this is only allowed
-      // when mt == dual_mt and mt is a host type; in this case we do not
-      // actually update the dual
-      MFEM_VERIFY(mt == dual_mt && IsHostMemory(mt),
-                  "invalid (mt, dual_mt) pair: ("
-                  << MemoryTypeName[(int)mt] << ", "
-                  << MemoryTypeName[(int)dual_mt] << ')');
-   }
-}
-
 void MemoryManager::Configure(const MemoryType host_mt,
                               const MemoryType device_mt)
 {
-   MemoryManager::UpdateDualMemoryType(host_mt, device_mt);
-   MemoryManager::UpdateDualMemoryType(device_mt, host_mt);
    Init();
    host_mem_type = host_mt;
    device_mem_type = device_mt;
-   configured = true;
 }
 
+#ifdef MFEM_USE_UMPIRE
+void MemoryManager::SetUmpireAllocatorNames(const char *h_name,
+                                            const char *d_name)
+{
+   h_umpire_name = h_name;
+   d_umpire_name = d_name;
+}
+#endif
+
 void MemoryManager::Destroy()
 {
    MFEM_VERIFY(exists, "MemoryManager has already been destroyed!");
@@ -1508,7 +1293,6 @@ void MemoryManager::Destroy()
    host_mem_type = MemoryType::HOST;
    device_mem_type = MemoryType::HOST;
    exists = false;
-   configured = false;
 }
 
 void MemoryManager::RegisterCheck(void *ptr)
@@ -1595,36 +1379,18 @@ void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr)
 MemoryManager mm;
 
 bool MemoryManager::exists = false;
-bool MemoryManager::configured = false;
+
+#ifdef MFEM_USE_UMPIRE
+const char* MemoryManager::h_umpire_name = "HOST";
+const char* MemoryManager::d_umpire_name = "DEVICE";
+#endif
 
 MemoryType MemoryManager::host_mem_type = MemoryType::HOST;
 MemoryType MemoryManager::device_mem_type = MemoryType::HOST;
 
-MemoryType MemoryManager::dual_map[MemoryTypeSize] =
-{
-   /* HOST            */  MemoryType::DEVICE,
-   /* HOST_32         */  MemoryType::DEVICE,
-   /* HOST_64         */  MemoryType::DEVICE,
-   /* HOST_DEBUG      */  MemoryType::DEVICE_DEBUG,
-   /* HOST_UMPIRE     */  MemoryType::DEVICE_UMPIRE,
-   /* HOST_PINNED     */  MemoryType::DEVICE,
-   /* MANAGED         */  MemoryType::MANAGED,
-   /* DEVICE          */  MemoryType::HOST,
-   /* DEVICE_DEBUG    */  MemoryType::HOST_DEBUG,
-   /* DEVICE_UMPIRE   */  MemoryType::HOST_UMPIRE,
-   /* DEVICE_UMPIRE_2 */  MemoryType::HOST_UMPIRE
-};
-
-#ifdef MFEM_USE_UMPIRE
-const char * MemoryManager::h_umpire_name = "MFEM_HOST";
-const char * MemoryManager::d_umpire_name = "MFEM_DEVICE";
-const char * MemoryManager::d_umpire_2_name = "MFEM_DEVICE_2";
-#endif
-
-
 const char *MemoryTypeName[MemoryTypeSize] =
 {
-   "host-std", "host-32", "host-64", "host-debug", "host-umpire", "host-pinned",
+   "host-std", "host-32", "host-64", "host-debug", "host-umpire",
 #if defined(MFEM_USE_CUDA)
    "cuda-uvm",
    "cuda",
@@ -1637,14 +1403,11 @@ const char *MemoryTypeName[MemoryTypeSize] =
 #endif
    "device-debug",
 #if defined(MFEM_USE_CUDA)
-   "cuda-umpire",
-   "cuda-umpire-2",
+   "cuda-umpire"
 #elif defined(MFEM_USE_HIP)
-   "hip-umpire",
-   "hip-umpire-2",
+   "hip-umpire"
 #else
-   "device-umpire",
-   "device-umpire-2",
+   "device-umpire"
 #endif
 };