small updates

.github/workflows/stale.yml

@@ -29,47 +29,3 @@ jobs:
         operations-per-run: 500
         exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
         exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"
-
-  # Stale action for PRs with "in-review" label.
-  stale-in-review-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 150 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "in-review"
-        days-before-pr-stale: 150
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500
-
-  # Stale action for PRs with "WIP" label.
-  stale-wip-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 300 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "WIP"
-        days-before-pr-stale: 300
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500

.gitignore

@@ -208,13 +208,10 @@ miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
-miniapps/electromagnetics/lorentz
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_[0-9]*
-miniapps/electromagnetics/Lorentz_[0-9]*
-miniapps/electromagnetics/Lorentz.dat
 
 miniapps/gslib/field-diff
 miniapps/gslib/field-interp
@@ -414,6 +411,15 @@ miniapps/tribol/contact-patch-test
 miniapps/diag-smoothers/abs-l1-jacobi
 miniapps/diag-smoothers/mg-abs-l1-jacobi
 
+miniapps/hdg/hdg_advection
+miniapps/hdg/hdg_advectionp
+miniapps/hdg/hdg_poisson
+miniapps/hdg/hdg_poissonp
+miniapps/hdg/mesh*
+miniapps/hdg/sol*
+miniapps/hdg/*.mesh
+miniapps/hdg/*.gf
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

CHANGELOG

@@ -91,10 +91,6 @@ New and updated examples and miniapps
 - Added a new miniapp (tools/gridfunction-bounds) to compute piecewise linear
   bounds on a given high-order grid function.
 
-- Added a new miniapp (electromagnetics/lorentz) which computes the trajectory
-  of a charged particle, subject to Lorentz forces, in electrostatic and/or
-  magnetostatic fields as computed by the volta or tesla miniapps.
-
 API changes:
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to

config/cmake/modules/FindHYPRE.cmake

@@ -197,6 +197,16 @@ if (HYPRE_FOUND AND HYPRE_USING_HIP)
   message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
 endif()
 
+# Hypre+Umpire check
+if (HYPRE_FOUND AND (HYPRE_USING_CUDA OR HYPRE_USING_HIP) AND NOT MFEM_USE_UMPIRE)
+  message(WARNING
+"===============================================================
+ Detected GPU-enabled HYPRE build without Umpire support.
+ This is not recommended for performance reasons!
+ Consider rebuilding HYPRE with Umpire support.
+===============================================================")
+endif()
+
 find_package_handle_standard_args(HYPRE
   REQUIRED_VARS HYPRE_LIBRARIES HYPRE_INCLUDE_DIRS HYPRE_VERSION
 )

config/defaults.mk

@@ -57,7 +57,7 @@ CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
 CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
+ -arch=$(CUDA_ARCH)
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=

data/mfem.mesh

@@ -1,593 +0,0 @@
-MFEM mesh v1.0
-
-# Created by: Pointwise
-
-# MFEM Geometry Types:
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-
-dimension
-2
-
-elements
-160
-1 3 1 164 163 0
-1 3 164 165 162 163
-1 3 2 166 164 1
-1 3 166 132 165 164
-1 3 3 167 166 2
-1 3 167 131 132 166
-1 3 4 168 167 3
-1 3 168 130 131 167
-1 3 5 169 168 4
-1 3 169 129 130 168
-1 3 6 170 169 5
-1 3 170 128 129 169
-1 3 171 172 170 6
-1 3 172 127 128 170
-1 3 124 125 172 171
-1 3 125 126 127 172
-1 3 162 165 173 161
-1 3 165 132 133 173
-1 3 161 173 174 160
-1 3 173 133 134 174
-1 3 160 174 175 159
-1 3 174 134 135 175
-1 3 6 7 176 171
-1 3 7 8 177 176
-1 3 171 176 123 124
-1 3 176 177 122 123
-1 3 159 175 178 158
-1 3 175 135 136 178
-1 3 158 178 179 157
-1 3 178 136 137 179
-1 3 157 179 180 156
-1 3 179 137 138 180
-1 3 122 177 181 121
-1 3 177 8 182 181
-1 3 8 9 183 182
-1 3 9 10 184 183
-1 3 10 11 185 184
-1 3 11 12 186 185
-1 3 12 13 187 186
-1 3 13 14 15 187
-1 3 121 181 119 120
-1 3 181 182 118 119
-1 3 182 183 117 118
-1 3 183 184 188 117
-1 3 184 185 109 188
-1 3 185 186 108 109
-1 3 186 187 189 108
-1 3 187 15 16 189
-1 3 109 110 190 188
-1 3 110 111 191 190
-1 3 111 112 113 191
-1 3 188 190 116 117
-1 3 190 191 115 116
-1 3 191 113 114 115
-1 3 189 192 107 108
-1 3 192 193 106 107
-1 3 193 194 105 106
-1 3 194 195 104 105
-1 3 195 196 103 104
-1 3 16 17 192 189
-1 3 17 18 193 192
-1 3 18 19 194 193
-1 3 19 20 195 194
-1 3 20 21 196 195
-1 3 97 98 197 96
-1 3 98 99 198 197
-1 3 99 100 199 198
-1 3 100 101 200 199
-1 3 101 102 201 200
-1 3 102 103 202 201
-1 3 103 196 203 202
-1 3 196 21 22 203
-1 3 96 197 204 95
-1 3 197 198 39 204
-1 3 198 199 38 39
-1 3 199 200 205 38
-1 3 200 201 32 205
-1 3 201 202 31 32
-1 3 202 203 206 31
-1 3 203 22 23 206
-1 3 32 33 207 205
-1 3 33 34 35 207
-1 3 205 207 37 38
-1 3 207 35 36 37
-1 3 39 40 208 204
-1 3 40 41 209 208
-1 3 41 42 210 209
-1 3 42 43 211 210
-1 3 43 44 212 211
-1 3 204 208 94 95
-1 3 208 209 93 94
-1 3 209 210 92 93
-1 3 210 211 91 92
-1 3 211 212 90 91
-1 3 90 212 213 89
-1 3 212 44 214 213
-1 3 44 45 215 214
-1 3 45 46 216 215
-1 3 46 47 217 216
-1 3 47 48 218 217
-1 3 48 49 219 218
-1 3 49 50 51 219
-1 3 89 213 87 88
-1 3 213 214 86 87
-1 3 214 215 85 86
-1 3 215 216 84 85
-1 3 216 217 83 84
-1 3 217 218 82 83
-1 3 218 219 220 82
-1 3 219 51 52 220
-1 3 53 221 220 52
-1 3 221 81 82 220
-1 3 54 222 221 53
-1 3 222 80 81 221
-1 3 55 223 222 54
-1 3 223 79 80 222
-1 3 26 27 224 25
-1 3 27 28 29 224
-1 3 25 224 225 24
-1 3 224 29 30 225
-1 3 24 225 206 23
-1 3 225 30 31 206
-1 3 154 155 226 153
-1 3 155 156 180 226
-1 3 153 226 227 152
-1 3 226 180 138 227
-1 3 152 227 228 151
-1 3 227 138 139 228
-1 3 151 228 229 150
-1 3 228 139 140 229
-1 3 150 229 230 149
-1 3 229 140 141 230
-1 3 149 230 231 148
-1 3 230 141 142 231
-1 3 148 231 232 147
-1 3 231 142 143 232
-1 3 147 232 145 146
-1 3 232 143 144 145
-1 3 56 233 223 55
-1 3 233 78 79 223
-1 3 57 234 233 56
-1 3 234 77 78 233
-1 3 58 235 234 57
-1 3 235 76 77 234
-1 3 61 236 59 60
-1 3 236 235 58 59
-1 3 62 237 236 61
-1 3 237 76 235 236
-1 3 63 238 237 62
-1 3 238 75 76 237
-1 3 64 239 238 63
-1 3 239 74 75 238
-1 3 65 240 239 64
-1 3 240 73 74 239
-1 3 66 241 240 65
-1 3 241 72 73 240
-1 3 67 242 241 66
-1 3 242 71 72 241
-1 3 68 69 242 67
-1 3 69 70 71 242
-
-boundary
-164
-3 1 0 1
-3 1 1 2
-3 1 2 3
-3 1 3 4
-3 1 4 5
-3 1 5 6
-3 1 6 7
-3 1 7 8
-3 1 8 9
-3 1 9 10
-3 1 10 11
-3 1 11 12
-3 1 12 13
-3 1 13 14
-3 1 16 17
-3 1 17 18
-3 1 18 19
-3 1 19 20
-3 1 20 21
-3 1 21 22
-3 1 22 23
-3 1 23 24
-3 1 24 25
-3 1 25 26
-3 1 26 27
-3 1 27 28
-3 1 28 29
-3 1 29 30
-3 1 30 31
-3 1 31 32
-3 1 32 33
-3 1 33 34
-3 1 34 35
-3 1 35 36
-3 1 36 37
-3 1 37 38
-3 1 38 39
-3 1 39 40
-3 1 40 41
-3 1 41 42
-3 1 42 43
-3 1 43 44
-3 1 49 50
-3 1 48 49
-3 1 47 48
-3 1 46 47
-3 1 45 46
-3 1 44 45
-3 1 52 53
-3 1 53 54
-3 1 54 55
-3 1 57 58
-3 1 56 57
-3 1 55 56
-3 1 60 61
-3 1 61 62
-3 1 62 63
-3 1 63 64
-3 1 64 65
-3 1 65 66
-3 1 66 67
-3 1 67 68
-3 1 75 76
-3 1 74 75
-3 1 73 74
-3 1 72 73
-3 1 71 72
-3 1 70 71
-3 1 76 77
-3 1 77 78
-3 1 78 79
-3 1 81 82
-3 1 80 81
-3 1 79 80
-3 1 82 83
-3 1 83 84
-3 1 84 85
-3 1 85 86
-3 1 86 87
-3 1 87 88
-3 1 94 95
-3 1 93 94
-3 1 92 93
-3 1 91 92
-3 1 90 91
-3 1 96 97
-3 1 95 96
-3 1 97 98
-3 1 98 99
-3 1 99 100
-3 1 100 101
-3 1 101 102
-3 1 102 103
-3 1 107 108
-3 1 106 107
-3 1 105 106
-3 1 104 105
-3 1 103 104
-3 1 108 109
-3 1 109 110
-3 1 110 111
-3 1 111 112
-3 1 112 113
-3 1 113 114
-3 1 114 115
-3 1 115 116
-3 1 116 117
-3 1 119 120
-3 1 118 119
-3 1 117 118
-3 1 131 132
-3 1 130 131
-3 1 129 130
-3 1 128 129
-3 1 127 128
-3 1 126 127
-3 1 132 133
-3 1 133 134
-3 1 134 135
-3 1 137 138
-3 1 136 137
-3 1 135 136
-3 1 138 139
-3 1 139 140
-3 1 140 141
-3 1 141 142
-3 1 142 143
-3 1 143 144
-3 1 147 148
-3 1 146 147
-3 1 153 154
-3 1 152 153
-3 1 151 152
-3 1 150 151
-3 1 149 150
-3 1 148 149
-3 1 156 157
-3 1 157 158
-3 1 158 159
-3 1 161 162
-3 1 160 161
-3 1 159 160
-2 1 69 70
-2 1 68 69
-3 1 88 89
-3 1 89 90
-3 1 121 122
-3 1 120 121
-3 1 123 124
-3 1 122 123
-3 1 125 126
-3 1 124 125
-1 1 144 145
-1 1 145 146
-3 1 15 16
-3 1 14 15
-3 1 50 51
-3 1 51 52
-3 1 59 60
-3 1 58 59
-3 1 154 155
-3 1 155 156
-3 1 163 0
-3 1 162 163
-
-vertices
-243
-2
-4 4
-4 3.5
-4 3
-4 2.5
-4 2
-4 1.5
-4 1
-4.5 1
-5 1
-5 1.5
-5 2
-5 2.5
-5 3
-5 3.5
-5 4
-5.500 4
-6 4
-6.500 4
-7 4
-7.5 4
-8 4
-8.5 4
-9 4
-9.5 4
-10 4
-10.5 4
-11 4
-11 3.5
-11 3
-10.5 3
-10 3
-9.5 3
-9.5 2.5
-10 2.5
-10.5 2.5
-10.5 2
-10.5 1.5
-10 1.5
-9.5 1.5
-9.5 1
-10 1
-10.5 1
-11 1
-11.5 1
-12 1
-12 1.5
-12 2
-12 2.5
-12 3
-12 3.5
-12 4
-12.5 4
-13 4
-13.333 3.75
-13.666 3.5
-14.000 3.25
-14.333 3.5
-14.666 3.75
-15.000 4
-15.500 4
-16.000 4
-16.000 3.5
-16.000 3
-16.000 2.5
-16.000 2
-16.000 1.5
-16.000 1
-16.000 0.5
-16.000 0
-15.500 0
-15.000 0
-15.000 0.5000000000000002
-15.000 1
-15.000 1.5
-15.000 2
-15.000 2.5
-15.000 3
-14.666 2.75
-14.333 2.5
-14.000 2.25
-13.666 2.5
-13.333 2.75
-13 3
-13 2.5
-13 2
-13 1.5
-13 1
-13 0.500
-13 0
-12.5 0
-12 0
-11.5 0
-11 0
-10.5 0
-10 0
-9.5 0
-9 0
-8.5 0
-8.5 0.5
-8.5 1
-8.5 1.5
-8.5 2
-8.5 2.5
-8.5 3
-8 3
-7.5 3
-7 3
-6.500 3
-6 3
-6 2.5
-6.5 2.5
-7 2.5
-7.5 2.5
-7.5 2
-7.5 1.5
-7.000 1.5
-6.5 1.5
-6 1.5
-6 1
-6 0.5
-6 0
-5.5 0
-5 0
-4.5 0
-4 0
-3.5 0
-3 0
-3 0.500
-3 1
-3 1.5
-3 2
-3 2.5
-3 3
-2.666 2.75
-2.333 2.5
-2.000 2.25
-1.666 2.5
-1.333 2.75
-1.000 3
-1.000 2.5
-1.000 2
-1.000 1.5
-1.000 1
-1.000 0.5000
-1.000 0
-0.5000 0
-0.0000 0
-0.0000 0.5
-0.0000 1
-0.0000 1.5
-0.0000 2
-0.0000 2.5
-0.0000 3
-0.0000 3.5
-0.0000 4
-0.5000 4
-1.000 4
-1.333 3.75
-1.666 3.5
-2.000 3.25
-2.333 3.5
-2.666 3.75
-3 4
-3.5 4
-3.5 3.5
-3 3.5
-3.5 3
-3.5 2.5
-3.5 2
-3.5 1.5
-3.5 1
-4 0.5
-3.5 0.5
-2.666 3.25
-2.333 3
-2.000 2.75
-4.5 0.5
-5 0.5
-1.666 3
-1.333 3.25
-1.000 3.5
-5.5 0.5
-5.500 1
-5.500 1.5
-5.500 2
-5.500 2.5
-5.500 3
-5.500 3.5
-6 2
-6 3.5
-6.5 2
-7 2
-6.5 3.5
-7 3.5
-7.5 3.5
-8 3.5
-8.5 3.5
-9 0.5
-9 1
-9 1.5
-9 2
-9 2.5
-9 3
-9 3.5
-9.5 0.5
-9.5 2
-9.5 3.5
-10 2
-10 0.5
-10.5 0.5
-11 0.5
-11.5 0.5
-12 0.5
-12.5 0.500
-12.5 1
-12.5 1.5
-12.5 2
-12.5 2.5
-12.5 3
-12.5 3.5
-13 3.5
-13.333 3.250
-13.666 3
-14.000 2.75
-10.5 3.5
-10 3.5
-0.500 3.5
-0.500 3
-0.500 2.5
-0.500 2
-0.500 1.5
-0.500 1
-0.500 0.5
-14.333 3
-14.666 3.25
-15.000 3.5
-15.500 3.5
-15.500 3
-15.500 2.5
-15.500 2
-15.500 1.5
-15.500 1
-15.500 0.5

doc/CodeDocumentation.dox

@@ -202,7 +202,6 @@ namespace mfem {
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="lorentz_8cpp_source.html">Lorentz</a>: simple particle tracking code based on the Lorentz force
  * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
  * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces

fem/bilinearform.cpp

@@ -103,6 +103,11 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
    boundary_face_integs = bf->boundary_face_integs;
    boundary_face_integs_marker = bf->boundary_face_integs_marker;
 
+   /* HDG */
+   hdgintbfi = bf->hdgintbfi;
+   hdgbdrbfi = bf->hdgbdrbfi;
+
+
    AllocMat();
 }
 
@@ -273,6 +278,27 @@ void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
    boundary_face_integs_marker.Append(&bdr_marker);
 }
 
+/* HDG */
+void BilinearForm::AddHDGInteriorFaceIntegrator (BilinearFormIntegrator * bfi)
+{
+   hdgintbfi.Append (bfi);
+}
+
+/* HDG */
+void BilinearForm::AddHDGBoundaryFaceIntegrator (BilinearFormIntegrator * bfi)
+{
+   hdgbdrbfi.Append (bfi);
+   skeleton_boundary_face_integs_marker.Append(NULL);
+}
+
+/* HDG */
+void BilinearForm::AddHDGBoundaryFaceIntegrator (BilinearFormIntegrator * bfi,
+                                                 Array<int> &bdr_marker)
+{
+   hdgbdrbfi.Append (bfi);
+   skeleton_boundary_face_integs_marker.Append(&bdr_marker);
+}
+
 void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat) const
 {
    if (element_matrices)
@@ -749,6 +775,96 @@ void BilinearForm::Assemble(int skip_zeros)
       }
    }
 
+   /* HDG */
+   // Skeleton interior face integrals for HDG
+   if (hdgintbfi.Size())
+   {
+      FaceElementTransformations *ftr;
+      const FiniteElement *face_fe;
+      int nfaces = mesh->GetNumFaces();
+
+      // loop over all the edges
+      for (int i = 0; i < nfaces; i++)
+      {
+         ftr = mesh->GetInteriorFaceTransformations(i); // the transformation of the face
+         fes->GetFaceVDofs(i, vdofs);   // the defrees of freedom related to the face
+         face_fe = fes->GetFaceElement(
+                      i);   // point face_fe to the FiniteElement over the edge
+         if (ftr != NULL)
+         {
+            for (int k = 0; k < hdgintbfi.Size();
+                 k++) // Loop over the related interals, but there is only one hdgintbfi right now
+            {
+               hdgintbfi[k] -> AssembleFaceMatrix (*face_fe, *ftr,
+                                                   elemmat); // call AssembleFaceMatrix over the face
+               mat -> AddSubMatrix (vdofs, vdofs, elemmat,
+                                    skip_zeros);  // assemble the local matrix to the global one, skipping the zeros
+            }
+         }
+      }
+   }
+
+   /* HDG */
+   // Skeleton boundary face integrals for HDG
+   if (hdgbdrbfi.Size())
+   {
+      FaceElementTransformations *ftr;
+      const FiniteElement *face_fe;
+      // Which boundary attributes need to be processed?
+      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
+                                 mesh->bdr_attributes.Max() : 0);
+      bdr_attr_marker = 0;
+      for (int k = 0; k < hdgbdrbfi.Size(); k++)
+      {
+         if (skeleton_boundary_face_integs_marker[k] == NULL)
+         {
+            bdr_attr_marker = 1;
+            break;
+         }
+         Array<int> &bdr_marker = *skeleton_boundary_face_integs_marker[k];
+         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
+                     "invalid boundary marker for boundary face integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < bdr_attr_marker.Size(); i++)
+         {
+            bdr_attr_marker[i] |= bdr_marker[i];
+         }
+      }
+
+
+      int nbdrfaces = fes->GetNBE();
+
+      // loop over all the edges
+      for (int i = 0; i < nbdrfaces; i++)
+      {
+         const int bdr_attr = mesh->GetBdrAttribute(i);
+         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
+
+         int face = mesh->GetBdrElementFaceIndex(i);
+         ftr = mesh->GetBdrFaceTransformations(i); // the transformation of the face
+
+         if (ftr != NULL)
+         {
+            fes->GetFaceVDofs(face, vdofs);   // the defrees of freedom related to the face
+            face_fe = fes->GetFaceElement(
+                         face);   // point face_fe to the FiniteElement over the edge
+            for (int k = 0; k < hdgbdrbfi.Size();
+                 k++) // Loop over the related interals, but there is only one hdgbdrbfi right now
+            {
+               if (skeleton_boundary_face_integs_marker[k] &&
+                   (*skeleton_boundary_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }
+
+               hdgbdrbfi[k] -> AssembleFaceMatrix (*face_fe, *ftr,
+                                                   elemmat); // call AssembleFaceMatrix over the face
+               mat -> AddSubMatrix (vdofs, vdofs, elemmat,
+                                    skip_zeros);  // assemble the local matrix to the global one, skipping the zeros
+            }
+         }
+      }
+   }
+
+
 #ifdef MFEM_USE_LEGACY_OPENMP
    if (free_element_matrices)
    {
@@ -1296,6 +1412,10 @@ BilinearForm::~BilinearForm()
       { delete interior_face_integs[k]; }
       for (k=0; k < boundary_face_integs.Size(); k++)
       { delete boundary_face_integs[k]; }
+      /* HDG */
+      // Delete skeleton integrals for HDG
+      for (k=0; k < hdgintbfi.Size(); k++) { delete hdgintbfi[k]; }
+      for (k=0; k < hdgbdrbfi.Size(); k++) { delete hdgbdrbfi[k]; }
    }
 }
 

fem/bilinearform.hpp

@@ -119,6 +119,15 @@ protected:
    Array<BilinearFormIntegrator*> boundary_face_integs;
    Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.
 
+   /* HDG */
+   /// Set of HDG skeleton face Integrators over interior face to be applied.
+   Array<BilinearFormIntegrator*> hdgintbfi;
+
+   /* HDG */
+   /// Set of HDG skeleton face Integrators over boundary face to be applied.
+   Array<BilinearFormIntegrator*> hdgbdrbfi;
+   Array<Array<int>*> skeleton_boundary_face_integs_marker;
+
    mutable DenseMatrix elemmat;
    mutable Array<int>  vdofs;
 
@@ -288,6 +297,14 @@ public:
    Array<Array<int>*> *GetBFBFI_Marker()
    { return &boundary_face_integs_marker; }
 
+   /* HDG */
+   // Array of the HDG type bilinear form integrators, right now there is only one
+   Array<BilinearFormIntegrator*> *GetHDGIntBFI() { return &hdgintbfi; }
+
+   /* HDG */
+   // Array of the HDG type bilinear form integrators, right now there is only one
+   Array<BilinearFormIntegrator*> *GetHDGBdrBFI() { return &hdgbdrbfi; }
+
    /// Returns a reference to: $ M_{ij} $
    const real_t &operator()(int i, int j) { return (*mat)(i,j); }
 
@@ -437,6 +454,17 @@ public:
    void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                              Array<int> &bdr_marker);
 
+   /* HDG */
+   /// Adds HDG Interior Integrator.
+   void AddHDGInteriorFaceIntegrator(BilinearFormIntegrator *bfi);
+
+   /* HDG */
+   /// Adds new HDG Boundary Integrator.
+   void AddHDGBoundaryFaceIntegrator(BilinearFormIntegrator *bfi);
+
+   void AddHDGBoundaryFaceIntegrator(BilinearFormIntegrator *bfi,
+                                     Array<int> &bdr_marker);
+
    /// Sets all sparse values of $ M $ and $ M_e $ to 'a'.
    void operator=(const real_t a)
    {

fem/bilininteg.cpp

@@ -220,6 +220,76 @@ void BilinearFormIntegrator::AssembleFaceMatrix(
               " Integrator class.");
 }
 
+/* HDG optimized integrators starts */
+void BilinearFormIntegrator::AssembleElementMatrix2FES(const FiniteElement
+                                                       &fe_q,
+                                                       const FiniteElement &fe_u,
+                                                       ElementTransformation &Trans,
+                                                       DenseMatrix &elmat1)
+{
+   MFEM_ABORT("AssembleElementMatrix2FES is not implemented for this"
+              " Integrator class.");
+}
+
+void BilinearFormIntegrator::AssembleFaceMatrixOneElement1and1FES(
+   const FiniteElement &fe_u,
+   const FiniteElement &face_fe,
+   FaceElementTransformations &Trans,
+   const int elem1or2,
+   const bool onlyB,
+   DenseMatrix &elmat1,
+   DenseMatrix &elmat2,
+   DenseMatrix &elmat3,
+   DenseMatrix &elmat4)
+{
+   MFEM_ABORT("AssembleFaceMatrixOneElement1and1FES is not implemented for this"
+              " Integrator class.");
+}
+
+void BilinearFormIntegrator::AssembleFaceMatrixOneElement2and1FES(
+   const FiniteElement &fe_q,
+   const FiniteElement &fe_u,
+   const FiniteElement &face_fe,
+   FaceElementTransformations &Trans,
+   const int elem1or2,
+   const bool onlyB,
+   DenseMatrix &elmat1,
+   DenseMatrix &elmat2,
+   DenseMatrix &elmat3,
+   DenseMatrix &elmat4)
+{
+   MFEM_ABORT("AssembleFaceMatrixOneElement2and1FES is not implemented for this"
+              " Integrator class.");
+}
+
+void BilinearFormIntegrator::AssembleFaceMatrixOneElement2and2FES(
+   const FiniteElement &fe_q,
+   const FiniteElement &fe_u,
+   const FiniteElement &face_fe,
+   const FiniteElement &face_fe2,
+   FaceElementTransformations &Trans,
+   const int elem1or2,
+   const bool onlyB,
+   DenseMatrix &elmat1,
+   DenseMatrix &elmat2,
+   DenseMatrix &elmat3,
+   DenseMatrix &elmat4)
+{
+   MFEM_ABORT("AssembleFaceMatrixOneElement2and2FES is not implemented for this"
+              " Integrator class.");
+}
+/* HDG optimized integrators ends */
+
+/* HDG */
+void BilinearFormIntegrator::AssembleFaceMatrix(const FiniteElement &face_fe,
+                                                FaceElementTransformations &Trans,
+                                                DenseMatrix &elmat)
+{
+   MFEM_ABORT("AssembleFaceMatrix(const FiniteElement &face_fe,"
+              " FaceElementTransformations &Trans,"
+              " DenseMatrix &elmat) is not implemented for this");
+}
+
 void BilinearFormIntegrator::AssembleTraceFaceMatrix (int elem,
                                                       const FiniteElement &trial_face_fe,
                                                       const FiniteElement &test_fe1,

fem/bilininteg.hpp

@@ -194,6 +194,54 @@ public:
                                    FaceElementTransformations &Trans,
                                    DenseMatrix &elmat);
 
+   /* HDG */
+   /* For the case when there are 2 finite element spaces - such as LHDG */
+   virtual void AssembleElementMatrix2FES(const FiniteElement &fe_q,
+                                          const FiniteElement &fe_u,
+                                          ElementTransformation &Trans,
+                                          DenseMatrix &elmat1);
+
+   /* HDG */
+   /* For the optimized HDG calculations  */
+   /* 1 element based 1 face based FES */
+   virtual void AssembleFaceMatrixOneElement1and1FES(const FiniteElement &fe_u,
+                                                     const FiniteElement &face_fe,
+                                                     FaceElementTransformations &Trans,
+                                                     const int elem1or2,
+                                                     const bool onlyB,
+                                                     DenseMatrix &elmat1,
+                                                     DenseMatrix &elmat2,
+                                                     DenseMatrix &elmat3,
+                                                     DenseMatrix &elmat4);
+   /* 2 element based 1 face based FES */
+   virtual void AssembleFaceMatrixOneElement2and1FES(const FiniteElement &fe_q,
+                                                     const FiniteElement &fe_u,
+                                                     const FiniteElement &face_fe,
+                                                     FaceElementTransformations &Trans,
+                                                     const int elem1or2,
+                                                     const bool onlyB,
+                                                     DenseMatrix &elmat1,
+                                                     DenseMatrix &elmat2,
+                                                     DenseMatrix &elmat3,
+                                                     DenseMatrix &elmat4);
+   /* 2 element based 2 face based FES */
+   virtual void AssembleFaceMatrixOneElement2and2FES(const FiniteElement &fe_q,
+                                                     const FiniteElement &fe_u,
+                                                     const FiniteElement &face_fe,
+                                                     const FiniteElement &face_fe2,
+                                                     FaceElementTransformations &Trans,
+                                                     const int elem1or2,
+                                                     const bool onlyB,
+                                                     DenseMatrix &elmat1,
+                                                     DenseMatrix &elmat2,
+                                                     DenseMatrix &elmat3,
+                                                     DenseMatrix &elmat4);
+   // Assemble a local matrix over an edge, HDG skeleton integral
+   virtual void AssembleFaceMatrix(const FiniteElement &face_fe,
+                                   FaceElementTransformations &Trans,
+                                   DenseMatrix &elmat);
+   /* End of HDG integrators */
+
    /** Abstract method used for assembling TraceFaceIntegrators for
        DPG weak formulations. */
    virtual void AssembleTraceFaceMatrix(int elem,

fem/coefficient.cpp

@@ -2060,4 +2060,42 @@ CoefficientVector::~CoefficientVector()
    delete qf;
 }
 
+/* HDG */
+double ComputeMean(Coefficient &coeff, Mesh &mesh,
+                   const IntegrationRule *irs[])
+{
+   double norm = 0.0;
+   ElementTransformation *tr;
+
+   for (int i = 0; i < mesh.GetNE(); i++)
+   {
+      tr = mesh.GetElementTransformation(i);
+      const IntegrationRule &ir = *irs[mesh.GetElementType(i)];
+      for (int j = 0; j < ir.GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         tr->SetIntPoint(&ip);
+         double val = coeff.Eval(*tr, ip);
+
+         norm += ip.weight * tr->Weight() * val;
+      }
+   }
+   return norm;
+}
+
+#ifdef MFEM_USE_MPI
+double ComputeGlobalMean(Coefficient &coeff, ParMesh &pmesh,
+                         const IntegrationRule *irs[])
+{
+   double loc_norm = ComputeMean(coeff, pmesh, irs);
+   double glob_norm = 0;
+
+   MPI_Comm comm = pmesh.GetComm();
+
+   MPI_Allreduce(&loc_norm, &glob_norm, 1, MPI_DOUBLE, MPI_SUM, comm);
+
+   return glob_norm;
+}
+#endif
+
 }

fem/coefficient.hpp

@@ -2544,6 +2544,19 @@ real_t ComputeGlobalLpNorm(real_t p, VectorCoefficient &coeff, ParMesh &pmesh,
                            const IntegrationRule *irs[]);
 #endif
 
+/* HDG */
+/** Compute the mean of a function f.
+    $ | f |_{mean} = ( \int_\Omega f d\Omega) $ */
+double ComputeMean(Coefficient &coeff, Mesh &mesh,
+                   const IntegrationRule *irs[]);
+
+#ifdef MFEM_USE_MPI
+/** Compute the mean of a function f.
+    $ | f |_{mean} = ( \int_\Omega f d\Omega) $ */
+double ComputeGlobalMean(Coefficient &coeff, ParMesh &pmesh,
+                         const IntegrationRule *irs[]);
+#endif
+
 }
 
 #endif

fem/fe/fe_base.cpp

@@ -133,6 +133,20 @@ void FiniteElement::GetTransferMatrix(const FiniteElement &fe,
    MFEM_ABORT("method is not overloaded");
 }
 
+/* HDG */
+void FiniteElement::Project (
+   Coefficient &coeff, FaceElementTransformations &Trans, Vector &dofs) const
+{
+   mfem_error ("FiniteElement::Project (...) (skeleton) is not overloaded !");
+}
+
+/* HDG */
+void FiniteElement::Project (
+   VectorCoefficient &coeff, FaceElementTransformations &Trans, Vector &dofs) const
+{
+   mfem_error ("FiniteElement::Project (...) (skeleton - VectorCoefficient) is not overloaded !");
+}
+
 void FiniteElement::Project(
    Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {
@@ -803,6 +817,52 @@ void NodalFiniteElement::GetLocalRestriction(ElementTransformation &Trans,
    R.Threshold(1e-12);
 }
 
+/* HDG */
+void NodalFiniteElement::Project (
+   Coefficient &coeff, FaceElementTransformations &Trans, Vector &dofs) const
+{
+   for (int i = 0; i < dof; i++)
+   {
+      const IntegrationPoint &ip = Nodes.IntPoint(i);
+      // some coefficients expect that Trans.IntPoint is the same
+      // as the second argument of Eval
+      Trans.Face->SetIntPoint(&ip);
+      dofs(i) = coeff.Eval (*Trans.Face, ip);
+      if (map_type == INTEGRAL)
+      {
+         dofs(i) *= Trans.Face->Weight();
+      }
+   }
+}
+
+/* HDG */
+void NodalFiniteElement::Project (
+   VectorCoefficient &vc, FaceElementTransformations &Trans, Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
+
+   Vector x (vc.GetVDim());
+
+   for (int i = 0; i < dof; i++)
+   {
+      const IntegrationPoint &ip = Nodes.IntPoint(i);
+      // some coefficients expect that Trans.IntPoint is the same
+      // as the second argument of Eval
+      Trans.Face->SetIntPoint(&ip);
+      vc.Eval (x, *Trans.Face, ip);
+      if (map_type == INTEGRAL)
+      {
+         x *= Trans.Face->Weight();
+      }
+      for (int j = 0; j < x.Size(); j++)
+      {
+         dofs(dof*j+i) = x(j);
+      }
+   }
+}
+
+
+
 void NodalFiniteElement::Project(
    Coefficient &coeff, ElementTransformation &Trans, Vector &dofs) const
 {

fem/fe/fe_base.hpp

@@ -238,6 +238,8 @@ public:
 };
 
 class ElementTransformation;
+/* HDG */
+class FaceElementTransformations;
 class Coefficient;
 class VectorCoefficient;
 class MatrixCoefficient;
@@ -523,6 +525,15 @@ public:
                                   ElementTransformation &Trans,
                                   DenseMatrix &I) const;
 
+   /* HDG */
+   /** Given a coefficient and a transformation, compute its projection
+       (approximation) in the local finite dimensional skeleton space in terms
+       of the degrees of freedom. */
+   virtual void Project (Coefficient &coeff,
+                         FaceElementTransformations &Trans, Vector &dofs) const;
+   /* HDG */
+   virtual void Project (VectorCoefficient &coeff,
+                         FaceElementTransformations &Trans, Vector &dofs) const;
    /** @brief Given a coefficient and a transformation, compute its projection
        (approximation) in the local finite dimensional space in terms
        of the degrees of freedom. */
@@ -764,6 +775,13 @@ public:
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override;
 
+   /* HDG */
+   virtual void Project (Coefficient &coeff,
+                         FaceElementTransformations &Trans, Vector &dofs) const override;
+   /* HDG */
+   virtual void Project (VectorCoefficient &vc,
+                         FaceElementTransformations &Trans, Vector &dofs) const override;
+
    // (mc.height x mc.width) @ DOFs -> (Dof x mc.width x mc.height) in dofs
    void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const override;

fem/fe_coll.cpp

@@ -2456,7 +2456,8 @@ RT_FECollection::RT_FECollection(const int order, const int dim,
       const char *cb_name = BasisType::Name(cb_type); // this may abort
       MFEM_ABORT("unknown closed BasisType: " << cb_name);
    }
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
+       ob_type != BasisType::IntegratedGLL)
    {
       const char *ob_name = BasisType::Name(ob_type); // this may abort
       MFEM_ABORT("unknown open BasisType: " << ob_name);
@@ -2783,7 +2784,8 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
    int cp_type = BasisType::GetQuadrature1D(cb_type);
 
    // Error checking
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
+       ob_type != BasisType::IntegratedGLL)
    {
       const char *ob_name = BasisType::Name(ob_type);
       MFEM_ABORT("Invalid open basis point type: " << ob_name);

fem/gridfunc.cpp

@@ -446,6 +446,52 @@ void GridFunction::GetNodalValues(int i, Array<real_t> &nval, int vdim) const
    }
 }
 
+/* HDG */
+double GridFunction::GetValueFacet(FaceElementTransformations &T,
+                                   const IntegrationPoint &ip, int vdim, Vector *tr)
+const
+{
+   if (tr)
+   {
+      T.SetIntPoint(&ip);
+      T.Transform(ip, *tr);
+   }
+
+   const FiniteElement * fe = NULL;
+   Array<int> dofs;
+
+   switch (T.ElementType)
+   {
+      case ElementTransformation::FACE:
+      {
+         fe = fes->GetFaceElement(T.ElementNo);
+         fes->GetFaceDofs(T.ElementNo, dofs);
+         break;
+      }
+      case ElementTransformation::BDR_FACE:
+      {
+         fe = fes->GetBE(T.ElementNo);
+         fes->GetBdrElementDofs(T.ElementNo, dofs);
+         break;
+      }
+   }
+
+   fes->DofsToVDofs(vdim-1, dofs);
+   Vector DofVal(dofs.Size()), LocVec;
+   if (fe->GetMapType() == FiniteElement::VALUE)
+   {
+      fe->CalcShape(ip, DofVal);
+   }
+   else
+   {
+      fe->CalcPhysShape(T, DofVal);
+   }
+   GetSubVector(dofs, LocVec);
+
+   return (DofVal * LocVec);
+
+}
+
 real_t GridFunction::GetValue(int i, const IntegrationPoint &ip, int vdim)
 const
 {
@@ -2261,6 +2307,42 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
    }
 }
 
+/* HDG */
+double GridFunction::ComputeMean(const IntegrationRule *irs[]) const
+{
+   double global_mean = 0.0;
+   double element_mean;
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Vector vals;
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      fe = fes->GetFE(i);
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         int intorder = 2*fe->GetOrder() + 1; // <----------
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      GetValues(i, *ir, vals);
+      T = fes->GetElementTransformation(i);
+      element_mean = 0.0;
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         T->SetIntPoint(&ip);
+         element_mean += ip.weight * T->Weight() * vals(j);
+      }
+      global_mean += element_mean;
+   }
+   return global_mean;
+}
+
 void GridFunction::ComputeMeans(AvgType type, Array<int> &zones_per_vdof)
 {
    switch (type)
@@ -2352,6 +2434,133 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
    }
 }
 
+/* HDG */
+void GridFunction::ProjectCoefficientSkeleton(Coefficient &coeff)
+{
+   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
+
+   // check if the GridFunction belongs to a skeletal FEColl
+   MFEM_ASSERT(bool(dynamic_cast<const DG_Interface_FECollection*>
+                    (fes->FEColl())) || bool (dynamic_cast<const H1_Trace_FECollection*>
+                                              (fes->FEColl())),
+               "Incorrect FEColl");
+
+   if (delta_c == NULL)
+   {
+      Mesh *mesh = fes->GetMesh();
+      int nfaces = mesh->GetNumFaces();
+
+      Array<int> vdofs;
+      Vector vals;
+
+      for (int i = 0; i < nfaces; i++)
+      {
+         fes->GetFaceVDofs(i, vdofs);
+         vals.SetSize(vdofs.Size());
+         fes->GetFaceElement(i)->Project(coeff, *mesh->GetFaceElementTransformations(i),
+                                         vals);
+         SetSubVector(vdofs, vals);
+      }
+   }
+   else
+   {
+      real_t integral;
+
+      ProjectDeltaCoefficient(*delta_c, integral);
+
+      (*this) *= (delta_c->Scale() / integral);
+   }
+}
+
+void GridFunction::ProjectCoefficientSkeleton(VectorCoefficient &vcoeff)
+{
+   Mesh *mesh = fes->GetMesh();
+   int nfaces = mesh->GetNumFaces();
+   Array<int> vdofs;
+   Vector vals;
+
+   // check if the GridFunction belongs to a skeletal FEColl
+   MFEM_ASSERT(bool(dynamic_cast<const DG_Interface_FECollection*>
+                    (fes->FEColl())) || bool (dynamic_cast<const H1_Trace_FECollection*>
+                                              (fes->FEColl())),
+               "Incorrect FEColl");
+
+   for (int i = 0; i < nfaces; i++)
+   {
+      fes->GetFaceVDofs(i, vdofs);
+      vals.SetSize(vdofs.Size());
+      fes->GetFaceElement(i)->Project(vcoeff, *mesh->GetFaceElementTransformations(i),
+                                      vals);
+      SetSubVector(vdofs, vals);
+   }
+}
+
+void GridFunction::ProjectCoefficientSkeletonBdr(Coefficient &coeff)
+{
+   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
+
+   // check if the GridFunction belongs to a skeletal FEColl
+   MFEM_ASSERT(bool(dynamic_cast<const DG_Interface_FECollection*>
+                    (fes->FEColl())) || bool (dynamic_cast<const H1_Trace_FECollection*>
+                                              (fes->FEColl())),
+               "Incorrect FEColl");
+
+   if (delta_c == NULL)
+   {
+      Mesh *mesh = fes->GetMesh();
+      int nbdrfaces = mesh->GetNBE();
+      Array<int> vdofs;
+      Vector vals;
+
+      for (int i = 0; i < nbdrfaces; i++)
+      {
+         int face = mesh->GetBdrElementFaceIndex(i);
+         fes->GetFaceVDofs(face, vdofs);
+         vals.SetSize(vdofs.Size());
+         fes->GetFaceElement(face)->Project(coeff,
+                                            *mesh->GetFaceElementTransformations(face),
+                                            vals);
+         SetSubVector(vdofs, vals);
+      }
+   }
+   else
+   {
+      real_t integral;
+
+      ProjectDeltaCoefficient(*delta_c, integral);
+
+      (*this) *= (delta_c->Scale() / integral);
+   }
+}
+
+void GridFunction::ProjectCoefficientSkeletonBdr(VectorCoefficient &vcoeff)
+{
+   Mesh *mesh = fes->GetMesh();
+   int nbdrfaces = mesh->GetNBE();
+   Array<int> vdofs;
+   Vector vals;
+
+   // check if the GridFunction belongs to a skeletal FEColl
+   MFEM_ASSERT(bool(dynamic_cast<const DG_Interface_FECollection*>
+                    (fes->FEColl())) || bool (dynamic_cast<const H1_Trace_FECollection*>
+                                              (fes->FEColl())),
+               "Incorrect FEColl");
+
+   for (int i = 0; i < nbdrfaces; i++)
+   {
+      int face = mesh->GetBdrElementFaceIndex(i);
+      fes->GetFaceVDofs(face, vdofs);
+      vals.SetSize(vdofs.Size());
+      fes->GetFaceElement(face)->Project(vcoeff,
+                                         *mesh->GetFaceElementTransformations(face),
+                                         vals);
+      SetSubVector(vdofs, vals);
+   }
+
+}
+
+/* HDG ends */
+
 void GridFunction::ProjectCoefficient(Coefficient &coeff)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
@@ -3365,6 +3574,69 @@ real_t GridFunction::ComputeW11Error(
    return error;
 }
 
+/// To compute \| mean(u) - mean(u_h) \|_p
+double GridFunction::ComputeMeanLpError(const double p, Coefficient &exsol,
+                                        const IntegrationRule *irs[]) const
+{
+   double error = 0.0;
+   double aux, local_error, local_size ;
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Vector vals;
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      fe = fes->GetFE(i);
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         int intorder = 2*fe->GetOrder() + 1; // <----------
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      GetValues(i, *ir, vals);
+      T = fes->GetElementTransformation(i);
+      local_error = local_size = 0.0 ;
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         T->SetIntPoint(&ip);
+         aux = (vals(j) - exsol.Eval(*T, ip));
+         local_error += ip.weight * T->Weight() * aux;
+         local_size += ip.weight * T->Weight();
+      }
+
+      if (p < numeric_limits<double>::infinity())
+      {
+         aux = pow(fabs(local_error), p) / pow(local_size, p-1.);
+         error += aux;
+      }
+      else
+      {
+         aux = fabs(local_error) / fabs(local_size);
+         error = std::max(error, aux);
+      }
+   }
+
+   if (p < numeric_limits<double>::infinity())
+   {
+      // negative quadrature weights may cause the error to be negative
+      if (error < 0.)
+      {
+         error = -pow(-error, 1./p);
+      }
+      else
+      {
+         error = pow(error, 1./p);
+      }
+   }
+
+   return error;
+}
+
 real_t GridFunction::ComputeLpError(const real_t p, Coefficient &exsol,
                                     Coefficient *weight,
                                     const IntegrationRule *irs[],

fem/gridfunc.hpp

@@ -23,6 +23,8 @@
 #include <limits>
 #include <ostream>
 #include <string>
+/* HDG */
+//#include "lininteg.hpp"
 
 namespace mfem
 {
@@ -286,6 +288,12 @@ public:
                            DenseMatrix &vals, DenseMatrix &tr) const;
    ///@}
 
+   /* HDG */
+   double GetValueFacet(FaceElementTransformations &T,
+                        const IntegrationPoint &ip,
+                        int vdim = 1,
+                        Vector *tr = NULL) const;
+
    void GetLaplacians(int i, const IntegrationRule &ir, Vector &laps,
                       int vdim = 1) const;
 
@@ -417,6 +425,14 @@ public:
        projection matrix. */
    void ProjectGridFunction(const GridFunction &src);
 
+   /* HDG */
+   void ProjectCoefficientSkeleton(Coefficient &coeff);
+   void ProjectCoefficientSkeleton(VectorCoefficient &vcoeff);
+   //   void ProjectCoefficientSkeletonBdr(Coefficient &coeff,
+   //       Array<int> &bdr_attr_marker);
+   void ProjectCoefficientSkeletonBdr(Coefficient &coeff);
+   void ProjectCoefficientSkeletonBdr(VectorCoefficient &vcoeff);
+
    /** @brief Project @a coeff Coefficient to @a this GridFunction. The
        projection computation depends on the choice of the FiniteElementSpace
        #fes. Note that this is usually interpolation at the degrees of freedom
@@ -493,6 +509,10 @@ protected:
                                            const Array<int> &bdr_attr,
                                            Array<int> &values_counter);
 
+   /* HDG */
+   /* Compute the mean of a GridFunction */
+   double ComputeMean(const IntegrationRule *irs[]) const;
+
    // Complete the computation of averages; called e.g. after
    // AccumulateAndCountZones().
    void ComputeMeans(AvgType type, Array<int> &zones_per_vdof);
@@ -671,6 +691,7 @@ public:
    ///       function uses the absolute values of the element-wise integrals.
    ///       This may lead to results which are not entirely consistent with
    ///       such integration rules.
+
    virtual real_t ComputeGradError(VectorCoefficient *exgrad,
                                    const IntegrationRule *irs[] = NULL) const;
 
@@ -1137,6 +1158,7 @@ public:
    ///       function uses the absolute values of the element-wise integrals.
    ///       This may lead to results which are not entirely consistent with
    ///       such integration rules.
+
    virtual real_t ComputeLpError(const real_t p, Coefficient &exsol,
                                  Coefficient *weight = NULL,
                                  const IntegrationRule *irs[] = NULL,
@@ -1492,6 +1514,12 @@ public:
                                        ) const
    { ComputeElementLpErrors(infinity(), exsol, error, NULL, NULL, irs); }
 
+   /* HDG */
+   double ComputeMeanLpError(const double p, Coefficient &exsol,
+                             const IntegrationRule *irs[] = NULL) const;
+
+
+
    virtual void ComputeFlux(BilinearFormIntegrator &blfi,
                             GridFunction &flux,
                             bool wcoef = true, int subdomain = -1);

fem/intrules.cpp

@@ -947,7 +947,6 @@ int Quadrature1D::CheckOpen(int type)
       case OpenUniform:
       case ClosedUniform:
       case OpenHalfUniform:
-      case ClosedGL:
          return type; // all types can work as open
       default:
          return Invalid;

fem/linearform.cpp

@@ -105,6 +105,25 @@ void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
    interior_face_integs.Append(lfi);
 }
 
+/* HDG */
+void LinearForm::AddSktBoundaryNeumannIntegrator(LinearFormIntegrator * lfi)
+{
+   bdrsklneufi.Append (lfi);
+   bdrsklneufi_marker.Append(NULL); // NULL -> all attributes are active
+}
+void LinearForm::AddSktBoundaryNeumannIntegrator(LinearFormIntegrator * lfi,
+                                                 Array<int> &bdr_attr_marker)
+{
+   bdrsklneufi.Append (lfi);
+   bdrsklneufi_marker.Append(
+      &bdr_attr_marker); // NULL -> all attributes are active
+}
+/* HDG */
+void LinearForm::AddSktInteriorFaceIntegrator(LinearFormIntegrator * lfi)
+{
+   interiorsklfi.Append (lfi);
+}
+
 bool LinearForm::SupportsDevice() const
 {
    // return false for NURBS meshes, so we don’t convert it to non-NURBS
@@ -338,6 +357,77 @@ void LinearForm::Assemble()
          }
       }
    }
+
+   /* HDG */
+   if (bdrsklneufi.Size())
+   {
+      FaceElementTransformations *ftr;
+      const FiniteElement *face_fe;
+      int nbdrfaces = fes->GetNBE();
+      Mesh *mesh = fes -> GetMesh();
+
+      for (int i = 0; i < nbdrfaces; i++)
+      {
+         int face = mesh->GetBdrElementFaceIndex(i);
+         ftr = mesh->GetBdrFaceTransformations(
+                  i); // the transformation of the face
+         //          fes->GetBdrElementVDofs(i, vdofs);   // the degrees of freedom related to the face
+         fes->GetFaceVDofs(face, vdofs);   // the degrees of freedom related to the face
+         face_fe = fes->GetFaceElement(
+                      face);   // point face_fe to the FiniteElement over the edge
+
+         if (ftr != NULL)
+         {
+            for (int k = 0; k < bdrsklneufi.Size(); k++) // Loop over the related interals
+            {
+               int compute = 0;
+               if (bdrsklneufi_marker[k] == NULL)
+               {
+                  compute = 1;
+               }
+               else
+               {
+                  Array<int> &bdr_marker = *bdrsklneufi_marker[k];
+                  const int bdr_attr = mesh->GetBdrAttribute(i);
+                  if (bdr_marker[bdr_attr-1] == 1)
+                  {
+                     compute = 1;
+                  }
+               }
+               if (compute)
+               {
+                  bdrsklneufi[k] -> AssembleRHSElementVect (*face_fe, *ftr, elemvect);
+                  AddElementVector (vdofs, elemvect);
+               }
+            }
+         }
+
+      }
+   }
+
+   /* HDG */
+   if (interiorsklfi.Size())
+   {
+      Mesh *mesh = fes->GetMesh();
+
+      for (int k = 0; k < interiorsklfi.Size(); k++)
+      {
+         for (int i = 0; i < mesh->GetNumFaces(); i++)
+         {
+            FaceElementTransformations *tr = NULL;
+            tr = mesh->GetInteriorFaceTransformations (i);
+            if (tr != NULL)
+            {
+               fes->GetFaceVDofs(i, vdofs);   // the degrees of freedom related to the face
+
+               interiorsklfi[k]->
+               AssembleRHSElementVect(*fes->GetFaceElement(i),
+                                      *tr, elemvect);
+               AddElementVector (vdofs, elemvect);
+            }
+         }
+      }
+   }
 }
 
 void LinearForm::Update()
@@ -428,6 +518,8 @@ LinearForm::~LinearForm()
       { delete boundary_face_integs[k]; }
       for (k=0; k < interior_face_integs.Size(); k++)
       { delete interior_face_integs[k]; }
+      for (k=0; k < bdrsklneufi.Size(); k++)
+      { delete bdrsklneufi[k]; }
    }
 
    delete ext;

fem/linearform.hpp

@@ -71,6 +71,16 @@ protected:
    /// The reference coordinates where the centers of the delta functions lie
    Array<IntegrationPoint> domain_delta_integs_ip;
 
+   /* HDG */
+   /// Set of Boundary Skeleton Integrators .
+   Array<LinearFormIntegrator*> bdrsklneufi;
+   Array<Array<int>*>           bdrsklneufi_marker;
+
+   /* HDG */
+   /// Set of Interior Skeleton Integrators .
+   Array<LinearFormIntegrator*> interiorsklfi;
+
+
    /// If true, the delta locations are not (re)computed during assembly.
    bool HaveDeltaLocations()
    { return (domain_delta_integs_elem_id.Size() != 0); }
@@ -151,6 +161,17 @@ public:
    /// Adds new Boundary Face Integrator. Assumes ownership of @a lfi.
    void AddBdrFaceIntegrator(LinearFormIntegrator *lfi);
 
+   /* HDG */
+   /// Adds new Boundary Face Integrator with face number.
+   void AddSktBoundaryNeumannIntegrator (LinearFormIntegrator * lfi);
+
+   void AddSktBoundaryNeumannIntegrator (LinearFormIntegrator * lfi,
+                                         Array<int> &bdr_attr_marker);
+
+   /* HDG */
+   /// Access all integrators added with AddSktBoundaryNeumannIntegrator().
+   Array<LinearFormIntegrator*> *GetBDRSKTFLFI() { return &bdrsklneufi; }
+
    /** @brief Add new Boundary Face Integrator, restricted to the given boundary
        attributes.
 
@@ -190,6 +211,13 @@ public:
        corresponding pointer (to Array<int>) will be NULL. */
    Array<Array<int>*> *GetFLFI_Marker() { return &boundary_face_integs_marker; }
 
+   /* HDG */
+   void AddSktInteriorFaceIntegrator(LinearFormIntegrator * lfi);
+
+   /* HDG */
+   /// Access all integrators added with AddSktInteriorFaceIntegrator().
+   Array<LinearFormIntegrator*> *GetISKTFLFI() { return &interiorsklfi; }
+
    /// @brief Which assembly algorithm to use: the new device-compatible fast
    /// assembly (true), or the legacy CPU-only algorithm (false).
    /** If not set, the default value is false.  If used, this method must be

fem/lininteg.cpp

@@ -22,6 +22,13 @@ void LinearFormIntegrator::AssembleDevice(const FiniteElementSpace &fes,
    MFEM_ABORT("Not supported.");
 }
 
+
+void LinearFormIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   mfem_error("LinearFormIntegrator::AssembleRHSElementVect(..., ElementTransformation, ...)");
+}
+
 void LinearFormIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
 {
@@ -121,6 +128,8 @@ void DomainLFGradIntegrator::AssembleRHSElementVect(
    }
 }
 
+
+
 void DomainLFGradIntegrator::AssembleDeltaElementVect(
    const FiniteElement &fe, ElementTransformation &Trans, Vector &elvect)
 {

fem/lininteg.hpp

@@ -56,7 +56,6 @@ public:
    virtual ~LinearFormIntegrator() { }
 };
 
-
 /// Abstract class for integrators that support delta coefficients
 class DeltaLFIntegrator : public LinearFormIntegrator
 {

fem/nonlinearform.cpp

@@ -436,7 +436,7 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
    // In parallel, the result is in 'py' which is an alias for 'aux2'.
 }
 
-Operator &NonlinearForm::GetGradient(const Vector &x, bool finalize) const
+Operator &NonlinearForm::GetGradient(const Vector &x) const
 {
    if (ext)
    {
@@ -644,8 +644,6 @@ Operator &NonlinearForm::GetGradient(const Vector &x, bool finalize) const
       }
    }
 
-   if (!finalize) { return *Grad; }
-
    if (!Grad->Finalized())
    {
       Grad->Finalize(skip_zeros);
@@ -1205,14 +1203,7 @@ const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
       aux1.Update(block_offsets);
       for (int s = 0; s < fes.Size(); s++)
       {
-         if (P[s])
-         {
-            P[s]->Mult(bx.GetBlock(s), aux1.GetBlock(s));
-         }
-         else
-         {
-            aux1.GetBlock(s) = bx.GetBlock(s);
-         }
+         P[s]->Mult(bx.GetBlock(s), aux1.GetBlock(s));
       }
       return aux1;
    }
@@ -1241,16 +1232,11 @@ void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
       {
          cP[s]->MultTranspose(pby.GetBlock(s), by.GetBlock(s));
       }
-      else if (needs_prolongation)
-      {
-         by.GetBlock(s) = pby.GetBlock(s);
-      }
       by.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
    }
 }
 
-void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx,
-                                                bool finalize) const
+void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
 {
    const int skip_zeros = 0;
    Array<Array<int> *> vdofs(fes.Size());
@@ -1504,7 +1490,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx,
       }
    }
 
-   if (finalize && !Grads(0,0)->Finalized())
+   if (!Grads(0,0)->Finalized())
    {
       for (int i=0; i<fes.Size(); ++i)
       {
@@ -1543,23 +1529,7 @@ Operator &BlockNonlinearForm::GetGradient(const Vector &x) const
          for (int s2 = 0; s2 < fes.Size(); ++s2)
          {
             delete cGrads(s1, s2);
-            if (cP[s1] && cP[s2])
-            {
-               cGrads(s1, s2) = RAP(*cP[s1], *Grads(s1, s2), *cP[s2]);
-            }
-            else if (cP[s1])
-            {
-               cGrads(s1, s2) = TransposeMult(*cP[s1], *Grads(s1, s2));
-            }
-            else if (cP[s2])
-            {
-               cGrads(s1, s2) = mfem::Mult(*Grads(s1, s2), *cP[s2]);
-            }
-            else
-            {
-               cGrads(s1, s2) = NULL;
-               continue;
-            }
+            cGrads(s1, s2) = RAP(*cP[s1], *Grads(s1, s2), *cP[s2]);
             mGrads(s1, s2) = cGrads(s1, s2);
          }
       }

fem/nonlinearform.hpp

@@ -217,12 +217,7 @@ public:
        In general, @a x may have non-homogeneous essential boundary values.
 
        The state @a x must be a true-dof vector. */
-   Operator &GetGradient(const Vector &x) const override { return GetGradient(x, true); }
-
-   /** @brief Compute the gradient Operator of the NonlinearForm corresponding
-       to the state @a x with optional finalization and elimintaion. */
-   /** @see GetGradient(const Vector &) */
-   Operator &GetGradient(const Vector &x, bool finalize) const;
+   Operator &GetGradient(const Vector &x) const override;
 
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
@@ -313,7 +308,7 @@ protected:
    void MultBlocked(const BlockVector &bx, BlockVector &by) const;
 
    /// Specialized version of GetGradient() for BlockVector
-   void ComputeGradientBlocked(const BlockVector &bx, bool finalize = true) const;
+   void ComputeGradientBlocked(const BlockVector &bx) const;
 
 public:
    /// Construct an empty BlockNonlinearForm. Initialize with SetSpaces().

fem/pbilinearform.cpp

@@ -151,15 +151,6 @@ void ParBilinearForm::ParallelRAP(SparseMatrix &loc_A, OperatorHandle &A,
    }
 }
 
-HypreParMatrix *ParBilinearForm::ParallelAssembleInternalMatrix()
-{
-   if (p_mat.Ptr() == NULL)
-   {
-      ParallelAssemble(p_mat, mat);
-   }
-   return p_mat.As<HypreParMatrix>();
-}
-
 void ParBilinearForm::ParallelAssemble(OperatorHandle &A, SparseMatrix *A_local)
 {
    A.Clear();
@@ -271,9 +262,39 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
    }
 }
 
+/* HDG */
+void ParBilinearForm::AssembleSharedHDGFaces(int skip_zeros)
+{
+   ParMesh *pmesh = pfes->GetParMesh();
+   FaceElementTransformations *ftr;
+   const FiniteElement *face_fe;
+   Array<int> vdofs;
+   DenseMatrix elemmat;
+
+   int nfaces = pmesh->GetNSharedFaces();
+   for (int i = 0; i < nfaces; i++)
+   {
+      int FaceNo = pmesh->GetSharedFace(i);
+      ftr = pmesh->GetSharedFaceTransformations(i);
+      pfes->GetFaceVDofs(FaceNo, vdofs);
+      face_fe = pfes->GetFaceElement(FaceNo);
+      //face_fe = pfes->GetFaceNbrFaceFE(FaceNo);
+      if (ftr != NULL)
+      {
+         for (int k = 0; k < hdgintbfi.Size(); k++)
+         {
+            hdgintbfi[k]->AssembleFaceMatrix(*face_fe, *ftr, elemmat);
+            elemmat *= 0.5;
+            mat->AddSubMatrix(vdofs, vdofs, elemmat, skip_zeros);
+         }
+      }
+   }
+}
+
 void ParBilinearForm::Assemble(int skip_zeros)
 {
-   if (interior_face_integs.Size())
+   /* HDG */
+   if (interior_face_integs.Size() || hdgintbfi.Size())
    {
       pfes->ExchangeFaceNbrData();
       if (!ext && mat == NULL)
@@ -288,6 +309,13 @@ void ParBilinearForm::Assemble(int skip_zeros)
    {
       AssembleSharedFaces(skip_zeros);
    }
+
+   /* HDG */
+   if (!ext && hdgintbfi.Size() > 0)
+   {
+      AssembleSharedHDGFaces(skip_zeros);
+   }
+
 }
 
 void ParBilinearForm::AssembleDiagonal(Vector &diag) const
@@ -342,15 +370,6 @@ void ParBilinearForm
    A.EliminateRowsCols(dof_list, X, B);
 }
 
-void ParBilinearForm::ParallelEliminateEssentialBC(
-   const Array<int> &bdr_attr_is_ess, const HypreParVector &X, HypreParVector &B)
-{
-   Array<int> dof_list;
-   pfes->GetEssentialTrueDofs(bdr_attr_is_ess, dof_list);
-
-   p_mat.As<HypreParMatrix>()->EliminateRowsCols(dof_list, X, B);
-}
-
 HypreParMatrix *ParBilinearForm::
 ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
                              HypreParMatrix &A) const
@@ -362,26 +381,6 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
    return A.EliminateRowsCols(dof_list);
 }
 
-void ParBilinearForm::ParallelEliminateEssentialBC(const Array<int>
-                                                   &bdr_attr_is_ess)
-{
-   Array<int> tdofs_list;
-   pfes->GetEssentialTrueDofs(bdr_attr_is_ess, tdofs_list);
-
-   ParallelEliminateTDofs(tdofs_list);
-}
-
-void ParBilinearForm::ParallelEliminateTDofs(const Array<int> &tdofs_list)
-{
-   p_mat_e.EliminateRowsCols(p_mat, tdofs_list);
-}
-
-void ParBilinearForm::ParallelEliminateTDofsInRHS(
-   const Array<int> &tdofs_list, const Vector &x, Vector &b)
-{
-   p_mat.EliminateBC(p_mat_e, tdofs_list, x, b);
-}
-
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const real_t a)
 const
 {
@@ -523,7 +522,7 @@ void ParBilinearForm::FormLinearSystem(
       HypreParVector true_X(pfes), true_B(pfes);
       P.MultTranspose(b, true_B);
       R.Mult(x, true_X);
-      ParallelEliminateTDofsInRHS(ess_tdof_list, true_X, true_B);
+      p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B);
       R.MultTranspose(true_B, b);
       hybridization->ReduceRHS(true_B, B);
       X.SetSize(B.Size());
@@ -536,11 +535,17 @@ void ParBilinearForm::FormLinearSystem(
       B.SetSize(X.Size());
       P.MultTranspose(b, B);
       R.Mult(x, X);
-      ParallelEliminateTDofsInRHS(ess_tdof_list, X, B);
+      p_mat.EliminateBC(p_mat_e, ess_tdof_list, X, B);
       if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
    }
 }
 
+void ParBilinearForm::EliminateVDofsInRHS(
+   const Array<int> &vdofs, const Vector &x, Vector &b)
+{
+   p_mat.EliminateBC(p_mat_e, vdofs, x, b);
+}
+
 void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
                                        OperatorHandle &A)
 {
@@ -585,7 +590,7 @@ void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
          mat = NULL;
          delete mat_e;
          mat_e = NULL;
-         ParallelEliminateTDofs(ess_tdof_list);
+         p_mat_e.EliminateRowsCols(p_mat, ess_tdof_list);
       }
       if (hybridization)
       {
@@ -647,180 +652,36 @@ void ParBilinearForm::Update(FiniteElementSpace *nfes)
    p_mat_e.Clear();
 }
 
-void ParMixedBilinearForm::pAllocMat()
-{
-   const int trial_nbr_size = trial_pfes->GetFaceNbrVSize();
-   const int test_nbr_size = test_pfes->GetFaceNbrVSize();
 
-   if (keep_nbr_block)
-   {
-      mat = new SparseMatrix(height + test_nbr_size, width + trial_nbr_size);
-   }
-   else
-   {
-      mat = new SparseMatrix(height, width + trial_nbr_size);
-   }
+HypreParMatrix *ParMixedBilinearForm::ParallelAssemble()
+{
+   // construct the block-diagonal matrix A
+   HypreParMatrix *A =
+      new HypreParMatrix(trial_pfes->GetComm(),
+                         test_pfes->GlobalVSize(),
+                         trial_pfes->GlobalVSize(),
+                         test_pfes->GetDofOffsets(),
+                         trial_pfes->GetDofOffsets(),
+                         mat);
+
+   HypreParMatrix *rap = RAP(test_pfes->Dof_TrueDof_Matrix(), A,
+                             trial_pfes->Dof_TrueDof_Matrix());
+
+   delete A;
+
+   return rap;
 }
 
-void ParMixedBilinearForm::AssembleSharedFaces(int skip_zeros)
+void ParMixedBilinearForm::ParallelAssemble(OperatorHandle &A)
 {
-   ParMesh *pmesh = trial_pfes->GetParMesh();
-   FaceElementTransformations *T;
-   Array<int> tr_vdofs1, tr_vdofs2, tr_vdofs_all;
-   Array<int> te_vdofs1, te_vdofs2, te_vdofs_all;
-   DenseMatrix elemmat;
-
-   int nfaces = pmesh->GetNSharedFaces();
-   for (int i = 0; i < nfaces; i++)
-   {
-      T = pmesh->GetSharedFaceTransformations(i);
-      int Elem2NbrNo = T->Elem2No - pmesh->GetNE();
-      trial_pfes->GetElementVDofs(T->Elem1No, tr_vdofs1);
-      test_pfes->GetElementVDofs(T->Elem1No, te_vdofs1);
-      trial_pfes->GetFaceNbrElementVDofs(Elem2NbrNo, tr_vdofs2);
-      test_pfes->GetFaceNbrElementVDofs(Elem2NbrNo, te_vdofs2);
-
-      tr_vdofs1.Copy(tr_vdofs_all);
-      for (int j = 0; j < tr_vdofs2.Size(); j++)
-      {
-         if (tr_vdofs2[j] >= 0)
-         {
-            tr_vdofs2[j] += width;
-         }
-         else
-         {
-            tr_vdofs2[j] -= width;
-         }
-      }
-      tr_vdofs_all.Append(tr_vdofs2);
-
-      if (keep_nbr_block)
-      {
-         te_vdofs1.Copy(te_vdofs_all);
-         for (int j = 0; j < te_vdofs2.Size(); j++)
-         {
-            if (te_vdofs2[j] >= 0)
-            {
-               te_vdofs2[j] += height;
-            }
-            else
-            {
-               te_vdofs2[j] -= height;
-            }
-         }
-         te_vdofs_all.Append(te_vdofs2);
-      }
-
-      for (int k = 0; k < interior_face_integs.Size(); k++)
-      {
-         interior_face_integs[k]->
-         AssembleFaceMatrix(*trial_pfes->GetFE(T->Elem1No),
-                            *test_pfes->GetFE(T->Elem1No),
-                            *trial_pfes->GetFaceNbrFE(Elem2NbrNo),
-                            *test_pfes->GetFaceNbrFE(Elem2NbrNo),
-                            *T, elemmat);
-         if (keep_nbr_block)
-         {
-            mat->AddSubMatrix(te_vdofs_all, tr_vdofs_all, elemmat, skip_zeros);
-         }
-         else
-         {
-            mat->AddSubMatrix(te_vdofs1, tr_vdofs_all, elemmat, skip_zeros);
-         }
-      }
-   }
-}
-
-void ParMixedBilinearForm::Assemble(int skip_zeros)
-{
-   if (interior_face_integs.Size())
-   {
-      trial_pfes->ExchangeFaceNbrData();
-      test_pfes->ExchangeFaceNbrData();
-      if (!ext && mat == NULL)
-      {
-         pAllocMat();
-      }
-   }
-
-   MixedBilinearForm::Assemble(skip_zeros);
-
-   if (!ext && interior_face_integs.Size() > 0)
-   {
-      AssembleSharedFaces(skip_zeros);
-   }
-}
-
-HypreParMatrix *ParMixedBilinearForm::ParallelAssembleInternalMatrix()
-{
-   if (p_mat.Ptr() == NULL)
-   {
-      ParallelAssemble(p_mat, mat);
-   }
-   return p_mat.As<HypreParMatrix>();
-}
-
-HypreParMatrix *ParMixedBilinearForm::ParallelAssemble(SparseMatrix *m)
-{
-   OperatorHandle Mh(Operator::Hypre_ParCSR);
-   ParallelAssemble(Mh, m);
-   Mh.SetOperatorOwner(false);
-   return Mh.As<HypreParMatrix>();
-}
-
-void ParMixedBilinearForm::ParallelAssemble(OperatorHandle &A,
-                                            SparseMatrix *A_local)
-{
-   A.Clear();
-
-   if (A_local == NULL) { return; }
-   MFEM_VERIFY(A_local->Finalized(), "the local matrix must be finalized");
-
-   OperatorHandle dA(A.Type()), hdA;
-
-   if (interior_face_integs.Size() == 0)
-   {
-      // construct the rectangular block-diagonal matrix dA
-      dA.MakeRectangularBlockDiag(trial_pfes->GetComm(),
-                                  test_pfes->GlobalVSize(),
-                                  trial_pfes->GlobalVSize(),
-                                  test_pfes->GetDofOffsets(),
-                                  trial_pfes->GetDofOffsets(),
-                                  A_local);
-   }
-   else
-   {
-      // handle the case when 'a' contains off-diagonal
-      const int lvrows = test_pfes->GetVSize();
-      const int lvcols = trial_pfes->GetVSize();
-      const HYPRE_BigInt *face_nbr_glob_lcol = trial_pfes->GetFaceNbrGlobalDofMap();
-      const HYPRE_BigInt lcol_offset = trial_pfes->GetMyDofOffset();
-
-      Array<HYPRE_BigInt> glob_J(A_local->NumNonZeroElems());
-      const int *J = A_local->GetJ();
-      for (int i = 0; i < glob_J.Size(); i++)
-      {
-         if (J[i] < lvcols)
-         {
-            glob_J[i] = J[i] + lcol_offset;
-         }
-         else
-         {
-            glob_J[i] = face_nbr_glob_lcol[J[i] - lvcols];
-         }
-      }
-
-      // TODO - construct dA directly in the A format
-      hdA.Reset(
-         new HypreParMatrix(trial_pfes->GetComm(), lvrows, test_pfes->GlobalVSize(),
-                            trial_pfes->GlobalVSize(), A_local->GetI(), glob_J,
-                            A_local->GetData(), test_pfes->GetDofOffsets(),
-                            trial_pfes->GetDofOffsets()));
-      // - hdA owns the new HypreParMatrix
-      // - the above constructor copies all input arrays
-      glob_J.DeleteAll();
-      dA.ConvertFrom(hdA);
-   }
+   // construct the rectangular block-diagonal matrix dA
+   OperatorHandle dA(A.Type());
+   dA.MakeRectangularBlockDiag(trial_pfes->GetComm(),
+                               test_pfes->GlobalVSize(),
+                               trial_pfes->GlobalVSize(),
+                               test_pfes->GetDofOffsets(),
+                               trial_pfes->GetDofOffsets(),
+                               mat);
 
    OperatorHandle P_test(A.Type()), P_trial(A.Type());
 
@@ -846,44 +707,6 @@ void ParMixedBilinearForm::TrueAddMult(const Vector &x, Vector &y,
    test_pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Yaux, 1.0, y);
 }
 
-void ParMixedBilinearForm::ParallelEliminateTrialEssentialBC(
-   const Array<int> &bdr_attr_is_ess)
-{
-   Array<int> trial_tdof_list;
-   trial_pfes->GetEssentialTrueDofs(bdr_attr_is_ess, trial_tdof_list);
-
-   ParallelEliminateTrialTDofs(trial_tdof_list);
-}
-
-void ParMixedBilinearForm::ParallelEliminateTrialTDofs(
-   const Array<int> &trial_tdof_list)
-{
-   HypreParMatrix *temp = p_mat.As<HypreParMatrix>()->EliminateCols(
-                             trial_tdof_list);
-   p_mat_e.Reset(temp, true);
-}
-
-void ParMixedBilinearForm::ParallelEliminateTrialTDofsInRHS(
-   const Array<int> &trial_tdof_list, const Vector &x, Vector &b)
-{
-   p_mat_e.As<HypreParMatrix>()->Mult(-1.0, x, 1.0, b);
-}
-
-void ParMixedBilinearForm::ParallelEliminateTestEssentialBC(
-   const Array<int> &bdr_attr_is_ess)
-{
-   Array<int> test_tdof_list;
-   test_pfes->GetEssentialTrueDofs(bdr_attr_is_ess, test_tdof_list);
-
-   ParallelEliminateTestTDofs(test_tdof_list);
-}
-
-void ParMixedBilinearForm::ParallelEliminateTestTDofs(
-   const Array<int> &test_tdof_list)
-{
-   p_mat.As<HypreParMatrix>()->EliminateRows(test_tdof_list);
-}
-
 void ParMixedBilinearForm::FormRectangularSystemMatrix(
    const Array<int>
    &trial_tdof_list,
@@ -904,8 +727,10 @@ void ParMixedBilinearForm::FormRectangularSystemMatrix(
       mat = NULL;
       delete mat_e;
       mat_e = NULL;
-      ParallelEliminateTrialTDofs(trial_tdof_list);
-      ParallelEliminateTestTDofs(test_tdof_list);
+      HypreParMatrix *temp =
+         p_mat.As<HypreParMatrix>()->EliminateCols(trial_tdof_list);
+      p_mat.As<HypreParMatrix>()->EliminateRows(test_tdof_list);
+      p_mat_e.Reset(temp, true);
    }
 
    A = p_mat;
@@ -935,7 +760,7 @@ void ParMixedBilinearForm::FormRectangularLinearSystem(
    test_P->MultTranspose(b, B);
    trial_R->Mult(x, X);
 
-   ParallelEliminateTrialTDofsInRHS(trial_tdof_list, X, B);
+   p_mat_e.As<HypreParMatrix>()->Mult(-1.0, X, 1.0, B);
    B.SetSubVector(test_tdof_list, 0.0);
 }
 

fem/pbilinearform.hpp

@@ -43,6 +43,8 @@ protected:
 
    void AssembleSharedFaces(int skip_zeros = 1);
 
+   /* HDG */
+   void AssembleSharedHDGFaces(int skip_zeros = 1);
 private:
    /// Copy construction is not supported; body is undefined.
    ParBilinearForm(const ParBilinearForm &);
@@ -73,7 +75,7 @@ public:
    /** When set to true and the ParBilinearForm has interior face integrators,
        the local SparseMatrix will include the rows (in addition to the columns)
        corresponding to face-neighbor dofs. The default behavior is to disregard
-       those rows. Must be called before the first Assemble() call. */
+       those rows. Must be called before the first Assemble call. */
    void KeepNbrBlock(bool knb = true) { keep_nbr_block = knb; }
 
    /** @brief Set the operator type id for the parallel matrix/operator when
@@ -101,14 +103,6 @@ public:
        diagonal for this case. */
    void AssembleDiagonal(Vector &diag) const override;
 
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix is the internal one, owned by the form. It is not
-       reassembled if it has been already constructed. If FormSystemMatrix()
-       has been called before, it is the system matrix with eliminated
-       essential DOFs, otherwise the parallel matrix is assembled here without
-       the elimination process. */
-   HypreParMatrix *ParallelAssembleInternalMatrix();
-
    /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
    /** The returned matrix has to be deleted by the caller. */
    HypreParMatrix *ParallelAssemble() { return ParallelAssemble(mat); }
@@ -154,13 +148,6 @@ public:
                                      const HypreParVector &X,
                                      HypreParVector &B) const;
 
-   /// Eliminate essential boundary DOFs from the parallel system matrix.
-   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. */
-   void ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                     const HypreParVector &X,
-                                     HypreParVector &B);
-
    /// Eliminate essential boundary DOFs from a parallel assembled matrix @a A.
    /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
        the essential part of the boundary. The eliminated part is stored in a
@@ -172,12 +159,6 @@ public:
    HypreParMatrix *ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
                                                 HypreParMatrix &A) const;
 
-   /// Eliminate essential boundary DOFs from the parallel system matrix.
-   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. This method relies on
-       ParallelEliminateTDofs(const Array<int> &), see it for details. */
-   void ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess);
-
    /// Eliminate essential true DOFs from a parallel assembled matrix @a A.
    /** Given a list of essential true dofs and the parallel assembled matrix
        @a A, eliminate the true dofs from the matrix, storing the eliminated
@@ -190,28 +171,6 @@ public:
                                           HypreParMatrix &A) const
    { return A.EliminateRowsCols(tdofs_list); }
 
-   /// Eliminate essential true DOFs from the parallel system matrix.
-   /** Given a list of essential true dofs, eliminate the true dofs from
-       the parallel assembled system matrix, storing the eliminated part
-       internally. This method works in conjunction with
-       ParallelEliminateTDofsInRHS() and allows elimination of boundary
-       conditions in multiple right-hand sides. */
-   void ParallelEliminateTDofs(const Array<int> &tdofs_list);
-
-   /** @brief Use the stored eliminated part of the parallel system matrix for
-       elimination of boundary conditions in the r.h.s. */
-   /** Given a list of essential true dofs, eliminate the true dofs from the
-       right-hand side @a b using the solution vector @a x and the previously
-       stored eliminated part of the parallel assembled system matrix produced
-       by ParallelEliminateTDofs(const Array<int> &). */
-   void ParallelEliminateTDofsInRHS(const Array<int> &tdofs, const Vector &x,
-                                    Vector &b);
-
-   /// @deprecated Use ParallelEliminateTDofsInRHS() instead.
-   MFEM_DEPRECATED void EliminateVDofsInRHS(const Array<int> &vdofs,
-                                            const Vector &x, Vector &b)
-   { ParallelEliminateTDofsInRHS(vdofs, x, b); }
-
    /** @brief Compute @a y += @a a (P^t A P) @a x, where @a x and @a y are
        vectors on the true dofs. */
    void TrueAddMult(const Vector &x, Vector &y, const real_t a = 1.0) const;
@@ -281,6 +240,8 @@ public:
 
    void Update(FiniteElementSpace *nfes = NULL) override;
 
+   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x, Vector &b);
+
    virtual ~ParBilinearForm() { }
 };
 
@@ -298,13 +259,6 @@ protected:
    /// Matrix and eliminated matrix
    OperatorHandle p_mat, p_mat_e;
 
-   bool keep_nbr_block;
-
-   // Allocate mat - called when (mat == NULL && fbfi.Size() > 0)
-   void pAllocMat();
-
-   void AssembleSharedFaces(int skip_zeros = 1);
-
 private:
    /// Copy construction is not supported; body is undefined.
    ParMixedBilinearForm(const ParMixedBilinearForm &);
@@ -324,7 +278,6 @@ public:
    {
       trial_pfes = trial_fes;
       test_pfes  = test_fes;
-      keep_nbr_block = false;
    }
 
    /** @brief Create a ParMixedBilinearForm on the given FiniteElementSpace%s
@@ -344,89 +297,15 @@ public:
    {
       trial_pfes = trial_fes;
       test_pfes  = test_fes;
-      keep_nbr_block = false;
    }
 
-   /** When set to true and the ParMixedBilinearForm has interior face
-       integrators, the local SparseMatrix will include the rows (in addition
-       to the columns) corresponding to face-neighbor dofs. The default
-       behavior is to disregard those rows. Must be called before the first
-       Assemble() call. */
-   void KeepNbrBlock(bool knb = true) { keep_nbr_block = knb; }
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
    /// Returns the matrix assembled on the true dofs, i.e. P_test^t A P_trial.
-   /** The returned matrix is the internal one, owned by the form. It is not
-       reassembled if it has been already constructed. If
-       FormRectangularSystemMatrix() has been called before, it is the system
-       matrix with eliminated essential DOFs, otherwise the parallel matrix is
-       assembled here without the elimination process. */
-   HypreParMatrix *ParallelAssembleInternalMatrix();
-
-   /// Returns the matrix assembled on the true dofs, i.e. P_test^t A P_trial.
-   /** The returned matrix has to be deleted by the caller. */
-   HypreParMatrix *ParallelAssemble() { return ParallelAssemble(mat); }
-
-   /** @brief Returns the eliminated matrix assembled on the true dofs, i.e.
-       P_test^t A_local P_trial. */
-   /** The returned matrix has to be deleted by the caller. */
-   HypreParMatrix *ParallelAssembleElim() { return ParallelAssemble(mat_e); }
-
-   /** @brief Return the matrix @a m assembled on the true dofs, i.e. P_test^t
-       A_local P_trial. */
-   /** The returned matrix has to be deleted by the caller. */
-   HypreParMatrix *ParallelAssemble(SparseMatrix *m);
+   HypreParMatrix *ParallelAssemble();
 
    /** @brief Returns the matrix assembled on the true dofs, i.e.
        @a A = P_test^t A_local P_trial, in the format (type id) specified by
        @a A. */
-   void ParallelAssemble(OperatorHandle &A) { ParallelAssemble(A, mat); }
-
-   /** Returns the eliminated matrix assembled on the true dofs, i.e.
-       @a A_elim = P^t A_elim_local P in the format (type id) specified by @a A.
-    */
-   void ParallelAssembleElim(OperatorHandle &A_elim)
-   { ParallelAssemble(A_elim, mat_e); }
-
-   /** Returns the matrix @a A_local assembled on the true dofs, i.e.
-       @a A = P_test^t A_local P_trial in the format (type id) specified by
-       @a A. */
-   void ParallelAssemble(OperatorHandle &A, SparseMatrix *A_local);
-
-   /// Eliminate essential boundary trial DOFs from the parallel system matrix.
-   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. This method relies on
-       ParallelEliminateTrialTDofs(const Array<int> &), see it for details. */
-   void ParallelEliminateTrialEssentialBC(const Array<int> &bdr_attr_is_ess);
-
-   /// Eliminate essential trial true DOFs from the parallel system matrix.
-   /** Given a list of essential trial true dofs, eliminate the trial true dofs
-       from the parallel assembled system matrix, storing the eliminated part
-       internally. This method works in conjunction with
-       ParallelEliminateTrialTDofsInRHS() and allows elimination of boundary
-       conditions in multiple right-hand sides. */
-   void ParallelEliminateTrialTDofs(const Array<int> &trial_tdof_list);
-
-   /** @brief Use the stored eliminated part of the parallel system matrix for
-       elimination of boundary conditions in the r.h.s. */
-   /** Given a list of essential trial true dofs, eliminate the trial true dofs
-       from the right-hand side @a B using the solution vector @a X and the
-       previously stored eliminated part of the parallel assembled system
-       matrix produced by ParallelEliminateTrialTDofs(const Array<int> &). */
-   void ParallelEliminateTrialTDofsInRHS(const Array<int> &trial_tdof_list,
-                                         const Vector &X, Vector &B);
-
-   /// Eliminate essential boundary test DOFs from the parallel system matrix.
-   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. */
-   void ParallelEliminateTestEssentialBC(const Array<int> &bdr_attr_is_ess);
-
-   /// Eliminate essential test true DOFs from the parallel system matrix.
-   /** Given a list of essential test true dofs, eliminate the test true dofs
-       from the parallel assembled system matrix. */
-   void ParallelEliminateTestTDofs(const Array<int> &test_tdof_list);
+   void ParallelAssemble(OperatorHandle &A);
 
    using MixedBilinearForm::FormRectangularSystemMatrix;
    using MixedBilinearForm::FormRectangularLinearSystem;

fem/pgridfunc.cpp

@@ -565,6 +565,50 @@ void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
    }
 }
 
+/* HDG */
+void ParGridFunction::ProjectCoefficientSkeleton(Coefficient &coeff)
+{
+   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
+
+   if (delta_c == NULL)
+   {
+      GridFunction::ProjectCoefficientSkeleton(coeff);
+   }
+   else
+   {
+      real_t loc_integral, glob_integral;
+
+      ProjectDeltaCoefficient(*delta_c, loc_integral);
+
+      MPI_Allreduce(&loc_integral, &glob_integral, 1, MPI_DOUBLE, MPI_SUM,
+                    pfes->GetComm());
+
+      (*this) *= (delta_c->Scale() / glob_integral);
+   }
+}
+
+/* HDG */
+void ParGridFunction::ProjectCoefficientSkeletonBdr(Coefficient &coeff)
+{
+   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
+
+   if (delta_c == NULL)
+   {
+      GridFunction::ProjectCoefficientSkeletonBdr(coeff);
+   }
+   else
+   {
+      real_t loc_integral, glob_integral;
+
+      ProjectDeltaCoefficient(*delta_c, loc_integral);
+
+      MPI_Allreduce(&loc_integral, &glob_integral, 1, MPI_DOUBLE, MPI_SUM,
+                    pfes->GetComm());
+
+      (*this) *= (delta_c->Scale() / glob_integral);
+   }
+}
+
 void ParGridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff)
 {
    // local maximal element attribute for each dof
@@ -1215,6 +1259,15 @@ void ParGridFunction::SaveAsOne(std::ostream &os) const
    delete [] nrdofs;
 }
 
+/* HDG */
+double GlobalMean(double loc_mean, MPI_Comm comm)
+{
+   double glob_mean;
+   MPI_Allreduce(&loc_mean, &glob_mean, 1, MPI_DOUBLE, MPI_SUM, comm);
+
+   return glob_mean;
+}
+
 real_t GlobalLpNorm(const real_t p, real_t loc_norm, MPI_Comm comm)
 {
    real_t glob_norm;

fem/pgridfunc.hpp

@@ -45,6 +45,11 @@ namespace mfem
 ///       such integration rules.
 real_t GlobalLpNorm(const real_t p, real_t loc_norm, MPI_Comm comm);
 
+/* HDG */
+/* Compute the mean of a coefficient in parallel */
+double GlobalMean(double loc_mean, MPI_Comm comm);
+
+
 /// Class for parallel grid function
 class ParGridFunction : public GridFunction
 {
@@ -259,6 +264,12 @@ public:
    using GridFunction::ProjectCoefficient;
    void ProjectCoefficient(Coefficient &coeff) override;
 
+   /* HDG */
+   using GridFunction::ProjectCoefficientSkeleton;
+   void ProjectCoefficientSkeleton(Coefficient &coeff);
+   using GridFunction::ProjectCoefficientSkeletonBdr;
+   void ProjectCoefficientSkeletonBdr(Coefficient &coeff);
+
    using GridFunction::ProjectDiscCoefficient;
    /** @brief Project a discontinuous vector coefficient as a grid function on
        a continuous finite element space. The values in shared dofs are
@@ -433,6 +444,7 @@ public:
    ///       function uses the absolute values of the element-wise integrals.
    ///       This may lead to results which are not entirely consistent with
    ///       such integration rules.
+
    real_t ComputeDGFaceJumpError(Coefficient *exsol,
                                  Coefficient *ell_coeff,
                                  JumpScaling jump_scaling,
@@ -577,6 +589,12 @@ public:
                              p, exsol, weight, v_weight, irs), pfes->GetComm());
    }
 
+   /* HDG */
+   double ComputeMean(const IntegrationRule *irs[] = NULL) const
+   {
+      return GlobalMean(GridFunction::ComputeMean(irs), pfes->GetComm());
+   }
+
    void ComputeFlux(BilinearFormIntegrator &blfi,
                     GridFunction &flux,
                     bool wcoef = true, int subdomain = -1) override;

fem/plinearform.cpp

@@ -52,6 +52,11 @@ void ParLinearForm::Assemble()
       pfes->ExchangeFaceNbrData();
       AssembleSharedFaces();
    }
+   if (interiorsklfi.Size())
+   {
+      pfes->ExchangeFaceNbrData();
+      AssembleSharedHDGFaces();
+   }
 }
 
 bool ParLinearForm::SupportsDevice() const
@@ -93,6 +98,37 @@ void ParLinearForm::AssembleSharedFaces()
    }
 }
 
+void ParLinearForm::AssembleSharedHDGFaces()
+{
+   Array<int> vdofs;
+   Vector elemvect;
+
+   if (interiorsklfi.Size())
+   {
+      ParMesh *pmesh = pfes->GetParMesh();
+
+      for (int k = 0; k < interiorsklfi.Size(); k++)
+      {
+         for (int i = 0; i < pmesh->GetNSharedFaces(); i++)
+         {
+            FaceElementTransformations *tr = NULL;
+            tr = pmesh->GetSharedFaceTransformations(i);
+            if (tr != NULL)
+            {
+               int face_idx = pmesh->GetSharedFace(i);
+               fes->GetFaceVDofs(face_idx,
+                                 vdofs);   // the degrees of freedom related to the face
+
+               interiorsklfi[k]->
+               AssembleRHSElementVect(*fes->GetFaceElement(face_idx),
+                                      *tr, elemvect);
+               AddElementVector (vdofs, elemvect);
+            }
+         }
+      }
+   }
+}
+
 void ParLinearForm::ParallelAssemble(Vector &tv)
 {
    const Operator* prolong = pfes->GetProlongationMatrix();

fem/plinearform.hpp

@@ -124,6 +124,8 @@ public:
 
    void AssembleSharedFaces();
 
+   void AssembleSharedHDGFaces();
+
    /// Assemble the vector on the true dofs, i.e. P^t v.
    void ParallelAssemble(Vector &tv);
 

fem/pnonlinearform.cpp

@@ -105,59 +105,6 @@ const SparseMatrix &ParNonlinearForm::GetLocalGradient(const Vector &x) const
    return *Grad;
 }
 
-void ParNonlinearForm::GradientSharedFaces(const Vector &x,
-                                           int skip_zeros) const
-{
-   ParFiniteElementSpace *pfes = ParFESpace();
-   ParMesh *pmesh = pfes->GetParMesh();
-   FaceElementTransformations *T;
-   Array<int> vdofs1, vdofs2, vdofs_all;
-   DenseMatrix elemmat;
-   Vector el_x, nbr_x, face_x;
-   const Vector &px = Prolongate(x);
-
-   ParGridFunction pgf(pfes, const_cast<Vector&>(px), 0);
-   pgf.ExchangeFaceNbrData();
-
-   int nfaces = pmesh->GetNSharedFaces();
-   for (int i = 0; i < nfaces; i++)
-   {
-      T = pmesh->GetSharedFaceTransformations(i);
-      int Elem2NbrNo = T->Elem2No - pmesh->GetNE();
-
-      pfes->GetElementVDofs(T->Elem1No, vdofs1);
-      pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2);
-      face_x.SetSize(vdofs1.Size() + vdofs2.Size());
-
-      el_x.MakeRef(face_x, 0, vdofs1.Size());
-      pgf.GetSubVector(vdofs1, el_x);
-
-      nbr_x.MakeRef(face_x, vdofs1.Size(), vdofs2.Size());
-      pgf.FaceNbrData().GetSubVector(vdofs2, nbr_x);
-
-      vdofs1.Copy(vdofs_all);
-      for (int j = 0; j < vdofs2.Size(); j++)
-      {
-         if (vdofs2[j] >= 0)
-         {
-            vdofs2[j] += height;
-         }
-         else
-         {
-            vdofs2[j] -= height;
-         }
-      }
-      vdofs_all.Append(vdofs2);
-      for (int k = 0; k < fnfi.Size(); k++)
-      {
-         fnfi[k]->AssembleFaceGrad(*pfes->GetFE(T->Elem1No),
-                                   *pfes->GetFaceNbrFE(Elem2NbrNo),
-                                   *T, face_x, elemmat);
-         Grad->AddSubMatrix(vdofs1, vdofs_all, elemmat, skip_zeros);
-      }
-   }
-}
-
 Operator &ParNonlinearForm::GetGradient(const Vector &x) const
 {
    if (NonlinearForm::ext) { return NonlinearForm::GetGradient(x); }
@@ -165,61 +112,19 @@ Operator &ParNonlinearForm::GetGradient(const Vector &x) const
    ParFiniteElementSpace *pfes = ParFESpace();
 
    pGrad.Clear();
-   OperatorHandle dA(pGrad.Type()), Ph(pGrad.Type()), hdA;
 
-   if (fnfi.Size())
+   NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
+
+   OperatorHandle dA(pGrad.Type()), Ph(pGrad.Type());
+
+   if (fnfi.Size() == 0)
    {
-      const int skip_zeros = 0;
-
-      pfes->ExchangeFaceNbrData();
-      if (Grad == NULL)
-      {
-         int nbr_size = pfes->GetFaceNbrVSize();
-         Grad = new SparseMatrix(pfes->GetVSize(), pfes->GetVSize() + nbr_size);
-      }
-
-      NonlinearForm::GetGradient(x, false); // (re)assemble Grad, no b.c.
-
-      GradientSharedFaces(x, skip_zeros);
-
-      Grad->Finalize(skip_zeros);
-
-      // handle the case when 'a' contains off-diagonal
-      int lvsize = pfes->GetVSize();
-      const HYPRE_BigInt *face_nbr_glob_ldof = pfes->GetFaceNbrGlobalDofMap();
-      HYPRE_BigInt ldof_offset = pfes->GetMyDofOffset();
-
-      Array<HYPRE_BigInt> glob_J(Grad->NumNonZeroElems());
-      int *J = Grad->GetJ();
-      for (int i = 0; i < glob_J.Size(); i++)
-      {
-         if (J[i] < lvsize)
-         {
-            glob_J[i] = J[i] + ldof_offset;
-         }
-         else
-         {
-            glob_J[i] = face_nbr_glob_ldof[J[i] - lvsize];
-         }
-      }
-
-      // TODO - construct dA directly in the A format
-      hdA.Reset(
-         new HypreParMatrix(pfes->GetComm(), lvsize, pfes->GlobalVSize(),
-                            pfes->GlobalVSize(), Grad->GetI(), glob_J,
-                            Grad->GetData(), pfes->GetDofOffsets(),
-                            pfes->GetDofOffsets()));
-      // - hdA owns the new HypreParMatrix
-      // - the above constructor copies all input arrays
-      glob_J.DeleteAll();
-      dA.ConvertFrom(hdA);
+      dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
+                             pfes->GetDofOffsets(), Grad);
    }
    else
    {
-      NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
-
-      dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
-                             pfes->GetDofOffsets(), Grad);
+      MFEM_ABORT("TODO: assemble contributions from shared face terms");
    }
 
    // RAP the local gradient dA.
@@ -366,70 +271,7 @@ void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 
    if (fnfi.Size() > 0)
    {
-      // Terms over shared interior faces in parallel.
-      ParMesh *pmesh = ParFESpace(0)->GetParMesh();
-      FaceElementTransformations *tr;
-
-      Array<Array<int> *>vdofs(fes.Size());
-      Array<Array<int> *>vdofs2(fes.Size());
-      Array<Vector *> el_x(fes.Size());
-      Array<const Vector *> el_x_const(fes.Size());
-      Array<Vector *> el_y(fes.Size());
-      Array<const FiniteElement *> fe(fes.Size());
-      Array<const FiniteElement *> fe2(fes.Size());
-      Array<ParGridFunction *> pgfs(fes.Size());
-      for (int s=0; s<fes.Size(); ++s)
-      {
-         el_x_const[s] = el_x[s] = new Vector();
-         el_y[s] = new Vector();
-         vdofs[s] = new Array<int>;
-         vdofs2[s] = new Array<int>;
-         pgfs[s] = new ParGridFunction(const_cast<ParFiniteElementSpace*>(ParFESpace(s)),
-                                       xs.GetBlock(s));
-         pgfs[s]->ExchangeFaceNbrData();
-      }
-
-      const int n_shared_faces = pmesh->GetNSharedFaces();
-      for (int i = 0; i < n_shared_faces; i++)
-      {
-         tr = pmesh->GetSharedFaceTransformations(i, true);
-         int Elem2NbrNo = tr->Elem2No - pmesh->GetNE();
-
-         for (int s=0; s<fes.Size(); ++s)
-         {
-            const ParFiniteElementSpace *pfes = ParFESpace(s);
-            fe[s] = pfes->GetFE(tr->Elem1No);
-            fe2[s] = pfes->GetFaceNbrFE(Elem2NbrNo);
-
-            pfes->GetElementVDofs(tr->Elem1No, *(vdofs[s]));
-            pfes->GetFaceNbrElementVDofs(Elem2NbrNo, *(vdofs2[s]));
-
-            el_x[s]->SetSize(vdofs[s]->Size() + vdofs2[s]->Size());
-            xs.GetBlock(s).GetSubVector(*(vdofs[s]), el_x[s]->GetData());
-            pgfs[s]->FaceNbrData().GetSubVector(*(vdofs2[s]),
-                                                el_x[s]->GetData() + vdofs[s]->Size());
-         }
-
-         for (int k = 0; k < fnfi.Size(); ++k)
-         {
-            fnfi[k]->AssembleFaceVector(fe, fe2, *tr, el_x_const, el_y);
-
-            for (int s=0; s<fes.Size(); ++s)
-            {
-               if (el_y[s]->Size() == 0) { continue; }
-               ys.GetBlock(s).AddElementVector(*(vdofs[s]), *el_y[s]);
-            }
-         }
-      }
-
-      for (int s=0; s<fes.Size(); ++s)
-      {
-         delete pgfs[s];
-         delete vdofs2[s];
-         delete vdofs[s];
-         delete el_y[s];
-         delete el_x[s];
-      }
+      MFEM_ABORT("TODO: assemble contributions from shared face terms");
    }
 
    for (int s=0; s<fes.Size(); ++s)
@@ -486,106 +328,6 @@ void ParBlockNonlinearForm::SetGradientType(Operator::Type tid)
    }
 }
 
-void ParBlockNonlinearForm::GradientSharedFaces(const BlockVector &xs,
-                                                int skip_zeros) const
-{
-   // Terms over shared interior faces in parallel.
-   ParMesh *pmesh = ParFESpace(0)->GetParMesh();
-   FaceElementTransformations *tr;
-
-   Array<Array<int> *>vdofs(fes.Size());
-   Array<Array<int> *>vdofs2(fes.Size());
-   Array<Array<int> *>vdofs_all(fes.Size());
-   Array<Vector *> el_x(fes.Size());
-   Array<const Vector *> el_x_const(fes.Size());
-   Array2D<DenseMatrix *> elmats(fes.Size(), fes.Size());
-   Array<const FiniteElement *> fe(fes.Size());
-   Array<const FiniteElement *> fe2(fes.Size());
-   Array<ParGridFunction *> pgfs(fes.Size());
-
-   for (int s1=0; s1<fes.Size(); ++s1)
-   {
-      el_x_const[s1] = el_x[s1] = new Vector();
-      vdofs[s1] = new Array<int>;
-      vdofs2[s1] = new Array<int>;
-      vdofs_all[s1] = new Array<int>;
-      pgfs[s1] = new ParGridFunction(
-         const_cast<ParFiniteElementSpace*>(ParFESpace(s1)),
-         const_cast<Vector&>(xs.GetBlock(s1)));
-      pgfs[s1]->ExchangeFaceNbrData();
-      for (int s2=0; s2<fes.Size(); ++s2)
-      {
-         elmats(s1,s2) = new DenseMatrix();
-      }
-   }
-
-   const int n_shared_faces = pmesh->GetNSharedFaces();
-   for (int i = 0; i < n_shared_faces; i++)
-   {
-      tr = pmesh->GetSharedFaceTransformations(i, true);
-      int Elem2NbrNo = tr->Elem2No - pmesh->GetNE();
-
-      for (int s=0; s<fes.Size(); ++s)
-      {
-         const ParFiniteElementSpace *pfes = ParFESpace(s);
-         fe[s] = pfes->GetFE(tr->Elem1No);
-         fe2[s] = pfes->GetFaceNbrFE(Elem2NbrNo);
-
-         pfes->GetElementVDofs(tr->Elem1No, *(vdofs[s]));
-         pfes->GetFaceNbrElementVDofs(Elem2NbrNo, *(vdofs2[s]));
-
-         el_x[s]->SetSize(vdofs[s]->Size() + vdofs2[s]->Size());
-         xs.GetBlock(s).GetSubVector(*(vdofs[s]), el_x[s]->GetData());
-         pgfs[s]->FaceNbrData().GetSubVector(*(vdofs2[s]),
-                                             el_x[s]->GetData() + vdofs[s]->Size());
-
-         vdofs[s]->Copy(*vdofs_all[s]);
-
-         const int lvsize = pfes->GetVSize();
-         for (int j = 0; j < vdofs2[s]->Size(); j++)
-         {
-            if ((*vdofs2[s])[j] >= 0)
-            {
-               (*vdofs2[s])[j] += lvsize;
-            }
-            else
-            {
-               (*vdofs2[s])[j] -= lvsize;
-            }
-         }
-         vdofs_all[s]->Append(*(vdofs2[s]));
-      }
-
-      for (int k = 0; k < fnfi.Size(); ++k)
-      {
-         fnfi[k]->AssembleFaceGrad(fe, fe2, *tr, el_x_const, elmats);
-
-         for (int s1=0; s1<fes.Size(); ++s1)
-         {
-            for (int s2=0; s2<fes.Size(); ++s2)
-            {
-               if (elmats(s1,s2)->Height() == 0) { continue; }
-               Grads(s1,s2)->AddSubMatrix(*vdofs[s1], *vdofs_all[s2],
-                                          *elmats(s1,s2), skip_zeros);
-            }
-         }
-      }
-   }
-
-   for (int s1=0; s1<fes.Size(); ++s1)
-   {
-      delete pgfs[s1];
-      delete vdofs_all[s1];
-      delete vdofs2[s1];
-      delete vdofs[s1];
-      delete el_x[s1];
-      for (int s2=0; s2<fes.Size(); ++s2)
-      {
-         delete elmats(s1,s2);
-      }
-   }
-}
-
 BlockOperator & ParBlockNonlinearForm::GetGradient(const Vector &x) const
 {
    if (pBlockGrad == NULL)
@@ -605,155 +347,49 @@ BlockOperator & ParBlockNonlinearForm::GetGradient(const Vector &x) const
       }
    }
 
-   // xs_true is not modified, so const_cast is okay
-   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
-   xs.Update(block_offsets);
-
-   for (int s=0; s<fes.Size(); ++s)
-   {
-      fes[s]->GetProlongationMatrix()->Mult(
-         xs_true.GetBlock(s), xs.GetBlock(s));
-   }
+   GetLocalGradient(x); // gradients are stored in 'Grads'
 
    if (fnfi.Size() > 0)
    {
-      const int skip_zeros = 0;
-
-      for (int s=0; s<fes.Size(); ++s)
-      {
-         const_cast<ParFiniteElementSpace*>(pfes[s])->ExchangeFaceNbrData();
-      }
-
-      for (int s1=0; s1<fes.Size(); ++s1)
-      {
-         for (int s2=0; s2<fes.Size(); ++s2)
-         {
-            if (Grads(s1,s2) == NULL)
-            {
-               int nbr_size = pfes[s2]->GetFaceNbrVSize();
-               Grads(s1,s2) = new SparseMatrix(pfes[s1]->GetVSize(),
-                                               pfes[s2]->GetVSize() + nbr_size);
-            }
-         }
-      }
-
-      // (re)assemble Grad without b.c. into 'Grads'
-      BlockNonlinearForm::ComputeGradientBlocked(xs, false);
-
-      GradientSharedFaces(xs, skip_zeros);
-
-      // finalize the gradients
-      for (int s1=0; s1<fes.Size(); ++s1)
-         for (int s2=0; s2<fes.Size(); ++s2)
-         {
-            Grads(s1,s2)->Finalize(skip_zeros);
-         }
-
-      for (int s1=0; s1<fes.Size(); ++s1)
-      {
-         for (int s2=0; s2<fes.Size(); ++s2)
-         {
-            OperatorHandle hdA;
-            OperatorHandle dA(phBlockGrad(s1,s2)->Type()),
-                           Ph(phBlockGrad(s1,s2)->Type()),
-                           Rh(phBlockGrad(s1,s2)->Type());
-
-            // handle the case when 'a' contains off-diagonal
-            int lvsize = pfes[s2]->GetVSize();
-            const HYPRE_BigInt *face_nbr_glob_ldof =
-               const_cast<ParFiniteElementSpace*>(pfes[s2])->GetFaceNbrGlobalDofMap();
-            HYPRE_BigInt ldof_offset = pfes[s2]->GetMyDofOffset();
-
-            Array<HYPRE_BigInt> glob_J(Grads(s1,s2)->NumNonZeroElems());
-            int *J = Grads(s1,s2)->GetJ();
-            for (int i = 0; i < glob_J.Size(); i++)
-            {
-               if (J[i] < lvsize)
-               {
-                  glob_J[i] = J[i] + ldof_offset;
-               }
-               else
-               {
-                  glob_J[i] = face_nbr_glob_ldof[J[i] - lvsize];
-               }
-            }
-
-            // TODO - construct dA directly in the A format
-            hdA.Reset(
-               new HypreParMatrix(pfes[s2]->GetComm(), pfes[s1]->GetVSize(),
-                                  pfes[s1]->GlobalVSize(), pfes[s2]->GlobalVSize(),
-                                  Grads(s1,s2)->GetI(), glob_J, Grads(s1,s2)->GetData(),
-                                  pfes[s1]->GetDofOffsets(), pfes[s2]->GetDofOffsets()));
-            // - hdA owns the new HypreParMatrix
-            // - the above constructor copies all input arrays
-            glob_J.DeleteAll();
-            dA.ConvertFrom(hdA);
-
-            if (s1 == s2)
-            {
-               Ph.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix());
-               phBlockGrad(s1,s1)->MakePtAP(dA, Ph);
-
-               OperatorHandle Ae;
-               Ae.EliminateRowsCols(*phBlockGrad(s1,s1), *ess_tdofs[s1]);
-            }
-            else
-            {
-               Rh.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix());
-               Ph.ConvertFrom(pfes[s2]->Dof_TrueDof_Matrix());
-
-               phBlockGrad(s1,s2)->MakeRAP(Rh, dA, Ph);
-
-               phBlockGrad(s1,s2)->EliminateRows(*ess_tdofs[s1]);
-               phBlockGrad(s1,s2)->EliminateCols(*ess_tdofs[s2]);
-            }
-
-            pBlockGrad->SetBlock(s1, s2, phBlockGrad(s1,s2)->Ptr());
-         }
-      }
+      MFEM_ABORT("TODO: assemble contributions from shared face terms");
    }
-   else
+
+   for (int s1=0; s1<fes.Size(); ++s1)
    {
-      // (re)assemble Grad without b.c. into 'Grads'
-      BlockNonlinearForm::ComputeGradientBlocked(xs);
-
-      for (int s1=0; s1<fes.Size(); ++s1)
+      for (int s2=0; s2<fes.Size(); ++s2)
       {
-         for (int s2=0; s2<fes.Size(); ++s2)
+         OperatorHandle dA(phBlockGrad(s1,s2)->Type()),
+                        Ph(phBlockGrad(s1,s2)->Type()),
+                        Rh(phBlockGrad(s1,s2)->Type());
+
+         if (s1 == s2)
          {
-            OperatorHandle dA(phBlockGrad(s1,s2)->Type()),
-                           Ph(phBlockGrad(s1,s2)->Type()),
-                           Rh(phBlockGrad(s1,s2)->Type());
+            dA.MakeSquareBlockDiag(pfes[s1]->GetComm(), pfes[s1]->GlobalVSize(),
+                                   pfes[s1]->GetDofOffsets(), Grads(s1,s1));
+            Ph.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix());
+            phBlockGrad(s1,s1)->MakePtAP(dA, Ph);
 
-            if (s1 == s2)
-            {
-               dA.MakeSquareBlockDiag(pfes[s1]->GetComm(), pfes[s1]->GlobalVSize(),
-                                      pfes[s1]->GetDofOffsets(), Grads(s1,s1));
-               Ph.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix());
-               phBlockGrad(s1,s1)->MakePtAP(dA, Ph);
-
-               OperatorHandle Ae;
-               Ae.EliminateRowsCols(*phBlockGrad(s1,s1), *ess_tdofs[s1]);
-            }
-            else
-            {
-               dA.MakeRectangularBlockDiag(pfes[s1]->GetComm(),
-                                           pfes[s1]->GlobalVSize(),
-                                           pfes[s2]->GlobalVSize(),
-                                           pfes[s1]->GetDofOffsets(),
-                                           pfes[s2]->GetDofOffsets(),
-                                           Grads(s1,s2));
-               Rh.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix());
-               Ph.ConvertFrom(pfes[s2]->Dof_TrueDof_Matrix());
-
-               phBlockGrad(s1,s2)->MakeRAP(Rh, dA, Ph);
-
-               phBlockGrad(s1,s2)->EliminateRows(*ess_tdofs[s1]);
-               phBlockGrad(s1,s2)->EliminateCols(*ess_tdofs[s2]);
-            }
-
-            pBlockGrad->SetBlock(s1, s2, phBlockGrad(s1,s2)->Ptr());
+            OperatorHandle Ae;
+            Ae.EliminateRowsCols(*phBlockGrad(s1,s1), *ess_tdofs[s1]);
          }
+         else
+         {
+            dA.MakeRectangularBlockDiag(pfes[s1]->GetComm(),
+                                        pfes[s1]->GlobalVSize(),
+                                        pfes[s2]->GlobalVSize(),
+                                        pfes[s1]->GetDofOffsets(),
+                                        pfes[s2]->GetDofOffsets(),
+                                        Grads(s1,s2));
+            Rh.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix());
+            Ph.ConvertFrom(pfes[s2]->Dof_TrueDof_Matrix());
+
+            phBlockGrad(s1,s2)->MakeRAP(Rh, dA, Ph);
+
+            phBlockGrad(s1,s2)->EliminateRows(*ess_tdofs[s1]);
+            phBlockGrad(s1,s2)->EliminateCols(*ess_tdofs[s2]);
+         }
+
+         pBlockGrad->SetBlock(s1, s2, phBlockGrad(s1,s2)->Ptr());
       }
    }
 

fem/pnonlinearform.hpp

@@ -29,8 +29,6 @@ protected:
    mutable ParGridFunction X, Y;
    mutable OperatorHandle pGrad;
 
-   void GradientSharedFaces(const Vector &x, int skip_zeros = 1) const;
-
 public:
    ParNonlinearForm(ParFiniteElementSpace *pf);
 
@@ -83,8 +81,6 @@ protected:
    mutable Array2D<OperatorHandle *> phBlockGrad;
    mutable BlockOperator *pBlockGrad;
 
-   void GradientSharedFaces(const BlockVector &xs, int skip_zeros) const;
-
 public:
    /// Computes the energy of the system
    real_t GetEnergy(const Vector &x) const override;

general/array.cpp

@@ -0,0 +1,214 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+
+// Abstract array data type
+
+#include "array.hpp"
+#include "../general/forall.hpp"
+#include <fstream>
+#include <type_traits>
+
+namespace mfem
+{
+
+template <class T>
+void Array<T>::Print(std::ostream &os, int width) const
+{
+   for (int i = 0; i < size; i++)
+   {
+      os << data[i];
+      if ( !((i+1) % width) || i+1 == size )
+      {
+         os << '\n';
+      }
+      else
+      {
+         os << " ";
+      }
+   }
+}
+
+template <class T>
+void Array<T>::Save(std::ostream &os, int fmt) const
+{
+   if (fmt == 0)
+   {
+      os << size << '\n';
+   }
+   for (int i = 0; i < size; i++)
+   {
+      os << operator[](i) << '\n';
+   }
+}
+
+template <class T>
+void Array<T>::Load(std::istream &in, int fmt)
+{
+   if (fmt == 0)
+   {
+      int new_size;
+      in >> new_size;
+      SetSize(new_size);
+   }
+   for (int i = 0; i < size; i++)
+   {
+      in >> operator[](i);
+   }
+}
+
+template <class T>
+T Array<T>::Max() const
+{
+   MFEM_ASSERT(size > 0, "Array is empty with size " << size);
+
+   T max = operator[](0);
+   for (int i = 1; i < size; i++)
+   {
+      if (max < operator[](i))
+      {
+         max = operator[](i);
+      }
+   }
+
+   return max;
+}
+
+template <class T>
+T Array<T>::Min() const
+{
+   MFEM_ASSERT(size > 0, "Array is empty with size " << size);
+
+   T min = operator[](0);
+   for (int i = 1; i < size; i++)
+   {
+      if (operator[](i) < min)
+      {
+         min = operator[](i);
+      }
+   }
+
+   return min;
+}
+
+// Partial Sum
+template <class T>
+void Array<T>::PartialSum()
+{
+   T sum = static_cast<T>(0);
+   for (int i = 0; i < size; i++)
+   {
+      sum+=operator[](i);
+      operator[](i) = sum;
+   }
+}
+
+template <class T>
+void Array<T>::Abs()
+{
+   static_assert(std::is_arithmetic<T>::value, "Use with arithmetic types!");
+   const bool useDevice = UseDevice();
+   const int N = size;
+   auto y = ReadWrite(useDevice);
+   mfem::forall_switch(useDevice, N, [=] MFEM_HOST_DEVICE (int i)
+   {
+      y[i] = std::abs(y[i]);
+   });
+}
+
+// Sum
+template <class T>
+T Array<T>::Sum() const
+{
+   T sum = static_cast<T>(0);
+   for (int i = 0; i < size; i++)
+   {
+      sum+=operator[](i);
+   }
+
+   return sum;
+}
+
+template <class T>
+int Array<T>::IsSorted() const
+{
+   T val_prev = operator[](0), val;
+   for (int i = 1; i < size; i++)
+   {
+      val=operator[](i);
+      if (val < val_prev)
+      {
+         return 0;
+      }
+      val_prev = val;
+   }
+
+   return 1;
+}
+
+template <class T>
+bool Array<T>::IsConstant() const
+{
+   if (size < 2) { return true; }
+   const T v0 = data[0];
+   for (int i = 1; i < size; i++)
+   {
+      if (data[i] != v0)
+      {
+         return false;
+      }
+   }
+
+   return true;
+}
+
+template <class T>
+void Array2D<T>::Load(const char *filename, int fmt)
+{
+   std::ifstream in;
+   in.open(filename, std::ifstream::in);
+   MFEM_VERIFY(in.is_open(), "File " << filename << " does not exist.");
+   Load(in, fmt);
+   in.close();
+}
+
+template <class T>
+void Array2D<T>::Print(std::ostream &os, int width_)
+{
+   int height = this->NumRows();
+   int width  = this->NumCols();
+
+   for (int i = 0; i < height; i++)
+   {
+      os << "[row " << i << "]\n";
+      for (int j = 0; j < width; j++)
+      {
+         os << (*this)(i,j);
+         if ( (j+1) == width_ || (j+1) % width_ == 0 )
+         {
+            os << '\n';
+         }
+         else
+         {
+            os << ' ';
+         }
+      }
+   }
+}
+
+template class Array<char>;
+template class Array<int>;
+template class Array<long long>;
+template class Array<real_t>;
+template class Array2D<int>;
+template class Array2D<real_t>;
+
+} // namespace mfem

general/array.hpp

@@ -16,13 +16,9 @@
 #include "mem_manager.hpp"
 #include "device.hpp"
 #include "error.hpp"
-#include "forall.hpp"
 #include "globals.hpp"
-#include "reducers.hpp"
-#include "scan.hpp"
 
 #include <iostream>
-#include <fstream>
 #include <cstdlib>
 #include <cstring>
 #include <algorithm>
@@ -139,8 +135,6 @@ public:
    /// Return the device flag of the Memory object used by the Array
    bool UseDevice() const { return data.UseDevice(); }
 
-   void UseDevice(bool use_dev) { data.UseDevice(use_dev); }
-
    /// Return true if the data will be deleted by the Array
    inline bool OwnsData() const { return data.OwnsHostPtr(); }
 
@@ -281,11 +275,11 @@ public:
 
    /** @brief Find the maximal element in the array, using the comparison
        operator `<` for class T. */
-   inline T Max() const;
+   T Max() const;
 
    /** @brief Find the minimal element in the array, using the comparison
        operator `<` for class T. */
-   inline T Min() const;
+   T Min() const;
 
    /// Sorts the array in ascending order. This requires operator< to be defined for T.
    void Sort() { std::sort((T*)data, data + size); }
@@ -303,22 +297,22 @@ public:
    }
 
    /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
-   inline int IsSorted() const;
+   int IsSorted() const;
 
    /// Does the Array have Size zero.
    bool IsEmpty() const { return Size() == 0; }
 
    /// Return true if all entries of the array are the same.
-   inline bool IsConstant() const;
+   bool IsConstant() const;
 
    /// Fill the entries of the array with the cumulative sum of the entries.
-   inline void PartialSum();
+   void PartialSum();
 
    /// Replace each entry of the array with its absolute value.
-   inline void Abs();
+   void Abs();
 
    /// Return the sum of all the array entries using the '+'' operator for class 'T'.
-   inline T Sum() const;
+   T Sum() const;
 
    /// Set all entries of the array to the provided constant.
    inline void operator=(const T &a);
@@ -803,14 +797,8 @@ template <typename T> template <typename CT>
 inline Array<T> &Array<T>::operator=(const Array<CT> &src)
 {
    SetSize(src.Size());
-
-   const bool use_dev = UseDevice() || src.UseDevice();
-   const auto x = src.Read(use_dev);
-   auto y = Write(use_dev);
-   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (int i)
-   {
-      y[i] = x[i];
-   });
+   for (int i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
+   return *this;
 }
 
 template <class T>
@@ -1026,24 +1014,19 @@ template <class T>
 inline void Array<T>::GetSubArray(int offset, int sa_size, Array<T> &sa) const
 {
    sa.SetSize(sa_size);
-   const bool use_dev = UseDevice() || sa.UseDevice();
-   const auto x = Read(use_dev);
-   auto y = sa.Write(use_dev);
-   mfem::forall_switch(use_dev, sa_size, [=] MFEM_HOST_DEVICE (int i)
+   for (int i = 0; i < sa_size; i++)
    {
-      y[i] = x[offset + i];
-   });
+      sa[i] = (*this)[offset+i];
+   }
 }
 
 template <class T>
 inline void Array<T>::operator=(const T &a)
 {
-   const bool use_dev = UseDevice();
-   auto x = Write(use_dev);
-   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (int i)
+   for (int i = 0; i < size; i++)
    {
-      x[i] = a;
-   });
+      data[i] = a;
+   }
 }
 
 template <class T>
@@ -1052,153 +1035,6 @@ inline void Array<T>::Assign(const T *p)
    data.CopyFromHost(p, Size());
 }
 
-template <class T>
-inline void Array<T>::Print(std::ostream &os, int width) const
-{
-   for (int i = 0; i < size; i++)
-   {
-      os << data[i];
-      if ( !((i+1) % width) || i+1 == size )
-      {
-         os << '\n';
-      }
-      else
-      {
-         os << " ";
-      }
-   }
-}
-
-template <class T>
-inline void Array<T>::Save(std::ostream &os, int fmt) const
-{
-   if (fmt == 0)
-   {
-      os << size << '\n';
-   }
-   for (int i = 0; i < size; i++)
-   {
-      os << operator[](i) << '\n';
-   }
-}
-
-template <class T>
-void Array<T>::Load(std::istream &in, int fmt)
-{
-   if (fmt == 0)
-   {
-      int new_size;
-      in >> new_size;
-      SetSize(new_size);
-   }
-   for (int i = 0; i < size; i++)
-   {
-      in >> operator[](i);
-   }
-}
-
-template <class T>
-inline T Array<T>::Max() const
-{
-   MFEM_ASSERT(size > 0, "Array is empty with size " << size);
-
-   T max = operator[](0);
-   for (int i = 1; i < size; i++)
-   {
-      if (max < operator[](i))
-      {
-         max = operator[](i);
-      }
-   }
-
-   return max;
-}
-
-template <class T>
-inline T Array<T>::Min() const
-{
-   MFEM_ASSERT(size > 0, "Array is empty with size " << size);
-
-   T min = operator[](0);
-   for (int i = 1; i < size; i++)
-   {
-      if (operator[](i) < min)
-      {
-         min = operator[](i);
-      }
-   }
-
-   return min;
-}
-
-// Partial Sum
-template <class T>
-inline void Array<T>::PartialSum()
-{
-   auto data_ptr = ReadWrite(UseDevice());
-   InclusiveScan(UseDevice(), data_ptr, data_ptr, size);
-}
-
-template <class T>
-inline void Array<T>::Abs()
-{
-   static_assert(std::is_arithmetic<T>::value, "Use with arithmetic types!");
-   const bool useDevice = UseDevice();
-   const int N = size;
-   auto y = ReadWrite(useDevice);
-   mfem::forall_switch(useDevice, N, [=] MFEM_HOST_DEVICE (int i)
-   {
-      y[i] = std::abs(y[i]);
-   });
-}
-
-// Sum
-template <class T>
-inline T Array<T>::Sum() const
-{
-   T sum = static_cast<T>(0);
-   if (size > 0)
-   {
-      const auto m_data = Read(UseDevice());
-      reduce(size, sum, [=] MFEM_HOST_DEVICE(int i, T &r) { r += m_data[i]; },
-      /*  */ SumReducer<T> {}, UseDevice());
-   }
-   return sum;
-}
-
-template <class T>
-inline int Array<T>::IsSorted() const
-{
-   T val_prev = operator[](0), val;
-   for (int i = 1; i < size; i++)
-   {
-      val=operator[](i);
-      if (val < val_prev)
-      {
-         return 0;
-      }
-      val_prev = val;
-   }
-
-   return 1;
-}
-
-template <class T>
-inline bool Array<T>::IsConstant() const
-{
-   if (size < 2) { return true; }
-   const T v0 = data[0];
-   for (int i = 1; i < size; i++)
-   {
-      if (data[i] != v0)
-      {
-         return false;
-      }
-   }
-
-   return true;
-}
-
 
 template <class T>
 inline const T &Array2D<T>::operator()(int i, int j) const
@@ -1238,40 +1074,6 @@ inline T *Array2D<T>::operator[](int i)
    return &array1d[i*N];
 }
 
-template <class T>
-void Array2D<T>::Load(const char *filename, int fmt)
-{
-   std::ifstream in;
-   in.open(filename, std::ifstream::in);
-   MFEM_VERIFY(in.is_open(), "File " << filename << " does not exist.");
-   Load(in, fmt);
-   in.close();
-}
-
-template <class T>
-void Array2D<T>::Print(std::ostream &os, int width_)
-{
-   int height = this->NumRows();
-   int width  = this->NumCols();
-
-   for (int i = 0; i < height; i++)
-   {
-      os << "[row " << i << "]\n";
-      for (int j = 0; j < width; j++)
-      {
-         os << (*this)(i,j);
-         if ( (j+1) == width_ || (j+1) % width_ == 0 )
-         {
-            os << '\n';
-         }
-         else
-         {
-            os << ' ';
-         }
-      }
-   }
-}
-
 
 template <class T>
 inline void Swap(Array2D<T> &a, Array2D<T> &b)

general/reducers.hpp

@@ -12,6 +12,7 @@
 #ifndef MFEM_REDUCERS_HPP
 #define MFEM_REDUCERS_HPP
 
+#include "array.hpp"
 #include "forall.hpp"
 
 #include <cmath>
@@ -513,33 +514,6 @@ template<class B, class R> struct reduction_kernel
       }
    }
 };
-
-template <class T>
-class ReductionWorkspace
-{
-   Memory<T> workspace;
-
-   static ReductionWorkspace &Instance()
-   {
-      static ReductionWorkspace instance;
-      return instance;
-   }
-
-   ~ReductionWorkspace() { workspace.Delete(); }
-
-public:
-   static T *Get(int num_blocks)
-   {
-      ReductionWorkspace &instance = Instance();
-      if (instance.workspace.Capacity() < num_blocks)
-      {
-         instance.workspace.Delete();
-         instance.workspace.New(num_blocks, MemoryType::HOST_PINNED);
-      }
-      return instance.workspace;
-   }
-};
-
 }
 
 /**
@@ -555,7 +529,8 @@ public:
  @tparam T value_type to operate on
  */
 template <class T, class B, class R>
-void reduce(int N, T &res, B &&body, const R &reducer, bool use_dev)
+void reduce(int N, T &res, B &&body, const R &reducer, bool use_dev,
+            Array<T> &workspace)
 {
    if (N == 0)
    {
@@ -592,7 +567,13 @@ void reduce(int N, T &res, B &&body, const R &reducer, bool use_dev)
 
       red_type red{nullptr, std::forward<B>(body), reducer, N, items_per_thread};
       // allocate res to fit block_size entries
-      auto work = internal::ReductionWorkspace<T>::Get(nblocks);
+      auto mt = workspace.GetMemory().GetMemoryType();
+      if (mt != MemoryType::HOST_PINNED && mt != MemoryType::MANAGED)
+      {
+         mt = MemoryType::HOST_PINNED;
+      }
+      workspace.SetSize(nblocks, mt);
+      auto work = workspace.HostWrite();
       red.work = work;
       forall_2D(nblocks, block_size, 1, std::move(red));
       // wait for results

general/scan.hpp

@@ -28,37 +28,8 @@
 
 namespace mfem
 {
-
-namespace internal
-{
-class ScanWorkspace
-{
-   Memory<std::byte> workspace;
-   static ScanWorkspace &Instance()
-   {
-      static ScanWorkspace instance;
-      return instance;
-   }
-   ~ScanWorkspace() { workspace.Delete(); }
-public:
-   static std::byte *Get(int num_bytes)
-   {
-      ScanWorkspace &instance = Instance();
-      if (Size() < num_bytes)
-      {
-         instance.workspace.Delete();
-         instance.workspace.New(num_bytes);
-      }
-      return instance.workspace.Write(MemoryClass::DEVICE, Size());
-   }
-   static int Size()
-   {
-      return Instance().workspace.Capacity();
-   }
-};
-}
-
-/// Equivalent to InclusiveScan(use_dev, d_in, d_out, num_items, std::plus<>{})
+/// Equivalent to InclusiveScan(use_dev, d_in, d_out, num_items, workspace,
+/// std::plus<>{})
 template <class InputIt, class OutputIt>
 void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items)
 {
@@ -66,12 +37,12 @@ void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items)
 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
    if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
-      using internal::ScanWorkspace;
-      size_t bytes = ScanWorkspace::Size();
-      if (bytes > 0)
+      static Array<std::byte> workspace;
+      size_t bytes = workspace.Size();
+      if (bytes)
       {
          auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                       ScanWorkspace::Get(bytes), bytes, d_in, d_out, num_items);
+                       workspace.Write(), bytes, d_in, d_out, num_items);
 #if defined(MFEM_USE_CUDA)
          if (err == cudaSuccess)
          {
@@ -86,12 +57,11 @@ void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items)
       }
       // try allocating a larger buffer
       bytes = 0;
-      // get size of buffer
       MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
                         nullptr, bytes, d_in, d_out, num_items));
-      // resize buffer (in ScanWorkspace::Get) and try again
+      workspace.SetSize(bytes);
       MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                        ScanWorkspace::Get(bytes), bytes, d_in, d_out, num_items));
+                        workspace.Write(), bytes, d_in, d_out, num_items));
       return;
    }
 #endif
@@ -131,13 +101,12 @@ void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
    if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
-      using internal::ScanWorkspace;
-      size_t bytes = ScanWorkspace::Size();
-      if (bytes > 0)
+      static Array<std::byte> workspace;
+      size_t bytes = workspace.Size();
+      if (bytes)
       {
          auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                       ScanWorkspace::Get(bytes), bytes, d_in, d_out, scan_op,
-                       num_items);
+                       workspace.Write(), bytes, d_in, d_out, scan_op, num_items);
 #if defined(MFEM_USE_CUDA)
          if (err == cudaSuccess)
          {
@@ -154,9 +123,9 @@ void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
       bytes = 0;
       MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
                         nullptr, bytes, d_in, d_out, scan_op, num_items));
+      workspace.SetSize(bytes);
       MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                        ScanWorkspace::Get(bytes), bytes, d_in, d_out, scan_op,
-                        num_items));
+                        workspace.Write(), bytes, d_in, d_out, scan_op, num_items));
       return;
    }
 #endif
@@ -195,13 +164,13 @@ void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
    if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
-      using internal::ScanWorkspace;
-      size_t bytes = ScanWorkspace::Size();
+      static Array<std::byte> workspace;
+      size_t bytes = workspace.Size();
       if (bytes)
       {
          auto err = MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                       ScanWorkspace::Get(bytes), bytes, d_in, d_out, scan_op,
-                       init_value, num_items);
+                       workspace.Write(), bytes, d_in, d_out, scan_op, init_value,
+                       num_items);
 #if defined(MFEM_USE_CUDA)
          if (err == cudaSuccess)
          {
@@ -218,9 +187,10 @@ void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
       bytes = 0;
       MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
                         nullptr, bytes, d_in, d_out, scan_op, init_value, num_items));
+      workspace.SetSize(bytes);
       MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                        ScanWorkspace::Get(bytes), bytes, d_in, d_out, scan_op,
-                        init_value, num_items));
+                        workspace.Write(), bytes, d_in, d_out, scan_op, init_value,
+                        num_items));
       return;
    }
 #endif
@@ -243,7 +213,7 @@ void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
 }
 
 /// Equivalent to ExclusiveScan(use_dev, d_in, d_out, num_items, init_value,
-/// std::plus<>{})
+/// workspace, std::plus<>{})
 template <class InputIt, class OutputIt, class T>
 void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
                    T init_value)

linalg/batched/magma.cpp

@@ -167,7 +167,7 @@ void MagmaBatchedLinAlg::Invert(DenseTensor &A) const
    magma_int_t status;
 
    status = MFEM_MAGMA_PREFIX(getrf_batched)(
-               n, n, d_LU_ptrs, n, d_P_ptrs, info_array.Write(), n_mat,
+               n, n, d_A_ptrs, n, d_P_ptrs, info_array.Write(), n_mat,
                Magma::Queue());
    MFEM_VERIFY(status == MAGMA_SUCCESS, "");
 

linalg/hypre.cpp

@@ -561,8 +561,7 @@ void CopyMemory(Memory<T> &src, Memory<T> &dst, MemoryClass dst_mc,
     this function. In particular, @a dst should be empty or deleted before
     calling this function. */
 template <typename SrcT, typename DstT>
-void CopyConvertMemory(const Memory<SrcT> &src, MemoryClass dst_mc,
-                       Memory<DstT> &dst)
+void CopyConvertMemory(Memory<SrcT> &src, MemoryClass dst_mc, Memory<DstT> &dst)
 {
    auto capacity = src.Capacity();
    dst.New(capacity, GetMemoryType(dst_mc));
@@ -843,8 +842,8 @@ static int GetPartitioningArraySize(MPI_Comm comm)
 ///
 /// Both @a row and @a col are partitioning arrays, whose length is returned by
 /// GetPartitioningArraySize(), see @ref hypre_partitioning_descr.
-static bool RowAndColStartsAreEqual(MPI_Comm comm, const HYPRE_BigInt *rows,
-                                    const HYPRE_BigInt *cols)
+static bool RowAndColStartsAreEqual(MPI_Comm comm, HYPRE_BigInt *rows,
+                                    HYPRE_BigInt *cols)
 {
    const int part_size = GetPartitioningArraySize(comm);
    bool are_equal = true;
@@ -1132,7 +1131,7 @@ HypreParMatrix::HypreParMatrix(
 HypreParMatrix::HypreParMatrix(MPI_Comm comm,
                                HYPRE_BigInt *row_starts,
                                HYPRE_BigInt *col_starts,
-                               const SparseMatrix *sm_a)
+                               SparseMatrix *sm_a)
 {
    MFEM_ASSERT(sm_a != NULL, "invalid input");
    MFEM_VERIFY(!HYPRE_AssumedPartitionCheck(),
@@ -1146,7 +1145,7 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
 
    hypre_CSRMatrixSetDataOwner(csr_a,0);
    MemoryIJData mem_a;
-   CopyCSR(const_cast<SparseMatrix*>(sm_a), mem_a, csr_a, false);
+   CopyCSR(sm_a, mem_a, csr_a, false);
    hypre_CSRMatrixSetRownnz(csr_a);
 
    // NOTE: this call creates a matrix on host even when device support is
@@ -1308,11 +1307,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
 HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
                                HYPRE_BigInt glob_nrows,
                                HYPRE_BigInt glob_ncols,
-                               const int *I,
-                               const HYPRE_BigInt *J,
-                               const real_t *data,
-                               const HYPRE_BigInt *rows,
-                               const HYPRE_BigInt *cols)
+                               int *I, HYPRE_BigInt *J,
+                               real_t *data,
+                               HYPRE_BigInt *rows,
+                               HYPRE_BigInt *cols)
 {
    Init();
 

linalg/hypre.hpp

@@ -565,7 +565,7 @@ public:
        partitioning arrays @a row_starts and @a col_starts. */
    HypreParMatrix(MPI_Comm comm, HYPRE_BigInt *row_starts,
                   HYPRE_BigInt *col_starts,
-                  const SparseMatrix *a); // constructor with 4 arguments, v2
+                  SparseMatrix *a); // constructor with 4 arguments, v2
 
    /// Creates boolean block-diagonal rectangular parallel matrix.
    /** The new HypreParMatrix does not take ownership of any of the input
@@ -594,9 +594,9 @@ public:
        arrays (so they can be deleted). See @ref hypre_partitioning_descr "here"
        for a description of the partitioning arrays @a rows and @a cols. */
    HypreParMatrix(MPI_Comm comm, int nrows, HYPRE_BigInt glob_nrows,
-                  HYPRE_BigInt glob_ncols, const int *I, const HYPRE_BigInt *J,
-                  const real_t *data, const HYPRE_BigInt *rows,
-                  const HYPRE_BigInt *cols); // constructor with 9 arguments
+                  HYPRE_BigInt glob_ncols, int *I, HYPRE_BigInt *J,
+                  real_t *data, HYPRE_BigInt *rows,
+                  HYPRE_BigInt *cols); // constructor with 9 arguments
 
    /** @brief Copy constructor for a ParCSR matrix which creates a deep copy of
        structure and data from @a P. */

linalg/sparsemat.cpp

@@ -46,12 +46,6 @@
 #define MFEM_GPUSPARSE_ALG HIPSPARSE_CSRMV_ALG1
 #endif // defined(MFEM_USE_CUDA)
 
-#if defined(MFEM_USE_SINGLE)
-#define MFEM_CUDA_or_HIP_REAL_T MFEM_CUDA_or_HIP(_R_32F)
-#elif defined(MFEM_USE_DOUBLE)
-#define MFEM_CUDA_or_HIP_REAL_T MFEM_CUDA_or_HIP(_R_64F)
-#endif
-
 namespace mfem
 {
 
@@ -63,10 +57,8 @@ int SparseMatrix::SparseMatrixCount = 0;
 /// @cond Suppress_Doxygen_warnings
 MFEM_cu_or_hip(sparseHandle_t) SparseMatrix::handle = nullptr;
 /// @endcond
-#ifndef MFEM_CUDA_1897_WORKAROUND
 size_t SparseMatrix::bufferSize = 0;
 void * SparseMatrix::dBuffer = nullptr;
-#endif
 #endif // MFEM_USE_CUDA_OR_HIP
 
 void SparseMatrix::InitGPUSparse()
@@ -472,67 +464,109 @@ void SparseMatrix::SortColumnIndices()
    }
 
 #ifdef MFEM_USE_CUDA_OR_HIP
-   if (Device::Allows(Backend::CUDA_MASK) || Device::Allows(Backend::HIP_MASK))
+   if ( Device::Allows( Backend::CUDA_MASK ))
    {
-      const int m = Height();
-      const int n = Width();
+#if defined(MFEM_USE_CUDA)
+      size_t pBufferSizeInBytes = 0;
+      void *pBuffer = NULL;
+
+      const int n = Height();
+      const int m = Width();
       const int nnzA = J.Capacity();
-      const int *d_ia = ReadI();
-      int *d_ja = ReadWriteJ();
+      real_t * d_a_sorted = ReadWriteData();
+      const int * d_ia = ReadI();
+      int * d_ja_sorted = ReadWriteJ();
+      csru2csrInfo_t sortInfoA;
 
-      // Get size of temporary buffer needed to sort the column indices,
-      // allocate the temporary buffer.
-      size_t pBufferSizeInBytes;
-      MFEM_cu_or_hip(sparseXcsrsort_bufferSizeExt)(handle, m, n, nnzA, d_ia,
-                                                   d_ja, &pBufferSizeInBytes);
-      void *pBuffer = MFEM_Cu_or_Hip(MemAlloc)(&pBuffer, pBufferSizeInBytes);
+      cusparseMatDescr_t matA_descr;
+      cusparseCreateMatDescr( &matA_descr );
+      cusparseSetMatIndexBase( matA_descr, CUSPARSE_INDEX_BASE_ZERO );
+      cusparseSetMatType( matA_descr, CUSPARSE_MATRIX_TYPE_GENERAL );
 
-      // Create matrix descriptor, will have default values
-      // CUSPARSE_INDEX_BASE_ZERO and CUSPARSE_MATRIX_TYPE_GENERAL.
-      MFEM_cu_or_hip(sparseMatDescr_t) matA_descr;
-      MFEM_cu_or_hip(sparseCreateMatDescr)(&matA_descr);
+      cusparseCreateCsru2csrInfo( &sortInfoA );
 
-      // Initialize permutation to identity
-      Array<int> P(nnzA);
-      int *d_P = P.Write();
-      mfem::forall(nnzA, [=] MFEM_HOST_DEVICE (int i) { d_P[i] = i; });
+#ifdef MFEM_USE_SINGLE
+      cusparseScsru2csr_bufferSizeExt( handle, n, m, nnzA, d_a_sorted, d_ia,
+                                       d_ja_sorted, sortInfoA,
+                                       &pBufferSizeInBytes);
+#elif defined MFEM_USE_DOUBLE
+      cusparseDcsru2csr_bufferSizeExt( handle, n, m, nnzA, d_a_sorted, d_ia,
+                                       d_ja_sorted, sortInfoA,
+                                       &pBufferSizeInBytes);
+#else
+      MFEM_ABORT("Floating point type undefined");
+#endif
 
-      // Sort the column indices. The array d_ja will now be sorted. The
-      // permutation required to sort the values will be returned in d_P.
-      MFEM_cu_or_hip(sparseXcsrsort)(handle, m, n, nnzA, matA_descr, d_ia, d_ja,
-                                     d_P, pBuffer);
+      CuMemAlloc( &pBuffer, pBufferSizeInBytes );
 
-      // Create a copy of the unsorted matrix values.
-      real_t *d_a = ReadWriteData();
-      void *d_a_unsorted = MFEM_Cu_or_Hip(MemAlloc)(&d_a_unsorted,
-                                                    nnzA * sizeof(real_t));
-      MFEM_Cu_or_Hip(MemcpyDtoD)(d_a_unsorted, d_a, nnzA * sizeof(real_t));
+#ifdef MFEM_USE_SINGLE
+      cusparseScsru2csr( handle, n, m, nnzA, matA_descr, d_a_sorted, d_ia,
+                         d_ja_sorted, sortInfoA, pBuffer);
+#elif defined MFEM_USE_DOUBLE
+      cusparseDcsru2csr( handle, n, m, nnzA, matA_descr, d_a_sorted, d_ia,
+                         d_ja_sorted, sortInfoA, pBuffer);
+#else
+      MFEM_ABORT("Floating point type undefined");
+#endif
 
-      // Create the (input) dense vector with the unsorted values.
-      MFEM_cu_or_hip(sparseDnVecDescr_t) d_a_dense;
-      MFEM_cu_or_hip(sparseCreateDnVec)(&d_a_dense, nnzA, d_a_unsorted,
-                                        MFEM_CUDA_or_HIP_REAL_T);
-
-      // Create the (output) sparse vector that will have the sorted values.
-      MFEM_cu_or_hip(sparseSpVecDescr_t) d_a_sparse;
-      MFEM_cu_or_hip(sparseCreateSpVec)(&d_a_sparse, nnzA, nnzA, d_P, d_a,
-                                        MFEM_CU_or_HIP(SPARSE_INDEX_32I),
-                                        MFEM_CU_or_HIP(SPARSE_INDEX_BASE_ZERO),
-                                        MFEM_CUDA_or_HIP_REAL_T);
-
-      // Sort the matrix values using the permutation vector.
-      MFEM_cu_or_hip(sparseGather)(handle, d_a_dense, d_a_sparse);
-
-      // The above calls may be asynchronous, so we need to wait for them to
-      // finish before we can free memory.
+      // The above call is (at least in some cases) asynchronous, so we need to
+      // wait for it to finish before we can free device temporaries.
       MFEM_STREAM_SYNC;
 
-      MFEM_cu_or_hip(sparseDestroyDnVec)(d_a_dense);
-      MFEM_cu_or_hip(sparseDestroySpVec)(d_a_sparse);
-      MFEM_cu_or_hip(sparseDestroyMatDescr)(matA_descr);
+      cusparseDestroyCsru2csrInfo( sortInfoA );
+      cusparseDestroyMatDescr( matA_descr );
 
-      MFEM_Cu_or_Hip(MemFree)(d_a_unsorted);
-      MFEM_Cu_or_Hip(MemFree)(pBuffer);
+      CuMemFree( pBuffer );
+#endif
+   }
+   else if ( Device::Allows( Backend::HIP_MASK ))
+   {
+#if defined(MFEM_USE_HIP)
+      size_t pBufferSizeInBytes = 0;
+      void *pBuffer = NULL;
+      int *P = NULL;
+
+      const int n = Height();
+      const int m = Width();
+      const int nnzA = J.Capacity();
+      real_t * d_a_sorted = ReadWriteData();
+      const int * d_ia = ReadI();
+      int * d_ja_sorted = ReadWriteJ();
+
+      hipsparseMatDescr_t descrA;
+      hipsparseCreateMatDescr( &descrA );
+      // FIXME: There is not in-place version of csr sort in hipSPARSE currently, so we make
+      //        a temporary copy of the data for gthr, sort that, and then copy the sorted values
+      //        back to the array being returned. Where there is an in-place version available,
+      //        we should use it.
+      Array< real_t > a_tmp( nnzA );
+      real_t *d_a_tmp = a_tmp.Write();
+
+      hipsparseXcsrsort_bufferSizeExt(handle, n, m, nnzA, d_ia, d_ja_sorted,
+                                      &pBufferSizeInBytes);
+
+      HipMemAlloc( &pBuffer, pBufferSizeInBytes );
+      HipMemAlloc( (void**)&P, nnzA * sizeof(int) );
+
+      hipsparseCreateIdentityPermutation(handle, nnzA, P);
+      hipsparseXcsrsort(handle, n, m, nnzA, descrA, d_ia, d_ja_sorted, P, pBuffer);
+
+#if defined(MFEM_USE_SINGLE)
+      hipsparseSgthr(handle, nnzA, d_a_sorted, d_a_tmp, P,
+                     HIPSPARSE_INDEX_BASE_ZERO);
+#elif defined(MFEM_USE_DOUBLE)
+      hipsparseDgthr(handle, nnzA, d_a_sorted, d_a_tmp, P,
+                     HIPSPARSE_INDEX_BASE_ZERO);
+#else
+      MFEM_ABORT("Unsupported floating point type!");
+#endif
+
+      A.CopyFrom( a_tmp.GetMemory(), nnzA );
+      hipsparseDestroyMatDescr( descrA );
+
+      HipMemFree( pBuffer );
+      HipMemFree( P );
+#endif
    }
    else
 #endif // MFEM_USE_CUDA_OR_HIP
@@ -787,15 +821,27 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const real_t a) const
             MFEM_CU_or_HIP(SPARSE_INDEX_32I),
             MFEM_CU_or_HIP(SPARSE_INDEX_32I),
             MFEM_CU_or_HIP(SPARSE_INDEX_BASE_ZERO),
-            MFEM_CUDA_or_HIP_REAL_T);
+#ifdef MFEM_USE_SINGLE
+            MFEM_CUDA_or_HIP(_R_32F));
+#else
+            MFEM_CUDA_or_HIP(_R_64F));
+#endif
 
          // Create handles for input/output vectors
          MFEM_cu_or_hip(sparseCreateDnVec)(&vecX_descr,
                                            x.Size(),
                                            const_cast<real_t *>(d_x),
-                                           MFEM_CUDA_or_HIP_REAL_T);
+#ifdef MFEM_USE_SINGLE
+                                           MFEM_CUDA_or_HIP(_R_32F));
+#else
+                                           MFEM_CUDA_or_HIP(_R_64F));
+#endif
          MFEM_cu_or_hip(sparseCreateDnVec)(&vecY_descr, y.Size(), d_y,
-                                           MFEM_CUDA_or_HIP_REAL_T);
+#ifdef MFEM_USE_SINGLE
+                                           MFEM_CUDA_or_HIP(_R_32F));
+#else
+                                           MFEM_CUDA_or_HIP(_R_64F));
+#endif
 #else
          cusparseCreateMatDescr(&matA_descr);
          cusparseSetMatIndexBase(matA_descr, CUSPARSE_INDEX_BASE_ZERO);
@@ -814,7 +860,11 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const real_t a) const
          vecX_descr,
          &beta,
          vecY_descr,
-         MFEM_CUDA_or_HIP_REAL_T,
+#ifdef MFEM_USE_SINGLE
+         MFEM_CUDA_or_HIP(_R_32F),
+#else
+         MFEM_CUDA_or_HIP(_R_64F),
+#endif
          MFEM_GPUSPARSE_ALG,
          &newBufferSize);
 
@@ -841,7 +891,11 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const real_t a) const
          vecX_descr,
          &beta,
          vecY_descr,
-         MFEM_CUDA_or_HIP_REAL_T,
+#ifdef MFEM_USE_SINGLE
+         MFEM_CUDA_or_HIP(_R_32F),
+#else
+         MFEM_CUDA_or_HIP(_R_64F),
+#endif
          MFEM_GPUSPARSE_ALG,
          dBuffer);
 #else
@@ -4318,14 +4372,6 @@ SparseMatrix::~SparseMatrix()
 #ifdef MFEM_USE_CUDA_OR_HIP
    if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
-#ifdef MFEM_CUDA_1897_WORKAROUND
-      if (dBuffer)
-      {
-         MFEM_Cu_or_Hip(MemFree)(dBuffer);
-         dBuffer = nullptr;
-         bufferSize = 0;
-      }
-#endif
       if (SparseMatrixCount==1)
       {
          if (handle)
@@ -4333,14 +4379,12 @@ SparseMatrix::~SparseMatrix()
             MFEM_cu_or_hip(sparseDestroy)(handle);
             handle = nullptr;
          }
-#ifndef MFEM_CUDA_1897_WORKAROUND
          if (dBuffer)
          {
             MFEM_Cu_or_Hip(MemFree)(dBuffer);
             dBuffer = nullptr;
             bufferSize = 0;
          }
-#endif
       }
       SparseMatrixCount--;
    }

linalg/sparsemat.hpp

@@ -98,17 +98,9 @@ protected:
 #ifdef MFEM_USE_CUDA_OR_HIP
    // common for hipSPARSE and cuSPARSE
    static int SparseMatrixCount;
-   mutable bool initBuffers = false;
-
-#if defined(MFEM_USE_CUDA) && CUDA_VERSION >= 12300 && CUDA_VERSION < 12602
-   // Workaround for bug CUSPARSE-1897
-#define MFEM_CUDA_1897_WORKAROUND
-   mutable size_t bufferSize = 0;
-   mutable void *dBuffer = nullptr;
-#else
    static size_t bufferSize;
    static void *dBuffer;
-#endif
+   mutable bool initBuffers = false;
 
 #if defined(MFEM_USE_CUDA)
    cusparseStatus_t status;

linalg/vector.cpp

@@ -92,6 +92,18 @@ struct LpReducer
    }
 };
 
+static Array<real_t>& vector_workspace()
+{
+   static Array<real_t> instance;
+   return instance;
+}
+
+static Array<DevicePair<real_t, real_t>> &Lpvector_workspace()
+{
+   static Array<DevicePair<real_t, real_t>> instance;
+   return instance;
+}
+
 Vector::Vector(const Vector &v)
 {
    const int s = v.Size();
@@ -979,7 +991,7 @@ real_t Vector::Norml2() const
          }
       }
    },
-   L2Reducer{}, UseDevice());
+   L2Reducer{}, UseDevice(), Lpvector_workspace());
    // final answer
    return res.second * sqrt(res.first);
 }
@@ -994,7 +1006,7 @@ real_t Vector::Normlinf() const
    {
       r = fmax(r, fabs(m_data[i]));
    },
-   MaxReducer<real_t> {}, UseDevice());
+   MaxReducer<real_t> {}, UseDevice(), vector_workspace());
    return res;
 }
 
@@ -1008,7 +1020,7 @@ real_t Vector::Norml1() const
    {
       r += fabs(m_data[i]);
    },
-   SumReducer<real_t> {}, UseDevice());
+   SumReducer<real_t> {}, UseDevice(), vector_workspace());
    return res;
 }
 
@@ -1051,7 +1063,7 @@ real_t Vector::Normlp(real_t p) const
             }
          }
       },
-      LpReducer{p}, UseDevice());
+      LpReducer{p}, UseDevice(), Lpvector_workspace());
       // final answer
       return res.second * pow(res.first, 1.0 / p);
    } // end if p < infinity()
@@ -1084,7 +1096,7 @@ real_t Vector::operator*(const Vector &v) const
       {
          r += m_data[i] * v_data[i];
       },
-      SumReducer<real_t> {}, use_dev);
+      SumReducer<real_t> {}, use_dev, vector_workspace());
       return res;
    };
 
@@ -1155,7 +1167,7 @@ real_t Vector::Min() const
       {
          r = fmin(r, m_data[i]);
       },
-      MinReducer<real_t> {}, use_dev);
+      MinReducer<real_t> {}, use_dev, vector_workspace());
       return res;
    };
 
@@ -1201,7 +1213,7 @@ real_t Vector::Max() const
       {
          r = fmax(r, m_data[i]);
       },
-      MaxReducer<real_t> {}, use_dev);
+      MaxReducer<real_t> {}, use_dev, vector_workspace());
       return res;
    };
 
@@ -1236,7 +1248,7 @@ real_t Vector::Sum() const
    {
       r += m_data[i];
    },
-   SumReducer<real_t> {}, UseDevice());
+   SumReducer<real_t> {}, UseDevice(), vector_workspace());
    return res;
 }
 

makefile

@@ -125,7 +125,8 @@ EXAMPLE_TEST_DIRS := examples
 
 MINIAPP_SUBDIRS = common electromagnetics meshing navier performance tools \
  toys nurbs gslib adjoint solvers shifted mtop parelag tribol autodiff dfem \
- hooke multidomain dpg hdiv-linear-solver spde diag-smoothers
+ hooke multidomain dpg hdiv-linear-solver spde diag-smoothers hdg
+
 MINIAPP_DIRS := $(addprefix miniapps/,$(MINIAPP_SUBDIRS))
 MINIAPP_TEST_DIRS := $(filter-out %/common,$(MINIAPP_DIRS))
 MINIAPP_USE_COMMON := $(addprefix miniapps/,electromagnetics meshing tools \

mesh/mesh.cpp

@@ -1616,7 +1616,7 @@ Element::Type Mesh::GetFaceElementType(int Face) const
 
 Array<int> Mesh::GetFaceToBdrElMap() const
 {
-   Array<int> face_to_be(Dim == 2 ? NumOfEdges : NumOfFaces);
+   Array<int> face_to_be(GetNumFaces());
    face_to_be = -1;
    for (int i = 0; i < NumOfBdrElements; i++)
    {

mesh/mesh.hpp

@@ -1604,6 +1604,10 @@ public:
    void GetElementVertices(int i, Array<int> &v) const
    { elements[i]->GetVertices(v); }
 
+   /// HDG:sets the indices of the vertices of element i.
+   void SetElementVertices(int i, Array<int> &v) const
+   { elements[i]->SetVertices(v); }
+
    /// Returns the indices of the vertices of boundary element i.
    void GetBdrElementVertices(int i, Array<int> &v) const
    { boundary[i]->GetVertices(v); }
@@ -1936,7 +1940,6 @@ public:
                                   IsoparametricTransformation &ElTr2) const;
 
    /// @}
-
    /// @anchor mfem_Mesh_geom_factors
    /// @name Access the coordinate transformation at quadrature points
    ///

mesh/mesh_readers.cpp

@@ -773,7 +773,7 @@ struct BufferReader : BufferReaderBase
          int header_entry_size = HeaderEntrySize();
          int nblocks = ReadHeaderEntry(header_buf);
          header_buf += header_entry_size;
-         std::vector<size_t> header(nblocks + 2);
+         std::vector<int> header(nblocks + 2);
          for (int i=0; i<nblocks+2; ++i)
          {
             header[i] = ReadHeaderEntry(header_buf);
@@ -792,7 +792,7 @@ struct BufferReader : BufferReaderBase
             dest_ptr += dest_len;
             source_ptr += source_len;
          }
-         MFEM_VERIFY(size_t(sizeof(F)*n) == (dest_ptr - dest_start),
+         MFEM_VERIFY(int(sizeof(F)*n) == (dest_ptr - dest_start),
                      "AppendedData: wrong data size");
          buf = uncompressed_data.data();
 #else

miniapps/electromagnetics/CMakeLists.txt

@@ -34,11 +34,6 @@ if (MFEM_USE_MPI)
    EXTRA_HEADERS maxwell_solver.hpp ${MFEM_MINIAPPS_COMMON_HEADERS}
    LIBRARIES mfem-common)
 
- add_mfem_miniapp(lorentz
-   MAIN lorentz.cpp
-   EXTRA_HEADERS ${MFEM_MINIAPPS_COMMON_HEADERS}
-   LIBRARIES mfem-common)
-
   # Add the corresponding tests to the "test" target
   if (MFEM_ENABLE_TESTING)
     add_test(NAME tesla_np=4

miniapps/electromagnetics/lorentz.cpp

@@ -1,571 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-//
-//           -----------------------------------------------------
-//           Lorentz Miniapp:  Simple Lorentz Force Particle Mover
-//           -----------------------------------------------------
-//
-// This miniapp computes the trajectory of a single charged particle subject to
-// Lorentz forces.
-//
-//                           dp/dt = q (E + v x B)
-//
-// The method used is the explicit Boris algortihm which conserves phase space
-// volume for long term accuracy.
-//
-// The electric and magnetic fields are read from VisItDataCollection objects
-// such as those produced by the Volta and Tesla miniapps. It is notable that
-// these two fields do not need to be defined on the same mesh. Of course, the
-// particle trajectory can only be computed on the intersection of the two
-// domains. The starting point of the path must be chosen within in this
-// intersection and the trajectory will terminate when it leaves the
-// intersection or reaches a specified time duration.
-//
-// Note that the VisItDataCollection objects must have been stored using the
-// parallel format e.g. visit_dc.SetFormat(DataCollection::PARALLEL_FORMAT);.
-// Without this optional format specifier the vector field lookups will fail.
-//
-// Compile with: make lorentz
-//
-// Sample runs:
-//
-//   Free particle moving with constant velocity
-//      mpirun -np 4 lorentz -p0 '1 1 1'
-//
-//   Particle accelerating in a constant electric field
-//      mpirun -np 4 volta -m ../../data/inline-hex.mesh -dbcs '1 6' -dbcv '0 1'
-//      mpirun -np 4 lorentz -er Volta-AMR-Parallel -x0 '0.5 0.5 0.9' -p0 '1 0 0'
-//
-//   Particle accelerating in a constant magnetic field
-//      mpirun -np 4 tesla -m ../../data/inline-hex.mesh -ubbc '0 0 1'
-//      mpirun -np 4 lorentz -br Tesla-AMR-Parallel -x0 '0.1 0.5 0.1' -p0 '0 0.4 0.1' -tf 9
-//
-//   Magnetic mirror effect near a charged sphere and a bar magnet
-//      mpirun -np 4 volta -m ../../data/ball-nurbs.mesh -dbcs 1 -cs '0 0 0 0.1 2e-11' -rs 2 -maxit 4
-//      mpirun -np 4 tesla -m ../../data/fichera.mesh -maxit 4 -rs 3 -bm '-0.1 -0.1 -0.1 0.1 0.1 0.1 0.1 -1e10'
-//      mpirun -np 4 lorentz -er Volta-AMR-Parallel -ec 4 -br Tesla-AMR-Parallel -bc 4 -x0 '0.8 0 0' -p0 '-8 -4 4' -q -10 -tf 0.2 -dt 1e-3 -rf 1e-6
-//
-// This miniapp demonstrates the use of the ParMesh::FindPoints functionality
-// to evaluate field data from stored DataCollection objects.  While this
-// miniapp is far from a full particle-in-cell (PIC) code it does demonstrate
-// some of the building blocks that might be used to construct the particle
-// mover portion of a PIC code.
-
-#include "mfem.hpp"
-#include "../common/fem_extras.hpp"
-#include "../common/pfem_extras.hpp"
-#include "electromagnetics.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-using namespace mfem::common;
-using namespace mfem::electromagnetics;
-
-typedef DataCollection::FieldMapType fields_t;
-
-/// This class implements the Boris algorithm as described in the
-/// article `Why is Boris algorithm so good?` by H. Qin et al in
-/// Physics of Plasmas, Volume 20 Issue 8, August 2013,
-/// https://doi.org/10.1063/1.4818428.
-class BorisAlgorithm
-{
-private:
-   real_t charge_;
-   real_t mass_;
-
-   ParMesh         *E_pmesh_;
-   ParGridFunction *E_field_;
-
-   ParMesh         *B_pmesh_;
-   ParGridFunction *B_field_;
-
-   mutable Array<int>              elem_id_;
-   mutable Array<IntegrationPoint> ip_;
-
-   mutable Vector E_;
-   mutable Vector B_;
-   mutable Vector pxB_;
-   mutable Vector pm_;
-   mutable Vector pp_;
-
-   // Returns true if a usable V has been found. If @a pgf is NULL, V = 0 is
-   // returned as a default value.
-   bool GetValue(ParMesh *pmesh, ParGridFunction *pgf, Vector q, Vector &V)
-   {
-      DenseMatrix point(q.GetData(), 3, 1);
-
-      int pt_found =
-         (pmesh != NULL) ? pmesh->FindPoints(point, elem_id_, ip_, false) : -1;
-
-      // We have a mesh but the point was not found. The path must be outside
-      // the domain of interest.
-      if (pmesh != NULL && pt_found <= 0) { return false; }
-
-      int pt_root = -1;
-
-      if (pt_found > 0 && elem_id_[0] >= 0 && pgf != NULL)
-      {
-         pt_root = pmesh->GetMyRank();
-
-         pgf->GetVectorValue(elem_id_[0], ip_[0], V);
-      }
-      else
-      {
-         pt_root = 0;
-         V = 0.0;
-      }
-
-      // Determine processor which found the field point
-      int glb_pt_root = -1;
-      MPI_Allreduce(&pt_root, &glb_pt_root, 1,
-                    MPI_INT, MPI_MAX, MPI_COMM_WORLD);
-
-      // Send the field value to the root processor
-      if (pmesh != NULL && elem_id_[0] >= 0 && glb_pt_root != 0)
-      {
-         MPI_Send(V.GetData(), 3, MPITypeMap<real_t>::mpi_type,
-                  0, 1030, MPI_COMM_WORLD);
-      }
-
-      // Receive the field value on the root processor
-      if (Mpi::Root() && pmesh != NULL && glb_pt_root != 0)
-      {
-         MPI_Status status;
-         MPI_Recv(V.GetData(), 3, MPITypeMap<real_t>::mpi_type,
-                  glb_pt_root, 1030, MPI_COMM_WORLD, &status);
-      }
-      return true;
-   }
-
-public:
-   BorisAlgorithm(ParGridFunction *E_gf,
-                  ParGridFunction *B_gf,
-                  real_t charge, real_t mass)
-      : charge_(charge), mass_(mass),
-        E_field_(E_gf),
-        B_field_(B_gf),
-        E_(3), B_(3), pxB_(3), pm_(3), pp_(3)
-   {
-      E_pmesh_ = (E_field_) ? E_field_->ParFESpace()->GetParMesh() : NULL;
-      B_pmesh_ = (B_field_) ? B_field_->ParFESpace()->GetParMesh() : NULL;
-   }
-
-   bool Step(Vector &q, Vector &p, real_t &t, real_t &dt)
-   {
-      // Locate current point in each mesh, evaluate the fields, and collect
-      // field values on the root processor.
-      if (!GetValue(E_pmesh_, E_field_, q, E_)) { return false; }
-      if (!GetValue(B_pmesh_, B_field_, q, B_)) { return false; }
-
-      // Compute updated position and momentum using the Boris algorithm
-      if (Mpi::Root())
-      {
-         // Compute half of the contribution from q E
-         add(p, 0.5 * dt * charge_, E_, pm_);
-
-         // Compute the contributiobn from q p x B
-         const real_t B2 = B_ * B_;
-
-         // ... along pm x B
-         const real_t a1 = 4.0 * dt * charge_ * mass_;
-         pm_.cross3D(B_, pxB_);
-         pp_.Set(a1, pxB_);
-
-         // ... along pm
-         const real_t a2 = 4.0 * mass_ * mass_ -
-                           dt * dt * charge_ * charge_ * B2;
-         pp_.Add(a2, pm_);
-
-         // ... along B
-         const real_t a3 = 2.0 * dt * dt * charge_ * charge_ * (B_ * p);
-         pp_.Add(a3, B_);
-
-         // scale by common denominator
-         const real_t a4 = 4.0 * mass_ * mass_ +
-                           dt * dt * charge_ * charge_ * B2;
-         pp_ /= a4;
-
-         // Update the momentum
-         add(pp_, 0.5 * dt * charge_, E_, p);
-
-         // Update the position
-         q.Add(dt / mass_, p);
-      }
-
-      // Update the time
-      t += dt;
-
-      // Broadcast the updated position
-      MPI_Bcast(q.GetData(), 3, MPITypeMap<real_t>::mpi_type,
-                0, MPI_COMM_WORLD);
-
-      // Broadcast the updated momentum
-      MPI_Bcast(p.GetData(), 3, MPITypeMap<real_t>::mpi_type,
-                0, MPI_COMM_WORLD);
-
-      return true;
-   }
-};
-
-// Open the named VisItDataCollection and read the named field.
-// Returns pointers to the two new objects.
-int ReadGridFunction(const char * coll_name, const char * field_name,
-                     int pad_digits_cycle, int pad_digits_rank, int cycle,
-                     VisItDataCollection *&dc, ParGridFunction *& gf);
-
-// By default the initial position will be the center of the intersection
-// of the bounding boxes of the meshes containing the E and B fields.
-void SetInitialPosition(VisItDataCollection *E_dc,
-                        VisItDataCollection *B_dc,
-                        Vector &x_init);
-
-// Build a quadrilateral mesh approximating the trajectory as a
-// ribbon. One edge of the ribbon follows the trajectory of the
-// particle. The opposite edge is offset by the acceleration vector
-// (scaled by a constant called the r_factor).
-Mesh MakeTrajectoryMesh(int step, real_t m, real_t dt, real_t r_factor,
-                        const DenseMatrix &pos_data,
-                        const DenseMatrix &mom_data);
-
-// Prints the program's logo to the given output stream
-void display_banner(ostream & os);
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init(argc, argv);
-   Hypre::Init();
-
-   if ( Mpi::Root() ) { display_banner(cout); }
-
-   const char *E_coll_name = "";
-   const char *E_field_name = "E";
-   int E_cycle = 10;
-   int E_pad_digits_cycle = 6;
-   int E_pad_digits_rank = 6;
-
-   const char *B_coll_name = "";
-   const char *B_field_name = "B";
-   int B_cycle = 10;
-   int B_pad_digits_cycle = 6;
-   int B_pad_digits_rank = 6;
-
-   real_t q = 1.0;
-   real_t m = 1.0;
-   real_t dt = 1e-2;
-   real_t t_init = 0.0;
-   real_t t_final = 1.0;
-   real_t r_factor = -1.0;
-   Vector x_init;
-   Vector p_init;
-   int visport = 19916;
-   bool visualization = true;
-   bool visit = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&E_coll_name, "-er", "--e-root-file",
-                  "Set the VisIt data collection E field root file prefix.");
-   args.AddOption(&E_field_name, "-ef", "--e-field-name",
-                  "Set the VisIt data collection E field name");
-   args.AddOption(&E_cycle, "-ec", "--e-cycle",
-                  "Set the E field cycle index to read.");
-   args.AddOption(&E_pad_digits_cycle, "-epdc", "--e-pad-digits-cycle",
-                  "Number of digits in E field cycle.");
-   args.AddOption(&E_pad_digits_rank, "-epdr", "--e-pad-digits-rank",
-                  "Number of digits in E field MPI rank.");
-   args.AddOption(&B_coll_name, "-br", "--b-root-file",
-                  "Set the VisIt data collection B field root file prefix.");
-   args.AddOption(&B_field_name, "-bf", "--b-field-name",
-                  "Set the VisIt data collection B field name");
-   args.AddOption(&B_cycle, "-bc", "--b-cycle",
-                  "Set the B field cycle index to read.");
-   args.AddOption(&B_pad_digits_cycle, "-bpdc", "--b-pad-digits-cycle",
-                  "Number of digits in B field cycle.");
-   args.AddOption(&B_pad_digits_rank, "-bpdr", "--b-pad-digits-rank",
-                  "Number of digits in B field MPI rank.");
-   args.AddOption(&q, "-q", "--charge",
-                  "Particle charge.");
-   args.AddOption(&m, "-m", "--mass",
-                  "Particle mass.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time Step.");
-   args.AddOption(&t_init, "-ti", "--initial-time",
-                  "Initial Time.");
-   args.AddOption(&t_final, "-tf", "--final-time",
-                  "Final Time.");
-   args.AddOption(&x_init, "-x0", "--initial-position",
-                  "Initial position.");
-   args.AddOption(&p_init, "-p0", "--initial-momentum",
-                  "Initial momentum.");
-   args.AddOption(&r_factor, "-rf", "--ribbon-factor",
-                  "Scale factor for ribbon width (rf * (p1-p0) / (m * dt) "
-                  "where p0 and p1 are computed momenta).");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit", "-no-visit", "--no-visit",
-                  "Enable or disable VisIt visualization.");
-   args.AddOption(&visport, "-p", "--send-port", "Socket for GLVis.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (Mpi::Root())
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (r_factor <= 0.0)
-   {
-      r_factor = dt;
-   }
-   if (Mpi::Root())
-   {
-      args.PrintOptions(cout);
-   }
-
-   VisItDataCollection *E_dc = NULL;
-   ParGridFunction     *E_gf = NULL;
-
-   if (strcmp(E_coll_name, ""))
-   {
-      if (ReadGridFunction(E_coll_name, E_field_name, E_pad_digits_cycle,
-                           E_pad_digits_rank, E_cycle, E_dc, E_gf))
-      {
-         mfem::out << "Error loading E field" << endl;
-         return 1;
-      }
-   }
-
-   VisItDataCollection *B_dc = NULL;
-   ParGridFunction     *B_gf = NULL;
-
-   if (strcmp(B_coll_name, ""))
-   {
-      if (ReadGridFunction(B_coll_name, B_field_name, B_pad_digits_cycle,
-                           B_pad_digits_rank, B_cycle, B_dc, B_gf))
-      {
-         mfem::out << "Error loading B field" << endl;
-         return 1;
-      }
-   }
-
-   if (x_init.Size() < 3)
-   {
-      SetInitialPosition(E_dc, B_dc, x_init);
-   }
-   if (p_init.Size() < 3)
-   {
-      p_init.SetSize(3); p_init = 0.0;
-   }
-   if (Mpi::Root())
-   {
-      mfem::out << "Initial position: "; x_init.Print(mfem::out);
-      mfem::out << "Initial momentum: "; p_init.Print(mfem::out);
-   }
-
-   BorisAlgorithm boris(E_gf, B_gf, q, m);
-   Vector pos(x_init);
-   Vector mom(p_init);
-
-   ofstream ofs("Lorentz.dat");
-   ofs.precision(14);
-
-   int nsteps = 1 + (int)ceil((t_final - t_init) / dt);
-   DenseMatrix pos_data(3, nsteps);
-   DenseMatrix mom_data(3, nsteps + 1);
-   mom_data.SetCol(0, p_init);
-
-   if (Mpi::Root())
-   {
-      mfem::out << "Maximum number of steps: " << nsteps << endl;
-   }
-
-   int step = -1;
-   real_t t = t_init;
-   do
-   {
-      if (Mpi::Root())
-      {
-         ofs << t
-             << '\t' << pos[0] << '\t' << pos[1] << '\t' << pos[2]
-             << '\t' << mom[0] << '\t' << mom[1] << '\t' << mom[2]
-             << '\n';
-      }
-      step++;
-
-      pos_data.SetCol(step, pos);
-      mom_data.SetCol(step + 1, mom);
-   }
-   while (boris.Step(pos, mom, t, dt) && step < nsteps - 1);
-
-   if (Mpi::Root() && (visit || visualization))
-   {
-      Mesh trajectory = MakeTrajectoryMesh(step, m, dt, r_factor,
-                                           pos_data, mom_data);
-
-      L2_FECollection    fec_l2(0, 2);
-      FiniteElementSpace fes_l2(&trajectory, &fec_l2);
-      GridFunction traj_time(&fes_l2);
-      for (int i=0; i<step; i++)
-      {
-         traj_time[i] = dt * i;
-      }
-
-      if (visit)
-      {
-         VisItDataCollection visit_dc("Lorentz", &trajectory);
-         visit_dc.RegisterField("Time", &traj_time);
-         visit_dc.SetCycle(step);
-         visit_dc.SetTime(step * dt);
-         visit_dc.Save();
-      }
-
-      if (visualization)
-      {
-         socketstream traj_sock;
-         traj_sock.precision(8);
-
-         char vishost[] = "localhost";
-
-         int Wx = 0, Wy = 0; // window position
-         int Ww = 350, Wh = 350; // window size
-
-         VisualizeField(traj_sock, vishost, visport,
-                        traj_time, "Trajectory", Wx, Wy, Ww, Wh);
-      }
-   }
-   if (Mpi::Root())
-   {
-      mfem::out << "Number of steps taken: " << step << endl;
-   }
-
-   // Clean up
-   delete E_dc;
-   delete B_dc;
-}
-
-// Print the Lorentz ascii logo to the given ostream
-void display_banner(ostream & os)
-{
-   os << "   ____                                __          "
-      << endl
-      << "  |    |    ___________   ____   _____/  |_________"
-      << endl
-      << "  |    |   /  _ \\_  __ \\_/ __ \\ /    \\   __\\___   /"
-      << endl
-      << "  |    |__(  <_> )  | \\/\\  ___/|   |  \\  |  /    / "
-      << endl
-      << "  |_______ \\____/|__|    \\___  >___|  /__| /_____ \\"
-      << endl
-      << "          \\/                 \\/     \\/           \\/"
-      << endl << flush;
-}
-
-int ReadGridFunction(const char * coll_name, const char * field_name,
-                     int pad_digits_cycle, int pad_digits_rank, int cycle,
-                     VisItDataCollection *&dc, ParGridFunction *& gf)
-{
-   dc = new VisItDataCollection(MPI_COMM_WORLD, coll_name);
-   dc->SetPadDigitsCycle(pad_digits_cycle);
-   dc->SetPadDigitsRank(pad_digits_rank);
-   dc->Load(cycle);
-
-   if (dc->Error() != DataCollection::No_Error)
-   {
-      mfem::out << "Error loading VisIt data collection: "
-                << coll_name << endl;
-      return 1;
-   }
-
-   if (dc->GetMesh()->Dimension() < 3)
-   {
-      mfem::out << "Field must be defined on a three dimensional mesh"
-                << endl;
-      return 1;
-   }
-
-   if (dc->HasField(field_name))
-   {
-      gf = dc->GetParField(field_name);
-   }
-
-   return 0;
-}
-
-void SetInitialPosition(VisItDataCollection *E_dc,
-                        VisItDataCollection *B_dc,
-                        Vector &x_init)
-{
-   x_init.SetSize(3); x_init = 0.0;
-
-   if (E_dc != NULL || B_dc != NULL)
-   {
-      Vector E_p_min(3); E_p_min = -infinity();
-      Vector E_p_max(3); E_p_max = infinity();
-      if (E_dc != NULL)
-      {
-         ParMesh * E_pmesh = dynamic_cast<ParMesh*>(E_dc->GetMesh());
-         E_pmesh->GetBoundingBox(E_p_min, E_p_max);
-      }
-
-      Vector B_p_min(3); B_p_min = -infinity();
-      Vector B_p_max(3); B_p_max = infinity();
-      if (B_dc != NULL)
-      {
-         ParMesh *B_pmesh = dynamic_cast<ParMesh*>(B_dc->GetMesh());
-         B_pmesh->GetBoundingBox(B_p_min, B_p_max);
-      }
-
-      for (int d = 0; d<3; d++)
-      {
-         const real_t p_min = std::max(E_p_min[d], B_p_min[d]);
-         const real_t p_max = std::min(E_p_max[d], B_p_max[d]);
-         x_init[d] = 0.5 * (p_min + p_max);
-      }
-   }
-}
-
-Mesh MakeTrajectoryMesh(int step, real_t m, real_t dt, real_t r_factor,
-                        const DenseMatrix &pos_data,
-                        const DenseMatrix &mom_data)
-{
-   Mesh trajectory(2, 2 * (step + 1), step, 0, 3);
-
-   for (int i=0; i<=step; i++)
-   {
-      trajectory.AddVertex(pos_data(0,i), pos_data(1,i), pos_data(2,i));
-
-      real_t dpx = (mom_data(0, i + 1) - mom_data(0, i)) / (m * dt);
-      real_t dpy = (mom_data(1, i + 1) - mom_data(1, i)) / (m * dt);
-      real_t dpz = (mom_data(2, i + 1) - mom_data(2, i)) / (m * dt);
-
-      trajectory.AddVertex(pos_data(0,i) + r_factor * dpx,
-                           pos_data(1,i) + r_factor * dpy,
-                           pos_data(2,i) + r_factor * dpz);
-   }
-
-   int v[4];
-   for (int i=0; i<step; i++)
-   {
-      v[0] = 2 * i;
-      v[1] = 2 * (i + 1);
-      v[2] = 2 * (i + 1) + 1;
-      v[3] = 2 * i + 1;
-
-      trajectory.AddQuad(v);
-   }
-
-   trajectory.FinalizeQuadMesh(1);
-
-   return trajectory;
-}

miniapps/electromagnetics/makefile

@@ -21,7 +21,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_MINIAPPS =
-PAR_MINIAPPS = volta tesla maxwell joule lorentz
+PAR_MINIAPPS = volta tesla maxwell joule
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
 else
@@ -51,10 +51,6 @@ all: $(MINIAPPS)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $@.o $@_solver.o $(COMMON_LIB) \
 	  $(MFEM_LIBS)
 
-lorentz: %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) | lib-common
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<)
-	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $@.o $(COMMON_LIB) $(MFEM_LIBS)
-
 # Rules for compiling miniapp dependencies
 $(addsuffix _solver.o,$(MINIAPPS)): \
 %.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
@@ -85,7 +81,7 @@ include $(MFEM_TEST_MK)
 
 # Testing: Specific execution options
 RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-volta-test-par: volta-test-1 volta-test-2 volta-test-3
+volta-test-par: volta-test-1 volta-test-2
 volta-test-1: volta
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
 	-maxit 2 -dbcs 1 -dbcg -ds '0.0 0.0 0.0 0.2 8.0')
@@ -93,29 +89,15 @@ volta-test-2: volta
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
 	-maxit 2 -m ../../data/square-disc.mesh \
 	-dbcs '1 2 3 4 5 6 7 8' -dbcv '0 0 0 0 1 1 1 1')
-volta-test-3: volta
-	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-maxit 2 -m ../../data/inline-hex.mesh -dbcs '1 6' -dbcv '0 1')
-tesla-test-par: tesla-test-1 tesla-test-2
-tesla-test-1: tesla
+tesla-test-par: tesla
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
 	-maxit 2 -cr '0 0 -0.2 0 0 0.2 0.2 0.4 1')
-tesla-test-2: tesla
-	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-maxit 2 -m ../../data/inline-hex.mesh -ubbc '0 0 1')
 maxwell-test-par: maxwell
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
 	-abcs '-1' -dp '-0.3 0.0 0.0 0.3 0.0 0.0 0.1 1 .5 .5')
 joule-test-par: joule
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
 	-m cylinder-hex.mesh -p rod -tf 3)
-lorentz-test-par: lorentz-test-1 lorentz-test-2
-lorentz-test-1: lorentz volta-test-3
-	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-er Volta-AMR-Parallel -ec 2 -x0 '0.5 0.5 0.9' -p0 '1 0 0')
-lorentz-test-2: lorentz tesla-test-2
-	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-br Tesla-AMR-Parallel -bc 2 -x0 '0.1 0.5 0.1' -p0 '0 0.4 0.1' -tf 9)
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 
@@ -130,4 +112,4 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -rf Volta-AMR* Tesla-AMR* Maxwell-Parallel* Joule_* Lorentz*
+	@rm -rf Volta-AMR* Tesla-AMR* Maxwell-Parallel* Joule_*

miniapps/electromagnetics/tesla.cpp

@@ -253,7 +253,6 @@ int main(int argc, char *argv[])
 
    // Initialize VisIt visualization
    VisItDataCollection visit_dc("Tesla-AMR-Parallel", &pmesh);
-   visit_dc.SetFormat(DataCollection::PARALLEL_FORMAT);
 
    if ( visit )
    {

miniapps/electromagnetics/volta.cpp

@@ -266,7 +266,6 @@ int main(int argc, char *argv[])
 
    // Initialize VisIt visualization
    VisItDataCollection visit_dc("Volta-AMR-Parallel", &pmesh);
-   visit_dc.SetFormat(DataCollection::PARALLEL_FORMAT);
 
    if ( visit )
    {

miniapps/hdg/HDGBilinearForm.hpp

@@ -0,0 +1,330 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+// Implementation of class HDGBilinearForm
+//
+// Contributed by: T. Horvath: Oakland University
+//                 S. Rhebergen, A. Sivas: University of Waterloo
+
+#ifndef MFEM_HDGBILINEARFORM
+#define MFEM_HDGBILINEARFORM
+
+#include "../../config/config.hpp"
+#include "../../linalg/linalg.hpp"
+#include "../../fem/fespace.hpp"
+#include "../../fem/gridfunc.hpp"
+#include "../../fem/linearform.hpp"
+#include "../../fem/bilininteg.hpp"
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+using namespace std;
+using namespace mfem;
+
+class HDGBilinearForm
+{
+protected:
+   /// FE spaces on which the form lives.
+   Array<FiniteElementSpace*> volume_fes, skeletal_fes;
+
+   int NVolumeFES, NSkeletalFES;
+
+   bool parallel;
+
+   /// Sparse matrix to be assembled
+   Array<SparseMatrix*> mat;
+
+   /// Right hand side vector to be assembled.
+   Array<Vector*> rhs_SC;
+
+   /// Table that contains the faces for all elements
+   Table *el_to_face;
+
+   /// List that separates the interior edges from the shared edges
+   Array<int> ess_dofs, Edge_to_SharedEdge;
+
+   /// HDG Integrators
+   Array<BilinearFormIntegrator*> hdg_dbfi;
+   Array<BilinearFormIntegrator*> hdg_fbfi;
+
+   /// Dense matrices to be used for computing the integrals
+   DenseMatrix elemmat1, elemmat2, elemmat3, elemmat4;
+
+   /// Vectors to store A and B, the corresponding offsets and the number
+   /// of elements on which A and B will be stored
+   Array<int> A_offsets, B_offsets;
+   real_t *A_data, *B_data;
+   int elements_A, elements_B;
+
+
+   // may be used in the construction of derived classes
+   HDGBilinearForm()
+   {
+      for (int i =0; i<NVolumeFES; i++)
+      {
+         delete volume_fes[i];
+      }
+      for (int i =0; i<NSkeletalFES; i++)
+      {
+         delete skeletal_fes[i];
+         delete rhs_SC[i];
+      }
+      for (int i =0; i<NSkeletalFES*NSkeletalFES; i++)
+      {
+         delete mat[i];
+      }
+      NVolumeFES = 0;
+      NSkeletalFES = 0;
+      volume_fes = NULL;
+      skeletal_fes = NULL;
+      parallel = false;
+      el_to_face = NULL;
+      A_data = NULL;
+      B_data = NULL;
+      elements_A = elements_B = 0;
+   }
+
+public:
+   /// Creates bilinear form associated with FE spaces *_fes1 and _fes2.
+   HDGBilinearForm(Array<FiniteElementSpace*> &_fes1,
+                   Array<FiniteElementSpace*> &_fes2,
+                   bool _parallel = false);
+
+   // Advection-reaction test case without FES arrays
+   HDGBilinearForm(FiniteElementSpace *_fes1,
+                   FiniteElementSpace *_fes2,
+                   bool _parallel = false);
+
+   // Diffusion test case without FES arrays
+   HDGBilinearForm(FiniteElementSpace *_fes1,
+                   FiniteElementSpace *_fes2,
+                   FiniteElementSpace *_fes3,
+                   bool _parallel = false);
+
+   // Arrays of the HDG domain integrators
+   Array<BilinearFormIntegrator*> *GetHDG_DBFI()
+   {
+      return &hdg_dbfi;
+   }
+
+   // Arrays of the HDG face integrators
+   Array<BilinearFormIntegrator*> *GetHDG_FBFI()
+   {
+      return &hdg_fbfi;
+   }
+
+   /// Finalizes the matrix
+   virtual void Finalize(int skip_zeros = 1);
+
+   // Gets all dofs for a given element (goes over all volume FES)
+   void GetInteriorVDofs(int i, Array<int> &vdofs) const;
+
+   // Gets the values of volume GFs at all dofs for a given element (goes over all volume FES)
+   void GetInteriorSubVector(const Array<GridFunction*> &rhs_gridfunctions,
+                             int i, int ndof, Vector &SubVector) const;
+
+   // Gets the values of a skeletal GFs at all dofs for a given element (goes over all skeletal FES)
+   void GetFaceSubVector(const Array<GridFunction*> &face_gridfunctions,
+                         int i, int ndof, Vector &SubVector) const;
+
+   // Sets the values of volume GFs at all dofs for a given element (goes over all volume FES)
+   void SetInteriorSubVector(Array<GridFunction*> &sol_gridfunctions,
+                             int i, int ndof, Vector &SubVector);
+
+   // Gets all dofs for a given element (goes over all skeletal FES)
+   // includes an array that counts the number of dofs per FES
+   void GetFaceVDofs(int i, Array<int> &vdofs, Array<int> &dof_length) const;
+
+   // Gets all dofs for a given element (goes over all volume FES)
+   void GetFaceVDofs(int i, Array<int> &vdofs) const;
+
+   /// Returns the sparse Schur complement matrix
+   // For block systems: the matrices are counted row-wise
+   const SparseMatrix *SpMatSC(int m = 0) const
+   {
+      MFEM_VERIFY(mat[m], "mat is NULL and can't be dereferenced");
+      return mat[m];
+   }
+
+   /// Returns the
+   // For block systems: the matrices are counted row-wise
+   SparseMatrix *SpMatSC(int m = 0)
+   {
+      MFEM_VERIFY(mat[m], "mat is NULL and can't be dereferenced");
+      return mat[m];
+   }
+
+   /// Returns a constant reference to the right hand side vector
+   const Vector *VectorSC(int m = 0) const
+   {
+      return rhs_SC[m];
+   }
+
+   /// Returns a reference to the right hand side vector
+   Vector *VectorSC(int m = 0)
+   {
+      return rhs_SC[m];
+   }
+
+
+   /// Adds new HDG Integrators (domain terms).
+   void AddHDGDomainIntegrator(BilinearFormIntegrator *bfi);
+
+   /// Adds new HDG Integrators (facet terms).
+   void AddHDGFaceIntegrator(BilinearFormIntegrator *bfi);
+
+   /// Allocates the vectors for the part of A and B that will be stored
+   void Allocate(const Array<int> &bdr_attr_is_ess,
+                 const real_t memA = 0.0, const real_t memB = 0.0);
+
+   /// Assembles the Schur complement - general approach, with FES arrays.
+   // rhs_F is the volume right hand side gf array
+   // rhs_G is the skeleton right hand side gf array
+   // sol is the array of gfs containing the known boundary conditions
+   void AssembleSC(Array<GridFunction*> rhs_F,
+                   Array<GridFunction*> rhs_G,
+                   const Array<int> &bdr_attr_is_ess,
+                   Array<GridFunction*> sol,
+                   int skip_zeros = 1);
+
+   /// Assembles the Schur complement - for the hdg_advection.cpp and hdg_advectionp.cpp test cases
+   void AssembleSC(GridFunction *F,
+                   const real_t memA = 0.0, const real_t memB = 0.0,
+                   int skip_zeros = 1);
+
+   /// Assembles the Schur complement - for the hdg_poisson.cpp and hdg_poissonp.cpp test cases
+   void AssembleSC(GridFunction *F1,
+                   GridFunction *F2,
+                   Array<int> &bdr_attr_is_ess,
+                   GridFunction &sol,
+                   const real_t memA = 0.0, const real_t memB = 0.0,
+                   int skip_zeros = 1);
+
+   /** Eliminate the boundary condition
+    * Currently only used for the diffusion case, but written in a way that it can be used for mutiple skeleton veriables
+    * Eliminates the row and also the column (to keep the SPD matrix for diffusion)
+    */
+   void Eliminate_BC(const Array<int> &vdofs_e1, const Array<int> &vdofs_e1_length,
+                     const int ndof_u,
+                     Array<GridFunction*> sol, Vector *rhs_Volume, Vector *rhs_Skeleton,
+                     DenseMatrix *B_local, DenseMatrix *C_local, DenseMatrix *D_local);
+
+   /* To calculate the inverse of the local matrix A
+    * Can be optimized is some parts are easy to implement (such as the vector mass integrator A11 for the diffusion case)
+    */
+   DenseMatrix CalculateInverse(DenseMatrix A_local);
+
+   /// Computes domain based integrators
+   void compute_domain_integrals(const int elem, DenseMatrix *A_local);
+
+   /// Computes face based integrators
+   void compute_face_integrals(const int elem,
+                               const int edge,
+                               const int isshared,
+                               const bool reconstruct_only,
+                               DenseMatrix *A_local,
+                               DenseMatrix *B_local,
+                               DenseMatrix *C_local,
+                               DenseMatrix *D_local);
+
+   /// Reconstructs u from the facet unknowns - general approach, with FES arrays.
+   // Volume_GF is the volume right hand side gf array
+   // Skeleton_GF is the skeleton solution gf array
+   // u is the volume solution gf array
+   void Reconstruct(Array<GridFunction*> Volume_GF,
+                    Array<GridFunction*> Skeleton_GF,
+                    Array<GridFunction*> u);
+
+   /// Reconstructs u from the facet unknowns - for the hdg_advection.cpp and hdg_advectionp.cpp test cases
+   void Reconstruct(GridFunction *F,
+                    GridFunction *ubar,
+                    GridFunction *u);
+
+   /// Reconstructs u and q from the facet unknowns - for the hdg_poisson.cpp and hdg_poissonp.cpp test cases
+   void Reconstruct(GridFunction *R,
+                    GridFunction *F,
+                    GridFunction *ubar,
+                    GridFunction *q,
+                    GridFunction *u);
+
+   /**
+    Depending on 'assemble' it either assembles the Schur complement, or
+    reconstruct the volume unknowns from the facet solution.
+    For the asssebly part:
+     - Vol_GF is the array of the volume equations right hand side grid functions (might be all zero)
+     - Skel_GF is the array of the skeletal equations right hand side grid functions (might be all zero)
+     - bdr_sol_sol_GF is the projection of the exact solution to the facet unkowns (used only on the boundary for elimination)
+     - bdr_attr_is_ess is the array is essential bouddaries for all facet eqautions (might be empty)
+    For the reconstructions part:
+     - Vol_GF is the array of the volume equations right hand side grid functions (might be all zero)
+     - Skel_GF is the array of the skeletal solutions
+     - bdr_sol_sol_GF is the array of the volume unknowns (to be recontructed)
+
+    */
+   void AssembleReconstruct(Array<GridFunction*> Vol_GF,
+                            Array<GridFunction*> Skel_GF,
+                            const Array<int> &bdr_attr_is_ess,
+                            Array<GridFunction*> bdr_sol_sol_GF,
+                            bool assemble = true,
+                            const real_t memA = 0.0, const real_t memB = 0.0,
+                            int skip_zeros = 1);
+
+
+   /// Updates the spaces
+   virtual void Update(FiniteElementSpace *nfes1 = NULL,
+                       FiniteElementSpace *nfes2 = NULL);
+
+   // Add the vector v_add to the right hand side of the Shur complement system. Works even for block Shur complement systems.
+   void AddToRHS(Array<int> &skeletal_vdofs, Array<int> &skeletal_vdof_length,
+                 Vector v_add);
+
+   // Add the matrix dm_add to the Schur complement system. Works even for block Shur complement systems.
+   void AddToMat(Array<int> &skeletal_vdofs_edge_i,
+                 Array<int> &skeletal_vdof_length_edge_i,
+                 Array<int> &skeletal_vdofs_edge_j, Array<int> &skeletal_vdof_length_edge_j,
+                 DenseMatrix dm_add, int skip_zeros);
+
+   /// Destroys bilinear form.
+   virtual ~HDGBilinearForm();
+
+#ifdef MFEM_USE_MPI
+   // Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   // For block systems: the matrices are counted row-wise
+   HypreParMatrix *ParallelAssembleSC(int i = 0)
+   {
+      return ParallelAssembleSC(i,mat[i]);
+   }
+   // Return the matrix m assembled on the true dofs, i.e. P^t A P
+   HypreParMatrix *ParallelAssembleSC(int i, SparseMatrix *m);
+
+   // Return the right hand side vector
+   HypreParVector *ParallelVectorSC(int i = 0);
+
+   // Return the right hand side vector
+   void ParallelVectorSC(int i, Vector &tv);
+
+   // 2025 Sept begins
+   HypreParMatrix *ParallelAssemble(int i, SparseMatrix *m);
+
+   void ParallelAssemble(OperatorHandle &A, int i = 0) { ParallelAssemble(i, A, mat[i]); }
+
+   /** Returns the matrix @a A_local assembled on the true dofs, i.e.
+      @a A = P^t A_local P in the format (type id) specified by @a A. */
+   // needs input i to know which part of the Schur complement are we working on
+   void ParallelAssemble(int i, OperatorHandle &A, SparseMatrix *m);
+   // 2025 Sept ends
+
+#endif
+};
+
+#endif

miniapps/hdg/hdg_advection.cpp

@@ -0,0 +1,470 @@
+//                        MFEM EDG/HDG example
+//
+// Compile with: make advection
+//
+// Sample runs:  hdg_advection -o 1 -r 1 -tr 4 -no-vis
+//            hdg_advection -o 5 -r 1 -tr 4 -no-vis
+//            hdg_advection -o 1 -r 4 -tr 1
+//            hdg_advection -o 5 -r 4 -tr 1
+//            hdg_advection -o 1 -r 1 -tr 4 -no-vis -m ../data/inline-tri.mesh
+//            hdg_advection -o 5 -r 1 -tr 4 -no-vis -m ../data/inline-tri.mesh
+//            hdg_advection -o 1 -r 5 -tr 1 -m ../data/inline-tri.mesh
+//            hdg_advection -o 5 -r 5 -tr 1 -m ../data/inline-tri.mesh
+//
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               finite element discretization of the advection-reaction problem
+//               mu u + a.grad(u) = f with inhomogeneous Neumann boundary conditions.
+//               Specifically, we discretize using a HDG space of the
+//               specified order.
+//
+// The weak form is: seek (u,ubar) such that for all (v, vbar)
+//
+// \mu (u,v)   + (v, a.grad(u) - < 1, [zeta a.n u v] >  + < ubar, [zeta a.n v] > = (f, w)
+// < ubar, [zeta a.n v] >  +
+//       < 1, [zeta a.n ubar vbar] > + < 1, [(1-zeta) a.n ubar vbar >_{\Gamma_N} = < g, vbar >
+//
+// where (.,.) is the d-dimensional L2 product, <.,.> is the d-1 dimensional L2 product,
+// zeta = 1 for inflow boundaries, and 0 otherwise.
+//
+// The discretization is based on the paper:
+//
+// G. N. Wells, Analysis of an interface stabilized finite element method: the advection-diffusion-reaction equation, SIAM J. Numer. Anal., 2011, 49:1, 87--109.
+//
+// Contributed by: T. Horvath, Oakland University
+//                 S. Rhebergen, A. Sivas, University of Waterloo
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+#include "HDGBilinearForm.hpp"
+#include "hdg_integrators.hpp"
+
+using namespace std;
+using namespace mfem;
+
+//---------------------------------------------------------------------
+// Exact solution and r.h.s.. See below for implementation.
+real_t u_exact(const Vector &x);
+real_t f_rhs  (const Vector &x);
+void advection_function(const Vector &x, Vector &v);
+int dim;
+//---------------------------------------------------------------------
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-tri.mesh";
+   int order = 1;
+   int total_ref_levels = 2;
+   int initial_ref_levels = 0;
+   bool visualization = true;
+   bool save = true;
+   bool hdg = true;
+   real_t memA = 0.0;
+   real_t memB = 0.0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree > 1).");
+   args.AddOption(&initial_ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly for the initial calculation.");
+   args.AddOption(&total_ref_levels, "-tr", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&save, "-save", "--save-files", "-no-save",
+                  "--no-save-files",
+                  "Enable or disable file saving.");
+   args.AddOption(&hdg, "-hdg", "--hybrid", "-edg",
+                  "--embedded",
+                  "HDG / EDG option.");
+   args.AddOption(&memA, "-memA", "--memoryA",
+                  "Storage of A.");
+   args.AddOption(&memB, "-memB", "--memoryB",
+                  "Storage of B.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (order < 1)
+   {
+      cout << "Polynomial order should be > 0. Changing to order 1.";
+      order = 1;
+   }
+
+   // memA, memB \in [0,1], memB <= memA
+   if (memB > memA)
+   {
+      std::cout << "memB cannot be more than memA. Resetting to be equal" << std::endl
+                << std::flush;
+      memA = memB;
+   }
+   if (memA > 1.0)
+   {
+      std::cout << "memA cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memA = 1.0;
+   }
+   else if (memA < 0.0)
+   {
+      std::cout << "memA cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memA = 0.0;
+   }
+   if (memB > 1.0)
+   {
+      std::cout << "memB cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memB = 1.0;
+   }
+   else if (memB < 0.0)
+   {
+      std::cout << "memB cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memB = 0.0;
+   }
+
+   // 2. Read the mesh from the given mesh file. Refine it up to the initial_ref_levels.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   if (mesh->Nonconforming())
+   {
+      cout << "The current implementation does not support Nonconforming meshes. Terminating"
+           << endl << flush;
+      return 1;
+   }
+
+   dim = mesh->Dimension();
+
+   for (int ii=0; ii<initial_ref_levels; ii++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 3. Define the vectors that will contain the errors and the iteration count at every refinement level
+   Vector l2errors(total_ref_levels);
+   Array<int> iterativeMethodIts(total_ref_levels);
+
+   // 4. Define the finite element spaces on the mesh.
+   FiniteElementCollection *Uh_fec(new DG_FECollection(order, dim));
+   FiniteElementCollection *Uhbar_fec = NULL;
+   if (hdg)
+   {
+      Uhbar_fec = new DG_Interface_FECollection(order, dim);
+   }
+   else
+   {
+      Uhbar_fec = new H1_Trace_FECollection(order, dim);
+   }
+
+   // Finite element spaces:
+   // Uh_space is the DG space on elements
+   // ubar_space is the DG space on faces
+   FiniteElementSpace *Uh_space(new FiniteElementSpace(mesh, Uh_fec));
+   FiniteElementSpace *Uhbar_space(new FiniteElementSpace(mesh, Uhbar_fec));
+
+   // 5. Define the coefficients
+   ConstantCoefficient mu(1.0); // reaction constant
+   // Given boundary condition / exact solution
+   FunctionCoefficient ucoeff(u_exact);
+   // Given advection vector:
+   VectorFunctionCoefficient advection(dim, advection_function);
+
+   // 6. Define the different forms and gridfunctions.
+
+   // We apply static condensation to the system
+   //
+   // [ A   B ] [  u   ] = [ F ]
+   // [ C   D ] [ ubar ]   [ H ]
+   //
+   // Eliminating u we find the global system
+   //
+   // S ubar = G
+   //
+   // where S = - C A^{-1} B + D and G = -C A^{-1} F + H.
+   // Having solved this system for ubar, we can compute u from
+   //
+   // u = A^{-1} (F - B ubar)
+
+   // Set up the linear form fform(.) which corresponds to the right-hand
+   // side of the linear system, which in this case is (f, phi_i) and
+   // phi_i are the basis functions in the finite element Uh_space.
+   FunctionCoefficient fcoeff(f_rhs);
+   LinearForm *fform(new LinearForm);
+   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
+
+   // Set up the linear form gform(.) which corresponds to the right-hand
+   // side of the linear system, which in this case is <g, bar_phi_i>_{Gamma_N} and
+   // bar_phi_i are the basis functions in the finite element Uhbar_space.
+   LinearForm *gform(new LinearForm);
+   gform->AddSktBoundaryNeumannIntegrator(new HDGInflowLFIntegrator(ucoeff,
+                                                                    advection));
+
+   // Set up the bilinear form for the whole system. HDGBilinearForm2 can compute
+   // the Schur complement locally for a 2x2 problem.
+   HDGBilinearForm *AVarf(new HDGBilinearForm(Uh_space, Uhbar_space));
+   AVarf->AddHDGDomainIntegrator(
+      new HDGDomainIntegratorAdvection(mu, advection));
+   AVarf->AddHDGFaceIntegrator(
+      new HDGFaceIntegratorAdvection(advection));
+
+   GridFunction ubar(Uhbar_space);
+   GridFunction u(Uh_space);
+
+   for (int ref_levels = 0; ref_levels < total_ref_levels; ref_levels++)
+   {
+      // 7. Define the right hand side vectors
+      int dimUh   = Uh_space->GetVSize();
+      int dimUhbar = Uhbar_space->GetVSize();
+
+      std::cout << "***********************************************************\n";
+      std::cout << "dim(Uh) = " << dimUh << "\n";
+      std::cout << "dim(Uhbar) = " << dimUhbar << "\n";
+      std::cout << "***********************************************************\n";
+
+      Vector rhs_F(dimUh);
+      Vector rhs_G(dimUhbar);
+      Vector UBAR(dimUhbar);
+
+      // 8. Assemble the RHS and the bilinear forms
+      fform->Update(Uh_space, rhs_F, 0);
+      fform->Assemble(); // This is a vector
+
+      gform->Update(Uhbar_space, rhs_G, 0);
+      gform->Assemble(); // This is a vector
+
+      // Compute and Finalize the Schur complement
+      GridFunction *F = new GridFunction(Uh_space, rhs_F);
+      AVarf->AssembleSC(F, memA, memB);
+      AVarf->Finalize();
+
+      SparseMatrix *SC = AVarf->SpMatSC();
+
+      Vector *rhs_SC = AVarf->VectorSC();
+
+      // AVarf->VectorSC() provides -C*A^{-1} F, but the RHS for the
+      // Schur complement is  G - C*A^{-1} F
+      *rhs_SC += rhs_G;
+
+      // 9. Solve the Schur complement system
+      const int maxIter(1000);
+      const real_t rtol(1.e-15);
+      const real_t atol(0.0);
+      const int PrintLevel = -1;
+      GSSmoother M(*SC, 1, 1);
+      BiCGSTABSolver solver;
+      solver.SetAbsTol(atol);
+      solver.SetRelTol(rtol);
+      solver.SetMaxIter(maxIter);
+      solver.SetOperator(*SC);
+      solver.SetPrintLevel(PrintLevel);
+      solver.SetPreconditioner(M);
+      ubar = 0.0;
+      chrono.Clear();
+      chrono.Start();
+      solver.Mult(*rhs_SC, ubar);
+      chrono.Stop();
+
+      if (solver.GetConverged())
+         std::cout << "Iterative method converged in "
+                   << solver.GetNumIterations()
+                   << " iterations with a residual norm of "
+                   << solver.GetFinalNorm() << ".\n";
+      else
+         std::cout << "Iterative method did not converge in "
+                   << solver.GetNumIterations()
+                   << " iterations. Residual norm is "
+                   << solver.GetFinalNorm() << ".\n";
+
+      std::cout << "Iterative method solver took "
+                << chrono.RealTime() << "s. \n";
+      iterativeMethodIts[ref_levels] = solver.GetNumIterations();
+
+      // Delete the SC matrix to save memory
+      SC = NULL;
+
+      // 10. Reconstruction
+      // Reconstruct the solution u from the facet solution ubar
+      AVarf->Reconstruct(F, &ubar, &u);
+
+      // 11. Compute the discretization error
+      const int order_quad = max(2, 2*order+2);
+      const IntegrationRule *irs[Geometry::NumGeom];
+      for (int i=0; i < Geometry::NumGeom; ++i)
+      {
+         irs[i] = &(IntRules.Get(i, order_quad));
+      }
+
+      const real_t err_u  = u.ComputeL2Error(ucoeff, irs);
+      l2errors(ref_levels) = fabs(
+                                err_u); // fabs() to avoid negative values that ComputeL2Error can create
+
+      // 12. Save the mesh and the solution.
+      if (save)
+      {
+         ofstream mesh_ofs("refined.mesh");
+         mesh_ofs.precision(8);
+         mesh->Print(mesh_ofs);
+
+         ofstream u_ofs("sol_u.gf");
+         u_ofs.precision(8);
+         u.Save(u_ofs);
+
+         ofstream ubar_ofs("sol_lambda.gf");
+         ubar_ofs.precision(8);
+         ubar.Save(ubar_ofs);
+      }
+
+      // 13. Send the solution by socket to a GLVis server.
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream u_sock(vishost, visport);
+         u_sock.precision(8);
+         u_sock << "solution\n" << *mesh << u << flush;
+      }
+
+      // 14. Refine the mesh to increase the resolution and update the spaces and the forms.
+      mesh->UniformRefinement();
+
+      Uh_space->Update(0);
+      Uhbar_space->Update(0);
+
+      AVarf->Update();
+
+      u.Update();
+      ubar.Update();
+
+      delete F;
+   }
+
+   // 15. Print the results and compute the rates
+   std::cout << "\n\n---------------------------------\n";
+   std::cout << "level  l2errors  order iterations\n";
+   std::cout << "---------------------------------\n";
+   for (int ref_levels = 0; ref_levels < total_ref_levels; ref_levels++)
+   {
+      if (ref_levels == 0)
+      {
+         std::cout << "  " << ref_levels << "   "
+                   << std::setprecision(2) << std::scientific
+                   << l2errors(ref_levels)
+                   << "  " << "-    " << "    "
+                   << iterativeMethodIts[ref_levels] << std::endl;
+      }
+      else
+      {
+         const real_t conv_order = log(l2errors(ref_levels)/l2errors(ref_levels-1))/log(
+                                      0.5);
+         std::cout << "  " << ref_levels << "   "
+                   << std::setprecision(2) << std::scientific
+                   << l2errors(ref_levels)
+                   << "  " << std::setprecision(4) << std::fixed
+                   << conv_order << "   "
+                   << iterativeMethodIts[ref_levels] << std::endl;
+      }
+   }
+   std::cout << "\n\n";
+
+   // 16. Free the used memory.
+   delete mesh;
+   delete Uh_fec;
+   delete Uhbar_fec;
+   delete Uh_space;
+   delete Uhbar_space;
+   delete fform;
+   delete gform;
+   delete AVarf;
+
+   std::cout << "Done." << std::endl ;
+
+   return 0;
+}
+//---------------------------------------------------------------------
+// Exact solution
+real_t u_exact(const Vector &x)
+{
+   real_t ue = 0.0;
+   const real_t xx = x(0);
+   const real_t yy = x(1);
+   if (dim == 2)
+   {
+      ue = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+   }
+   else if (dim == 3)
+   {
+      const real_t zz = x(2);
+      ue = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+   }
+
+   return ue;
+}
+//---------------------------------------------------------------------
+// Rhs function
+real_t f_rhs(const Vector &x)
+{
+   real_t rhs = 0.0;
+   const real_t ax = 0.8;
+   const real_t ay = 0.6;
+   const real_t mu = 1.0;
+   const real_t xx = x(0);
+   const real_t yy = x(1);
+
+   if (dim == 2)
+   {
+      const real_t uu = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+      const real_t dudx = 0.125 * M_PI * (1.0+yy) * (1.0+yy)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+      const real_t dudy =  0.25 * M_PI * (1.0+xx) * (1.0+yy)
+                           * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+
+      rhs = mu * uu + ax * dudx + ay * dudy;
+   }
+
+   if (dim == 3)
+   {
+      const real_t az = 0.7;
+      const real_t zz = x(2);
+      const real_t uu = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+      const real_t dudx = 0.125 * M_PI * (1.0+yy) * (1.0+zz)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+
+      const real_t dudy = 0.125 * M_PI * (1.0+xx) * (1.0+zz)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+
+      const real_t dudz = 0.125 * M_PI * (1.0+xx) * (1.0+yy)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+
+      rhs = mu* uu + ax * dudx + ay * dudy + az * dudz;
+   }
+
+   return rhs;
+}
+//---------------------------------------------------------------------
+// Advection vector
+void advection_function(const Vector &x, Vector &v)
+{
+   if (dim == 3)
+   {
+      v(0) = 0.8;
+      v(1) = 0.6;
+      v(2) = 0.7;
+   }
+   else if (dim == 2)
+   {
+      v(0) = 0.8;
+      v(1) = 0.6;
+   }
+}

miniapps/hdg/hdg_advectionp.cpp

@@ -0,0 +1,606 @@
+//                        MFEM EDG/HDG example
+//
+// Compile with: make advectionp
+//
+// Sample runs:  mpirun -np 1 hdg_advectionp -o 1 -r 4 -tr 1 -no-vis // test scalability
+//            mpirun -np 2 hdg_advectionp -o 1 -r 4 -tr 1 -no-vis // test scalability
+//            mpirun -np 4 hdg_advectionp -o 1 -r 4 -tr 1 -no-vis // test scalability
+//            mpirun -np 4 hdg_advectionp -o 1 -r 0 -tr 3 -no-vis // test conv rates
+//            mpirun -np 4 hdg_advectionp -o 1 -r 2 -tr 1 -no-vis // test scalability
+//            mpirun -np 2 hdg_advectionp -o 5 -r 4 -tr 1 -no-vis // test conv rates
+//            mpirun -np 2 hdg_advectionp -o 5 -r 5 -tr 1 -m ../data/inline-tri.mesh -no-vis // test
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               finite element discretization of the advection-reaction problem
+//               mu u + a.grad(u) = f with inhomogeneous Neumann boundary conditions.
+//               Specifically, we discretize using a HDG space of the
+//               specified order.
+//
+// The weak form is: seek (u,ubar) such that for all (v, vbar)
+//
+// \mu (u,v)   + (v, a.grad(u))  - < 1, [zeta a.n u v] >  + < ubar, [zeta a.n v] > = (f, w)
+// < ubar, [zeta a.n v] >  +
+//       < 1, [zeta a.n ubar vbar] > + < 1, [(1-zeta) a.n ubar vbar >_{\Gamma_N} = < g, vbar >
+//
+// where (.,.) is the d-dimensional L2 product, <.,.> is the d-1 dimensional L2 product.
+//
+// The discretization is based on the paper:
+//
+// G. N. Wells, Analysis of an interface stabilized finite element method: the advection-diffusion-reaction equation, SIAM J. Numer. Anal., 2011, 49:1, 87--109.
+//
+// Contributed by: T. Horvath, S. Rhebergen, A. Sivas
+//             University of Waterloo
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+#include "HDGBilinearForm.hpp"
+#include "hdg_integrators.hpp"
+
+using namespace std;
+using namespace mfem;
+
+//---------------------------------------------------------------------
+// Exact solution and r.h.s.. See below for implementation.
+real_t u_exact(const Vector &x);
+real_t f_rhs  (const Vector &x);
+void advection_function(const Vector &x, Vector &v);
+int dim;
+//---------------------------------------------------------------------
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Initialize MPI.
+   // 1. Initialize MPI.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   real_t assemblyTime, solveTime, reconstructTime;
+   real_t GassemblyTime, GsolveTime, GreconstructTime;
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-tri.mesh";
+   int order          = 1;
+   int initial_ref_levels = 0;
+   int total_ref_levels  = 2;
+   bool visualization   = true;
+   bool save          = true;
+   bool hdg = true;
+   real_t memA = 0.0;
+   real_t memB = 0.0;
+   bool petsc = false;
+   bool verbose = (myid == 0);
+   const char *petscrc_file = "";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree > 1).");
+   args.AddOption(&initial_ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly for the initial calculation.");
+   args.AddOption(&total_ref_levels, "-tr", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&save, "-save", "--save-files", "-no-save",
+                  "--no-save-files",
+                  "Enable or disable file saving.");
+   args.AddOption(&hdg, "-hdg", "--hybrid", "-edg",
+                  "--embedded",
+                  "HDG / EDG option.");
+   args.AddOption(&memA, "-memA", "--memoryA",
+                  "Storage of A.");
+   args.AddOption(&memB, "-memB", "--memoryB",
+                  "Storage of B.");
+   args.AddOption(&petsc, "-petsc", "--use-petsc",
+                  "-no-petsc", "--no-use-petsc",
+                  "Enable or disable SC solver.");
+   args.AddOption(&petscrc_file, "-petscopts", "--petscopts",
+                  "PetscOptions file to use.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+#ifdef MFEM_USE_PETSC
+   // We initialize PETSc
+   MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL);
+#endif
+
+#ifndef MFEM_USE_PETSC
+   if (petsc)
+   {
+      std::cout << "MFEM does not use PETSc. Change the solver to hypre" << std::endl
+                << std::flush;
+      petsc = false;
+   }
+#endif
+
+   if (order < 1)
+   {
+      cout << "Polynomial order should be > 0. Changing to order 1.";
+      order = 1;
+   }
+
+   if (memB > memA)
+   {
+      std::cout << "memB cannot be more than memA. Resetting to be equal" << std::endl
+                << std::flush;
+      memA = memB;
+   }
+   if (memA > 1.0)
+   {
+      std::cout << "memA cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memA = 1.0;
+   }
+   else if (memA < 0.0)
+   {
+      std::cout << "memA cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memA = 0.0;
+   }
+   if (memB > 1.0)
+   {
+      std::cout << "memB cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memB = 1.0;
+   }
+   else if (memB < 0.0)
+   {
+      std::cout << "memB cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memB = 0.0;
+   }
+
+   // 3. Read the mesh from the given mesh file. Refine it up to the initial_ref_levels.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   if (mesh->Nonconforming())
+   {
+      if (verbose)
+      {
+         cout << "The current implementation does not support Nonconforming meshes. Terminating"
+              << endl << flush;
+      }
+#ifdef MFEM_USE_PETSC
+      MFEMFinalizePetsc();
+#endif
+      MPI_Finalize();
+      return 1;
+   }
+
+
+   for (int ii=0; ii<initial_ref_levels; ii++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // Define a parallel mesh. The serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 4. Define the vectors that will contain the errors and the iteration count at every refinement level
+   Vector l2errors(total_ref_levels);
+   Array<int> iterativeMethodIts(total_ref_levels);
+
+   // Define parallel finite element spaces on the parallel mesh.
+   // Uh_space is the DG space on elements
+   // ubar_space is the DG space on faces
+
+   if (order < 1)
+   {
+      cout << "Polynomial order should be > 0. Changing to order 1.";
+      order = 1;
+   }
+
+   // 5. Define the finite element spaces on the mesh.
+   FiniteElementCollection *Uh_fec(new DG_FECollection(order, dim));
+   FiniteElementCollection *Uhbar_fec = NULL;
+   if (hdg)
+   {
+      Uhbar_fec = new DG_Interface_FECollection(order, dim);
+   }
+   else
+   {
+      Uhbar_fec = new H1_Trace_FECollection(order, dim);
+   }
+
+   ParFiniteElementSpace *Uh_space   = new ParFiniteElementSpace(pmesh, Uh_fec);
+   ParFiniteElementSpace *Uhbar_space = new ParFiniteElementSpace(pmesh,
+                                                                  Uhbar_fec);
+
+   // 6. Define the coefficients
+   FunctionCoefficient fcoeff(f_rhs);
+   FunctionCoefficient ucoeff(u_exact);
+   ConstantCoefficient mu(1.0); // reaction constant
+
+   // Given advection vector:
+   VectorFunctionCoefficient advection(dim, advection_function);
+
+   // 7. Define the different forms and gridfunctions.
+
+   // Set up the linear form fform(.) which corresponds to the right-hand
+   // side of the linear system, which in this case is (f, phi_i) and
+   // phi_i are the basis functions in the finite element Uh_space.
+
+   ParLinearForm *fform = new ParLinearForm(Uh_space);
+   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
+
+   // Set up the linear form gform(.) which corresponds to the right-hand
+   // side of the linear system, which in this case is <g, bar_phi_i>_{Gamma_N} and
+   // bar_phi_i are the basis functions in the finite element Uhbar_space.
+
+   ParLinearForm *gform = new ParLinearForm(Uhbar_space);
+   gform->AddSktBoundaryNeumannIntegrator(new HDGInflowLFIntegrator(ucoeff,
+                                                                    advection));
+
+   // We apply static condensation to the system
+   //
+   // [ A   B ] [  u   ] = [ F ]
+   // [ C   D ] [ ubar ]   [ H ]
+   //
+   // Eliminating u we find the global system
+   //
+   // S ubar = G
+   //
+   // where S = - C A^{-1} B + D and G = -C A^{-1} F + H.
+   // Having solved this system for ubar, we can compute u from
+   //
+   // u = A^{-1} (F - B ubar)
+
+   // Set up the bilinear form for the whole system. ParHDGBilinearForm2 can compute
+   // the Schur complement locally for a 2x2 problem.
+   HDGBilinearForm *AVarf(new HDGBilinearForm(Uh_space, Uhbar_space, true));
+   AVarf->AddHDGDomainIntegrator(
+      new HDGDomainIntegratorAdvection(mu, advection));
+   AVarf->AddHDGFaceIntegrator(
+      new HDGFaceIntegratorAdvection(advection));
+
+   ParGridFunction u(Uh_space);
+   ParGridFunction ubar(Uhbar_space);
+
+   for (int ref_levels = 0; ref_levels < total_ref_levels; ref_levels++)
+   {
+      // 8. Define the right hand side vectors
+      HYPRE_Int dimUh   = Uh_space->GlobalTrueVSize();
+      HYPRE_Int dimUhbar = Uhbar_space->GlobalTrueVSize();
+
+      if (verbose)
+      {
+         std::cout << "****************************************************\n";
+         std::cout << "dim(Uh)      = " << dimUh << "\n";
+         std::cout << "dim(Uhbar)   = " << dimUhbar << "\n";
+         std::cout << "dim(Uh+Uhbar) = " << dimUh + dimUhbar << "\n";
+         std::cout << "****************************************************\n";
+      }
+
+      ubar = 0.0;
+      HypreParVector *UBAR = ubar.ParallelProject();
+
+      // 10. Assemble the RHS and the bilinear forms
+      Vector rhs_F(Uh_space->GlobalVSize());
+      Vector rhs_H(Uhbar_space->GlobalVSize());
+
+      // Linear forms
+      fform->Update(Uh_space);
+      fform->Assemble();
+      gform->Update(Uhbar_space);
+      gform->Assemble();
+
+      HypreParVector *trueF;
+      trueF = fform->ParallelAssemble();
+
+      HypreParVector *trueG;
+      trueG = gform->ParallelAssemble();
+      // Create a ParGridFunction from the right hand side.
+      ParGridFunction *F = new ParGridFunction(Uh_space, trueF);
+
+      chrono.Clear();
+      chrono.Start();
+      AVarf->AssembleSC(F, memA, memB);
+      chrono.Stop();
+      AVarf->Finalize();
+
+      assemblyTime = chrono.RealTime();
+
+      HypreParMatrix *SC = AVarf->ParallelAssembleSC();
+
+      HypreParVector *rhs_SC = AVarf->ParallelVectorSC();
+      // AVarf->ParallelVectorSC() provides -C*A^{-1} F, but the RHS for the
+      // Schur complement is  G - C*A^{-1} F
+      rhs_SC->Add(1.0, *trueG);
+
+      real_t tol = 1.0e-12;
+      int maxIter = 1000;
+      int PrintLevel = -1;
+
+      // 12. Solve the linear system
+      if (petsc)
+      {
+#ifdef MFEM_USE_PETSC
+         // Solver using PETSc
+         //=======================
+         PetscLinearSolver *petsc_solver;
+         PetscPreconditioner *petsc_precon= NULL;
+         petsc_solver = new PetscLinearSolver(MPI_COMM_WORLD, "solver_");
+         petsc_precon = new PetscPreconditioner(MPI_COMM_WORLD,*SC,"solver_");
+         petsc_solver->SetOperator(*SC);
+         petsc_solver->SetPreconditioner(*petsc_precon);
+         petsc_solver->SetTol(tol);
+         petsc_solver->SetAbsTol(0.0);
+         petsc_solver->SetMaxIter(maxIter);
+         petsc_solver->SetPrintLevel(PrintLevel);
+         chrono.Clear();
+         chrono.Start();
+         petsc_solver->Mult(*rhs_SC, *UBAR);
+         chrono.Stop();
+
+         if (verbose)
+         {
+            if (petsc_solver->GetConverged())
+               std::cout << "Solver converged in " << petsc_solver->GetNumIterations()
+                         << " iterations with a residual norm of " << petsc_solver->GetFinalNorm() <<
+                         ".\n";
+            else
+               std::cout << "Solver did not converge in " << petsc_solver->GetNumIterations()
+                         << " iterations. Residual norm is " << petsc_solver->GetFinalNorm() << ".\n";
+
+            std::cout << "Solver solver took " << chrono.RealTime() << "s. \n";
+         }
+         delete petsc_solver;
+         delete petsc_precon;
+#endif
+      }
+      else
+      {
+         HypreSolver *pdiag = new HypreDiagScale(*SC);
+         HypreGMRES *itsolver = new HypreGMRES(*SC);
+         itsolver->SetTol(tol);
+         itsolver->SetMaxIter(maxIter);
+         itsolver->SetPrintLevel(PrintLevel);
+         itsolver->SetPreconditioner(*pdiag);
+         chrono.Clear();
+         chrono.Start();
+         itsolver->Mult(*rhs_SC, *UBAR);
+         chrono.Stop();
+
+         int numIterations = 0;
+         itsolver->GetNumIterations(numIterations);
+         if (verbose)
+         {
+            std::cout << "\nIterative method converged in "
+                      << numIterations << ".\n";
+
+            iterativeMethodIts[ref_levels] = numIterations;
+            std::cout << "Iterative solver took " << chrono.RealTime() << "s. \n";
+         }
+      }
+
+      // Delete the SC matrix to save memory
+      SC = NULL;
+      solveTime = chrono.RealTime();
+
+      ubar = ParGridFunction(Uhbar_space, UBAR);
+
+      chrono.Clear();
+      chrono.Start();
+      AVarf->Reconstruct(F, &ubar, &u);
+      chrono.Stop();
+
+      reconstructTime = chrono.RealTime();
+
+      const int order_quad = max(2, 2*order+2);
+      const IntegrationRule *irs[Geometry::NumGeom];
+      for (int i=0; i < Geometry::NumGeom; ++i)
+      {
+         irs[i] = &(IntRules.Get(i, order_quad));
+      }
+
+      const real_t err_u  = u.ComputeL2Error(ucoeff, irs);
+      if (verbose)
+      {
+         std::cout << "\nL2 error " << err_u << ".\n";
+      }
+      l2errors(ref_levels) = fabs(
+                                err_u); // fabs() to avoid negative values that ComputeL2Error can create
+
+      // 14. Save the mesh and the solution.
+      if (save)
+      {
+         ostringstream mesh_name, u_name, ubar_name;
+         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+         u_name << "sol_u." << setfill('0') << setw(6) << myid;
+         ubar_name << "sol_ubar." << setfill('0') << setw(6) << myid;
+
+         ofstream mesh_ofs(mesh_name.str().c_str());
+         mesh_ofs.precision(8);
+         pmesh->Print(mesh_ofs);
+
+         ofstream u_ofs(u_name.str().c_str());
+         u_ofs.precision(8);
+         u.Save(u_ofs);
+      }
+
+      // 15. Send the solution by socket to a GLVis server.
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream u_sock(vishost, visport);
+         u_sock << "parallel " << num_procs << " " << myid << "\n";
+         u_sock.precision(8);
+         u_sock << "solution\n" << *pmesh << u << "window_title 'Velocity'"
+                << endl;
+         // Make sure all ranks have sent their 'u' solution before initiating
+         // another set of GLVis connections (one from each rank):
+         MPI_Barrier(pmesh->GetComm());
+      }
+
+      // 16. Refine the mesh to increase the resolution and update the spaces and the forms. Print the runtimes
+      pmesh->UniformRefinement();
+
+      Uh_space->Update(0);
+      Uhbar_space->Update(0);
+
+      AVarf->Update();
+
+      u.Update();
+      ubar.Update();
+      MPI_Reduce(&assemblyTime,&GassemblyTime,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
+      MPI_Reduce(&solveTime,&GsolveTime,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
+      MPI_Reduce(&reconstructTime,&GreconstructTime,1,MPI_DOUBLE,MPI_MAX,0,
+                 MPI_COMM_WORLD);
+
+      if (verbose)
+      {
+         printf("\t Assembly time   = %.2f\n",GassemblyTime);
+         printf("\t Solve time      = %.2f\n",GsolveTime);
+         printf("\t Reconstruct time = %.2f\n",GreconstructTime);
+      }
+
+      delete F;
+   }
+
+   // 17. Print the results
+   if (verbose)
+   {
+      std::cout << "\n\n---------------------------------\n";
+      std::cout << "level  l2errors  order iterations\n";
+      std::cout << "---------------------------------\n";
+      for (int ref_levels = 0; ref_levels < total_ref_levels; ref_levels++)
+      {
+         if (ref_levels == 0)
+         {
+            std::cout << "  " << ref_levels << "   "
+                      << std::setprecision(2) << std::scientific
+                      << l2errors(ref_levels)
+                      << "  " << "-    " << "    "
+                      << iterativeMethodIts[ref_levels] << std::endl;
+         }
+         else
+         {
+            const real_t conv_order = log(l2errors(ref_levels)/l2errors(ref_levels-1))
+                                      /log(0.5);
+            std::cout << "  " << ref_levels << "   "
+                      << std::setprecision(2) << std::scientific
+                      << l2errors(ref_levels)
+                      << "  " << std::setprecision(4) << std::fixed
+                      << conv_order << "   "
+                      << iterativeMethodIts[ref_levels] << std::endl;
+         }
+      }
+      std::cout << "\n\n";
+   }
+
+
+   // 19. Free the used memory.
+   delete pmesh;
+   delete Uh_fec;
+   delete Uhbar_fec;
+   delete Uh_space;
+   delete Uhbar_space;
+   delete fform;
+   delete gform;
+   delete AVarf;
+
+#ifdef MFEM_USE_PETSC
+   MFEMFinalizePetsc();
+#endif
+   MPI_Finalize();
+
+   return 0;
+}
+//---------------------------------------------------------------------
+// Exact solution
+real_t u_exact(const Vector &x)
+{
+   real_t ue = 0.0;
+   const real_t xx = x(0);
+   const real_t yy = x(1);
+   if (dim == 2)
+   {
+      ue = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+   }
+   else if (dim == 3)
+   {
+      const real_t zz = x(2);
+      ue = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+   }
+
+   return ue;
+}
+//---------------------------------------------------------------------
+// Rhs function
+real_t f_rhs(const Vector &x)
+{
+   real_t rhs = 0.0;
+   const real_t ax = 0.8;
+   const real_t ay = 0.6;
+   const real_t mu = 1.0;
+   const real_t xx = x(0);
+   const real_t yy = x(1);
+
+   if (dim == 2)
+   {
+      const real_t uu = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+      const real_t dudx = 0.125 * M_PI * (1.0+yy) * (1.0+yy)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+      const real_t dudy =  0.25 * M_PI * (1.0+xx) * (1.0+yy)
+                           * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+yy));
+
+      rhs = mu * uu + ax * dudx + ay * dudy;
+   }
+
+   if (dim == 3)
+   {
+      const real_t az = 0.7;
+      const real_t zz = x(2);
+      const real_t uu = 1.0 + sin(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+      const real_t dudx = 0.125 * M_PI * (1.0+yy) * (1.0+zz)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+
+      const real_t dudy = 0.125 * M_PI * (1.0+xx) * (1.0+zz)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+
+      const real_t dudz = 0.125 * M_PI * (1.0+xx) * (1.0+yy)
+                          * cos(0.125 * M_PI * (1.0+xx) * (1.0+yy) * (1.0+zz));
+
+      rhs = mu * uu + ax * dudx + ay * dudy + az * dudz;
+   }
+
+   return rhs;
+}
+//---------------------------------------------------------------------
+// Advection vector
+void advection_function(const Vector &x, Vector &v)
+{
+   if (dim == 3)
+   {
+      v(0) = 0.8;
+      v(1) = 0.6;
+      v(2) = 0.7;
+   }
+   else if (dim == 2)
+   {
+      v(0) = 0.8;
+      v(1) = 0.6;
+   }
+}

miniapps/hdg/hdg_integrators.cpp

@@ -0,0 +1,615 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+// Implementation of Bilinear Form Integrators
+//
+// Contributed by: T. Horvath: Oakland University
+//                 S. Rhebergen, A. Sivas: University of Waterloo
+
+#include "../../fem/fem.hpp"
+#include <cmath>
+#include <algorithm>
+#include "hdg_integrators.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+void HDGDomainIntegratorAdvection::AssembleElementMatrix(
+   const FiniteElement &fe_u,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int ndof_u = fe_u.GetDof();
+   int dim  = fe_u.GetDim();
+   int spaceDim = Trans.GetSpaceDim();
+   bool square = (dim == spaceDim);
+
+   Vector vec1; // for the convection integral
+   vec2.SetSize(dim);
+   BdFidxT.SetSize(ndof_u);
+
+   dshape.SetSize (ndof_u, dim); // for nabla \tilde{u}
+   gshape.SetSize (ndof_u, dim); // for nabla u
+   Jadj.SetSize (dim);  // for the Jacobian
+   shapeu.SetSize (ndof_u);    // shape of u
+
+   // setting the sizes of the local element matrices
+   elmat.SetSize(ndof_u, ndof_u);
+
+   // setting the order of integration
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * fe_u.GetOrder() + 1;
+      ir = &IntRules.Get(fe_u.GetGeomType(), order);
+   }
+
+   elmat = 0.0;
+
+   // evaluate the advection vector at all integration point
+   avec->Eval(Adv_ir, Trans, *ir);
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      // shape functions
+      fe_u.CalcDShape (ip, dshape);
+      fe_u.CalcShape (ip, shapeu);
+
+      // calculate the Adjugate of the Jacobian
+      Trans.SetIntPoint (&ip);
+      CalcAdjugate(Trans.Jacobian(), Jadj);
+
+      real_t w = Trans.Weight();
+      w = ip.weight / (square ? w : w*w*w);
+      // AdjugateJacobian = / adj(J),         if J is square
+      //                    \ adj(J^t.J).J^t, otherwise
+
+      // Calculate the gradient of the function of the physical element
+      Mult (dshape, Jadj, gshape);
+
+      // get the advection at the current integration point
+      Adv_ir.GetColumnReference(i, vec1);
+      vec1 *= ip.weight; // so it will be (cu, nabla v)
+
+      // compute -(cu, nabla v)
+      Jadj.Mult(vec1, vec2);
+      dshape.Mult(vec2, BdFidxT);
+      AddMultVWt(shapeu, BdFidxT, elmat);
+
+      real_t massw = Trans.Weight() * ip.weight;
+
+      if (mass_coeff)
+      {
+         massw *= mass_coeff->Eval(Trans, ip);
+      }
+      AddMult_a_VVt(massw, shapeu, elmat);
+
+   }
+}
+//---------------------------------------------------------------------
+void HDGFaceIntegratorAdvection::AssembleFaceMatrixOneElement1and1FES(
+   const FiniteElement &fe_u,
+   const FiniteElement &face_fe,
+   FaceElementTransformations &Trans,
+   const int elem1or2,
+   const bool reconstruct_only,
+   DenseMatrix &elmat1,
+   DenseMatrix &elmat2,
+   DenseMatrix &elmat3,
+   DenseMatrix &elmat4)
+{
+   int dim, ndof, ndof_face;
+   real_t w;
+
+   dim = fe_u.GetDim();
+   ndof_face = face_fe.GetDof();
+
+   shape_face.SetSize(ndof_face);
+
+   normal.SetSize(dim);
+   normalJ.SetSize(dim);
+   invJ.SetSize(dim);
+   adv.SetSize(dim);
+
+   ndof = fe_u.GetDof();
+
+   shape.SetSize(ndof);
+   dshape.SetSize(ndof, dim);
+   dshape_normal.SetSize(ndof);
+
+   elmat1.SetSize(ndof);
+   elmat2.SetSize(ndof, ndof_face);
+   elmat3.SetSize(ndof_face, ndof);
+   elmat4.SetSize(ndof_face);
+
+   elmat1 = 0.0;
+   elmat2 = 0.0;
+   elmat3 = 0.0;
+   elmat4 = 0.0;
+
+   // Since we are using GetSharedFaceTransformations over the shard faces
+   // we can find the boundary elements by checking Trans.Elem2No
+   bool is_bdr = (Trans.Elem2No < 0);
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      // a simple choice for the integration order
+      int order;
+      order = 2*fe_u.GetOrder();
+
+      ir = &IntRules.Get(Trans.FaceGeom, order);
+   }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip; // element integration point
+
+      Trans.Face->SetIntPoint(&ip);
+      face_fe.CalcShape(ip, shape_face);
+
+      if (dim == 1)
+      {
+         normal(0) = 2*eip.x - 1.0;
+      }
+      else
+      {
+         CalcOrtho(Trans.Face->Jacobian(), normal);
+      }
+
+      Trans.Loc1.Transform(ip, eip);
+      Trans.Elem1->SetIntPoint(&eip);
+
+      avec->Eval(adv, *Trans.Elem1, eip);
+      real_t an = adv * normal;
+      real_t an_L = an;
+
+      real_t zeta_R = 0.0, zeta_L = 0.0, zeta = 0.0;
+      if (an < 0.0)
+      {
+         zeta_L = 1.0;
+      }
+
+      if (elem1or2 == 1)
+      {
+         zeta = zeta_L;
+      }
+      else
+      {
+         Trans.Loc2.Transform(ip, eip);
+         Trans.Elem2->SetIntPoint(&eip);
+
+         avec->Eval(adv, *Trans.Elem2, eip);
+         an = adv * normal;
+         an *= -1.;
+
+         zeta_R = 1.0 - zeta_L;
+         zeta = zeta_R;
+      }
+
+      fe_u.CalcShape(eip, shape);
+
+      w = ip.weight;
+
+      for (int i = 0; i < ndof; i++)
+      {
+         for (int j = 0; j < ndof; j++)
+         {
+            // - < 1, [zeta a.n u v] >
+            elmat1(i, j) -= w * zeta * an * shape(i) * shape(j);
+         }
+
+         for (int j = 0; j < ndof_face; j++)
+         {
+            if (!reconstruct_only)
+            {
+               // - < ubar, [(1-zeta) a.n v] >
+               elmat3(j, i) -= w * an * (1.-zeta) * shape(i) * shape_face(j);
+            }
+
+            // + < ubar, [zeta a.n v] >
+            elmat2(i, j) += w * zeta * an * shape(i) * shape_face(j);
+
+         }
+      }
+      if (!reconstruct_only)
+      {
+
+         for (int i = 0; i < ndof_face; i++)
+            for (int j = 0; j < ndof_face; j++)
+            {
+               // - < 1, [zeta a.n ubar vbar] > + < 1, [(1-zeta) a.n ubar vbar >_{\Gamma_N}
+               if (!is_bdr)
+               {
+                  if (elem1or2 == 1)
+                  {
+                     elmat4(i, j) += -w * zeta_L * an_L * shape_face(i) * shape_face(j);
+                  }
+                  else
+                  {
+                     elmat4(i, j) += - w * (1.0 - zeta_L) * (-an_L) * shape_face(i) * shape_face(j);
+                  }
+               }
+               else
+               {
+                  elmat4(i, j) += -w * zeta_L * an * shape_face(i) * shape_face(j)
+                                  + w * (1.0 - zeta_L) * an * shape_face(i) * shape_face(j);
+               }
+            }
+      }
+
+   }
+}
+
+//---------------------------------------------------------------------
+void HDGInflowLFIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   mfem_error("Not implemented \n");
+}
+
+void HDGInflowLFIntegrator::AssembleRHSElementVect(
+   const FiniteElement &face_S, FaceElementTransformations &Trans,
+   Vector &favect)
+{
+   int dim, ndof_face;
+   real_t w, uin;
+
+   dim = face_S.GetDim(); // This is face dimension which is 1 less than
+   dim += 1;              // space dimension so add 1 to face dim to
+   // get the space dim.
+   n_L.SetSize(dim);
+   Vector adv(dim);
+
+   ndof_face = face_S.GetDof();
+
+   shape_f.SetSize(ndof_face);
+   favect.SetSize(ndof_face);
+   favect = 0.0;
+
+   if (Trans.Elem2No >= 0)
+   {
+      // Interior face, do nothing
+   }
+   else
+   {
+      // Boundary face
+      const IntegrationRule *ir = IntRule;
+      if (ir == NULL)
+      {
+         int order = 2 * face_S.GetOrder();
+         if (face_S.GetMapType() == FiniteElement::VALUE)
+         {
+            order += Trans.Face->OrderW();
+         }
+
+         ir = &IntRules.Get(Trans.FaceGeom, order);
+      }
+
+      for (int p = 0; p < ir->GetNPoints(); p++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(p);
+         face_S.CalcShape(ip, shape_f);
+
+         IntegrationPoint eip_L;
+         Trans.Loc1.Transform(ip, eip_L);
+         Trans.Face->SetIntPoint(&ip);
+
+         avec->Eval(adv, *Trans.Elem1, eip_L);
+         uin = u_in->Eval(*Trans.Elem1, eip_L);
+
+         if (dim == 1)
+         {
+            n_L(0) = 2*eip_L.x - 1.0;
+         }
+         else
+         {
+            CalcOrtho(Trans.Face->Jacobian(), n_L);
+         }
+
+         real_t an_L = adv * n_L;
+
+         real_t zeta_L = 0.0;
+         if (an_L < 0.0)
+         {
+            zeta_L = 1.0;
+         }
+
+         w = ip.weight;
+
+         real_t gg = -uin * an_L * zeta_L;
+         for (int i = 0; i < ndof_face; i++)
+         {
+            favect(i) += w * gg * shape_f(i);
+         }
+      }
+   }
+}
+//---------------------------------------------------------------------
+////////////////////////////////////////////////////////////////////////////////////////////////////
+void HDGDomainIntegratorDiffusion::AssembleElementMatrix2FES(
+   const FiniteElement &fe_q,
+   const FiniteElement &fe_u,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   // get the number of degrees of freedoms and the dimension of the problem
+   int ndof_u = fe_u.GetDof();
+   int ndof_q = fe_q.GetDof();
+   int dim  = fe_q.GetDim();
+   real_t norm;
+
+   int vdim = dim ;
+
+   // set the vector and matrix sizes
+   dshape.SetSize (ndof_u, dim); // for nabla u_reference
+   gshape.SetSize (ndof_u, dim); // for nabla u
+   Jadj.SetSize (dim);  // the Jacobian
+   divshape.SetSize (vdim*ndof_u);  // divergence of q
+   shape.SetSize (ndof_q);  // shape of q (and u)
+
+   // for vector diffusion the matrix is built up from partial matrices
+   partelmat.SetSize(ndof_q);
+
+   DenseMatrix local_A11, local_A12, local_A21;
+
+   // setting the sizes of the local element matrices
+   local_A11.SetSize(dim*ndof_q, dim*ndof_q);
+   local_A12.SetSize(vdim*ndof_q, ndof_u);
+   local_A21.SetSize(ndof_u, vdim*ndof_q);
+
+   elmat.SetSize(dim*ndof_q + ndof_u);
+
+   local_A11 = 0.0;
+   local_A12 = 0.0;
+   local_A21 = 0.0;
+   elmat  = 0.0;
+
+   // setting the order of integration
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order1 = 2 * fe_q.GetOrder();
+      int order2 = 2 * fe_q.GetOrder() + Trans.OrderW();
+      int order = max(order1, order2);
+      ir = &IntRules.Get(fe_u.GetGeomType(), order);
+   }
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      // compute the shape and the gradient values on the reference element
+      fe_u.CalcDShape (ip, dshape);
+      fe_q.CalcShape (ip, shape);
+
+      // calculate the adjugate of the Jacobian
+      Trans.SetIntPoint (&ip);
+      CalcAdjugate(Trans.Jacobian(), Jadj);
+
+      // Calculate the gradient of the function of the physical element
+      Mult (dshape, Jadj, gshape);
+
+      // the weight is the product of the integral weight and the
+      // determinant of the Jacobian
+      norm = ip.weight * Trans.Weight();
+      MultVVt(shape, partelmat);
+
+      real_t c = ip.weight;
+
+      // transform the the matrix to divergence vector
+      gshape.GradToDiv (divshape);
+
+      // mulitply by 1.0/nu
+      partelmat *= 1.0/nu->Eval(Trans, ip);
+
+      shape *= c;
+      // compute the (u, \div v) term
+      AddMultVWt (shape, divshape, local_A21);
+
+      // assemble -(q, v) from the partial matrices
+      partelmat *= norm*(-1.0);
+      for (int k = 0; k < vdim; k++)
+      {
+         local_A11.AddMatrix(partelmat, ndof_q*k, ndof_q*k);
+      }
+
+   }
+
+   local_A12.Transpose(local_A21);
+
+   int block_size1 = dim*ndof_q;
+
+   elmat.CopyMN(local_A11, 0, 0);
+   elmat.CopyMN(local_A12, 0, block_size1);
+   elmat.CopyMN(local_A21, block_size1, 0);
+}
+
+
+void HDGFaceIntegratorDiffusion::AssembleFaceMatrixOneElement2and1FES(
+   const FiniteElement &fe_q,
+   const FiniteElement &fe_u,
+   const FiniteElement &face_fe,
+   FaceElementTransformations &Trans,
+   const int elem1or2,
+   const bool reconstruct_only,
+   DenseMatrix &elmat1,
+   DenseMatrix &elmat2,
+   DenseMatrix &elmat3,
+   DenseMatrix &elmat4)
+{
+   // Get DoF from faces and the dimension
+   int ndof_face = face_fe.GetDof();
+   int ndof_q, ndof_u;
+   int dim = fe_q.GetDim();
+   int vdim = dim;
+   int order;
+
+   DenseMatrix shape1_n_mtx;
+
+   // set the dofs for u and q
+   ndof_u = fe_u.GetDof();
+   ndof_q = fe_q.GetDof();
+
+   DenseMatrix local_B1, local_A22, local_B2, local_C1, local_C2, local_D;
+
+   // set the shape functions, the normal and the advection
+   shapeu.SetSize(ndof_u);
+   shapeq.SetSize(ndof_q);
+   shape_face.SetSize(ndof_face);
+   normal.SetSize(dim);
+
+   // set the proper size for the matrices
+   local_B1.SetSize(vdim*ndof_q, ndof_face);
+   local_B1 = 0.0;
+   local_A22.SetSize(ndof_u, ndof_u);
+   local_A22 = 0.0;
+   local_B2.SetSize(ndof_u, ndof_face);
+   local_B2 = 0.0;
+   local_C1.SetSize(vdim*ndof_q, ndof_face);
+   local_C1 = 0.0;
+   local_C2.SetSize(ndof_u, ndof_face);
+   local_C2 = 0.0;
+   local_D.SetSize(ndof_face, ndof_face);
+   local_D = 0.0;
+
+   int sub_block_size1 = vdim*ndof_q;
+   int sub_block_size2 = ndof_u;
+
+   int block_size1 = sub_block_size1 + sub_block_size2;
+   int block_size2 = ndof_face;
+
+   elmat1.SetSize(block_size1);
+   elmat1 = 0.0;
+   elmat2.SetSize(block_size1, block_size2);
+   elmat2 = 0.0;
+   elmat3.SetSize(block_size2, block_size1);
+   elmat3 = 0.0;
+   elmat4.SetSize(block_size2);
+   elmat4 = 0.0;
+
+
+   shape1_n_mtx.SetSize(ndof_q,dim);
+   shape_dot_n.SetSize(ndof_q,dim);
+
+   // set the order of integration
+   // using the fact that q and u has the same order!
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = 2*max(max(fe_q.GetOrder(), fe_u.GetOrder()), face_fe.GetOrder());
+      order += 2;
+
+      // IntegrationRule depends on the Geometry of the face (pont, line, triangle, rectangular)
+      ir = &IntRules.Get(Trans.FaceGeom, order);
+   }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip; // integration point on the element
+
+      // Trace finite element shape function
+      Trans.Face->SetIntPoint(&ip);
+      face_fe.CalcShape(ip, shape_face);
+
+      // calculate the normal at the integration point
+      if (dim == 1)
+      {
+         normal(0) = 2*eip.x - 1.0;
+      }
+      else
+      {
+         CalcOrtho(Trans.Face->Jacobian(), normal);
+      }
+
+      if (elem1or2 == 1)
+      {
+         // Side 1 finite element shape function
+         Trans.Loc1.Transform(ip, eip);
+      }
+      else
+      {
+         // Side 2 finite element shape function
+         Trans.Loc2.Transform(ip, eip);
+      }
+
+      fe_u.CalcShape(eip, shapeu);
+      fe_q.CalcShape(eip, shapeq);
+      MultVWt(shapeq, normal, shape_dot_n) ;
+
+
+      // set the coefficients for the different terms
+      // if the normal is involved, Trans.Face->Weight() is not required
+      real_t w1 = ip.weight*(-1.0);
+
+      if (elem1or2 == 2)
+      {
+         w1 *=-1.0;
+      }
+
+      real_t w2 = tauD*Trans.Face->Weight()* ip.weight;
+
+      real_t w3 = -w2;
+
+      // local_B1 = < \lambda,\nu v\cdot n>
+      for (int i = 0; i < vdim; i++)
+         for (int k = 0; k < ndof_q; k++)
+            for (int j = 0; j < ndof_face; j++)
+            {
+               local_B1(i*ndof_q + k, j) += shape_face(j) * shape_dot_n(k,i) * w1;
+            }
+
+      // local_A22 =  < \tau u, w>
+      // local_B2= -< tau \lambda, w>
+      // local_C2 = -< tau \lambda, w>
+      for (int i = 0; i < ndof_u; i++)
+      {
+         for (int j = 0; j < ndof_u; j++)
+         {
+            local_A22(i, j) += w2 * shapeu(i) * shapeu(j);
+         }
+
+         for (int j = 0; j < ndof_face; j++)
+         {
+            local_B2(i, j) += w3 * shapeu(i) * shape_face(j);
+         }
+      }
+
+      if (!reconstruct_only)
+      {
+         // local_D = < \tau \lambda, \mu>
+
+         AddMult_a_VVt(w2, shape_face, local_D);
+      }
+   }
+
+   local_C1.Transpose(local_B1);
+   local_C2.Transpose(local_B2);
+
+   elmat1.CopyMN(local_A22, sub_block_size1, sub_block_size1);
+
+   elmat2.CopyMN(local_B1, 0, 0);
+   elmat2.CopyMN(local_B2, sub_block_size1, 0);
+
+   elmat3.CopyMN(local_C1, 0, 0);
+   elmat3.CopyMN(local_C2, 0, sub_block_size1);
+
+   elmat4 = local_D;
+}
+
+
+}
+;

miniapps/hdg/hdg_integrators.hpp

@@ -0,0 +1,223 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+// Implementation of Bilinear Form Integrators
+//
+// Contributed by: T. Horvath: Oakland University
+//                 S. Rhebergen, A. Sivas: University of Waterloo
+
+#ifndef MFEM_HDGINTEG
+#define MFEM_HDGINTEG
+
+#include "../../config/config.hpp"
+
+namespace mfem
+{
+
+//---------------------------------------------------------------------
+// Advection integrator: to compute all the domain based integrals
+//
+// The output is
+//
+// elemmat = mass_coeff (u,v) + (v, avec.grad(u))
+//
+// mass_coeff is the reaction coefficient
+// avec is the advection coefficient
+class HDGDomainIntegratorAdvection : public BilinearFormIntegrator
+{
+private:
+   Coefficient *mass_coeff;
+   VectorCoefficient *avec;
+
+   Vector shape1, shape2;
+   DenseMatrix shape1_n, shape2_n, partelmat;
+
+   Vector shapeq;
+   Vector shapeu;
+   Vector divshape, divshape_no_diffusion, vec2, BdFidxT;
+   DenseMatrix dshape;
+   DenseMatrix gshape;
+   DenseMatrix Jadj;
+   DenseMatrix Adv_ir;
+
+public:
+   HDGDomainIntegratorAdvection(Coefficient &mass, VectorCoefficient &_avec)
+      : mass_coeff(&mass), avec(&_avec)  {  }
+
+   using BilinearFormIntegrator::AssembleElementMatrix;
+   virtual void AssembleElementMatrix(const FiniteElement &fe_u,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat1);
+};
+
+// Advection integrator to compute all the face based integrals
+//
+// The output is
+//
+// elemmat1 = - < 1, [zeta avec.n u v] >
+// elemmat2 = < ubar, [zeta avec.n v] >
+// elemmat3 = < ubar, [zeta avec.n v] >
+// elemmat4 = < 1, [zeta avec.n ubar vbar] > + < 1, [(1-zeta) avec.n ubar vbar >_{\Gamma_N}
+//
+// avec is the advection coefficient
+class HDGFaceIntegratorAdvection : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient *avec;
+
+   Vector shape, normal, normalJ, dshape_normal, shape_face, adv;
+   DenseMatrix shape1_n, invJ, dshape, shape1_n_nodiff;
+
+public:
+   HDGFaceIntegratorAdvection(VectorCoefficient &_avec)
+      : avec(&_avec)  {  }
+
+   using BilinearFormIntegrator::AssembleFaceMatrixOneElement1and1FES;
+   virtual void AssembleFaceMatrixOneElement1and1FES(const FiniteElement &fe_u,
+                                                     const FiniteElement &face_fe,
+                                                     FaceElementTransformations &Trans,
+                                                     const int elem1or2,
+                                                     const bool reconstruct_only,
+                                                     DenseMatrix &elmat1,
+                                                     DenseMatrix &elmat2,
+                                                     DenseMatrix &elmat3,
+                                                     DenseMatrix &elmat4);
+
+};
+
+//---------------------------------------------------------------------
+/** Boundary linear integrator for imposing inflow boundary
+    conditions. Given the inflow data u_in, the linear form assembles the
+    following integral on the boundary:
+
+    + < g, vbar >
+
+    where g = - u_in * a.n * zeta and vbar is the test function. */
+class HDGInflowLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   Coefficient *u_in;
+   VectorCoefficient *avec;
+
+   // these are not thread-safe!
+   Vector shape_f, n_L;
+
+public:
+   HDGInflowLFIntegrator(Coefficient &_u, VectorCoefficient &_avec)
+   {
+      u_in = &_u;
+      avec = &_avec;
+   }
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect);
+
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       FaceElementTransformations &Tr,
+                                       Vector &elvect);
+};
+
+//---------------------------------------------------------------------
+//---------------------------------------------------------------------
+
+// Diffusion integrator: to compute all the domain based integrals
+//
+// The output is
+//
+//         [local_A11 local_A12]
+// elmat = [local_A21    0.0   ]
+//
+// local_A11 = -(\nu^{-1} q, v)
+// local_A12 = (u, div(v))
+// local_A21 = (div(q), w)
+//
+// local_A21 = local_A12^T
+//
+// \nu is the constant diffusion coefficient
+class HDGDomainIntegratorDiffusion : public BilinearFormIntegrator
+{
+private:
+   ConstantCoefficient *nu;
+
+   Vector shape, divshape;
+   DenseMatrix partelmat, dshape, gshape, Jadj;
+
+public:
+   HDGDomainIntegratorDiffusion(ConstantCoefficient &_nu)
+      : nu(&_nu) { }
+
+   using BilinearFormIntegrator::AssembleElementMatrix2FES;
+   virtual void AssembleElementMatrix2FES(const FiniteElement &fe_q,
+                                          const FiniteElement &fe_u,
+                                          ElementTransformation &Trans,
+                                          DenseMatrix &elmat);
+
+};
+
+// Diffusion integrator to compute all the face based integrals
+//
+// The output is
+//
+//          [ 0.0    0.0   ]
+// elmat1 = [ 0.0 local_A22 ]  - the face based integral for matrix A
+//
+//          [ local_B1 ]
+// elmat2 = [ local_B2 ]  - the face based integral for matrix B
+//
+// elmat3 = [ local_C1  local_C2 ]  - the face based integral for matrix C
+//
+// elmat4 = local_D  - the face based integral for matrix D
+//
+// where
+// local_B1 = < \lambda,v\cdot n>
+// local_A22 = < \tau u, w>
+// local_B2 = -< tau \lambda, w>
+// local_C1 = < \lambda, v\cdot n>
+// local_C2 = -< \tau \lambda, w>
+// local_D = < \tau \lambda, \mu>
+//
+// q_diff_coeff is the constant diffusion coefficient
+// local_C1 = local_B1^T
+// local_C2 = local_B2^T
+class HDGFaceIntegratorDiffusion : public BilinearFormIntegrator
+{
+private:
+   real_t tauD;
+
+   Vector shapeu, shapeq, normal, shape_face;
+   DenseMatrix shape_dot_n;
+
+public:
+   HDGFaceIntegratorDiffusion(real_t a)
+   {
+      tauD = a;
+   }
+
+   using BilinearFormIntegrator::AssembleFaceMatrixOneElement2and1FES;
+   virtual void AssembleFaceMatrixOneElement2and1FES(const FiniteElement &fe_q,
+                                                     const FiniteElement &fe_u,
+                                                     const FiniteElement &face_fe,
+                                                     FaceElementTransformations &Trans,
+                                                     const int elem1or2,
+                                                     const bool reconstruct_only,
+                                                     DenseMatrix &elmat1,
+                                                     DenseMatrix &elmat2,
+                                                     DenseMatrix &elmat3,
+                                                     DenseMatrix &elmat4);
+
+};
+
+
+}
+
+#endif

miniapps/hdg/hdg_poisson.cpp

@@ -0,0 +1,675 @@
+//                        MFEM Example Hybridizable DG
+//
+// Compile with: make hdg_poisson
+//
+// Sample runs:  hdg_poisson -o 1 -r 1 -tr 4 -no-vis
+//            hdg_poisson -o 5 -r 1 -tr 4 -no-vis
+//            hdg_poisson -o 1 -r 4 -tr 1
+//            hdg_poisson -o 5 -r 4 -tr 1
+//            hdg_poisson -o 1 -r 1 -tr 4 -no-vis -m ../data/inline-tri.mesh
+//            hdg_poisson -o 5 -r 1 -tr 4 -no-vis -m ../data/inline-tri.mesh
+//            hdg_poisson -o 1 -r 5 -tr 1 -m ../data/inline-tri.mesh
+//            hdg_poisson -o 5 -r 5 -tr 1 -m ../data/inline-tri.mesh
+//
+// Description:  This example code solves the 2D/3D diffusion problem
+//                     -\nu Delta u = f
+//               with Dirichlet boundary conditions, using HDG discretization.
+//
+// The methods approximates the solution u, the diffusive flux q = -\nu \nabla u,
+// and the restriction of u to the faces, denoted by lambda.
+//
+// The weak form is: seek (q,u,\lambda) such that for all (v, w, \mu)
+//
+// -\nu^{-1}(q, v)       + (u, div(v))       - <\lambda, v \cdot n>      = 0
+//  (div(q), w)          + <\tau u, w>       - <\tau \lambda, w>         = (f, w)
+// -<[[q \cdot n]], \mu> - <[[\tau u]], \mu> + <[[(\tau \lambda]], \mu>  = 0
+//
+// where [[.]] is the jump operator, (.,.) is the d-dimensional L2 product,
+// <.,.> is the d-1 dimensional L2 product.
+//
+// The discretization is based on the paper:
+//
+// N.C. Nguyen, J. Peraire, B. Cockburn, An implicit high-order hybridizable
+// discontinuous Galerkin method for linear convection–diffusion equations,
+// J. Comput. Phys., 2009, 228:9, 3232--3254.
+//
+// Contributed by: T. Horvath, Oakland University
+//                 S. Rhebergen, A. Sivas, University of Waterloo
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+#include "HDGBilinearForm.hpp"
+#include "hdg_integrators.hpp"
+
+
+using namespace std;
+using namespace mfem;
+
+// Define the analytical solution and forcing terms / boundary conditions
+real_t uFun_ex(const Vector & x);
+void qFun_ex(const Vector & x, Vector & q);
+real_t fFun(const Vector & x);
+real_t diff;
+
+// We can minimize the expression |\nu \nabla u_h^* + q_h |^2 over a single element K,
+// for p+1 degree u_h^*, with the constraint \int_K u_h^* = \int_K u_h, so the mean
+// of u_h^* is the same as the one of u_h.
+//
+// This results in the problem
+//
+// (nabla w_h, \nu \nabla u_h^*) = -(nabla w_h, q_h)
+// (1, u_h^*)                    = (1, u_h)
+//
+// Since the fist equation on its own would generate a singular problem
+// the last line of the system is rewritten by the second equation.
+//
+// This elementwise operation will provide a superconvergent solution
+// \|u-u_h\|_{L^2} < C h^{p+2} |u|_{p+1}
+class HDGPostProcessing
+{
+private:
+   GridFunction *q, *u;
+
+   FiniteElementSpace *fes;
+
+   Coefficient *diffcoeff;
+
+protected:
+   const IntegrationRule *IntRule;
+
+public:
+   HDGPostProcessing(FiniteElementSpace *f, GridFunction &_q, GridFunction &_u,
+                     Coefficient &_diffcoeff)
+      : q(&_q), u(&_u), fes(f), diffcoeff(&_diffcoeff)
+   {
+      IntRule = NULL;
+   }
+
+   void Postprocessing(GridFunction &u_postprocessed) ;
+};
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-tri.mesh";
+   int order = 1;
+   int initial_ref_levels = 0;
+   int total_ref_levels = 2;
+   bool visualization = true;
+   bool post = true;
+   bool save = true;
+   bool hdg = true;
+   real_t memA = 0.0;
+   real_t memB = 0.0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&initial_ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly for the initial calculation.");
+   args.AddOption(&total_ref_levels, "-tr", "--totalrefine",
+                  "Number of times to refine the mesh uniformly to get the convergence rates.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&post, "-post", "--postprocessing",
+                  "-no-post", "--no-postprocessing",
+                  "Enable or disable postprocessing.");
+   args.AddOption(&save, "-save", "--save-files", "-no-save",
+                  "--no-save-files",
+                  "Enable or disable file saving.");
+   args.AddOption(&hdg, "-hdg", "--hybrid", "-edg",
+                  "--embedded",
+                  "HDG / EDG option.");
+   args.AddOption(&memA, "-memA", "--memoryA",
+                  "Storage of A.");
+   args.AddOption(&memB, "-memB", "--memoryB",
+                  "Storage of B.");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // memA, memB \in [0,1], memB <= memA
+   if (memB > memA)
+   {
+      std::cout << "memB cannot be more than memA. Resetting to be equal" << std::endl
+                << std::flush;
+      memA = memB;
+   }
+   if (memA > 1.0)
+   {
+      std::cout << "memA cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memA = 1.0;
+   }
+   else if (memA < 0.0)
+   {
+      std::cout << "memA cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memA = 0.0;
+   }
+   if (memB > 1.0)
+   {
+      std::cout << "memB cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memB = 1.0;
+   }
+   else if (memB < 0.0)
+   {
+      std::cout << "memB cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memB = 0.0;
+   }
+
+   // 2. Read the mesh from the given mesh file. Refine it up to the initial_ref_levels.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   if (mesh->Nonconforming())
+   {
+      cout << "The current implementation does not support Nonconforming meshes. Terminating"
+           << endl << flush;
+      return 1;
+   }
+
+   for (int ii=0; ii<initial_ref_levels; ii++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 3. Vectors for the different discretization errors
+   Vector u_l2errors(total_ref_levels), q_l2errors(total_ref_levels),
+          mean_l2errors(total_ref_levels), u_star_l2errors(total_ref_levels);
+
+   // 4. Define a finite element collections and spaces on the mesh.
+   FiniteElementCollection *dg_coll(new DG_FECollection(order, dim));
+   FiniteElementCollection *face = NULL;
+   if (hdg)
+   {
+      face = new DG_Interface_FECollection(order, dim);
+   }
+   else
+   {
+      face = new H1_Trace_FECollection(order, dim);
+   }
+
+   // Finite element spaces:
+   // V_space is the vector valued DG space on elements for q_h
+   // W_space is the scalar DG space on elements for u_h
+   // M_space is the DG space on faces for lambda_h
+   FiniteElementSpace *V_space = new FiniteElementSpace(mesh, dg_coll, dim);
+   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, dg_coll);
+   FiniteElementSpace *M_space = new FiniteElementSpace(mesh, face);
+
+   // 5. Define the coefficients, the exact solutions, the right hand side and the diffusion coefficient along with the diffusion penalty parameter.
+   FunctionCoefficient fcoeff(fFun);
+
+   FunctionCoefficient ucoeff(uFun_ex);
+   VectorFunctionCoefficient qcoeff(dim, qFun_ex);
+
+   diff = 1.;
+   ConstantCoefficient diffusion(diff); // diffusion constant
+   real_t tau_D = 5.0;
+
+   // 6. Define the different forms and gridfunctions.
+   HDGBilinearForm *AVarf(new HDGBilinearForm(V_space, W_space, M_space));
+   AVarf->AddHDGDomainIntegrator(new HDGDomainIntegratorDiffusion(diffusion));
+   AVarf->AddHDGFaceIntegrator(new HDGFaceIntegratorDiffusion(tau_D));
+
+   GridFunction lambda_variable(M_space);
+   GridFunction q_variable(V_space), u_variable(W_space);
+
+   LinearForm *fform(new LinearForm);
+   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
+
+   for (int ref_levels = initial_ref_levels;
+        ref_levels < (initial_ref_levels + total_ref_levels); ref_levels++)
+   {
+      // 7. Compute the problem size and define the right hand side vectors
+      int dimV = V_space->GetVSize();
+      int dimW = W_space->GetVSize();
+      int dimM = M_space->GetVSize();
+
+      std::cout << "***********************************************************\n";
+      std::cout << "dim(W) = " << dimV << "\n";
+      std::cout << "dim(V) = " << dimW << "\n";
+      std::cout << "dim(M) = " << dimM << "\n";
+      std::cout << "dim(W+V+M) = " << dimV + dimW + dimM << "\n";
+      std::cout << "***********************************************************\n";
+
+      Vector rhs_R(dimV);
+      Vector rhs_F(dimW);
+      Vector V_aux(dimV);
+      Vector W_aux(dimW);
+
+      V_aux = 0.0;
+      W_aux = 0.0;
+      rhs_R = 0.0;
+
+      // 8. To eliminate the boundary conditions we project the BC to a grid function
+      // defined for the facet unknowns.
+      FunctionCoefficient lambda_coeff(uFun_ex);
+      lambda_variable.ProjectCoefficientSkeleton(lambda_coeff);
+
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // 9. Assemble the RHS and the Schur complement
+      fform->Update(W_space, rhs_F, 0);
+      fform->Assemble();
+
+      GridFunction *R = new GridFunction(V_space, rhs_R);
+      GridFunction *F = new GridFunction(W_space, rhs_F);
+      AVarf->AssembleSC(R, F, ess_bdr, lambda_variable, memA, memB);
+      AVarf->Finalize();
+
+      SparseMatrix* SC = AVarf->SpMatSC();
+
+      Vector* SC_RHS = AVarf->VectorSC();
+      // AVarf->VectorSC() provides -C*A^{-1} RF, the RHS for the
+      // Schur complement is  L - C*A^{-1} RF, but L is zero for this case.
+
+      // 10. Solve the Schur complement system
+      int maxIter(4000);
+      real_t rtol(1.e-13);
+      real_t atol(0.0);
+      GSSmoother M(*SC);
+      BiCGSTABSolver solver;
+      solver.SetAbsTol(atol);
+      solver.SetRelTol(rtol);
+      solver.SetMaxIter(maxIter);
+      solver.SetOperator(*SC);
+      solver.SetPrintLevel(-1);
+      solver.SetPreconditioner(M);
+      chrono.Clear();
+      chrono.Start();
+      solver.Mult(*SC_RHS, lambda_variable);
+      chrono.Stop();
+
+      if (solver.GetConverged())
+         std::cout << "Iterative method converged in " << solver.GetNumIterations()
+                   << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
+      else
+         std::cout << "Iterative method did not converge in " <<
+                   solver.GetNumIterations()
+                   << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
+      std::cout << "Iterative method solver took " << chrono.RealTime() << "s. \n";
+
+      // Delete the SC matrix to save memory
+      SC = NULL;
+
+      // 11. Reconstruction
+      // Reconstruct the solution u and q from the facet solution lambda
+      AVarf->Reconstruct(R, F, &lambda_variable, &q_variable, &u_variable);
+
+      // 12. Compute the discretization error
+      int order_quad = max(2, 2*order+2);
+      const IntegrationRule *irs[Geometry::NumGeom];
+      for (int i=0; i < Geometry::NumGeom; ++i)
+      {
+         irs[i] = &(IntRules.Get(i, order_quad));
+      }
+      real_t err_u    = u_variable.ComputeL2Error(ucoeff, irs);
+      real_t err_q    = q_variable.ComputeL2Error(qcoeff, irs);
+      real_t err_mean  = u_variable.ComputeMeanLpError(2.0, ucoeff, irs);
+
+      u_l2errors(ref_levels-initial_ref_levels) = fabs(err_u);
+      q_l2errors(ref_levels-initial_ref_levels) = fabs(err_q);
+      mean_l2errors(ref_levels-initial_ref_levels) = fabs(err_mean);
+
+      std::cout << "|| u_h - u_ex || = " << err_u << "\n";
+      std::cout << "|| q_h - q_ex || = " << err_q << "\n";
+      std::cout << "|| mean(u_h) - mean(u_ex) || = " << err_mean << "\n";
+
+      // 13. Save the mesh and the solution.
+      if (save)
+      {
+         ofstream mesh_ofs("ex_hdg.mesh");
+         mesh_ofs.precision(8);
+         mesh->Print(mesh_ofs);
+
+         ofstream q_variable_ofs("sol_q.gf");
+         q_variable_ofs.precision(8);
+         q_variable.Save(q_variable_ofs);
+
+         ofstream u_variable_ofs("sol_u.gf");
+         u_variable_ofs.precision(8);
+         u_variable.Save(u_variable_ofs);
+
+         ofstream lambda_variable_ofs("sol_lambda.gf");
+         lambda_variable_ofs.precision(8);
+         lambda_variable.Save(lambda_variable_ofs);
+      }
+
+      // 14. Send the solution by socket to a GLVis server.
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream u_sock(vishost, visport);
+         u_sock.precision(8);
+         u_sock << "solution\n" << *mesh << u_variable << "window_title 'Solution u'" <<
+                endl;
+
+         socketstream q_sock(vishost, visport);
+         q_sock.precision(8);
+         q_sock << "solution\n" << *mesh << q_variable << "window_title 'Solution q'" <<
+                endl;
+      }
+
+      // 15. Postprocessing
+      if (post)
+      {
+         FiniteElementCollection *dg_coll_pstar(new DG_FECollection(order+1, dim));
+         FiniteElementSpace *Vstar_space = new FiniteElementSpace(mesh, dg_coll_pstar);
+
+         GridFunction u_post(Vstar_space);
+
+         HDGPostProcessing *hdgpost(new HDGPostProcessing(Vstar_space, q_variable,
+                                                          u_variable, diffusion));
+
+         hdgpost->Postprocessing(u_post);
+
+         order_quad = max(2, 2*order+5);
+         for (int i=0; i < Geometry::NumGeom; ++i)
+         {
+            irs[i] = &(IntRules.Get(i, order_quad));
+         }
+         real_t err_u_post   = u_post.ComputeL2Error(ucoeff, irs);
+
+         u_star_l2errors(ref_levels-initial_ref_levels) = fabs(err_u_post);
+
+         std::cout << "|| u^*_h - u_ex || = " << err_u_post << "\n";
+
+         if (save)
+         {
+            ofstream u_post_ofs("sol_u_star.gf");
+            u_post_ofs.precision(8);
+            u_post.Save(u_post_ofs);
+         }
+
+         if (visualization)
+         {
+            char vishost[] = "localhost";
+            int  visport   = 19916;
+            socketstream u_star_sock(vishost, visport);
+            u_star_sock.precision(8);
+            u_star_sock << "solution\n" << *mesh << u_post <<
+                        "window_title 'Solution u_star'" << endl;
+         }
+
+         delete hdgpost;
+         delete Vstar_space;
+         delete dg_coll_pstar;
+      }
+
+      // 16. Refine the mesh to increase the resolution and update the spaces and the forms.
+      mesh->UniformRefinement();
+
+      V_space->Update(0);
+      W_space->Update(0);
+      M_space->Update(0);
+
+      AVarf->Update();
+      q_variable.Update();
+      u_variable.Update();
+      lambda_variable.Update();
+
+      delete R;
+      delete F;
+   }
+
+   // 17. Print the results
+   std::cout << "\n\n-----------------------\n";
+   std::cout <<
+             "level  u_l2errors  order   q_l2errors  order   mean_l2errors  order u_star_l2errors   order\n";
+   std::cout << "-----------------------\n";
+   for (int ref_levels = 0; ref_levels < total_ref_levels; ref_levels++)
+   {
+      if (ref_levels == 0)
+      {
+         std::cout << "  " << ref_levels << "    "
+                   << std::setprecision(2) << std::scientific << u_l2errors(ref_levels)
+                   << "   " << " -       "
+                   << std::setprecision(2) << std::scientific << q_l2errors(ref_levels)
+                   << "    " << " -       "
+                   << std::setprecision(2) << std::scientific << mean_l2errors(ref_levels)
+                   << "    " << " -       "
+                   << std::setprecision(2) << std::scientific << u_star_l2errors(ref_levels)
+                   << "    " << " -       " << std::endl;
+      }
+      else
+      {
+         real_t u_order     = log(u_l2errors(ref_levels)/u_l2errors(ref_levels-1))/log(
+                                 0.5);
+         real_t q_order     = log(q_l2errors(ref_levels)/q_l2errors(ref_levels-1))/log(
+                                 0.5);
+         real_t mean_order   = log(mean_l2errors(ref_levels)/mean_l2errors(
+                                      ref_levels-1))/log(0.5);
+         real_t u_star_order = log(u_star_l2errors(ref_levels)/u_star_l2errors(
+                                      ref_levels-1))/log(0.5);
+         std::cout << "  " << ref_levels << "    "
+                   << std::setprecision(2) << std::scientific << u_l2errors(ref_levels)
+                   << "  " << std::setprecision(4) << std::fixed << u_order
+                   << "    " << std::setprecision(2) << std::scientific << q_l2errors(ref_levels)
+                   << "   " << std::setprecision(4) << std::fixed << q_order
+                   << "    " << std::setprecision(2) << std::scientific << mean_l2errors(
+                      ref_levels)
+                   << "   " << std::setprecision(4) << std::fixed << mean_order
+                   << "    " << std::setprecision(2) << std::scientific << u_star_l2errors(
+                      ref_levels)
+                   << "   " << std::setprecision(4) << std::fixed << u_star_order << std::endl;
+      }
+   }
+   std::cout << "\n\n";
+
+   // 18. Free the used memory.
+   delete mesh;
+   delete V_space;
+   delete W_space;
+   delete M_space;
+   delete AVarf;
+   delete fform;
+   delete dg_coll;
+   delete face;
+
+   std::cout << "Done." << std::endl ;
+
+   return 0;
+}
+
+
+real_t uFun_ex(const Vector & x)
+{
+   real_t xi(x(0));
+   real_t yi(x(1));
+
+   int dim = x.Size();
+
+   switch (dim)
+   {
+      case 2:
+      {
+         return 1.0 + xi + sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi);
+         break;
+      }
+      case 3:
+      {
+         real_t zi(x(2));
+         return xi + sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*sin(2.0*M_PI*zi);
+         break;
+      }
+   }
+
+   return 0;
+}
+
+void qFun_ex(const Vector & x, Vector & q)
+{
+   real_t xi(x(0));
+   real_t yi(x(1));
+   int dim = x.Size();
+
+   switch (dim)
+   {
+      case 2:
+      {
+         q(0) = -diff*1.0 - diff*2.0*M_PI*cos(2.0*M_PI*xi)*sin(2.0*M_PI*yi);
+         q(1) =  0.0 - diff*2.0*M_PI*sin(2.0*M_PI*xi)*cos(2.0*M_PI*yi);
+         break;
+      }
+      case 3:
+      {
+         real_t zi(x(2));
+         q(0) = -diff*1.0 - diff*2.0*M_PI*cos(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*sin(
+                   2.0*M_PI*zi);
+         q(1) =  0.0 - diff*2.0*M_PI*sin(2.0*M_PI*xi)*cos(2.0*M_PI*yi)*sin(2.0*M_PI*zi);
+         q(2) =  0.0 - diff*2.0*M_PI*sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*cos(2.0*M_PI*zi);
+         break;
+      }
+   }
+}
+
+
+real_t fFun(const Vector & x)
+{
+   real_t xi(x(0));
+   real_t yi(x(1));
+   int dim = x.Size();
+
+   switch (dim)
+   {
+      case 2:
+      {
+         return diff*8.0*M_PI*M_PI*sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi);
+         break;
+      }
+      case 3:
+      {
+         real_t zi(x(2));
+         return diff*12.0*M_PI*M_PI*sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*sin(2.0*M_PI*zi);
+         break;
+      }
+   }
+
+   return 0;
+
+}
+
+// Postprocessing
+void HDGPostProcessing::Postprocessing(GridFunction &u_postprocessed)
+{
+   Mesh *mesh = fes->GetMesh();
+   Array<int>  vdofs;
+   Vector      elmat2, shape, RHS, to_RHS, vals, uvals;
+   real_t      RHS2;
+   DenseMatrix elmat, invdfdx, dshape, dshapedxt, qvals;
+
+   int  ndofs;
+   const FiniteElement *fe_elem;
+   ElementTransformation *Trans;
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      fes->GetElementVDofs(i, vdofs);
+      ndofs = vdofs.Size();
+      vals.SetSize(ndofs);
+      // elmat is the matrix for the -(nabla w_h, q_h) term
+      elmat.SetSize(ndofs);
+      // elmat 1 is the vector for the (1, u_h^*) term
+      elmat2.SetSize(ndofs);
+      shape.SetSize(ndofs);
+
+      RHS.SetSize(ndofs);
+      to_RHS.SetSize(ndofs);
+
+      elmat = 0.0;
+      elmat2 = 0.0;
+      RHS = 0.0;
+      RHS2 = 0.0;
+
+      fe_elem = fes->GetFE(i);
+      int dim = fe_elem->GetDim();
+      int spaceDim = dim;
+      invdfdx.SetSize(dim, spaceDim);
+      dshape.SetSize(ndofs, spaceDim);
+      dshapedxt.SetSize(ndofs, spaceDim);
+
+      Trans = mesh->GetElementTransformation(i);
+
+      const IntegrationRule *ir = IntRule;
+      if (ir == NULL)
+      {
+         int order = 3*fe_elem->GetOrder() + 3;
+         ir = &IntRules.Get(fe_elem->GetGeomType(), order);
+      }
+
+      // Get the values of u_h and q_h
+      u->GetValues(i, *ir, uvals);
+      q->GetVectorValues(*Trans, *ir, qvals);
+
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+
+         fe_elem->CalcDShape(ip, dshape);
+         fe_elem->CalcShape(ip, shape);
+
+         Trans->SetIntPoint(&ip);
+         // Compute invdfdx = / adj(J),       if J is square
+         //               \ adj(J^t.J).J^t, otherwise
+         CalcAdjugate(Trans->Jacobian(), invdfdx);
+         real_t w = Trans->Weight();
+         w = ip.weight / w;
+         w *= diffcoeff->Eval(*Trans, ip);
+         Mult(dshape, invdfdx, dshapedxt);
+
+         // compute the (nabla w_h, \nu \nabla u_h^*) term
+         AddMult_a_AAt(w, dshapedxt, elmat);
+
+         dshapedxt *= ip.weight ;
+
+         Vector qval_col;
+         qvals.GetColumn(j, qval_col);
+
+         // compute (nabla w_h, q_h)
+         dshapedxt.Mult(qval_col, to_RHS);
+
+         // subtract it from the rhs
+         RHS -= to_RHS;
+
+         // compute (1, u_h^*)
+         shape *= (Trans->Weight() * ip.weight);
+         elmat2 += shape;
+
+         // compute (1, u_h)
+         real_t rhs_weight = (Trans->Weight() * ip.weight);
+         RHS2  += (uvals(j)*rhs_weight);
+
+      }
+
+      // changing the last row and the last entry
+      for (int j = 0; j < ndofs; j++)
+      {
+         elmat(ndofs-1,j) = elmat2(j);
+      }
+      RHS(ndofs-1) = RHS2;
+
+      // solve the local problem
+      elmat.Invert();
+      elmat.Mult(RHS, vals);
+      u_postprocessed.SetSubVector(vdofs, vals);
+
+   }
+}

miniapps/hdg/hdg_poissonp.cpp

@@ -0,0 +1,831 @@
+//                  MFEM Example Hybridizable DG
+//
+// Compile with: make hdg_poissonp
+//
+// Sample runs:  mpirun -np 1 hdg_poissonp -o 1 -r 1 -tr 4 -no-vis
+//         mpirun -np 2 hdg_poissonp -o 5 -r 1 -tr 4 -no-vis
+//         mpirun -np 2 hdg_poissonp -o 1 -r 4 -tr 1
+//         mpirun -np 3 hdg_poissonp -o 5 -r 4 -tr 1
+//         mpirun -np 2 hdg_poissonp -o 1 -r 1 -tr 4 -no-vis -m ../data/inline-tri.mesh
+//         mpirun -np 2 hdg_poissonp -o 5 -r 1 -tr 4 -no-vis -m ../data/inline-tri.mesh
+//         mpirun -np 4 hdg_poissonp -o 1 -r 5 -tr 1 -m ../data/inline-tri.mesh
+//         mpirun -np 2 hdg_poissonp -o 5 -r 5 -tr 1 -m ../data/inline-tri.mesh
+//
+// Description:  This example code solves the 2D/3D diffusion problem
+//                -\nu Delta u = f
+//               with Dirichlet boundary conditions, using HDG discretization.
+//
+// The methods approximates the solution u, the diffusive flux q = -\nu \nabla u,
+// and the restriction of u to the faces, denoted by lambda.
+//
+// The weak form is: seek (q,u,\lambda) such that for all (v, w, \mu)
+//
+// -\nu^{-1}(q, v)       + (u, div(v))       - <\lambda, v \cdot n>      = 0
+//  (div(q), w)          + <\tau u, w>       - <\tau \lambda, w>         = (f, w)
+// -<[[q \cdot n]], \mu> - <[[\tau u]], \mu> + <[[(\tau \lambda]], \mu>  = 0
+//
+// where [[.]] is the jump operator, (.,.) is the d-dimensional L2 product,
+// <.,.> is the d-1 dimensional L2 product.
+//
+// The discretization is based on the paper:
+//
+// N.C. Nguyen, J. Peraire, B. Cockburn, An implicit high-order hybridizable
+// discontinuous Galerkin method for linear convection–diffusion equations,
+// J. Comput. Phys., 2009, 228:9, 3232--3254.
+//
+// Contributed by: T. Horvath, S. Rhebergen, A. Sivas
+//                 University of Waterloo
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+#include "HDGBilinearForm.hpp"
+#include "hdg_integrators.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Define the analytical solution and forcing terms / boundary conditions
+real_t uFun_ex(const Vector & x);
+void qFun_ex(const Vector & x, Vector & q);
+real_t fFun(const Vector & x);
+real_t diff;
+
+// We can minimize the expression |\nu \nabla u_h^* + q_h |^2 over a single element K,
+// for p+1 degree u_h^*, with the constraint \int_K u_h^* = \int_K u_h, so the mean
+// of u_h^* is the same as the one of u_h.
+//
+// This results in the problem
+//
+// (nabla w_h, \nu \nabla u_h^*) = -(nabla w_h, q_h)
+// (1, u_h^*)                    = (1, u_h)
+//
+// Since the fist equation on its own would generate a singular problem
+// the last line of the system is rewritten by the second equation.
+//
+// This elementwise operation will provide a superconvergent solution
+// \|u-u_h\|_{L^2} < C h^{p+2} |u|_{p+1}
+class pHDGPostProcessing
+{
+private:
+   ParGridFunction *q, *u;
+
+   ParFiniteElementSpace *pfes;
+
+   Coefficient *diffcoeff;
+protected:
+   const IntegrationRule *IntRule;
+
+public:
+   pHDGPostProcessing(ParFiniteElementSpace *f, ParGridFunction &_q,
+                      ParGridFunction &_u, Coefficient &_diffcoeff)
+      : q(&_q), u(&_u), pfes(f), diffcoeff(&_diffcoeff)
+   {
+      IntRule = NULL;
+   }
+
+   void Postprocessing(ParGridFunction &u_postprocessed) ;
+};
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Initialize MPI.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   real_t assemblyTime, solveTime, reconstructTime, pprocessTime;
+   real_t GassemblyTime, GsolveTime, GreconstructTime, GpprocessTime;
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-tri.mesh";
+   int order = 1;
+   int initial_ref_levels = 0;
+   int total_ref_levels = 2;
+   bool visualization = true;
+   bool verbose = (myid == 0);
+   bool post = true;
+   bool save = true;
+   bool hdg = true;
+   real_t memA = 0.0;
+   real_t memB = 0.0;
+   bool petsc = false;
+   const char *petscrc_file = "";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&initial_ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly for the initial calculation.");
+   args.AddOption(&total_ref_levels, "-tr", "--totalrefine",
+                  "Number of times to refine the mesh uniformly to get the convergence rates.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&post, "-post", "--postprocessing",
+                  "-no-post", "--no-postprocessing",
+                  "Enable or disable postprocessing.");
+   args.AddOption(&save, "-save", "--save-files", "-no-save",
+                  "--no-save-files",
+                  "Enable or disable file saving.");
+   args.AddOption(&hdg, "-hdg", "--hybrid", "-edg",
+                  "--embedded",
+                  "HDG / EDG option.");
+   args.AddOption(&memA, "-memA", "--memoryA",
+                  "Storage of A.");
+   args.AddOption(&memB, "-memB", "--memoryB",
+                  "Storage of B.");
+   args.AddOption(&petsc, "-petsc", "--use-petsc",
+                  "-no-petsc", "--no-use-petsc",
+                  "Enable or disable SC solver.");
+   args.AddOption(&petscrc_file, "-petscopts", "--petscopts",
+                  "PetscOptions file to use.");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (verbose)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (verbose)
+   {
+      args.PrintOptions(cout);
+   }
+
+#ifdef MFEM_USE_PETSC
+   // We initialize PETSc
+   MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL);
+#endif
+
+#ifndef MFEM_USE_PETSC
+   if (petsc)
+   {
+      std::cout << "MFEM does not use PETSc. Change the solver to hypre" << std::endl
+                << std::flush;
+      petsc = false;
+   }
+#endif
+
+   // memA, memB \in [0,1], memB <= memA
+   if (memB > memA)
+   {
+      std::cout << "memB cannot be more than memA. Resetting to be equal" << std::endl
+                << std::flush;
+      memA = memB;
+   }
+   if (memA > 1.0)
+   {
+      std::cout << "memA cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memA = 1.0;
+   }
+   else if (memA < 0.0)
+   {
+      std::cout << "memA cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memA = 0.0;
+   }
+   if (memB > 1.0)
+   {
+      std::cout << "memB cannot be more than 1. Resetting to 1" << std::endl <<
+                std::flush;
+      memB = 1.0;
+   }
+   else if (memB < 0.0)
+   {
+      std::cout << "memB cannot be less than 0. Resetting to 0." << std::endl <<
+                std::flush;
+      memB = 0.0;
+   }
+
+   // 3. Read the mesh from the given mesh file. Refine it up to the initial_ref_levels.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   if (mesh->Nonconforming())
+   {
+      if (verbose)
+      {
+         cout << "The current implementation does not support Nonconforming meshes. Terminating"
+              << endl << flush;
+      }
+#ifdef MFEM_USE_PETSC
+      MFEMFinalizePetsc();
+#endif
+      MPI_Finalize();
+      return 1;
+   }
+
+   for (int ii=0; ii<initial_ref_levels; ii++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 4. Vectors for the different discretization errors
+   Vector u_l2errors(total_ref_levels), q_l2errors(total_ref_levels),
+          mean_l2errors(total_ref_levels), u_star_l2errors(total_ref_levels);
+
+   // 5. Define a finite element collections and spaces on the mesh.
+   FiniteElementCollection *dg_coll(new DG_FECollection(order, dim));
+   FiniteElementCollection *face = NULL;
+   if (hdg)
+   {
+      face = new DG_Interface_FECollection(order, dim);
+   }
+   else
+   {
+      face = new H1_Trace_FECollection(order, dim);
+   }
+
+   // Finite element spaces:
+   // V_space is the vector valued DG space on elements for q_h
+   // W_space is the scalar DG space on elements for u_h
+   // M_space is the DG space on faces for lambda_h
+   ParFiniteElementSpace *V_space = new ParFiniteElementSpace(pmesh, dg_coll, dim);
+   ParFiniteElementSpace *W_space = new ParFiniteElementSpace(pmesh, dg_coll);
+   ParFiniteElementSpace *M_space = new ParFiniteElementSpace(pmesh, face);
+
+   // 6. Define the coefficients, the exact solutions, the right hand side and the diffusion coefficient along with the diffusion penalty parameter.
+   FunctionCoefficient fcoeff(fFun);
+
+   FunctionCoefficient ucoeff(uFun_ex);
+   VectorFunctionCoefficient qcoeff(dim, qFun_ex);
+
+   diff = 1.;
+   ConstantCoefficient diffusion(diff); // diffusion constant
+   real_t tau_D = 5.0;
+
+   // 7. Define the different forms and gridfunctions.
+   HDGBilinearForm *AVarf(new HDGBilinearForm(V_space, W_space, M_space, true));
+   AVarf->AddHDGDomainIntegrator(new HDGDomainIntegratorDiffusion(diffusion));
+   AVarf->AddHDGFaceIntegrator(new HDGFaceIntegratorDiffusion(tau_D));
+
+   ParGridFunction lambda(M_space);
+   ParGridFunction q_variable(V_space), u_variable(W_space);
+
+   ParLinearForm *fform(new ParLinearForm);
+   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
+
+   for (int ref_levels = initial_ref_levels;
+        ref_levels < (initial_ref_levels + total_ref_levels); ref_levels++)
+   {
+      // 8. Compute the problem size and define the right hand side vectors
+      HYPRE_Int dimV = V_space->GlobalTrueVSize();
+      HYPRE_Int dimW = W_space->GlobalTrueVSize();
+      HYPRE_Int dimM = M_space->GlobalTrueVSize();
+
+      if (verbose)
+      {
+         std::cout << "***********************************************************\n";
+         std::cout << "dim(V) = " << dimV << "\n";
+         std::cout << "dim(W) = " << dimW << "\n";
+         std::cout << "dim(M) = " << dimM << "\n";
+         std::cout << "dim(V+W+M) = " << dimV + dimW + dimM << "\n";
+         std::cout << "***********************************************************\n";
+      }
+
+      HypreParVector *trueR(new HypreParVector(V_space));
+      *trueR = 0.0;
+
+      HypreParVector *trueF;
+
+      // 9. To eliminate the boundary conditions we project the BC to a grid function
+      // defined for the facet unknowns.
+      lambda.ProjectCoefficientSkeleton(ucoeff);
+
+      HypreParVector *Lambda =  new HypreParVector(M_space);
+      lambda.ParallelProject(*Lambda);
+
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // 10. Assemble the RHS and the Schur complement
+      fform->Update(W_space);
+      fform->Assemble();
+
+      trueF = fform->ParallelAssemble();
+
+      // Creating a gridfunctions for the elimination of the boundary
+      ParGridFunction *R = new ParGridFunction(V_space, trueR);
+      ParGridFunction *F = new ParGridFunction(W_space, trueF);
+
+      chrono.Clear();
+      chrono.Start();
+      AVarf->AssembleSC(R, F, ess_bdr, lambda, memA, memB);
+      chrono.Stop();
+      AVarf->Finalize();
+
+      assemblyTime = chrono.RealTime();
+      HypreParMatrix *SC = AVarf->ParallelAssembleSC();
+
+      HypreParVector *rhs_SC = AVarf->ParallelVectorSC();
+      // AVarf->ParallelVectorSC() provides -C*A^{-1} RF, the RHS for the
+      // Schur complement is  L - C*A^{-1} RF, but L is zero for this case
+
+      // 11. Solve the Schur complement system
+      real_t tol = 1.0e-12;
+      int maxIter = 1000;
+      int PrintLevel = -1;
+
+      if (petsc)
+      {
+#ifdef MFEM_USE_PETSC
+         // Solver using PETSc
+         //=======================
+         PetscLinearSolver *petsc_solver;
+         PetscPreconditioner *petsc_precon= NULL;
+         petsc_solver = new PetscLinearSolver(MPI_COMM_WORLD, "solver_", 1, 0);
+         petsc_precon = new PetscPreconditioner(MPI_COMM_WORLD,*SC,"solver_");
+         petsc_solver->SetOperator(*SC);
+         petsc_solver->SetPreconditioner(*petsc_precon);
+         petsc_solver->SetTol(tol);
+         petsc_solver->SetAbsTol(0.0);
+         petsc_solver->SetMaxIter(maxIter);
+         petsc_solver->SetPrintLevel(PrintLevel);
+         chrono.Clear();
+         chrono.Start();
+         petsc_solver->Mult(*rhs_SC, *Lambda);
+         chrono.Stop();
+
+         if (verbose)
+         {
+            if (petsc_solver->GetConverged())
+               std::cout << "Solver converged in " << petsc_solver->GetNumIterations()
+                         << " iterations with a residual norm of " << petsc_solver->GetFinalNorm() <<
+                         ".\n";
+            else
+               std::cout << "Solver did not converge in " << petsc_solver->GetNumIterations()
+                         << " iterations. Residual norm is " << petsc_solver->GetFinalNorm() << ".\n";
+            std::cout << "Solver solver took " << chrono.RealTime() << "s. \n";
+         }
+
+         delete petsc_solver;
+         delete petsc_precon;
+#endif
+      }
+      else
+      {
+         HypreBoomerAMG *amg = new HypreBoomerAMG(*SC);
+         HyprePCG *pcg = new HyprePCG(*SC);
+         pcg->SetTol(tol);
+         pcg->SetMaxIter(maxIter);
+         amg->SetPrintLevel(PrintLevel);
+         pcg->SetPrintLevel(PrintLevel);
+         pcg->SetPreconditioner(*amg);
+         chrono.Clear();
+         chrono.Start();
+         pcg->Mult(*rhs_SC, *Lambda);
+         chrono.Stop();
+
+         int numIterations = 0;
+         pcg->GetNumIterations(numIterations);
+
+         if (verbose)
+         {
+            std::cout << "\nIterative method converged in "
+                      << numIterations << ".\n";
+
+            std::cout << "Iterative solver took " << chrono.RealTime() << "s. \n";
+         }
+      }
+
+      // Delete the SC matrix to save memory
+      SC = NULL;
+      solveTime = chrono.RealTime();
+
+      // 12. Reconstruction
+      // Create a gridfunction from the right hand side.
+      // It is mostly important for the parallel code,
+      // here it is done this way to make the 2 codes more similar
+      lambda = ParGridFunction(M_space, Lambda);
+
+      chrono.Clear();
+      chrono.Start();
+      AVarf->Reconstruct(R, F, &lambda, &q_variable, &u_variable);
+      chrono.Stop();
+
+      reconstructTime = chrono.RealTime();
+
+      // 13. Compute the discretization error
+      int order_quad = max(2, 2*order+1);
+      const IntegrationRule *irs[Geometry::NumGeom];
+      for (int i=0; i < Geometry::NumGeom; ++i)
+      {
+         irs[i] = &(IntRules.Get(i, order_quad));
+      }
+
+      real_t err_u   = u_variable.ComputeL2Error(ucoeff, irs);
+      real_t norm_u   = ComputeGlobalLpNorm(2., ucoeff, *pmesh, irs);
+      real_t err_q   = q_variable.ComputeL2Error(qcoeff, irs);
+      real_t norm_q   = ComputeGlobalLpNorm(2., qcoeff, *pmesh, irs);
+      real_t err_mean  = u_variable.ComputeMeanLpError(2.0, ucoeff, irs);
+
+      if (verbose)
+      {
+         std::cout << "|| u_h - u_ex || / || u_ex || = " << err_u / norm_u << "\n";
+         std::cout << "|| q_h - q_ex || / || q_ex || = " << err_q / norm_q << "\n";
+         std::cout << "|| u_h - u_ex || = " << err_u << "\n";
+         std::cout << "|| q_h - q_ex || = " << err_q << "\n";
+         std::cout << "|| mean(u_h) - mean(u_ex) || = " << err_mean << "\n";
+      }
+
+      u_l2errors(ref_levels-initial_ref_levels) = fabs(err_u);
+      q_l2errors(ref_levels-initial_ref_levels) = fabs(err_q);
+      mean_l2errors(ref_levels-initial_ref_levels) = fabs(err_mean);
+
+
+      // 14. Save the mesh and the solution.
+      if (save)
+      {
+         ostringstream mesh_name, u_name, q_name, lambda_name;
+         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+         u_name << "sol_u." << setfill('0') << setw(6) << myid;
+         q_name << "sol_q." << setfill('0') << setw(6) << myid;
+         lambda_name << "sol_lambda." << setfill('0') << setw(6) << myid;
+
+         ofstream mesh_ofs(mesh_name.str().c_str());
+         mesh_ofs.precision(8);
+         pmesh->Print(mesh_ofs);
+
+         ofstream u_ofs(u_name.str().c_str());
+         u_ofs.precision(8);
+         u_variable.Save(u_ofs);
+
+         ofstream q_ofs(q_name.str().c_str());
+         q_ofs.precision(8);
+         q_variable.Save(q_ofs);
+
+         ParGridFunction lambda_variable(M_space, Lambda);
+         ofstream lambda_ofs(lambda_name.str().c_str());
+         lambda_ofs.precision(8);
+         lambda_variable.Save(lambda_ofs);
+      }
+
+      // 15. Send the solution by socket to a GLVis server.
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream u_sock(vishost, visport);
+         u_sock << "parallel " << num_procs << " " << myid << "\n";
+         u_sock.precision(8);
+         u_sock << "solution\n" << *pmesh << u_variable << "window_title 'U'"
+                << endl;
+         // Make sure all ranks have sent their 'u' solution before initiating
+         // another set of GLVis connections (one from each rank):
+         MPI_Barrier(pmesh->GetComm());
+         socketstream q_sock(vishost, visport);
+         q_sock << "parallel " << num_procs << " " << myid << "\n";
+         q_sock.precision(8);
+         q_sock << "solution\n" << *pmesh << q_variable << "window_title 'Q'"
+                << endl;
+      }
+
+      // 16. Postprocessing
+      if (post)
+      {
+         FiniteElementCollection *dg_coll_pstar(new DG_FECollection(order+1, dim));
+         ParFiniteElementSpace *Vstar_space = new ParFiniteElementSpace(pmesh,
+                                                                        dg_coll_pstar);
+
+         ParGridFunction u_post(Vstar_space);
+
+         pHDGPostProcessing *hdgpost(new pHDGPostProcessing(Vstar_space, q_variable,
+                                                            u_variable, diffusion));
+
+         chrono.Clear();
+         chrono.Start();
+         hdgpost->Postprocessing(u_post);
+         chrono.Stop();
+
+         pprocessTime = chrono.RealTime();
+
+         order_quad = max(2, 2*order+5);
+         for (int i=0; i < Geometry::NumGeom; ++i)
+         {
+            irs[i] = &(IntRules.Get(i, order_quad));
+         }
+         real_t err_u_post   = u_post.ComputeL2Error(ucoeff, irs);
+
+         u_star_l2errors(ref_levels-initial_ref_levels) = fabs(err_u_post);
+
+         if (verbose)
+         {
+            std::cout << "|| u^*_h - u_ex || = " << err_u_post << "\n";
+         }
+
+         if (save)
+         {
+            ostringstream u_star_name;
+            u_star_name << "sol_u_star." << setfill('0') << setw(6) << myid;
+            ofstream u_star_ofs(u_star_name.str().c_str());
+            u_star_ofs.precision(8);
+            u_post.Save(u_star_ofs);
+         }
+
+         if (visualization)
+         {
+            char vishost[] = "localhost";
+            int  visport   = 19916;
+
+            MPI_Barrier(pmesh->GetComm());
+            socketstream u_star_sock(vishost, visport);
+            u_star_sock << "parallel " << num_procs << " " << myid << "\n";
+            u_star_sock.precision(8);
+            u_star_sock << "solution\n" << *pmesh << u_post << "window_title 'U_star'"
+                        << endl;
+         }
+
+         delete hdgpost;
+         delete Vstar_space;
+         delete dg_coll_pstar;
+      }
+
+      // 17. Refine the mesh to increase the resolution and update the spaces and the forms. Print the runtimes
+      pmesh->UniformRefinement();
+
+      V_space->Update(0);
+      W_space->Update(0);
+      M_space->Update(0);
+
+      AVarf->Update();
+      q_variable.Update();
+      u_variable.Update();
+      lambda.Update();
+      MPI_Reduce(&assemblyTime,&GassemblyTime,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
+      MPI_Reduce(&solveTime,&GsolveTime,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
+      MPI_Reduce(&reconstructTime,&GreconstructTime,1,MPI_DOUBLE,MPI_MAX,0,
+                 MPI_COMM_WORLD);
+      MPI_Reduce(&pprocessTime,&GpprocessTime,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
+
+      if (verbose)
+      {
+         printf("\t Assembly time   = %.2f\n",GassemblyTime);
+         printf("\t Solve time     = %.2f\n",GsolveTime);
+         printf("\t Reconstruct time = %.2f\n",GreconstructTime);
+         printf("\t Postprocess time = %.2f\n",GpprocessTime);
+      }
+
+      delete R;
+      delete F;
+   }
+
+   // 18. Print the results
+   if (verbose)
+   {
+      std::cout << "\n\n-----------------------\n";
+      std::cout <<
+                "level  u_l2errors  order   q_l2errors  order   mean_l2errors  order u_star_l2errors   order\n";
+      std::cout << "-----------------------\n";
+      for (int ref_levels = 0; ref_levels < total_ref_levels; ref_levels++)
+      {
+         if (ref_levels == 0)
+         {
+            std::cout << "  " << ref_levels << "   "
+                      << std::setprecision(2) << std::scientific << u_l2errors(ref_levels)
+                      << "   " << " -      "
+                      << std::setprecision(2) << std::scientific << q_l2errors(ref_levels)
+                      << "    " << " -      "
+                      << std::setprecision(2) << std::scientific << mean_l2errors(ref_levels)
+                      << "    " << " -      "
+                      << std::setprecision(2) << std::scientific << u_star_l2errors(ref_levels)
+                      << "    " << " -      " << std::endl;
+         }
+         else
+         {
+            real_t u_order    = log(u_l2errors(ref_levels)/u_l2errors(ref_levels-1))/log(
+                                   0.5);
+            real_t q_order    = log(q_l2errors(ref_levels)/q_l2errors(ref_levels-1))/log(
+                                   0.5);
+            real_t mean_order   = log(mean_l2errors(ref_levels)/mean_l2errors(
+                                         ref_levels-1))/log(0.5);
+            real_t u_star_order = log(u_star_l2errors(ref_levels)/u_star_l2errors(
+                                         ref_levels-1))/log(0.5);
+            std::cout << "  " << ref_levels << "   "
+                      << std::setprecision(2) << std::scientific << u_l2errors(ref_levels)
+                      << "  " << std::setprecision(4) << std::fixed << u_order
+                      << "   " << std::setprecision(2) << std::scientific << q_l2errors(ref_levels)
+                      << "   " << std::setprecision(4) << std::fixed << q_order
+                      << "   " << std::setprecision(2) << std::scientific << mean_l2errors(ref_levels)
+                      << "   " << std::setprecision(4) << std::fixed << mean_order
+                      << "   " << std::setprecision(2) << std::scientific << u_star_l2errors(
+                         ref_levels)
+                      << "   " << std::setprecision(4) << std::fixed << u_star_order << std::endl;
+         }
+      }
+   }
+
+   // 19. Free the used memory.
+   delete pmesh;
+   delete V_space;
+   delete W_space;
+   delete M_space;
+   delete AVarf;
+   delete fform;
+   delete dg_coll;
+   delete face;
+
+   if (verbose)
+   {
+      std::cout << "\n\nDone." << std::endl ;
+   }
+
+#ifdef MFEM_USE_PETSC
+   MFEMFinalizePetsc();
+#endif
+   MPI_Finalize();
+
+   return 0;
+}
+
+real_t uFun_ex(const Vector & x)
+{
+   real_t xi(x(0));
+   real_t yi(x(1));
+   int dim = x.Size();
+
+   switch (dim)
+   {
+      case 2:
+      {
+         return 1.0 + xi + sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi);
+         break;
+      }
+      case 3:
+      {
+         real_t zi(x(2));
+         return 1.0*xi + sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*sin(2.0*M_PI*zi);
+         break;
+      }
+   }
+
+   return 0;
+}
+
+void qFun_ex(const Vector & x, Vector & q)
+{
+   real_t xi(x(0));
+   real_t yi(x(1));
+   int dim = x.Size();
+
+   switch (dim)
+   {
+      case 2:
+      {
+         q(0) = -diff*1.0 - diff*2.0*M_PI*cos(2.0*M_PI*xi)*sin(2.0*M_PI*yi);
+         q(1) = 0.0 - diff*2.0*M_PI*sin(2.0*M_PI*xi)*cos(2.0*M_PI*yi);
+         break;
+      }
+      case 3:
+      {
+         real_t zi(x(2));
+         q(0) = -diff*1.0 - diff*2.0*M_PI*cos(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*sin(
+                   2.0*M_PI*zi);
+         q(1) = - diff*2.0*M_PI*sin(2.0*M_PI*xi)*cos(2.0*M_PI*yi)*sin(2.0*M_PI*zi);
+         q(2) = - diff*2.0*M_PI*sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*cos(2.0*M_PI*zi);
+         break;
+      }
+   }
+}
+
+
+real_t fFun(const Vector & x)
+{
+   real_t xi(x(0));
+   real_t yi(x(1));
+   int dim = x.Size();
+
+   switch (dim)
+   {
+      case 2:
+      {
+         return diff*8.0*M_PI*M_PI*sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi);
+         break;
+      }
+      case 3:
+      {
+         real_t zi(x(2));
+         return diff*12.0*M_PI*M_PI*sin(2.0*M_PI*xi)*sin(2.0*M_PI*yi)*sin(2.0*M_PI*zi);
+         break;
+      }
+   }
+
+   return 0;
+
+}
+
+void pHDGPostProcessing::Postprocessing(ParGridFunction &u_postprocessed)
+{
+   Mesh *mesh = pfes->GetMesh();
+   Array<int>  vdofs;
+   Vector      elmat2, shape, RHS, to_RHS, vals, uvals;
+   real_t      RHS2;
+   DenseMatrix elmat, invdfdx, dshape, dshapedxt, qvals;
+
+   int  ndofs;
+   const FiniteElement *fe_elem;
+   ElementTransformation *Trans;
+
+   for (int i = 0; i < pfes->GetNE(); i++)
+   {
+      pfes->GetElementVDofs(i, vdofs);
+      ndofs = vdofs.Size();
+      vals.SetSize(ndofs);
+      vals = 0.0;
+      elmat.SetSize(ndofs);
+      elmat2.SetSize(ndofs);
+      shape.SetSize(ndofs);
+
+      RHS.SetSize(ndofs);
+      to_RHS.SetSize(ndofs);
+
+      elmat = 0.0;
+      elmat2 = 0.0;
+      RHS = 0.0;
+      RHS2 = 0.0;
+
+      fe_elem = pfes->GetFE(i);
+      int dim = fe_elem->GetDim();
+      int spaceDim = dim;
+      Vector qval_col;
+      qval_col.SetSize(dim);
+      invdfdx.SetSize(dim, spaceDim);
+      dshape.SetSize(ndofs, spaceDim);
+      dshapedxt.SetSize(ndofs, spaceDim);
+
+      Trans = mesh->GetElementTransformation(i);
+
+      const IntegrationRule *ir = IntRule;
+      if (ir == NULL)
+      {
+         int order = 2*fe_elem->GetOrder() + 2;
+         ir = &IntRules.Get(fe_elem->GetGeomType(), order);
+      }
+
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+
+         fe_elem->CalcDShape(ip, dshape);
+         fe_elem->CalcShape(ip, shape);
+
+         Trans->SetIntPoint(&ip);
+         // Compute invdfdx = / adj(J),      if J is square
+         //            \ adj(J^t.J).J^t, otherwise
+         CalcAdjugate(Trans->Jacobian(), invdfdx);
+         real_t w = Trans->Weight();
+         w = ip.weight / w;
+         w *= diffcoeff->Eval(*Trans, ip);
+         Mult(dshape, invdfdx, dshapedxt);
+
+         AddMult_a_AAt(w, dshapedxt, elmat);
+
+         dshapedxt *= ip.weight ;
+
+         qval_col = 0.0;
+         for (int ii = 0; ii<dim; ii++)
+         {
+            qval_col(ii) = q->GetValue(i, ip, (ii+1));
+         }
+
+         dshapedxt.Mult(qval_col, to_RHS);
+
+         RHS -= to_RHS;
+
+         shape *= (Trans->Weight() * ip.weight);
+         elmat2 += shape;
+
+         real_t uvalsj;
+         uvalsj = u->GetValue(i, ip, 1);
+
+         real_t rhs_weight = (Trans->Weight() * ip.weight);
+         RHS2  += (uvalsj*rhs_weight);
+
+      }
+
+      // changing the last row and the last entry
+      for (int j = 0; j < ndofs; j++)
+      {
+         elmat(ndofs-1,j) = elmat2(j);
+      }
+      RHS(ndofs-1) = RHS2;
+
+      elmat.Invert();
+      elmat.Mult(RHS, vals);
+      u_postprocessed.SetSubVector(vdofs, vals);
+
+   }
+}

miniapps/hdg/makefile

@@ -0,0 +1,73 @@
+# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/hdg/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_MINIAPPS = hdg_advection hdg_poisson
+PAR_MINIAPPS = hdg_advectionp hdg_poissonp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+all: $(MINIAPPS)
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) HPP-CPP
+	$(MFEM_CXX) $(MFEM_FLAGS) $< *.o -o $@ $(MFEM_LIBS)
+
+HPP-CPP:
+	$(MFEM_CXX) $(MFEM_FLAGS) -c HDGBilinearForm.cpp
+	$(MFEM_CXX) $(MFEM_FLAGS) -c hdg_integrators.cpp
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), HDG miniapp)
+%-test-seq: %
+	@$(call mfem-test,$<,, HDG miniapp)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(MINIAPPS)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -rf mesh* sol*
+
+clean-exec:
+	@rm -rf *.mesh *.gf

miniapps/navier/navier_cht.cpp

@@ -461,7 +461,7 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 
    Kmat.Mult(u, z);
    z.Neg(); // z = -z
-   K->ParallelEliminateTDofsInRHS(ess_tdof_list, u, z);
+   K->EliminateVDofsInRHS(ess_tdof_list, u, z);
 
    M_solver.Mult(z, du_dt);
    du_dt.Print();
@@ -483,7 +483,7 @@ void ConductionOperator::ImplicitSolve(const real_t dt,
    MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
    Kmat.Mult(u, z);
    z.Neg();
-   K->ParallelEliminateTDofsInRHS(ess_tdof_list, u, z);
+   K->EliminateVDofsInRHS(ess_tdof_list, u, z);
 
    T_solver.Mult(z, du_dt);
    du_dt.SetSubVector(ess_tdof_list, 0.0);

miniapps/shifted/distance.cpp

@@ -87,8 +87,6 @@
 //
 //   Problem 4: level set: Union of doughnut and swiss cheese shapes
 //     mpirun -np 4 distance -m ../../data/inline-hex.mesh -rs 3 -o 2 -t 1.0 -p 4
-//   Problem 5: point source in mfem mesh.
-//     mpirun -np 4 distance -m ../../data/mfem.mesh -p 5 -rs 3 -t 300.0
 
 #include <fstream>
 #include <iostream>
@@ -235,8 +233,7 @@ int main(int argc, char *argv[])
                   "1: Circle / sphere level set in 2D / 3D\n\t"
                   "2: 2D sine-looking level set\n\t"
                   "3: Gyroid level set in 2D or 3D\n\t"
-                  "4: Combo of a doughnut and swiss cheese shapes in 3D.\n\t"
-                  "5: Point source in MFEM mesh.");
+                  "4: Combo of a doughnut and swiss cheese shapes in 3D.");
    args.AddOption(&rs_levels, "-rs", "--refine-serial",
                   "Number of times to refine the mesh uniformly in serial.");
    args.AddOption(&order, "-o", "--order",
@@ -302,11 +299,6 @@ int main(int argc, char *argv[])
       ls_coeff = new FunctionCoefficient(doughnut_cheese);
       smooth_steps = 0;
    }
-   else if (problem == 5)
-   {
-      ls_coeff = new DeltaCoefficient(0.0, 0.0, 1000.0);
-      smooth_steps = 0;
-   }
    else { MFEM_ABORT("Unrecognized -problem option."); }
 
    const real_t dx = AvgElementSize(pmesh);
@@ -314,7 +306,7 @@ int main(int argc, char *argv[])
    if (solver_type == 0)
    {
       auto ds = new HeatDistanceSolver(t_param * dx * dx);
-      if (problem == 0 || problem == 5)
+      if (problem == 0)
       {
          ds->transform = false;
       }
@@ -342,7 +334,7 @@ int main(int argc, char *argv[])
    // Smooth-out Gibbs oscillations from the input level set. The smoothing
    // parameter here is specified to be mesh dependent with length scale dx.
    ParGridFunction filt_gf(&pfes_s);
-   if (problem != 0 && problem != 5)
+   if (problem != 0)
    {
       real_t filter_weight = dx;
       // The normalization-based solver needs a more diffused input.

tests/unit/general/test_reduction.cpp

@@ -22,6 +22,7 @@ using namespace mfem;
 
 TEST_CASE("Reduce Sum", "[Reduction],[GPU]")
 {
+   Array<int> workspace;
    Array<int> a(1000);
    a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -35,7 +36,7 @@ TEST_CASE("Reduce Sum", "[Reduction],[GPU]")
       int res = 0;
       mfem::reduce(
       a.Size(), res, [=] MFEM_HOST_DEVICE(int i, int &r) { r += dptr[i]; },
-      SumReducer<int> {}, use_dev);
+      SumReducer<int> {}, use_dev, workspace);
       // correct for even-length summations
       int expected = (AsConst(a)[0] + AsConst(a)[a.Size() - 1]) * a.Size() / 2;
       CAPTURE(use_dev);
@@ -45,6 +46,7 @@ TEST_CASE("Reduce Sum", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce Mult", "[Reduction],[GPU]")
 {
+   Array<long long> workspace;
    Array<long long> a(64);
    a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -62,7 +64,7 @@ TEST_CASE("Reduce Mult", "[Reduction],[GPU]")
          mfem::reduce(
             a.Size(), res,
          [=] MFEM_HOST_DEVICE(int i, long long &r) { r *= dptr[i]; },
-         MultReducer<long long> {}, use_dev);
+         MultReducer<long long> {}, use_dev, workspace);
          long long expected = 0;
          CAPTURE(use_dev);
          REQUIRE(res == expected);
@@ -74,7 +76,7 @@ TEST_CASE("Reduce Mult", "[Reduction],[GPU]")
          mfem::reduce(
             a.Size(), res,
          [=] MFEM_HOST_DEVICE(int i, long long &r) { r *= dptr[i]; },
-         MultReducer<long long> {}, use_dev);
+         MultReducer<long long> {}, use_dev, workspace);
          long long expected = 21936950640377856;
          CAPTURE(use_dev);
          REQUIRE(res == expected);
@@ -84,6 +86,7 @@ TEST_CASE("Reduce Mult", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce BAnd", "[Reduction],[GPU]")
 {
+   Array<unsigned> workspace;
    Array<unsigned> a(10);
    SECTION("{ Bit unset }")
    {
@@ -105,7 +108,7 @@ TEST_CASE("Reduce BAnd", "[Reduction],[GPU]")
          mfem::reduce(
             a.Size(), res,
          [=] MFEM_HOST_DEVICE(int i, unsigned &r) { r &= dptr[i]; },
-         BAndReducer<unsigned> {}, use_dev);
+         BAndReducer<unsigned> {}, use_dev, workspace);
          CAPTURE(use_dev);
          REQUIRE(res == ((~1u) & ~(1u << unset_bit)));
          REQUIRE((res & (1u << unset_bit)) == 0);
@@ -129,7 +132,7 @@ TEST_CASE("Reduce BAnd", "[Reduction],[GPU]")
          mfem::reduce(
             a.Size(), res,
          [=] MFEM_HOST_DEVICE(int i, unsigned &r) { r &= dptr[i]; },
-         BAndReducer<unsigned> {}, use_dev);
+         BAndReducer<unsigned> {}, use_dev, workspace);
          CAPTURE(use_dev);
          REQUIRE(res == (1u << set_bit));
       }
@@ -138,6 +141,7 @@ TEST_CASE("Reduce BAnd", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce BOr", "[Reduction],[GPU]")
 {
+   Array<unsigned> workspace;
    Array<unsigned> a(0x210);
    a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -153,7 +157,7 @@ TEST_CASE("Reduce BOr", "[Reduction],[GPU]")
       mfem::reduce(
          a.Size(), res,
       [=] MFEM_HOST_DEVICE(int i, unsigned &r) { r |= dptr[i]; },
-      BOrReducer<unsigned> {}, use_dev);
+      BOrReducer<unsigned> {}, use_dev, workspace);
       CAPTURE(use_dev);
       REQUIRE(res == 0x3ffu);
    }
@@ -161,6 +165,7 @@ TEST_CASE("Reduce BOr", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce Min", "[Reduction],[GPU]")
 {
+   Array<int> workspace;
    Array<int> a(1000);
    auto hptr = a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -185,7 +190,7 @@ TEST_CASE("Reduce Min", "[Reduction],[GPU]")
             r = dptr[i];
          }
       },
-      MinReducer<int> {}, use_dev);
+      MinReducer<int> {}, use_dev, workspace);
       CAPTURE(use_dev);
       REQUIRE(res == -10);
    }
@@ -193,6 +198,7 @@ TEST_CASE("Reduce Min", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce Max", "[Reduction],[GPU]")
 {
+   Array<int> workspace;
    Array<int> a(1000);
    auto hptr = a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -217,7 +223,7 @@ TEST_CASE("Reduce Max", "[Reduction],[GPU]")
             r = dptr[i];
          }
       },
-      MaxReducer<int> {}, use_dev);
+      MaxReducer<int> {}, use_dev, workspace);
       CAPTURE(use_dev);
       REQUIRE(res == 999 - 10);
    }
@@ -225,6 +231,7 @@ TEST_CASE("Reduce Max", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce MinMax", "[Reduction],[GPU]")
 {
+   Array<DevicePair<int, int>> workspace;
    Array<int> a(1000);
    auto hptr = a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -255,7 +262,7 @@ TEST_CASE("Reduce MinMax", "[Reduction],[GPU]")
             r.second = dptr[i];
          }
       },
-      MinMaxReducer<int> {}, use_dev);
+      MinMaxReducer<int> {}, use_dev, workspace);
       CAPTURE(use_dev);
       REQUIRE(res.first == -10);
       REQUIRE(res.second == a.Size() - 11);
@@ -264,6 +271,7 @@ TEST_CASE("Reduce MinMax", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce ArgMin", "[Reduction],[GPU]")
 {
+   Array<DevicePair<double, int>> workspace;
    Array<double> a(1000);
    auto hptr = a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -289,7 +297,7 @@ TEST_CASE("Reduce ArgMin", "[Reduction],[GPU]")
             r.second = i;
          }
       },
-      ArgMinReducer<double, int> {}, use_dev);
+      ArgMinReducer<double, int> {}, use_dev, workspace);
       CAPTURE(use_dev);
       REQUIRE(res.first == -10);
       REQUIRE(res.second >= 0);
@@ -300,6 +308,7 @@ TEST_CASE("Reduce ArgMin", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce ArgMax", "[Reduction],[GPU]")
 {
+   Array<DevicePair<double, int>> workspace;
    Array<double> a(1000);
 
    auto hptr = a.HostReadWrite();
@@ -328,7 +337,7 @@ TEST_CASE("Reduce ArgMax", "[Reduction],[GPU]")
             r.second = i;
          }
       },
-      ArgMaxReducer<double, int> {}, use_dev);
+      ArgMaxReducer<double, int> {}, use_dev, workspace);
       CAPTURE(use_dev);
       REQUIRE(res.first == a.Size() - 11);
       REQUIRE(res.second >= 0);
@@ -339,6 +348,7 @@ TEST_CASE("Reduce ArgMax", "[Reduction],[GPU]")
 
 TEST_CASE("Reduce ArgMinMax", "[Reduction],[GPU]")
 {
+   Array<MinMaxLocScalar<double, int>> workspace;
    Array<double> a(1000);
    auto hptr = a.HostReadWrite();
    for (int i = 0; i < a.Size(); ++i)
@@ -373,7 +383,7 @@ TEST_CASE("Reduce ArgMinMax", "[Reduction],[GPU]")
             r.max_loc = i;
          }
       },
-      ArgMinMaxReducer<double, int> {}, use_dev);
+      ArgMinMaxReducer<double, int> {}, use_dev, workspace);
       CAPTURE(use_dev);
       REQUIRE(res.min_val == -10);
       REQUIRE(res.min_loc >= 0);

tests/unit/linalg/test_matrix_dense.cpp

@@ -373,7 +373,6 @@ TEST_CASE("Batched Linear Algebra",
    const int n_rhs = 2;
 
    DenseTensor A_batch(n, n, n_mat);
-   DenseTensor A_inv_batch(n, n, n_mat);
    Vector x_batch(n * n_rhs * n_mat), y_batch(n * n_rhs * n_mat);
    std::vector<DenseMatrix> As;
    std::vector<DenseMatrix> xs, ys;
@@ -405,7 +404,6 @@ TEST_CASE("Batched Linear Algebra",
       ys.back() = 0.0;
       AddMult_a(1.5, As.back(), xs.back(), ys.back());
       A_batch(i) = As.back();
-      A_inv_batch(i) = As.back();
    }
 
    // Test batched matrix-vector products
@@ -465,33 +463,6 @@ TEST_CASE("Batched Linear Algebra",
          }
       }
    }
-
-   // Test batched matrix inverse
-   BatchedLinAlg::Get(backend).Invert(A_inv_batch);
-   A_inv_batch.HostReadWrite();
-   Vector output_col(n);
-   Vector col;
-   for (int i = 0; i < n_mat; ++i)
-   {
-      DenseMatrix Ai_inv(A_inv_batch(i));
-      for (int j = 0; j < n; ++j)
-      {
-         output_col = 0.0;
-         As[i].GetColumnReference(j, col);
-         Ai_inv.Mult(col, output_col);
-         for (int k = 0; k < n; ++k)
-         {
-            if (j == k)
-            {
-               REQUIRE(output_col(k) == MFEM_Approx(1.0));
-            }
-            else
-            {
-               REQUIRE(output_col(k) == MFEM_Approx(0.0));
-            }
-         }
-      }
-   }
 }
 
 TEST_CASE("DenseTensor copy", "[DenseMatrix][DenseTensor]")

tests/unit/linalg/test_matrix_sparse.cpp

@@ -240,54 +240,4 @@ TEST_CASE("SparseMatrix printing", "[SparseMatrix]")
    }
 }
 
-TEST_CASE("SparseMatrix cuSPARSE Bug", "[SparseMatrix][GPU]")
-{
-   // This test case ensures that we have a functioning workaround for the bug
-   // CUSPARSE-1897. In versions of cuSPARSE before 12.8, the internal buffer
-   // used for cusparseSpMV must be the same when it is called with the same
-   // matrix.
-   //
-   // By default, MFEM uses one buffer, that is shared by all sparse matrices.
-   // In the code below, a buffer is created for A, then modified for B, then
-   // used again for A. Without the workaround, this fails with cuSPARSE version
-   // earlier than 12.8 (confirmed to fail with 12.4).
-
-   const int n = 100;
-   SparseMatrix A(n, n);
-   Vector d(n);
-   d.Randomize(1);
-   for (int i = 0; i < n; ++i)
-   {
-      A.Set(i, i, d[i]);
-   }
-   A.Finalize();
-
-   Vector x(n);
-   x = 1.0;
-
-   Vector y(n);
-   A.Mult(x, y);
-
-   {
-      SparseMatrix B(20, 20);
-      for (int i = 0; i < 20; ++i)
-      {
-         for (int j = 0; j < 20; ++j)
-         {
-            B.Set(i, j, 1.0);
-         }
-      }
-      B.Finalize();
-      Vector u(20);
-      u = 1.0;
-      Vector v(20);
-      B.Mult(u, v);
-   }
-
-   A.Mult(x, y);
-
-   y -= d;
-   REQUIRE(y.Normlinf() == MFEM_Approx(0.0));
-}
-
 } // namespace mfem