753 lines
26 KiB
C++
753 lines
26 KiB
C++
// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
|
|
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
|
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
|
//
|
|
// This file is part of the MFEM library. For more information and source code
|
|
// availability visit https://mfem.org.
|
|
//
|
|
// MFEM is free software; you can redistribute it and/or modify it under the
|
|
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
|
// CONTRIBUTING.md for details.
|
|
|
|
#include "../general/forall.hpp"
|
|
#include "bilininteg.hpp"
|
|
#include "gridfunc.hpp"
|
|
#include "qfunction.hpp"
|
|
#include "ceed/integrators/mass/mass.hpp"
|
|
#include "bilininteg_mass_pa.hpp"
|
|
|
|
using namespace std;
|
|
|
|
namespace mfem
|
|
{
|
|
|
|
// PA Mass Integrator
|
|
|
|
// PA Mass Assemble kernel
|
|
|
|
void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
|
|
{
|
|
const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
|
|
Device::GetDeviceMemoryType() : pa_mt;
|
|
|
|
// Assuming the same element type
|
|
fespace = &fes;
|
|
Mesh *mesh = fes.GetMesh();
|
|
if (mesh->GetNE() == 0) { return; }
|
|
const FiniteElement &el = *fes.GetFE(0);
|
|
ElementTransformation *T0 = mesh->GetElementTransformation(0);
|
|
const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
|
|
if (DeviceCanUseCeed())
|
|
{
|
|
delete ceedOp;
|
|
const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
|
|
fes.IsVariableOrder();
|
|
if (mixed)
|
|
{
|
|
ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
|
|
}
|
|
else
|
|
{
|
|
ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
|
|
}
|
|
return;
|
|
}
|
|
int map_type = el.GetMapType();
|
|
dim = mesh->Dimension();
|
|
ne = fes.GetMesh()->GetNE();
|
|
nq = ir->GetNPoints();
|
|
geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
|
|
GeometricFactors::JACOBIANS, mt);
|
|
maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
|
|
dofs1D = maps->ndof;
|
|
quad1D = maps->nqpt;
|
|
pa_data.SetSize(ne*nq, mt);
|
|
|
|
QuadratureSpace qs(*mesh, *ir);
|
|
CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
|
|
|
|
if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
|
|
if (dim==2)
|
|
{
|
|
const int NE = ne;
|
|
const int Q1D = quad1D;
|
|
const bool const_c = coeff.Size() == 1;
|
|
const bool by_val = map_type == FiniteElement::VALUE;
|
|
const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
|
|
const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
|
|
const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
|
|
Reshape(coeff.Read(), Q1D,Q1D,NE);
|
|
auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
|
|
MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
|
|
{
|
|
MFEM_FOREACH_THREAD(qx,x,Q1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(qy,y,Q1D)
|
|
{
|
|
const double J11 = J(qx,qy,0,0,e);
|
|
const double J12 = J(qx,qy,1,0,e);
|
|
const double J21 = J(qx,qy,0,1,e);
|
|
const double J22 = J(qx,qy,1,1,e);
|
|
const double detJ = (J11*J22)-(J21*J12);
|
|
const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
|
|
v(qx,qy,e) = W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
if (dim==3)
|
|
{
|
|
const int NE = ne;
|
|
const int Q1D = quad1D;
|
|
const bool const_c = coeff.Size() == 1;
|
|
const bool by_val = map_type == FiniteElement::VALUE;
|
|
const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
|
|
const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
|
|
const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
|
|
Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
|
|
auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
|
|
MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
|
|
{
|
|
MFEM_FOREACH_THREAD(qx,x,Q1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(qy,y,Q1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(qz,z,Q1D)
|
|
{
|
|
const double J11 = J(qx,qy,qz,0,0,e);
|
|
const double J21 = J(qx,qy,qz,1,0,e);
|
|
const double J31 = J(qx,qy,qz,2,0,e);
|
|
const double J12 = J(qx,qy,qz,0,1,e);
|
|
const double J22 = J(qx,qy,qz,1,1,e);
|
|
const double J32 = J(qx,qy,qz,2,1,e);
|
|
const double J13 = J(qx,qy,qz,0,2,e);
|
|
const double J23 = J(qx,qy,qz,1,2,e);
|
|
const double J33 = J(qx,qy,qz,2,2,e);
|
|
const double detJ = J11 * (J22 * J33 - J32 * J23) -
|
|
/* */ J21 * (J12 * J33 - J32 * J13) +
|
|
/* */ J31 * (J12 * J23 - J22 * J13);
|
|
const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
|
|
v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0>
|
|
static void PAMassAssembleDiagonal2D(const int NE,
|
|
const Array<double> &b,
|
|
const Vector &d,
|
|
Vector &y,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
MFEM_VERIFY(D1D <= MAX_D1D, "");
|
|
MFEM_VERIFY(Q1D <= MAX_Q1D, "");
|
|
auto B = Reshape(b.Read(), Q1D, D1D);
|
|
auto D = Reshape(d.Read(), Q1D, Q1D, NE);
|
|
auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
|
|
MFEM_FORALL(e, NE,
|
|
{
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
|
|
constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
double QD[MQ1][MD1];
|
|
for (int qx = 0; qx < Q1D; ++qx)
|
|
{
|
|
for (int dy = 0; dy < D1D; ++dy)
|
|
{
|
|
QD[qx][dy] = 0.0;
|
|
for (int qy = 0; qy < Q1D; ++qy)
|
|
{
|
|
QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
|
|
}
|
|
}
|
|
}
|
|
for (int dy = 0; dy < D1D; ++dy)
|
|
{
|
|
for (int dx = 0; dx < D1D; ++dx)
|
|
{
|
|
for (int qx = 0; qx < Q1D; ++qx)
|
|
{
|
|
Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
|
|
static void SmemPAMassAssembleDiagonal2D(const int NE,
|
|
const Array<double> &b_,
|
|
const Vector &d_,
|
|
Vector &y_,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int NBZ = T_NBZ ? T_NBZ : 1;
|
|
constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
|
|
MFEM_VERIFY(D1D <= MD1, "");
|
|
MFEM_VERIFY(Q1D <= MQ1, "");
|
|
auto b = Reshape(b_.Read(), Q1D, D1D);
|
|
auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
|
|
auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
|
|
MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
|
|
{
|
|
const int tidz = MFEM_THREAD_ID(z);
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int NBZ = T_NBZ ? T_NBZ : 1;
|
|
constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
|
|
MFEM_SHARED double B[MQ1][MD1];
|
|
MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
|
|
double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
|
|
if (tidz == 0)
|
|
{
|
|
MFEM_FOREACH_THREAD(d,y,D1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(q,x,Q1D)
|
|
{
|
|
B[q][d] = b(q,d);
|
|
}
|
|
}
|
|
}
|
|
MFEM_SYNC_THREAD;
|
|
MFEM_FOREACH_THREAD(qx,x,Q1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(dy,y,D1D)
|
|
{
|
|
QD[qx][dy] = 0.0;
|
|
for (int qy = 0; qy < Q1D; ++qy)
|
|
{
|
|
QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
|
|
}
|
|
}
|
|
}
|
|
MFEM_SYNC_THREAD;
|
|
MFEM_FOREACH_THREAD(dy,y,D1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(dx,x,D1D)
|
|
{
|
|
for (int qx = 0; qx < Q1D; ++qx)
|
|
{
|
|
// might need absolute values on next line
|
|
Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0>
|
|
static void PAMassAssembleDiagonal3D(const int NE,
|
|
const Array<double> &b,
|
|
const Vector &d,
|
|
Vector &y,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
MFEM_VERIFY(D1D <= MAX_D1D, "");
|
|
MFEM_VERIFY(Q1D <= MAX_Q1D, "");
|
|
auto B = Reshape(b.Read(), Q1D, D1D);
|
|
auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
|
|
auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
|
|
MFEM_FORALL(e, NE,
|
|
{
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
|
|
constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
double QQD[MQ1][MQ1][MD1];
|
|
double QDD[MQ1][MD1][MD1];
|
|
for (int qx = 0; qx < Q1D; ++qx)
|
|
{
|
|
for (int qy = 0; qy < Q1D; ++qy)
|
|
{
|
|
for (int dz = 0; dz < D1D; ++dz)
|
|
{
|
|
QQD[qx][qy][dz] = 0.0;
|
|
for (int qz = 0; qz < Q1D; ++qz)
|
|
{
|
|
QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (int qx = 0; qx < Q1D; ++qx)
|
|
{
|
|
for (int dz = 0; dz < D1D; ++dz)
|
|
{
|
|
for (int dy = 0; dy < D1D; ++dy)
|
|
{
|
|
QDD[qx][dy][dz] = 0.0;
|
|
for (int qy = 0; qy < Q1D; ++qy)
|
|
{
|
|
QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (int dz = 0; dz < D1D; ++dz)
|
|
{
|
|
for (int dy = 0; dy < D1D; ++dy)
|
|
{
|
|
for (int dx = 0; dx < D1D; ++dx)
|
|
{
|
|
double t = 0.0;
|
|
for (int qx = 0; qx < Q1D; ++qx)
|
|
{
|
|
t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
|
|
}
|
|
Y(dx, dy, dz, e) += t;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0>
|
|
static void SmemPAMassAssembleDiagonal3D(const int NE,
|
|
const Array<double> &b_,
|
|
const Vector &d_,
|
|
Vector &y_,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
|
|
MFEM_VERIFY(D1D <= MD1, "");
|
|
MFEM_VERIFY(Q1D <= MQ1, "");
|
|
auto b = Reshape(b_.Read(), Q1D, D1D);
|
|
auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
|
|
auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
|
|
MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
|
|
{
|
|
const int tidz = MFEM_THREAD_ID(z);
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
|
|
constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
MFEM_SHARED double B[MQ1][MD1];
|
|
MFEM_SHARED double QQD[MQ1][MQ1][MD1];
|
|
MFEM_SHARED double QDD[MQ1][MD1][MD1];
|
|
if (tidz == 0)
|
|
{
|
|
MFEM_FOREACH_THREAD(d,y,D1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(q,x,Q1D)
|
|
{
|
|
B[q][d] = b(q,d);
|
|
}
|
|
}
|
|
}
|
|
MFEM_SYNC_THREAD;
|
|
MFEM_FOREACH_THREAD(qx,x,Q1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(qy,y,Q1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(dz,z,D1D)
|
|
{
|
|
QQD[qx][qy][dz] = 0.0;
|
|
for (int qz = 0; qz < Q1D; ++qz)
|
|
{
|
|
QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
MFEM_SYNC_THREAD;
|
|
MFEM_FOREACH_THREAD(qx,x,Q1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(dz,z,D1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(dy,y,D1D)
|
|
{
|
|
QDD[qx][dy][dz] = 0.0;
|
|
for (int qy = 0; qy < Q1D; ++qy)
|
|
{
|
|
QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
MFEM_SYNC_THREAD;
|
|
MFEM_FOREACH_THREAD(dz,z,D1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(dy,y,D1D)
|
|
{
|
|
MFEM_FOREACH_THREAD(dx,x,D1D)
|
|
{
|
|
double t = 0.0;
|
|
for (int qx = 0; qx < Q1D; ++qx)
|
|
{
|
|
t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
|
|
}
|
|
Y(dx, dy, dz, e) += t;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
static void PAMassAssembleDiagonal(const int dim, const int D1D,
|
|
const int Q1D, const int NE,
|
|
const Array<double> &B,
|
|
const Vector &D,
|
|
Vector &Y)
|
|
{
|
|
if (dim == 2)
|
|
{
|
|
switch ((D1D << 4 ) | Q1D)
|
|
{
|
|
case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
|
|
case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
|
|
case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
|
|
case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
|
|
case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
|
|
case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
|
|
case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
|
|
case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
|
|
default: return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
|
|
}
|
|
}
|
|
else if (dim == 3)
|
|
{
|
|
switch ((D1D << 4 ) | Q1D)
|
|
{
|
|
case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
|
|
case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
|
|
case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
|
|
case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
|
|
case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
|
|
case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
|
|
case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
|
|
case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
|
|
case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
|
|
case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
|
|
case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
|
|
default: return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
|
|
}
|
|
}
|
|
MFEM_ABORT("Unknown kernel.");
|
|
}
|
|
|
|
void MassIntegrator::AssembleDiagonalPA(Vector &diag)
|
|
{
|
|
if (DeviceCanUseCeed())
|
|
{
|
|
ceedOp->GetDiagonal(diag);
|
|
}
|
|
else
|
|
{
|
|
PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
|
|
}
|
|
}
|
|
|
|
|
|
#ifdef MFEM_USE_OCCA
|
|
// OCCA PA Mass Apply 2D kernel
|
|
static void OccaPAMassApply2D(const int D1D,
|
|
const int Q1D,
|
|
const int NE,
|
|
const Array<double> &B,
|
|
const Array<double> &Bt,
|
|
const Vector &D,
|
|
const Vector &X,
|
|
Vector &Y)
|
|
{
|
|
occa::properties props;
|
|
props["defines/D1D"] = D1D;
|
|
props["defines/Q1D"] = Q1D;
|
|
const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
|
|
const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
|
|
const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
|
|
const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
|
|
occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
|
|
const occa_id_t id = std::make_pair(D1D,Q1D);
|
|
if (!Device::Allows(Backend::OCCA_CUDA))
|
|
{
|
|
static occa_kernel_t OccaMassApply2D_cpu;
|
|
if (OccaMassApply2D_cpu.find(id) == OccaMassApply2D_cpu.end())
|
|
{
|
|
const occa::kernel MassApply2D_CPU =
|
|
mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
|
|
"MassApply2D_CPU", props);
|
|
OccaMassApply2D_cpu.emplace(id, MassApply2D_CPU);
|
|
}
|
|
OccaMassApply2D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
|
|
}
|
|
else
|
|
{
|
|
static occa_kernel_t OccaMassApply2D_gpu;
|
|
if (OccaMassApply2D_gpu.find(id) == OccaMassApply2D_gpu.end())
|
|
{
|
|
const occa::kernel MassApply2D_GPU =
|
|
mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
|
|
"MassApply2D_GPU", props);
|
|
OccaMassApply2D_gpu.emplace(id, MassApply2D_GPU);
|
|
}
|
|
OccaMassApply2D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
|
|
}
|
|
}
|
|
|
|
// OCCA PA Mass Apply 3D kernel
|
|
static void OccaPAMassApply3D(const int D1D,
|
|
const int Q1D,
|
|
const int NE,
|
|
const Array<double> &B,
|
|
const Array<double> &Bt,
|
|
const Vector &D,
|
|
const Vector &X,
|
|
Vector &Y)
|
|
{
|
|
occa::properties props;
|
|
props["defines/D1D"] = D1D;
|
|
props["defines/Q1D"] = Q1D;
|
|
const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
|
|
const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
|
|
const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
|
|
const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
|
|
occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
|
|
const occa_id_t id = std::make_pair(D1D,Q1D);
|
|
if (!Device::Allows(Backend::OCCA_CUDA))
|
|
{
|
|
static occa_kernel_t OccaMassApply3D_cpu;
|
|
if (OccaMassApply3D_cpu.find(id) == OccaMassApply3D_cpu.end())
|
|
{
|
|
const occa::kernel MassApply3D_CPU =
|
|
mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
|
|
"MassApply3D_CPU", props);
|
|
OccaMassApply3D_cpu.emplace(id, MassApply3D_CPU);
|
|
}
|
|
OccaMassApply3D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
|
|
}
|
|
else
|
|
{
|
|
static occa_kernel_t OccaMassApply3D_gpu;
|
|
if (OccaMassApply3D_gpu.find(id) == OccaMassApply3D_gpu.end())
|
|
{
|
|
const occa::kernel MassApply3D_GPU =
|
|
mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
|
|
"MassApply3D_GPU", props);
|
|
OccaMassApply3D_gpu.emplace(id, MassApply3D_GPU);
|
|
}
|
|
OccaMassApply3D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
|
|
}
|
|
}
|
|
#endif // MFEM_USE_OCCA
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0>
|
|
static void PAMassApply2D(const int NE,
|
|
const Array<double> &b_,
|
|
const Array<double> &bt_,
|
|
const Vector &d_,
|
|
const Vector &x_,
|
|
Vector &y_,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
|
|
MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
|
|
|
|
const auto B = b_.Read();
|
|
const auto Bt = bt_.Read();
|
|
const auto D = d_.Read();
|
|
const auto X = x_.Read();
|
|
auto Y = y_.ReadWrite();
|
|
|
|
MFEM_FORALL(e, NE,
|
|
{
|
|
internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
|
|
});
|
|
}
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
|
|
static void SmemPAMassApply2D(const int NE,
|
|
const Array<double> &b_,
|
|
const Array<double> &bt_,
|
|
const Vector &d_,
|
|
const Vector &x_,
|
|
Vector &y_,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
MFEM_CONTRACT_VAR(bt_);
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int NBZ = T_NBZ ? T_NBZ : 1;
|
|
constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
|
|
MFEM_VERIFY(D1D <= MD1, "");
|
|
MFEM_VERIFY(Q1D <= MQ1, "");
|
|
const auto b = b_.Read();
|
|
const auto D = d_.Read();
|
|
const auto x = x_.Read();
|
|
auto Y = y_.ReadWrite();
|
|
MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
|
|
{
|
|
internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
|
|
});
|
|
}
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0>
|
|
static void PAMassApply3D(const int NE,
|
|
const Array<double> &b_,
|
|
const Array<double> &bt_,
|
|
const Vector &d_,
|
|
const Vector &x_,
|
|
Vector &y_,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
|
|
MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
|
|
|
|
const auto B = b_.Read();
|
|
const auto Bt = bt_.Read();
|
|
const auto D = d_.Read();
|
|
const auto X = x_.Read();
|
|
auto Y = y_.ReadWrite();
|
|
|
|
MFEM_FORALL(e, NE,
|
|
{
|
|
internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
|
|
});
|
|
}
|
|
|
|
template<int T_D1D = 0, int T_Q1D = 0>
|
|
static void SmemPAMassApply3D(const int NE,
|
|
const Array<double> &b_,
|
|
const Array<double> &bt_,
|
|
const Vector &d_,
|
|
const Vector &x_,
|
|
Vector &y_,
|
|
const int d1d = 0,
|
|
const int q1d = 0)
|
|
{
|
|
MFEM_CONTRACT_VAR(bt_);
|
|
const int D1D = T_D1D ? T_D1D : d1d;
|
|
const int Q1D = T_Q1D ? T_Q1D : q1d;
|
|
constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
|
|
constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
|
|
MFEM_VERIFY(D1D <= M1D, "");
|
|
MFEM_VERIFY(Q1D <= M1Q, "");
|
|
auto b = b_.Read();
|
|
auto d = d_.Read();
|
|
auto x = x_.Read();
|
|
auto y = y_.ReadWrite();
|
|
MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
|
|
{
|
|
internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
|
|
});
|
|
}
|
|
|
|
static void PAMassApply(const int dim,
|
|
const int D1D,
|
|
const int Q1D,
|
|
const int NE,
|
|
const Array<double> &B,
|
|
const Array<double> &Bt,
|
|
const Vector &D,
|
|
const Vector &X,
|
|
Vector &Y)
|
|
{
|
|
#ifdef MFEM_USE_OCCA
|
|
if (DeviceCanUseOcca())
|
|
{
|
|
if (dim == 2)
|
|
{
|
|
return OccaPAMassApply2D(D1D,Q1D,NE,B,Bt,D,X,Y);
|
|
}
|
|
if (dim == 3)
|
|
{
|
|
return OccaPAMassApply3D(D1D,Q1D,NE,B,Bt,D,X,Y);
|
|
}
|
|
MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
|
|
}
|
|
#endif // MFEM_USE_OCCA
|
|
const int id = (D1D << 4) | Q1D;
|
|
|
|
if (dim == 2)
|
|
{
|
|
switch (id)
|
|
{
|
|
case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
|
|
case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
|
|
case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
|
|
case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
|
|
case 0x35: return SmemPAMassApply2D<3,5,16>(NE,B,Bt,D,X,Y);
|
|
case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
|
|
case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
|
|
case 0x46: return SmemPAMassApply2D<4,6,8>(NE,B,Bt,D,X,Y);
|
|
case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
|
|
case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
|
|
case 0x57: return SmemPAMassApply2D<5,7,8>(NE,B,Bt,D,X,Y);
|
|
case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
|
|
case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
|
|
case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
|
|
case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
|
|
case 0x99: return SmemPAMassApply2D<9,9,2>(NE,B,Bt,D,X,Y);
|
|
default: return PAMassApply2D(NE,B,Bt,D,X,Y,D1D,Q1D);
|
|
}
|
|
}
|
|
else if (dim == 3)
|
|
{
|
|
switch (id)
|
|
{
|
|
case 0x22: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
|
|
case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
|
|
case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
|
|
case 0x26: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
|
|
case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
|
|
case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
|
|
case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
|
|
case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
|
|
case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
|
|
case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
|
|
case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
|
|
case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
|
|
case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
|
|
case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
|
|
case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
|
|
case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
|
|
case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
|
|
default: return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
|
|
}
|
|
}
|
|
mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
|
|
MFEM_ABORT("Unknown kernel.");
|
|
}
|
|
|
|
void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
|
|
{
|
|
if (DeviceCanUseCeed())
|
|
{
|
|
ceedOp->AddMult(x, y);
|
|
}
|
|
else
|
|
{
|
|
PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
|
|
}
|
|
}
|
|
|
|
void MassIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
|
|
{
|
|
// Mass integrator is symmetric
|
|
AddMultPA(x, y);
|
|
}
|
|
|
|
} // namespace mfem
|