1120 lines
35 KiB
C++
1120 lines
35 KiB
C++
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
|
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
|
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
|
//
|
|
// This file is part of the MFEM library. For more information and source code
|
|
// availability visit https://mfem.org.
|
|
//
|
|
// MFEM is free software; you can redistribute it and/or modify it under the
|
|
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
|
// CONTRIBUTING.md for details.
|
|
|
|
#include "../../config/config.hpp"
|
|
|
|
#ifdef MFEM_USE_MPI
|
|
|
|
#include <iostream>
|
|
#include <unordered_set>
|
|
#include <algorithm>
|
|
#include "psubmesh.hpp"
|
|
#include "pncsubmesh.hpp"
|
|
#include "submesh_utils.hpp"
|
|
#include "../segment.hpp"
|
|
|
|
namespace mfem
|
|
{
|
|
|
|
ParSubMesh ParSubMesh::CreateFromDomain(const ParMesh &parent,
|
|
const Array<int> &domain_attributes)
|
|
{
|
|
return ParSubMesh(parent, SubMesh::From::Domain, domain_attributes);
|
|
}
|
|
|
|
ParSubMesh ParSubMesh::CreateFromBoundary(const ParMesh &parent,
|
|
const Array<int> &boundary_attributes)
|
|
{
|
|
return ParSubMesh(parent, SubMesh::From::Boundary, boundary_attributes);
|
|
}
|
|
|
|
ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
|
|
const Array<int> &attributes) : parent_(parent), from_(from),
|
|
attributes_(attributes)
|
|
{
|
|
MyComm = parent.GetComm();
|
|
NRanks = parent.GetNRanks();
|
|
MyRank = parent.GetMyRank();
|
|
|
|
// This violation of const-ness may be justified in this instance because the
|
|
// exchange of face neighbor information only establishes or updates derived
|
|
// information without altering the primary mesh information, i.e., the
|
|
// topology, geometry, or region attributes.
|
|
const_cast<ParMesh&>(parent).ExchangeFaceNbrData();
|
|
|
|
if (from == SubMesh::From::Domain)
|
|
{
|
|
InitMesh(parent.Dimension(), parent.SpaceDimension(), 0, 0, 0);
|
|
|
|
std::tie(parent_vertex_ids_,
|
|
parent_element_ids_) = SubMeshUtils::AddElementsToMesh(parent_, *this,
|
|
attributes_);
|
|
}
|
|
else if (from == SubMesh::From::Boundary)
|
|
{
|
|
InitMesh(parent.Dimension() - 1, parent.SpaceDimension(), 0, 0, 0);
|
|
|
|
std::tie(parent_vertex_ids_,
|
|
parent_element_ids_) = SubMeshUtils::AddElementsToMesh(parent_, *this,
|
|
attributes_, true);
|
|
}
|
|
|
|
parent_to_submesh_vertex_ids_.SetSize(parent_.GetNV());
|
|
parent_to_submesh_vertex_ids_ = -1;
|
|
for (int i = 0; i < parent_vertex_ids_.Size(); i++)
|
|
{
|
|
parent_to_submesh_vertex_ids_[parent_vertex_ids_[i]] = i;
|
|
}
|
|
|
|
parent_to_submesh_element_ids_.SetSize(from == From::Boundary ? parent.GetNBE()
|
|
: parent.GetNE());
|
|
parent_to_submesh_element_ids_ = -1;
|
|
for (int i = 0; i < parent_element_ids_.Size(); i++)
|
|
{
|
|
parent_to_submesh_element_ids_[parent_element_ids_[i]] = i;
|
|
}
|
|
|
|
// Don't let boundary elements get generated automatically. This would
|
|
// generate boundary elements on each rank locally, which is topologically
|
|
// wrong for the distributed SubMesh.
|
|
FinalizeTopology(false);
|
|
|
|
if (parent.Nonconforming())
|
|
{
|
|
pncmesh = new ParNCSubMesh(*this, *parent.pncmesh, from, attributes);
|
|
pncsubmesh_ = dynamic_cast<ParNCSubMesh*>(pncmesh);
|
|
ncmesh = pncmesh;
|
|
InitFromNCMesh(*pncmesh);
|
|
pncmesh->OnMeshUpdated(this);
|
|
|
|
// Update the submesh to parent vertex mapping, NCSubMesh reordered the
|
|
// vertices so the map to parent is no longer valid.
|
|
parent_to_submesh_vertex_ids_ = -1;
|
|
for (int i = 0; i < parent_vertex_ids_.Size(); i++)
|
|
{
|
|
// vertex -> node -> parent node -> parent vertex
|
|
auto node = pncsubmesh_->vertex_nodeId[i];
|
|
auto parent_node = pncsubmesh_->parent_node_ids_[node];
|
|
auto parent_vertex = parent.pncmesh->GetNodeVertex(parent_node);
|
|
parent_vertex_ids_[i] = parent_vertex;
|
|
parent_to_submesh_vertex_ids_[parent_vertex] = i;
|
|
}
|
|
GenerateNCFaceInfo();
|
|
SetAttributes();
|
|
}
|
|
|
|
ReduceMeshGen();
|
|
DSTable v2v(parent_.GetNV());
|
|
parent_.GetVertexToVertexTable(v2v);
|
|
for (int i = 0; i < NumOfEdges; i++)
|
|
{
|
|
Array<int> lv;
|
|
GetEdgeVertices(i, lv);
|
|
|
|
// Find vertices/edge in parent mesh
|
|
int parent_edge_id = v2v(parent_vertex_ids_[lv[0]],
|
|
parent_vertex_ids_[lv[1]]);
|
|
parent_edge_ids_.Append(parent_edge_id);
|
|
}
|
|
|
|
parent_to_submesh_edge_ids_.SetSize(parent.GetNEdges());
|
|
parent_to_submesh_edge_ids_ = -1;
|
|
for (int i = 0; i < parent_edge_ids_.Size(); i++)
|
|
{
|
|
parent_to_submesh_edge_ids_[parent_edge_ids_[i]] = i;
|
|
}
|
|
|
|
if (Dim == 3)
|
|
{
|
|
parent_face_ids_ = SubMeshUtils::BuildFaceMap(parent_, *this,
|
|
parent_element_ids_);
|
|
|
|
parent_to_submesh_face_ids_.SetSize(parent.GetNFaces());
|
|
parent_to_submesh_face_ids_ = -1;
|
|
for (int i = 0; i < parent_face_ids_.Size(); i++)
|
|
{
|
|
parent_to_submesh_face_ids_[parent_face_ids_[i]] = i;
|
|
}
|
|
|
|
parent_face_ori_.SetSize(NumOfFaces);
|
|
for (int i = 0; i < NumOfFaces; i++)
|
|
{
|
|
Array<int> sub_vert;
|
|
GetFaceVertices(i, sub_vert);
|
|
|
|
Array<int> sub_par_vert(sub_vert.Size());
|
|
for (int j = 0; j < sub_vert.Size(); j++)
|
|
{
|
|
sub_par_vert[j] = parent_vertex_ids_[sub_vert[j]];
|
|
}
|
|
|
|
Array<int> par_vert;
|
|
parent.GetFaceVertices(parent_face_ids_[i], par_vert);
|
|
|
|
if (par_vert.Size() == 3)
|
|
{
|
|
parent_face_ori_[i] = GetTriOrientation(par_vert, sub_par_vert);
|
|
}
|
|
else
|
|
{
|
|
parent_face_ori_[i] = GetQuadOrientation(par_vert, sub_par_vert);
|
|
}
|
|
}
|
|
}
|
|
else if (Dim == 2)
|
|
{
|
|
parent_face_ori_.SetSize(NumOfElements);
|
|
|
|
for (int i = 0; i < NumOfElements; i++)
|
|
{
|
|
Array<int> sub_vert;
|
|
GetElementVertices(i, sub_vert);
|
|
|
|
Array<int> sub_par_vert(sub_vert.Size());
|
|
for (int j = 0; j < sub_vert.Size(); j++)
|
|
{
|
|
sub_par_vert[j] = parent_vertex_ids_[sub_vert[j]];
|
|
}
|
|
|
|
Array<int> par_vert;
|
|
int be_ori = 0;
|
|
if (from == SubMesh::From::Boundary)
|
|
{
|
|
parent.GetBdrElementVertices(parent_element_ids_[i], par_vert);
|
|
|
|
int f = -1;
|
|
parent.GetBdrElementFace(parent_element_ids_[i], &f, &be_ori);
|
|
}
|
|
else
|
|
{
|
|
parent.GetElementVertices(parent_element_ids_[i], par_vert);
|
|
}
|
|
|
|
if (par_vert.Size() == 3)
|
|
{
|
|
int se_ori = GetTriOrientation(par_vert, sub_par_vert);
|
|
parent_face_ori_[i] = ComposeTriOrientations(be_ori, se_ori);
|
|
}
|
|
else
|
|
{
|
|
int se_ori = GetQuadOrientation(par_vert, sub_par_vert);
|
|
parent_face_ori_[i] = ComposeQuadOrientations(be_ori, se_ori);
|
|
}
|
|
}
|
|
}
|
|
|
|
ListOfIntegerSets groups;
|
|
IntegerSet group;
|
|
// the first group is the local one
|
|
group.Recreate(1, &MyRank);
|
|
groups.Insert(group);
|
|
|
|
// Every rank containing elements of the ParSubMesh attributes now has a
|
|
// local ParSubMesh. We have to connect the local meshes and assign global
|
|
// boundaries correctly.
|
|
Array<int> rhvtx;
|
|
FindSharedVerticesRanks(rhvtx);
|
|
AppendSharedVerticesGroups(groups, rhvtx);
|
|
|
|
Array<int> rhe;
|
|
FindSharedEdgesRanks(rhe);
|
|
AppendSharedEdgesGroups(groups, rhe);
|
|
|
|
Array<int> rhq, rht;
|
|
if (Dim == 3)
|
|
{
|
|
FindSharedFacesRanks(rht, rhq);
|
|
AppendSharedFacesGroups(groups, rht, rhq);
|
|
}
|
|
|
|
|
|
// Build the group communication topology
|
|
gtopo.SetComm(MyComm);
|
|
gtopo.Create(groups, 822);
|
|
int ngroups = groups.Size()-1;
|
|
|
|
int nsverts, nsedges, nstrias, nsquads;
|
|
BuildVertexGroup(ngroups, rhvtx, nsverts);
|
|
BuildEdgeGroup(ngroups, rhe, nsedges);
|
|
if (Dim == 3)
|
|
{
|
|
BuildFaceGroup(ngroups, rht, nstrias, rhq, nsquads);
|
|
}
|
|
else
|
|
{
|
|
group_stria.MakeI(ngroups);
|
|
group_stria.MakeJ();
|
|
group_stria.ShiftUpI();
|
|
|
|
group_squad.MakeI(ngroups);
|
|
group_squad.MakeJ();
|
|
group_squad.ShiftUpI();
|
|
}
|
|
|
|
BuildSharedVerticesMapping(nsverts, rhvtx);
|
|
BuildSharedEdgesMapping(nsedges, rhe);
|
|
if (Dim == 3)
|
|
{
|
|
BuildSharedFacesMapping(nstrias, rht, nsquads, rhq);
|
|
}
|
|
|
|
ExchangeFaceNbrData();
|
|
|
|
SubMeshUtils::AddBoundaryElements(*this,
|
|
(from == SubMesh::From::Domain)
|
|
? FindGhostBoundaryElementAttributes()
|
|
: std::unordered_map<int,int> {});
|
|
|
|
if (Dim > 1)
|
|
{
|
|
if (!el_to_edge) { el_to_edge = new Table; }
|
|
NumOfEdges = GetElementToEdgeTable(*el_to_edge);
|
|
}
|
|
if (Dim > 2)
|
|
{
|
|
GetElementToFaceTable();
|
|
}
|
|
|
|
// If the parent ParMesh has nodes and therefore is defined on a higher order
|
|
// geometry, we define this ParSubMesh as a curved ParSubMesh and transfer
|
|
// the GridFunction from the parent ParMesh to the ParSubMesh.
|
|
const GridFunction *parent_nodes = parent_.GetNodes();
|
|
if (parent_nodes)
|
|
{
|
|
const FiniteElementSpace *parent_fes = parent_nodes->FESpace();
|
|
|
|
SetCurvature(
|
|
parent_fes->FEColl()->GetOrder(),
|
|
parent_fes->IsDGSpace(),
|
|
spaceDim,
|
|
parent_fes->GetOrdering());
|
|
|
|
const ParGridFunction* pn = dynamic_cast<const ParGridFunction*>
|
|
(parent_.GetNodes());
|
|
MFEM_ASSERT(pn,
|
|
"Internal error. Object is supposed to be ParGridFunction.");
|
|
|
|
ParGridFunction* n = dynamic_cast<ParGridFunction*>
|
|
(this->GetNodes());
|
|
MFEM_ASSERT(n,
|
|
"Internal error. Object is supposed to be ParGridFunction.");
|
|
|
|
Transfer(*pn, *n);
|
|
}
|
|
SetAttributes();
|
|
Finalize();
|
|
}
|
|
|
|
void ParSubMesh::FindSharedVerticesRanks(Array<int> &rhvtx)
|
|
{
|
|
// create a GroupCommunicator on the shared vertices
|
|
GroupCommunicator svert_comm(parent_.gtopo);
|
|
parent_.GetSharedVertexCommunicator(svert_comm);
|
|
// Number of shared vertices
|
|
int nsvtx = svert_comm.GroupLDofTable().Size_of_connections();
|
|
|
|
rhvtx.SetSize(nsvtx);
|
|
rhvtx = 0;
|
|
|
|
// On each rank of the group, locally determine if the shared vertex is in
|
|
// the SubMesh.
|
|
for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
const int group_sz = parent_.gtopo.GetGroupSize(g);
|
|
MFEM_VERIFY((unsigned int)group_sz <= 8*sizeof(int), // 32
|
|
"Group size too large. Groups with more than 32 ranks are not supported, yet.");
|
|
const int* group_lproc = parent_.gtopo.GetGroup(g);
|
|
|
|
const int* my_group_id_ptr = std::find(group_lproc, group_lproc+group_sz, 0);
|
|
MFEM_ASSERT(my_group_id_ptr != group_lproc+group_sz, "internal error");
|
|
|
|
const int my_group_id = my_group_id_ptr-group_lproc;
|
|
|
|
for (int gv = 0; gv < parent_.GroupNVertices(g); gv++, sv++)
|
|
{
|
|
int plvtx = parent_.GroupVertex(g, gv);
|
|
int submesh_vertex_id = parent_to_submesh_vertex_ids_[plvtx];
|
|
if (submesh_vertex_id != -1)
|
|
{
|
|
rhvtx[sv] |= 1 << my_group_id;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// Compute the sum on the root rank and broadcast the result to all ranks.
|
|
svert_comm.Reduce(rhvtx, GroupCommunicator::Sum);
|
|
svert_comm.Bcast<int>(rhvtx, 0);
|
|
}
|
|
|
|
void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
|
|
{
|
|
// create a GroupCommunicator on the shared edges
|
|
GroupCommunicator sedge_comm(parent_.gtopo);
|
|
parent_.GetSharedEdgeCommunicator(sedge_comm);
|
|
|
|
int nsedges = sedge_comm.GroupLDofTable().Size_of_connections();
|
|
|
|
// see rhvtx description
|
|
rhe.SetSize(nsedges);
|
|
rhe = 0;
|
|
|
|
// On each rank of the group, locally determine if the shared edge is in the
|
|
// SubMesh.
|
|
for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
const int group_sz = parent_.gtopo.GetGroupSize(g);
|
|
MFEM_VERIFY((unsigned int)group_sz <= 8*sizeof(int), // 32
|
|
"Group size too large. Groups with more than 32 ranks are not supported, yet.");
|
|
const int* group_lproc = parent_.gtopo.GetGroup(g);
|
|
|
|
const int* my_group_id_ptr = std::find(group_lproc, group_lproc+group_sz, 0);
|
|
MFEM_ASSERT(my_group_id_ptr != group_lproc+group_sz, "internal error");
|
|
|
|
// rank id inside this group
|
|
const int my_group_id = my_group_id_ptr-group_lproc;
|
|
|
|
for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
|
|
{
|
|
int ple = parent_.GroupEdge(g, ge);
|
|
int submesh_edge_id = parent_to_submesh_edge_ids_[ple];
|
|
if (submesh_edge_id != -1)
|
|
{
|
|
rhe[se] |= 1 << my_group_id;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// Compute the sum on the root rank and broadcast the result to all ranks.
|
|
sedge_comm.Reduce(rhe, GroupCommunicator::Sum);
|
|
sedge_comm.Bcast<int>(rhe, 0);
|
|
}
|
|
|
|
void ParSubMesh::FindSharedFacesRanks(Array<int>& rht, Array<int> &rhq)
|
|
{
|
|
GroupCommunicator stria_comm(parent_.gtopo);
|
|
parent_.GetSharedTriCommunicator(stria_comm);
|
|
int nstria = stria_comm.GroupLDofTable().Size_of_connections();
|
|
rht.SetSize(nstria);
|
|
rht = 0;
|
|
|
|
for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
MFEM_ASSERT(parent_.gtopo.GetGroupSize(g) == 2
|
|
|| parent_.GroupNTriangles(g) == 0,
|
|
parent_.gtopo.GetGroupSize(g) << ' ' << parent_.GroupNTriangles(g));
|
|
for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
|
|
{
|
|
// Group size of a shared face is always 2
|
|
int plt = parent_.GroupTriangle(g, gt);
|
|
int submesh_face_id = parent_to_submesh_face_ids_[plt];
|
|
if (submesh_face_id != -1)
|
|
{
|
|
rht[st] = 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Compute the sum on the root rank and broadcast the result to all ranks.
|
|
stria_comm.Reduce(rht, GroupCommunicator::Sum);
|
|
stria_comm.Bcast<int>(rht, 0);
|
|
|
|
GroupCommunicator squad_comm(parent_.gtopo);
|
|
parent_.GetSharedQuadCommunicator(squad_comm);
|
|
int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
|
|
rhq.SetSize(nsquad);
|
|
rhq = 0;
|
|
|
|
for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
MFEM_ASSERT(parent_.gtopo.GetGroupSize(g) == 2
|
|
|| parent_.GroupNQuadrilaterals(g) == 0,
|
|
parent_.gtopo.GetGroupSize(g) << ' ' << parent_.GroupNQuadrilaterals(g));
|
|
for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
|
|
{
|
|
// Group size of a shared face is always 2
|
|
int plq = parent_.GroupQuadrilateral(g, gq);
|
|
int submesh_face_id = parent_to_submesh_face_ids_[plq];
|
|
if (submesh_face_id != -1)
|
|
{
|
|
rhq[sq] = 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Compute the sum on the root rank and broadcast the result to all ranks.
|
|
squad_comm.Reduce(rhq, GroupCommunicator::Sum);
|
|
squad_comm.Bcast<int>(rhq, 0);
|
|
}
|
|
|
|
|
|
void ParSubMesh::AppendSharedVerticesGroups(ListOfIntegerSets &groups,
|
|
Array<int> &rhvtx)
|
|
{
|
|
IntegerSet group;
|
|
|
|
// g = 0 corresponds to the singleton group of each rank alone.
|
|
for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
const int group_sz = parent_.gtopo.GetGroupSize(g);
|
|
MFEM_VERIFY((unsigned int)group_sz <= 8*sizeof(int), // 32
|
|
"Group size too large. Groups with more than 32 ranks are not supported, yet.");
|
|
const int* group_lproc = parent_.gtopo.GetGroup(g);
|
|
|
|
const int* my_group_id_ptr = std::find(group_lproc, group_lproc+group_sz, 0);
|
|
MFEM_ASSERT(my_group_id_ptr != group_lproc+group_sz, "internal error");
|
|
|
|
const int my_group_id = my_group_id_ptr-group_lproc;
|
|
|
|
for (int gv = 0; gv < parent_.GroupNVertices(g); gv++, sv++)
|
|
{
|
|
// Returns the parents local vertex id
|
|
int plvtx = parent_.GroupVertex(g, gv);
|
|
int submesh_vtx = parent_to_submesh_vertex_ids_[plvtx];
|
|
|
|
// Reusing the `rhvtx` array as shared vertex to group array.
|
|
if (submesh_vtx == -1)
|
|
{
|
|
// parent shared vertex is not in SubMesh
|
|
rhvtx[sv] = -1;
|
|
}
|
|
else if (rhvtx[sv] & ~(1 << my_group_id))
|
|
{
|
|
// shared vertex is present on this rank and others
|
|
MFEM_ASSERT(rhvtx[sv] & (1 << my_group_id), "error again");
|
|
|
|
// determine which other ranks have the shared vertex
|
|
Array<int> &ranks = group;
|
|
ranks.SetSize(0);
|
|
for (int i = 0; i < group_sz; i++)
|
|
{
|
|
if ((rhvtx[sv] >> i) & 1)
|
|
{
|
|
ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[i]));
|
|
}
|
|
}
|
|
MFEM_ASSERT(ranks.Size() >= 2, "internal error");
|
|
|
|
rhvtx[sv] = groups.Insert(group) - 1;
|
|
}
|
|
else
|
|
{
|
|
// previously shared vertex is only present on this rank
|
|
rhvtx[sv] = -1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ParSubMesh::AppendSharedEdgesGroups(ListOfIntegerSets &groups,
|
|
Array<int> &rhe)
|
|
{
|
|
IntegerSet group;
|
|
|
|
for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
const int group_sz = parent_.gtopo.GetGroupSize(g);
|
|
MFEM_VERIFY((unsigned int)group_sz <= 8*sizeof(int), // 32
|
|
"Group size too large. Groups with more than 32 ranks are not supported, yet.");
|
|
const int* group_lproc = parent_.gtopo.GetGroup(g);
|
|
|
|
const int* my_group_id_ptr = std::find(group_lproc, group_lproc+group_sz, 0);
|
|
MFEM_ASSERT(my_group_id_ptr != group_lproc+group_sz, "internal error");
|
|
|
|
const int my_group_id = my_group_id_ptr-group_lproc;
|
|
|
|
for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
|
|
{
|
|
int ple = parent_.GroupEdge(g, ge);
|
|
int submesh_edge = parent_to_submesh_edge_ids_[ple];
|
|
|
|
// Reusing the `rhe` array as shared edge to group array.
|
|
if (submesh_edge == -1)
|
|
{
|
|
// parent shared edge is not in SubMesh
|
|
rhe[se] = -1;
|
|
}
|
|
else if (rhe[se] & ~(1 << my_group_id))
|
|
{
|
|
// shared edge is present on this rank and others
|
|
|
|
// determine which other ranks have the shared edge
|
|
Array<int> &ranks = group;
|
|
ranks.SetSize(0);
|
|
for (int i = 0; i < group_sz; i++)
|
|
{
|
|
if ((rhe[se] >> i) & 1)
|
|
{
|
|
ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[i]));
|
|
}
|
|
}
|
|
MFEM_ASSERT(ranks.Size() >= 2, "internal error");
|
|
|
|
rhe[se] = groups.Insert(group) - 1;
|
|
}
|
|
else
|
|
{
|
|
// previously shared edge is only present on this rank
|
|
rhe[se] = -1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
|
|
Array<int>& rht, Array<int> &rhq)
|
|
{
|
|
IntegerSet quad_group;
|
|
|
|
for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
const int* group_lproc = parent_.gtopo.GetGroup(g);
|
|
for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
|
|
{
|
|
const int group_sz = parent_.gtopo.GetGroupSize(g);
|
|
MFEM_ASSERT(group_sz == 2, "internal error");
|
|
|
|
int plq = parent_.GroupQuadrilateral(g, gq);
|
|
int submesh_face_id = parent_to_submesh_face_ids_[plq];
|
|
|
|
// Reusing the `rhq` array as shared face to group array.
|
|
if (submesh_face_id == -1)
|
|
{
|
|
// parent shared face is not in SubMesh
|
|
rhq[sq] = -1;
|
|
}
|
|
else if (rhq[sq] == group_sz)
|
|
{
|
|
// shared face is present on this rank and others
|
|
|
|
// There can only be two ranks in this group sharing faces. Add all
|
|
// ranks to a new communication group.
|
|
Array<int> &ranks = quad_group;
|
|
ranks.SetSize(0);
|
|
ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[0]));
|
|
ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[1]));
|
|
|
|
rhq[sq] = groups.Insert(quad_group) - 1;
|
|
}
|
|
else
|
|
{
|
|
// previously shared edge is only present on this rank
|
|
rhq[sq] = -1;
|
|
}
|
|
}
|
|
}
|
|
|
|
IntegerSet tria_group;
|
|
|
|
for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
const int* group_lproc = parent_.gtopo.GetGroup(g);
|
|
for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
|
|
{
|
|
const int group_sz = parent_.gtopo.GetGroupSize(g);
|
|
MFEM_ASSERT(group_sz == 2, "internal error");
|
|
|
|
int plt = parent_.GroupTriangle(g, gt);
|
|
int submesh_face_id = parent_to_submesh_face_ids_[plt];
|
|
|
|
// Reusing the `rht` array as shared face to group array.
|
|
if (submesh_face_id == -1)
|
|
{
|
|
// parent shared face is not in SubMesh
|
|
rht[st] = -1;
|
|
}
|
|
else if (rht[st] == group_sz)
|
|
{
|
|
// shared face is present on this rank and others
|
|
|
|
// There can only be two ranks in this group sharing faces. Add all
|
|
// ranks to a new communication group.
|
|
Array<int> &ranks = tria_group;
|
|
ranks.SetSize(0);
|
|
ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[0]));
|
|
ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[1]));
|
|
|
|
rht[st] = groups.Insert(tria_group) - 1;
|
|
}
|
|
else
|
|
{
|
|
// previously shared edge is only present on this rank
|
|
rht[st] = -1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void BuildGroup(Table &group, int ngroups, const Array<int>& rh, int &ns)
|
|
{
|
|
group.MakeI(ngroups);
|
|
for (int i = 0; i < rh.Size(); i++)
|
|
{
|
|
if (rh[i] >= 0)
|
|
{
|
|
group.AddAColumnInRow(rh[i]);
|
|
}
|
|
}
|
|
|
|
group.MakeJ();
|
|
ns = 0;
|
|
for (int i = 0; i < rh.Size(); i++)
|
|
{
|
|
if (rh[i] >= 0)
|
|
{
|
|
group.AddConnection(rh[i], ns++);
|
|
}
|
|
}
|
|
group.ShiftUpI();
|
|
}
|
|
|
|
void ParSubMesh::BuildVertexGroup(int ngroups, const Array<int>& rhvtx,
|
|
int& nsverts)
|
|
{
|
|
BuildGroup(group_svert, ngroups, rhvtx, nsverts);
|
|
}
|
|
|
|
void ParSubMesh::BuildEdgeGroup(int ngroups, const Array<int>& rhe,
|
|
int& nsedges)
|
|
{
|
|
BuildGroup(group_sedge, ngroups, rhe, nsedges);
|
|
}
|
|
|
|
void ParSubMesh::BuildFaceGroup(int ngroups, const Array<int>& rht,
|
|
int& nstrias, const Array<int>& rhq, int& nsquads)
|
|
{
|
|
BuildGroup(group_squad, ngroups, rhq, nsquads);
|
|
BuildGroup(group_stria, ngroups, rht, nstrias);
|
|
}
|
|
|
|
void ParSubMesh::BuildSharedVerticesMapping(const int nsverts,
|
|
const Array<int>& rhvtx)
|
|
{
|
|
svert_lvert.Reserve(nsverts);
|
|
|
|
for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int gv = 0; gv < parent_.GroupNVertices(g); gv++, sv++)
|
|
{
|
|
// Returns the parents local vertex id
|
|
int plvtx = parent_.GroupVertex(g, gv);
|
|
int submesh_vtx_id = parent_to_submesh_vertex_ids_[plvtx];
|
|
if ((submesh_vtx_id == -1) || (rhvtx[sv] == -1))
|
|
{
|
|
// parent shared vertex is not in SubMesh or is not shared
|
|
}
|
|
else
|
|
{
|
|
svert_lvert.Append(submesh_vtx_id);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ParSubMesh::BuildSharedEdgesMapping(const int sedges_ct,
|
|
const Array<int>& rhe)
|
|
{
|
|
shared_edges.Reserve(sedges_ct);
|
|
sedge_ledge.Reserve(sedges_ct);
|
|
|
|
for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
|
|
{
|
|
int ple, o;
|
|
parent_.GroupEdge(g, ge, ple, o);
|
|
int submesh_edge_id = parent_to_submesh_edge_ids_[ple];
|
|
if ((submesh_edge_id == -1) || rhe[se] == -1)
|
|
{
|
|
// parent shared edge is not in SubMesh or is not shared
|
|
}
|
|
else
|
|
{
|
|
Array<int> vert;
|
|
parent_.GetEdgeVertices(ple, vert);
|
|
// Swap order of vertices if orientation in parent group is -1
|
|
int v0 = parent_to_submesh_vertex_ids_[vert[(1-o)/2]];
|
|
int v1 = parent_to_submesh_vertex_ids_[vert[(1+o)/2]];
|
|
|
|
// The orienation of the shared edge relative to the local edge will
|
|
// be determined by whether v0 < v1 or v1 < v0
|
|
shared_edges.Append(new Segment(v0, v1, 1));
|
|
sedge_ledge.Append(submesh_edge_id);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
|
|
const Array<int>& rht,
|
|
const int nsquads, const Array<int>& rhq)
|
|
{
|
|
shared_trias.Reserve(nstrias);
|
|
shared_quads.Reserve(nsquads);
|
|
sface_lface.Reserve(nstrias + nsquads);
|
|
|
|
// sface_lface should list the triangular shared faces first followed by the
|
|
// quadrilateral shared faces.
|
|
for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
|
|
{
|
|
int plt, o;
|
|
parent_.GroupTriangle(g, gt, plt, o);
|
|
int submesh_face_id = parent_to_submesh_face_ids_[plt];
|
|
if ((submesh_face_id == -1) || rht[st] == -1)
|
|
{
|
|
// parent shared face is not in SubMesh or is not shared
|
|
}
|
|
else
|
|
{
|
|
Array<int> vert;
|
|
|
|
GetFaceVertices(submesh_face_id, vert);
|
|
|
|
int v0 = vert[0];
|
|
int v1 = vert[1];
|
|
int v2 = vert[2];
|
|
|
|
// See Mesh::GetTriOrientation for info on interpretting "o"
|
|
switch (o)
|
|
{
|
|
case 1:
|
|
std::swap(v0,v1);
|
|
break;
|
|
case 3:
|
|
std::swap(v2,v0);
|
|
break;
|
|
case 5:
|
|
std::swap(v1,v2);
|
|
break;
|
|
default:
|
|
// Do nothing
|
|
break;
|
|
}
|
|
|
|
shared_trias.Append(Vert3(v0, v1, v2));
|
|
sface_lface.Append(submesh_face_id);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
|
|
{
|
|
int plq, o;
|
|
parent_.GroupQuadrilateral(g, gq, plq, o);
|
|
int submesh_face_id = parent_to_submesh_face_ids_[plq];
|
|
if ((submesh_face_id == -1) || rhq[sq] == -1)
|
|
{
|
|
// parent shared face is not in SubMesh or is not shared
|
|
}
|
|
else
|
|
{
|
|
Array<int> vert;
|
|
GetFaceVertices(submesh_face_id, vert);
|
|
|
|
int v0 = vert[0];
|
|
int v1 = vert[1];
|
|
int v2 = vert[2];
|
|
int v3 = vert[3];
|
|
|
|
// See Mesh::GetQuadOrientation for info on interpreting "o"
|
|
switch (o)
|
|
{
|
|
case 1:
|
|
std::swap(v1,v3);
|
|
break;
|
|
case 3:
|
|
std::swap(v0,v1);
|
|
std::swap(v2,v3);
|
|
break;
|
|
case 5:
|
|
std::swap(v0,v2);
|
|
break;
|
|
case 7:
|
|
std::swap(v0,v3);
|
|
std::swap(v1,v2);
|
|
break;
|
|
default:
|
|
// Do nothing
|
|
break;
|
|
}
|
|
|
|
shared_quads.Append(Vert4(v0, v1, v2, v3));
|
|
sface_lface.Append(submesh_face_id);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::unordered_map<int, int>
|
|
ParSubMesh::FindGhostBoundaryElementAttributes() const
|
|
{
|
|
// Loop over shared faces in the parent mesh, find their attributes if they
|
|
// exist, and map to local faces in the submesh.
|
|
std::unordered_map<int,int> lface_boundary_attribute;
|
|
const auto &face_to_be = parent_.GetFaceToBdrElMap();
|
|
if (Dim == 3)
|
|
{
|
|
GroupCommunicator squad_comm(parent_.gtopo);
|
|
parent_.GetSharedQuadCommunicator(squad_comm);
|
|
int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
|
|
|
|
GroupCommunicator stria_comm(parent_.gtopo);
|
|
parent_.GetSharedTriCommunicator(stria_comm);
|
|
int nstria = stria_comm.GroupLDofTable().Size_of_connections();
|
|
|
|
Array<int> stba(nstria), sqba(nsquad);
|
|
Array<int> parent_ltface(nstria), parent_lqface(nsquad);
|
|
stba = 0; sqba = 0;
|
|
parent_ltface = -1; parent_lqface = -1;
|
|
for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
|
|
{
|
|
// Group size of a shared face is always 2
|
|
int plt = parent_.GroupTriangle(g, gt);
|
|
auto pbe = face_to_be[plt];
|
|
if (pbe >= 0)
|
|
{
|
|
stba[st] = parent_.GetBdrAttribute(pbe);
|
|
}
|
|
parent_ltface[st] = plt;
|
|
}
|
|
}
|
|
for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
|
|
{
|
|
// Group size of a shared face is always 2
|
|
int plq = parent_.GroupQuadrilateral(g, gq);
|
|
auto pbe = face_to_be[plq];
|
|
if (pbe >= 0)
|
|
{
|
|
sqba[sq] = parent_.GetBdrAttribute(pbe);
|
|
}
|
|
parent_lqface[sq] = plq;
|
|
}
|
|
}
|
|
#ifdef MFEM_DEBUG
|
|
auto pre_stba = stba;
|
|
auto pre_sqba = sqba;
|
|
#endif
|
|
stria_comm.Reduce(stba, GroupCommunicator::Sum);
|
|
stria_comm.Bcast<int>(stba, 0);
|
|
squad_comm.Reduce(sqba, GroupCommunicator::Sum);
|
|
squad_comm.Bcast<int>(sqba, 0);
|
|
#ifdef MFEM_DEBUG
|
|
{
|
|
Array<int> fail_indices;
|
|
fail_indices.Reserve(stba.Size());
|
|
for (int i = 0; i < stba.Size(); i++)
|
|
if (pre_stba[i] != 0 && pre_stba[i] != stba[i])
|
|
{
|
|
fail_indices.Append(i);
|
|
}
|
|
MFEM_ASSERT(fail_indices.Size() == 0, [&]()
|
|
{
|
|
std::stringstream msg;
|
|
msg << "More than one rank found attribute on shared tri face: ";
|
|
for (auto x : fail_indices)
|
|
{
|
|
msg << x << ' ';
|
|
}
|
|
return msg.str();
|
|
}());
|
|
}
|
|
|
|
{
|
|
Array<int> fail_indices;
|
|
fail_indices.Reserve(sqba.Size());
|
|
for (int i = 0; i < sqba.Size(); i++)
|
|
if (pre_sqba[i] != 0 && pre_sqba[i] != sqba[i])
|
|
{
|
|
fail_indices.Append(i);
|
|
}
|
|
MFEM_ASSERT(fail_indices.Size() == 0, [&]()
|
|
{
|
|
std::stringstream msg;
|
|
msg << "More than one rank found attribute on shared quad face: ";
|
|
for (auto x : fail_indices)
|
|
{
|
|
msg << x << ' ';
|
|
}
|
|
return msg.str();
|
|
}());
|
|
}
|
|
#endif
|
|
int nghost = 0;
|
|
for (auto x : stba)
|
|
if (x > 0) { ++nghost; }
|
|
|
|
for (auto x : sqba)
|
|
if (x > 0) { ++nghost; }
|
|
|
|
lface_boundary_attribute.reserve(nghost);
|
|
for (int i = 0; i < stba.Size(); i++)
|
|
if (stba[i] > 0)
|
|
{
|
|
MFEM_ASSERT(parent_ltface[i] > -1, i);
|
|
lface_boundary_attribute[parent_ltface[i]] = stba[i];
|
|
}
|
|
for (int i = 0; i < sqba.Size(); i++)
|
|
if (sqba[i] > 0)
|
|
{
|
|
MFEM_ASSERT(parent_lqface[i] > -1, i);
|
|
lface_boundary_attribute[parent_lqface[i]] = sqba[i];
|
|
}
|
|
}
|
|
else if (Dim == 2)
|
|
{
|
|
GroupCommunicator sedge_comm(parent_.gtopo);
|
|
parent_.GetSharedEdgeCommunicator(sedge_comm);
|
|
int nsedge = sedge_comm.GroupLDofTable().Size_of_connections();
|
|
|
|
Array<int> seba(nsedge), parent_ledge(nsedge);
|
|
seba = 0; parent_ledge = -1;
|
|
for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
|
|
{
|
|
// Group size of a shared edge is always 2
|
|
int ple = parent_.GroupEdge(g, ge);
|
|
auto pbe = face_to_be[ple];
|
|
if (pbe >= 0)
|
|
{
|
|
seba[se] = parent_.GetBdrAttribute(pbe);
|
|
}
|
|
parent_ledge[se] = ple;
|
|
}
|
|
}
|
|
|
|
#ifdef MFEM_DEBUG
|
|
auto pre_seba = seba;
|
|
#endif
|
|
sedge_comm.Reduce(seba, GroupCommunicator::Sum);
|
|
sedge_comm.Bcast<int>(seba, 0);
|
|
#ifdef MFEM_DEBUG
|
|
{
|
|
Array<int> fail_indices;
|
|
fail_indices.Reserve(seba.Size());
|
|
for (int i = 0; i < seba.Size(); i++)
|
|
if (pre_seba[i] != 0 && pre_seba[i] != seba[i])
|
|
{
|
|
fail_indices.Append(i);
|
|
}
|
|
MFEM_ASSERT(fail_indices.Size() == 0, [&]()
|
|
{
|
|
std::stringstream msg;
|
|
msg << "More than one rank found attribute on shared edge: ";
|
|
for (auto x : fail_indices)
|
|
{
|
|
msg << x << ' ';
|
|
}
|
|
return msg.str();
|
|
}());
|
|
}
|
|
#endif
|
|
int nghost = 0;
|
|
for (auto x : seba)
|
|
if (x > 0) { ++nghost; }
|
|
|
|
lface_boundary_attribute.reserve(nghost);
|
|
for (int i = 0; i < seba.Size(); i++)
|
|
if (seba[i] > 0)
|
|
{
|
|
MFEM_ASSERT(parent_ledge[i] > -1, i);
|
|
lface_boundary_attribute[parent_ledge[i]] = seba[i];
|
|
}
|
|
}
|
|
else if (Dim == 1)
|
|
{
|
|
GroupCommunicator svert_comm(parent_.gtopo);
|
|
parent_.GetSharedVertexCommunicator(svert_comm);
|
|
int nsvtx = svert_comm.GroupLDofTable().Size_of_connections();
|
|
|
|
Array<int> svba(nsvtx), parent_lvtx(nsvtx);
|
|
svba = 0; parent_lvtx = -1;
|
|
for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
|
|
{
|
|
for (int gv = 0; gv < parent_.GroupNVertices(g); gv++, sv++)
|
|
{
|
|
// Group size of a shared vertex is always 2
|
|
int plv = parent_.GroupVertex(g, gv);
|
|
auto pbe = face_to_be[plv];
|
|
if (pbe >= 0)
|
|
{
|
|
svba[sv] = parent_.GetBdrAttribute(pbe);
|
|
}
|
|
parent_lvtx[sv] = plv;
|
|
}
|
|
}
|
|
|
|
#ifdef MFEM_DEBUG
|
|
auto pre_svba = svba;
|
|
#endif
|
|
svert_comm.Reduce(svba, GroupCommunicator::Sum);
|
|
svert_comm.Bcast<int>(svba, 0);
|
|
#ifdef MFEM_DEBUG
|
|
{
|
|
Array<int> fail_indices;
|
|
fail_indices.Reserve(svba.Size());
|
|
for (int i = 0; i < svba.Size(); i++)
|
|
if (pre_svba[i] != 0 && pre_svba[i] != svba[i])
|
|
{
|
|
fail_indices.Append(i);
|
|
}
|
|
MFEM_ASSERT(fail_indices.Size() == 0, [&]()
|
|
{
|
|
std::stringstream msg;
|
|
msg << "More than one rank found attribute on shared vertex: ";
|
|
for (auto x : fail_indices)
|
|
{
|
|
msg << x << ' ';
|
|
}
|
|
return msg.str();
|
|
}());
|
|
}
|
|
#endif
|
|
int nghost = 0;
|
|
for (auto x : svba)
|
|
if (x > 0) { ++nghost; }
|
|
|
|
lface_boundary_attribute.reserve(nghost);
|
|
for (int i = 0; i < svba.Size(); i++)
|
|
if (svba[i] > 0)
|
|
{
|
|
MFEM_ASSERT(parent_lvtx[i] > -1, i);
|
|
lface_boundary_attribute[parent_lvtx[i]] = svba[i];
|
|
}
|
|
}
|
|
return lface_boundary_attribute;
|
|
}
|
|
|
|
|
|
void ParSubMesh::Transfer(const ParGridFunction &src, ParGridFunction &dst)
|
|
{
|
|
CreateTransferMap(src, dst).Transfer(src, dst);
|
|
}
|
|
|
|
ParTransferMap ParSubMesh::CreateTransferMap(const ParGridFunction &src,
|
|
const ParGridFunction &dst)
|
|
{
|
|
return ParTransferMap(src, dst);
|
|
}
|
|
|
|
} // namespace mfem
|
|
|
|
#endif // MFEM_USE_MPI
|