294 lines
10 KiB
C++
294 lines
10 KiB
C++
// MFEM Example 14 - Parallel Version
|
|
//
|
|
// Compile with: make ex14p
|
|
//
|
|
// Sample runs: mpirun -np 4 ex14p -m ../data/inline-quad.mesh -o 0
|
|
// mpirun -np 4 ex14p -m ../data/star.mesh -o 2
|
|
// mpirun -np 4 ex14p -m ../data/star-mixed.mesh -o 2
|
|
// mpirun -np 4 ex14p -m ../data/star-mixed.mesh -o 2 -k 0 -e 1
|
|
// mpirun -np 4 ex14p -m ../data/escher.mesh -s 1
|
|
// mpirun -np 4 ex14p -m ../data/fichera.mesh -s 1 -k 1
|
|
// mpirun -np 4 ex14p -m ../data/fichera-mixed.mesh -s 1 -k 1
|
|
// mpirun -np 4 ex14p -m ../data/square-disc-p2.vtk -o 2
|
|
// mpirun -np 4 ex14p -m ../data/square-disc-p3.mesh -o 3
|
|
// mpirun -np 4 ex14p -m ../data/square-disc-nurbs.mesh -o 1
|
|
// mpirun -np 4 ex14p -m ../data/disc-nurbs.mesh -rs 4 -o 2 -s 1 -k 0
|
|
// mpirun -np 4 ex14p -m ../data/pipe-nurbs.mesh -o 1
|
|
// mpirun -np 4 ex14p -m ../data/inline-segment.mesh -rs 5
|
|
// mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 3
|
|
// mpirun -np 4 ex14p -m ../data/amr-hex.mesh
|
|
// mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
|
|
// mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
|
|
//
|
|
// Device sample runs:
|
|
// mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
|
|
// mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -m ../data/fichera.mesh -o 3
|
|
//
|
|
// Description: This example code demonstrates the use of MFEM to define a
|
|
// discontinuous Galerkin (DG) finite element discretization of
|
|
// the Poisson problem -Delta u = 1 with homogeneous Dirichlet
|
|
// boundary conditions. Finite element spaces of any order,
|
|
// including zero on regular grids, are supported. The example
|
|
// highlights the use of discontinuous spaces and DG-specific face
|
|
// integrators.
|
|
//
|
|
// We recommend viewing examples 1 and 9 before viewing this
|
|
// example.
|
|
|
|
#include "mfem.hpp"
|
|
#include <fstream>
|
|
#include <iostream>
|
|
|
|
using namespace std;
|
|
using namespace mfem;
|
|
|
|
class CustomSolverMonitor : public IterativeSolverMonitor
|
|
{
|
|
private:
|
|
const ParMesh &pmesh;
|
|
ParGridFunction &pgf;
|
|
public:
|
|
CustomSolverMonitor(const ParMesh &pmesh_,
|
|
ParGridFunction &pgf_) :
|
|
pmesh(pmesh_),
|
|
pgf(pgf_) {}
|
|
|
|
void MonitorSolution(int i, real_t norm, const Vector &x, bool final) override
|
|
{
|
|
char vishost[] = "localhost";
|
|
int visport = 19916;
|
|
int num_procs, myid;
|
|
|
|
MPI_Comm_size(pmesh.GetComm(), &num_procs);
|
|
MPI_Comm_rank(pmesh.GetComm(), &myid);
|
|
|
|
pgf.SetFromTrueDofs(x);
|
|
|
|
socketstream sol_sock(vishost, visport);
|
|
sol_sock << "parallel " << num_procs << " " << myid << "\n";
|
|
sol_sock.precision(8);
|
|
sol_sock << "solution\n" << pmesh << pgf
|
|
<< "window_title 'Iteration no " << i << "'"
|
|
<< "keys rRjlc\n" << flush;
|
|
}
|
|
};
|
|
|
|
int main(int argc, char *argv[])
|
|
{
|
|
// 1. Initialize MPI and HYPRE.
|
|
Mpi::Init(argc, argv);
|
|
Hypre::Init();
|
|
|
|
// 2. Parse command-line options.
|
|
const char *mesh_file = "../data/star.mesh";
|
|
int ser_ref_levels = -1;
|
|
int par_ref_levels = 2;
|
|
int order = 1;
|
|
real_t sigma = -1.0;
|
|
real_t kappa = -1.0;
|
|
real_t eta = 0.0;
|
|
bool pa = false;
|
|
bool visualization = 1;
|
|
const char *device_config = "cpu";
|
|
|
|
OptionsParser args(argc, argv);
|
|
args.AddOption(&mesh_file, "-m", "--mesh",
|
|
"Mesh file to use.");
|
|
args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
|
|
"Number of times to refine the mesh uniformly in serial,"
|
|
" -1 for auto.");
|
|
args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
|
|
"Number of times to refine the mesh uniformly in parallel.");
|
|
args.AddOption(&order, "-o", "--order",
|
|
"Finite element order (polynomial degree) >= 0.");
|
|
args.AddOption(&sigma, "-s", "--sigma",
|
|
"One of the three DG penalty parameters, typically +1/-1."
|
|
" See the documentation of class DGDiffusionIntegrator.");
|
|
args.AddOption(&kappa, "-k", "--kappa",
|
|
"One of the three DG penalty parameters, should be positive."
|
|
" Negative values are replaced with (order+1)^2.");
|
|
args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
|
|
args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
|
|
"--no-partial-assembly", "Enable Partial Assembly.");
|
|
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
|
|
"--no-visualization",
|
|
"Enable or disable GLVis visualization.");
|
|
args.AddOption(&device_config, "-d", "--device",
|
|
"Device configuration string, see Device::Configure().");
|
|
args.Parse();
|
|
if (!args.Good())
|
|
{
|
|
if (Mpi::Root())
|
|
{
|
|
args.PrintUsage(cout);
|
|
}
|
|
return 1;
|
|
}
|
|
if (kappa < 0)
|
|
{
|
|
kappa = (order+1)*(order+1);
|
|
}
|
|
if (Mpi::Root())
|
|
{
|
|
args.PrintOptions(cout);
|
|
}
|
|
|
|
Device device(device_config);
|
|
if (Mpi::Root()) { device.Print(); }
|
|
|
|
// 3. Read the (serial) mesh from the given mesh file on all processors. We
|
|
// can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
|
|
// with the same code. NURBS meshes are projected to second order meshes.
|
|
Mesh mesh(mesh_file);
|
|
int dim = mesh.Dimension();
|
|
|
|
// 4. Refine the serial mesh on all processors to increase the resolution. In
|
|
// this example we do 'ser_ref_levels' of uniform refinement. By default,
|
|
// or if ser_ref_levels < 0, we choose it to be the largest number that
|
|
// gives a final mesh with no more than 50,000 elements.
|
|
{
|
|
if (ser_ref_levels < 0)
|
|
{
|
|
ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
|
|
}
|
|
for (int l = 0; l < ser_ref_levels; l++)
|
|
{
|
|
mesh.UniformRefinement();
|
|
}
|
|
}
|
|
if (mesh.NURBSext)
|
|
{
|
|
mesh.SetCurvature(max(order, 1));
|
|
}
|
|
|
|
// 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
|
|
// this mesh further in parallel to increase the resolution. Once the
|
|
// parallel mesh is defined, the serial mesh can be deleted.
|
|
ParMesh pmesh(MPI_COMM_WORLD, mesh);
|
|
mesh.Clear();
|
|
{
|
|
for (int l = 0; l < par_ref_levels; l++)
|
|
{
|
|
pmesh.UniformRefinement();
|
|
}
|
|
}
|
|
|
|
// 6. Define a parallel finite element space on the parallel mesh. Here we
|
|
// use discontinuous finite elements of the specified order >= 0.
|
|
const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
|
|
DG_FECollection fec(order, dim, bt);
|
|
ParFiniteElementSpace fespace(&pmesh, &fec);
|
|
HYPRE_BigInt size = fespace.GlobalTrueVSize();
|
|
if (Mpi::Root())
|
|
{
|
|
cout << "Number of unknowns: " << size << endl;
|
|
}
|
|
|
|
// 7. Set up the parallel linear form b(.) which corresponds to the
|
|
// right-hand side of the FEM linear system.
|
|
ParLinearForm b(&fespace);
|
|
ConstantCoefficient one(1.0);
|
|
ConstantCoefficient zero(0.0);
|
|
b.AddDomainIntegrator(new DomainLFIntegrator(one));
|
|
b.AddBdrFaceIntegrator(
|
|
new DGDirichletLFIntegrator(zero, one, sigma, kappa));
|
|
b.Assemble();
|
|
|
|
// 8. Define the solution vector x as a parallel finite element grid function
|
|
// corresponding to fespace. Initialize x with initial guess of zero.
|
|
ParGridFunction x(&fespace);
|
|
x = 0.0;
|
|
|
|
// 9. Set up the bilinear form a(.,.) on the finite element space
|
|
// corresponding to the Laplacian operator -Delta, by adding the Diffusion
|
|
// domain integrator and the interior and boundary DG face integrators.
|
|
// Note that boundary conditions are imposed weakly in the form, so there
|
|
// is no need for dof elimination. After serial and parallel assembly we
|
|
// extract the corresponding parallel matrix A.
|
|
ParBilinearForm a(&fespace);
|
|
a.AddDomainIntegrator(new DiffusionIntegrator(one));
|
|
a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
|
|
a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
|
|
if (eta > 0)
|
|
{
|
|
MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
|
|
a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
|
|
a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
|
|
}
|
|
if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
|
|
a.Assemble();
|
|
a.Finalize();
|
|
|
|
// 10. Define the parallel (hypre) matrix and vectors representing a(.,.),
|
|
// b(.) and the finite element approximation.
|
|
OperatorHandle A;
|
|
|
|
std::unique_ptr<HypreBoomerAMG> amg;
|
|
if (pa)
|
|
{
|
|
A.Reset(&a, false);
|
|
}
|
|
else
|
|
{
|
|
A.SetType(Operator::Hypre_ParCSR);
|
|
a.ParallelAssemble(A);
|
|
amg.reset(new HypreBoomerAMG(*A.As<HypreParMatrix>()));
|
|
}
|
|
|
|
// 11. Depending on the symmetry of A, define and apply a parallel PCG or
|
|
// GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
|
|
if (sigma == -1.0)
|
|
{
|
|
CGSolver cg(MPI_COMM_WORLD);
|
|
cg.SetRelTol(1e-12);
|
|
cg.SetMaxIter(500);
|
|
cg.SetPrintLevel(1);
|
|
cg.SetOperator(*A);
|
|
if (amg) { cg.SetPreconditioner(*amg); }
|
|
cg.Mult(b, x);
|
|
}
|
|
else
|
|
{
|
|
CustomSolverMonitor monitor(pmesh, x);
|
|
GMRESSolver gmres(MPI_COMM_WORLD);
|
|
gmres.SetAbsTol(0.0);
|
|
gmres.SetRelTol(1e-12);
|
|
gmres.SetMaxIter(500);
|
|
gmres.SetKDim(10);
|
|
gmres.SetPrintLevel(1);
|
|
gmres.SetOperator(*A);
|
|
if (amg) { gmres.SetPreconditioner(*amg); }
|
|
gmres.SetMonitor(monitor);
|
|
gmres.Mult(b, x);
|
|
}
|
|
|
|
// 12. Save the refined mesh and the solution in parallel. This output can
|
|
// be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
|
|
{
|
|
ostringstream mesh_name, sol_name;
|
|
mesh_name << "mesh." << setfill('0') << setw(6) << Mpi::WorldRank();
|
|
sol_name << "sol." << setfill('0') << setw(6) << Mpi::WorldRank();
|
|
|
|
ofstream mesh_ofs(mesh_name.str().c_str());
|
|
mesh_ofs.precision(8);
|
|
pmesh.Print(mesh_ofs);
|
|
|
|
ofstream sol_ofs(sol_name.str().c_str());
|
|
sol_ofs.precision(8);
|
|
x.Save(sol_ofs);
|
|
}
|
|
|
|
// 13. Send the solution by socket to a GLVis server.
|
|
if (visualization)
|
|
{
|
|
char vishost[] = "localhost";
|
|
int visport = 19916;
|
|
socketstream sol_sock(vishost, visport);
|
|
sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() << "\n";
|
|
sol_sock.precision(8);
|
|
sol_sock << "solution\n" << pmesh << x << flush;
|
|
}
|
|
|
|
return 0;
|
|
}
|