Merge branch 'master' into linearform-op-dev

examples/ex1p_vo.cpp

@@ -0,0 +1,348 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
+//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
+//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Device sample runs:
+//               mpirun -np 4 ex1p -pa -d cuda
+//               mpirun -np 4 ex1p -pa -d occa-cuda
+//               mpirun -np 4 ex1p -pa -d raja-omp
+//               mpirun -np 4 ex1p -pa -d ceed-cpu
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
+//             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//             * mpirun -np 4 ex1p -pa -d ceed-hip
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include "linalg/vector_operator.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class CoordCoefficient : public Coefficient
+{
+private:
+   int d;
+
+   mutable Vector x;
+
+public:
+   CoordCoefficient(int d) : d(d), x(3) {}
+
+   double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      if (d == -1) { return 1.0; }
+
+      T.Transform(ip, x);
+      return x[d];
+   }
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool pa = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   bool algebraic_ceed = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+#ifdef MFEM_USE_CEED
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
+                  "-no-a", "--no-algebraic",
+                  "Use algebraic Ceed solver");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh.UniformRefinement();
+      }
+   }
+
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   bool delete_fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+      delete_fec = true;
+   }
+   else if (pmesh.GetNodes())
+   {
+      fec = pmesh.GetNodes()->OwnFEC();
+      delete_fec = false;
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
+   }
+   ParFiniteElementSpace fespace(&pmesh, fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   ParVectorOperator vo(MPI_COMM_WORLD, myid, fespace.TrueVSize(), dim + 1);
+   {
+      for (int d=0; d <= dim; d++)
+      {
+         ParLinearForm bd(&fespace);
+         CoordCoefficient dCoef(d - 1);
+         bd.AddDomainIntegrator(new DomainLFIntegrator(dCoef));
+         bd.Assemble();
+
+         Vector *dv = new Vector(fespace.TrueVSize());
+         bd.ParallelAssemble(*dv);
+
+         vo.SetVector(d, dv, 1.0, true);
+      }
+   }
+
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   ParBilinearForm a(&fespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 12. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // 13. Solve the linear system A X = B.
+   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
+   //     * With partial assembly, use Jacobi smoothing, for now.
+   Solver *prec = NULL;
+   if (pa)
+   {
+      if (UsesTensorBasis(fespace))
+      {
+         if (algebraic_ceed)
+         {
+            prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
+         }
+         else
+         {
+            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         }
+      }
+   }
+   else
+   {
+      prec = new HypreBoomerAMG;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   if (prec) { cg.SetPreconditioner(*prec); }
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete prec;
+
+   {
+      Vector com((myid == 0) ? dim+1 : 0);
+      vo.Mult(X, com);
+
+      if (myid == 0)
+      {
+         cout << "Mass: " << com[0] << endl;
+         cout << "Center of mass: (";
+         for (int d=1; d<=dim; d++)
+         {
+            cout << com[d]/com[0];
+            if (d < dim) { cout << " ,"; }
+         }
+         cout << ")" << endl;
+      }
+   }
+
+   // 14. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   if (delete_fec)
+   {
+      delete fec;
+   }
+
+   return 0;
+}

examples/sundials/ex16.cpp

@@ -28,8 +28,10 @@
 //
 //               The example demonstrates the use of nonlinear operators (the
 //               class ConductionOperator defining C(u)), as well as their
-//               implicit time integration. By default, this example uses the
-//               SUNDIALS ODE solvers from CVODE and ARKODE.
+//               implicit time integration. Note that implementing the method
+//               ConductionOperator::ImplicitSolve is the only requirement for
+//               high-order implicit (SDIRK) time integration. By default, this
+//               example uses the SUNDIALS ODE solvers from CVODE and ARKODE.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -49,16 +51,15 @@ using namespace mfem;
  *  and K(u) is the diffusion operator with diffusivity depending on u:
  *  (\kappa + \alpha u).
  *
- *  Class ConductionOperator represents the above ODE operator as a
- *  TimeDependentOperator for use with native MFEM integrators and CVODE
- *  integrators, i.e., F(u, k, t) = G(u, t) with F(u, du/dt, t) = du/dt and
- *  G(u, t) = -K(u) u
+ *  Class ConductionOperatorOperator represents the above ODE operator in the
+ *  general form F(u, k, t) = G(u, t) where
  *
- *  Class ConductionOperator represents the above ODE operator as an
- *  ARKStepODE for use with ARKODE integrators, i.e., either M du/dt = -K(u) u
- *  (mass form) or du/dt = -inv(M) K(u) u (MFEM form)
+ *    1. F(u, du/dt, t) = du/dt            (ODE is expressed in EXPLICIT form)
+ *       G(u, t) = - inv(M) K(u) u
+ *    2. F(u, du/dt, t) = M du/dt          (ODE is expressed in IMPLICIT form)
+ *       G(u, t) = - K(u) u
  */
-class ConductionOperator : public TimeDependentOperator, public ARKStepODE
+class ConductionOperator : public TimeDependentOperator
 {
    FiniteElementSpace &fespace;
    Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
@@ -80,90 +81,50 @@ class ConductionOperator : public TimeDependentOperator, public ARKStepODE
 
    mutable Vector z; // auxiliary vector
 
-   const bool use_mass_form;
-
 public:
 
    ConductionOperator(FiniteElementSpace &f, const real_t alpha,
                       const real_t kappa, const Vector &u,
-                      const bool use_mass_form);
+                      const Type &ode_expression_type);
 
    // Compute K(u_n) for use as an approximation in - K(u) u
    void SetConductionTensor(const Vector &u);
 
-   // ********* methods for MFEM native time integrators *********
+   /** Compute G(u, t) as defined in the IMPLICIT expression form of the ODE
+    operator, i.e., @a v = - K(u_n) @a u. Note that K(u_n) is an
+      approximation to K(u). */
+   void ExplicitMult(const Vector &u, Vector &v) const override;
 
-   /** Solve for k in F(u, k, t) = G(u, t), i.e., @a k = - inv(M) K(u_n) @a u.
+   /** Solve for k in F(u, k, t) = G(u, t) for either EXPLICIT or IMPLICIT
+       expression forms of the ODE operator, i.e., @a k = - inv(M) K(u_n) @a u.
        Note that K(u_n) is an approximation to K(u). */
    void Mult(const Vector &u, Vector &k) const override;
 
-   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t), i.e.,
-       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u .
-       Note that K(u_n) is an approximation to K(u). */
+   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t) for either EXPLICIT
+       or IMPLICIT expression forms of the ODE operator, i.e.,
+       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u . Note that K(u_n) is an
+       approximation to K(u). */
    void ImplicitSolve(const real_t gam, const Vector &u, Vector &k) override;
 
-      // ********* methods for ARKODE time integrators *********
-
-   // TODO: add comments
-   int ARKSize() const override;
-
-   // TODO: add comments
-   bool ARKInMassForm() const override;
-
-   // TODO: add comments
-   void ARKEvaluateRHS(const Vector &u, const real_t t, Vector &result) const override;
-
-   // TODO: add comments
-   int ARKImplicitSetup(const Vector &u, const real_t t, const Vector &fu,
-                        int jok, int *jcur, real_t gam) override;
-
-   /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing either
-        1. @a r = G - F = inv(M) f(u) - k       (MFEM form)
-        1. @a r = G - F = f(u) - M k            (mass form)
-       */
-   int ARKImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
-
-   int ARKMassSetup(const real_t t) override;
-
-   int ARKMassSolve(const Vector &b, Vector &x, real_t tol) override;
-
-   int ARKMassMult(const Vector &x, Vector &v) override;
-
-   // ********* methods for CVODE time integrators *********
-   // note these methods merely call the corresponding ARKStepODE methods until
-   // the CVODESolver is refactored to use specialized interface like ARKStepODE
-
-   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F, i.e.,
+   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F for
+       either EXPLICIT or IMPLICIT expression forms of the ODE operator, i.e.,
        [M - @a gam Jf(u)] dk = G - F, where Jf(u) is an approximation of the
        Jacobian of -K(u) u. The approximation chosen here is Jf(u) = -K(u_n). */
    int SUNImplicitSetup(const Vector &u, const Vector &fu, int jok, int *jcur,
-                        real_t gam) override
-   {
-      return ARKImplicitSetup(u, 0.0, fu, jok, jcur, gam); // the ODE is autonomous
-   }
+                        real_t gam) override;
 
    /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing @a r = G - F = inv(M) f(u) - k. */
-   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override
-   {
-      return ARKImplicitSolve(r, dk, tol);
-   }
+       with the residual @a r providing either
+        1. @a r = G - F = inv(M) f(u) - k       (EXPLICIT expression form)
+        1. @a r = G - F = f(u) - M k            (IMPLICIT expression form)
+       */
+   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
 
-   int SUNMassSetup() override
-   {
-      return ARKMassSetup(0.0); // the ODE is autonomous
-   }
+   int SUNMassSetup() override;
 
-   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override
-   {
-      return ARKMassSolve(b, x, tol);
-   }
+   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override;
 
-   int SUNMassMult(const Vector &x, Vector &v) override
-   {
-      return ARKMassMult(x, v);
-   }
+   int SUNMassMult(const Vector &x, Vector &v) override;
 };
 
 real_t InitialTemperature(const Vector &x)
@@ -284,7 +245,16 @@ int main(int argc, char *argv[])
    u_gf.GetTrueDofs(u);
 
    // 6. Initialize the conduction ODE operator and the visualization.
-   ConductionOperator oper(fespace, alpha, kappa, u, use_mass_solver);
+   ConductionOperator::Type ode_expression_type;
+   if (use_mass_solver)
+   {
+      ode_expression_type = ConductionOperator::Type::IMPLICIT;
+   }
+   else
+   {
+      ode_expression_type = ConductionOperator::Type::EXPLICIT;
+   }
+   ConductionOperator oper(fespace, alpha, kappa, u, ode_expression_type);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -382,7 +352,7 @@ int main(int argc, char *argv[])
          }
          std::unique_ptr<ARKStepSolver> arkode(
             new ARKStepSolver(arkode_solver_type));
-         arkode->Init(&oper);
+         arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 11 || ode_solver_type == 14)
@@ -475,10 +445,9 @@ int main(int argc, char *argv[])
 ConductionOperator::ConductionOperator(FiniteElementSpace &fes,
                                        const real_t alpha, const real_t kappa,
                                        const Vector &u,
-                                       const bool use_mass_form)
-   : TimeDependentOperator(fes.GetTrueVSize(), 0.0),
-     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa), z(height),
-     use_mass_form(use_mass_form)
+                                       const Type &ode_expression_type)
+   : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
+     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa), z(height)
 {
    // specify a relative tolerance for all solves with MFEM integrators
    const real_t rel_tol = 1e-8;
@@ -505,16 +474,6 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &fes,
    SetConductionTensor(u);
 }
 
-int ConductionOperator::ARKSize() const
-{
-   return z.Size();
-}
-
-bool ConductionOperator::ARKInMassForm() const
-{
-   return use_mass_form;
-}
-
 void ConductionOperator::SetConductionTensor(const Vector &u)
 {
    // Compute K(u_n).
@@ -532,27 +491,17 @@ void ConductionOperator::SetConductionTensor(const Vector &u)
    K->FormSystemMatrix(ess_tdof_list, Kmat);
 }
 
-void ConductionOperator::ARKEvaluateRHS(const Vector &u, const real_t t,
-                                        Vector &result) const
+void ConductionOperator::ExplicitMult(const Vector &u, Vector &v) const
 {
-   if (use_mass_form) // compute -K(u_n) u.
-   {
-      Kmat.Mult(u, result);
-      result.Neg();
-   }
-   else // compute -inv(M) K(u_n) u
-   {
-      Kmat.Mult(u, z);
-      z.Neg();
-      M_solver.Mult(z, result);
-   }
+   // Compute - K(u_n) u.
+   Kmat.Mult(u, v);
+   v.Neg();
 }
 
 void ConductionOperator::Mult(const Vector &u, Vector &k) const
 {
    // Compute - inv(M) K(u_n) u.
-   Kmat.Mult(u, z);
-   z.Neg();
+   ExplicitMult(u, z);
    M_solver.Mult(z, k);
 }
 
@@ -560,16 +509,14 @@ void ConductionOperator::ImplicitSolve(const real_t gam, const Vector &u,
                                        Vector &k)
 {
    // Solve for k in M k = - K(u_n) [u + gam*k].
-   Kmat.Mult(u, z);
-   z.Neg();
+   ExplicitMult(u, z);
    T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
    T_solver.SetOperator(*T);
    T_solver.Mult(z, k);
 }
 
-int ConductionOperator::ARKImplicitSetup(const Vector &u, const real_t t,
-                                         const Vector &fu, int jok, int *jcur,
-                                         real_t gam)
+int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
+                                         int jok, int *jcur, real_t gam)
 {
    // Compute T = M + gamma K(u_n).
    T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
@@ -578,23 +525,23 @@ int ConductionOperator::ARKImplicitSetup(const Vector &u, const real_t t,
    return SUN_SUCCESS;
 }
 
-int ConductionOperator::ARKImplicitSolve(const Vector &r, Vector &dk,
+int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
                                          real_t tol)
 {
    // Solve the system [M + gamma K(u_n)] dk = - K(u_n) u - M k.
    // What value r is providing depends on the ODE expression form:
-   //   MFEM form: r = -inv(M) K(u_n) u - k
-   //   mass form: r = -K(u_n) u - M k
+   //   EXPLICIT form: r = -inv(M) K(u_n) u - k
+   //   IMPLICIT form: r = -K(u_n) u - M k
    T_solver.SetRelTol(tol);
-   if (use_mass_form)
-   {
-      T_solver.Mult(r, dk);
-   }
-   else
+   if (isExplicit())
    {
       Mmat.Mult(r, z);
       T_solver.Mult(z, dk);
    }
+   else
+   {
+      T_solver.Mult(r, dk);
+   }
    if (T_solver.GetConverged())
    {
       return SUN_SUCCESS;
@@ -605,13 +552,13 @@ int ConductionOperator::ARKImplicitSolve(const Vector &r, Vector &dk,
    }
 }
 
-int ConductionOperator::ARKMassSetup(const real_t t)
+int ConductionOperator::SUNMassSetup()
 {
    // Do nothing b/c mass solver was setup in constructor.
    return SUN_SUCCESS;
 }
 
-int ConductionOperator::ARKMassSolve(const Vector &b, Vector &x, real_t tol)
+int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
 {
    // Solve the system M x = b.
    M_solver.SetRelTol(tol);
@@ -626,7 +573,7 @@ int ConductionOperator::ARKMassSolve(const Vector &b, Vector &x, real_t tol)
    }
 }
 
-int ConductionOperator::ARKMassMult(const Vector &x, Vector &v)
+int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
 {
    // Compute M x.
    Mmat.Mult(x, v);

examples/sundials/ex16p.cpp

@@ -29,8 +29,10 @@
 //
 //               The example demonstrates the use of nonlinear operators (the
 //               class ConductionOperator defining C(u)), as well as their
-//               implicit time integration. By default, this example uses the
-//               SUNDIALS ODE solvers from CVODE and ARKODE.
+//               implicit time integration. Note that implementing the method
+//               ConductionOperator::ImplicitSolve is the only requirement for
+//               high-order implicit (SDIRK) time integration. By default, this
+//               example uses the SUNDIALS ODE solvers from CVODE and ARKODE.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -50,16 +52,15 @@ using namespace mfem;
  *  and K(u) is the diffusion operator with diffusivity depending on u:
  *  (\kappa + \alpha u).
  *
- *  Class ConductionOperator represents the above ODE operator as a
- *  TimeDependentOperator for use with native MFEM integrators and CVODE
- *  integrators, i.e., F(u, k, t) = G(u, t) with F(u, du/dt, t) = du/dt and
- *  G(u, t) = -K(u) u
+ *  Class ConductionOperatorOperator represents the above ODE operator in the
+ *  general form F(u, k, t) = G(u, t) where either
  *
- *  Class ConductionOperator represents the above ODE operator as an
- *  ARKStepODE for use with ARKODE integrators, i.e., either M du/dt = -K(u) u
- *  (mass form) or du/dt = -inv(M) K(u) u (MFEM form)
+ *    1. F(u, du/dt, t) = du/dt            (ODE is expressed in EXPLICIT form)
+ *       G(u, t) = - inv(M) K(u) u
+ *    2. F(u, du/dt, t) = M du/dt          (ODE is expressed in IMPLICIT form)
+ *       G(u, t) = - K(u) u
  */
-class ConductionOperator : public TimeDependentOperator, public ARKStepODE
+class ConductionOperator : public TimeDependentOperator
 {
    ParFiniteElementSpace &fespace;
    Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
@@ -81,90 +82,50 @@ class ConductionOperator : public TimeDependentOperator, public ARKStepODE
 
    mutable Vector z; // auxiliary vector
 
-   const bool use_mass_form;
-
 public:
 
    ConductionOperator(ParFiniteElementSpace &f, const real_t alpha,
                       const real_t kappa, const Vector &u,
-                      const bool use_mass_form);
+                      const Type &ode_expression_type);
 
    // Compute K(u_n) for use as an approximation in - K(u) u
    void SetConductionTensor(const Vector &u);
 
-   // ********* methods for MFEM native time integrators *********
+   /** Compute G(u, t) as defined in the IMPLICIT expression form of the ODE
+       operator, i.e., @a v = - K(u_n) @a u. Note that K(u_n) is an
+       approximation to K(u). */
+   void ExplicitMult(const Vector &u, Vector &v) const override;
 
-   /** Solve for k in F(u, k, t) = G(u, t), i.e., @a k = - inv(M) K(u_n) @a u.
+   /** Solve for k in F(u, k, t) = G(u, t) for either EXPLICIT or IMPLICIT
+       expression forms of the ODE operator, i.e., @a k = - inv(M) K(u_n) @a u.
        Note that K(u_n) is an approximation to K(u). */
    void Mult(const Vector &u, Vector &k) const override;
 
-   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t), i.e.,
-       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u .
-       Note that K(u_n) is an approximation to K(u). */
+   /** Solve for k in F(u + gam*k, k, t) = G(u + gam*k, t) for either EXPLICIT
+       or IMPLICIT expression forms of the ODE operator, i.e.,
+       [ M + @a gam K(u_n) ] @a k = - K(u_n) @a u . Note that K(u_n) is an
+       approximation to K(u). */
    void ImplicitSolve(const real_t gam, const Vector &u, Vector &k) override;
 
-   // ********* methods for ARKODE time integrators *********
-
-   // TODO: add comments
-   int ARKSize() const override;
-
-   // TODO: add comments
-   bool ARKInMassForm() const override;
-
-   // TODO: add comments
-   void ARKEvaluateRHS(const Vector &u, const real_t t, Vector &result) const override;
-
-   // TODO: add comments
-   int ARKImplicitSetup(const Vector &u, const real_t t, const Vector &fu,
-                        int jok, int *jcur, real_t gam) override;
-
-   /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing either
-        1. @a r = G - F = inv(M) f(u) - k       (MFEM form)
-        1. @a r = G - F = f(u) - M k            (mass form)
-       */
-   int ARKImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
-
-   int ARKMassSetup(const real_t t) override;
-
-   int ARKMassSolve(const Vector &b, Vector &x, real_t tol) override;
-
-   int ARKMassMult(const Vector &x, Vector &v) override;
-
-   // ********* methods for CVODE time integrators *********
-   // note these methods merely call the corresponding ARKStepODE methods until
-   // the CVODESolver is refactored to use specialized interface like ARKStepODE
-
-   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F, i.e.,
+   /** Setup to solve for dk in [dF/dk + gam*dF/du - gam*dG/du] dk = G - F for
+       either EXPLICIT or IMPLICIT expression forms of the ODE operator, i.e.,
        [M - @a gam Jf(u)] dk = G - F, where Jf(u) is an approximation of the
        Jacobian of -K(u) u. The approximation chosen here is Jf(u) = -K(u_n). */
    int SUNImplicitSetup(const Vector &u, const Vector &fu, int jok, int *jcur,
-                        real_t gam) override
-   {
-      return ARKImplicitSetup(u, 0.0, fu, jok, jcur, gam); // the ODE is autonomous
-   }
+                        real_t gam) override;
 
    /** Solve for @a dk in the system in SUNImplicitSetup to the given tolerance,
-       with the residual @a r providing @a r = G - F = inv(M) f(u) - k. */
-   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override
-   {
-      return ARKImplicitSolve(r, dk, tol);
-   }
+       with the residual @a r providing either
+        1. @a r = G - F = inv(M) f(u) - k       (EXPLICIT expression form)
+        1. @a r = G - F = f(u) - M k            (IMPLICIT expression form)
+       */
+   int SUNImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
 
-   int SUNMassSetup() override
-   {
-      return ARKMassSetup(0.0); // the ODE is autonomous
-   }
+   int SUNMassSetup() override;
 
-   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override
-   {
-      return ARKMassSolve(b, x, tol);
-   }
+   int SUNMassSolve(const Vector &b, Vector &x, real_t tol) override;
 
-   int SUNMassMult(const Vector &x, Vector &v) override
-   {
-      return ARKMassMult(x, v);
-   }
+   int SUNMassMult(const Vector &x, Vector &v) override;
 };
 
 real_t InitialTemperature(const Vector &x)
@@ -312,7 +273,16 @@ int main(int argc, char *argv[])
    u_gf.GetTrueDofs(u);
 
    // 8. Initialize the conduction ODE operator and the visualization.
-   ConductionOperator oper(fespace, alpha, kappa, u, use_mass_solver);
+   ConductionOperator::Type ode_expression_type;
+   if (use_mass_solver)
+   {
+      ode_expression_type = ConductionOperator::Type::IMPLICIT;
+   }
+   else
+   {
+      ode_expression_type = ConductionOperator::Type::EXPLICIT;
+   }
+   ConductionOperator oper(fespace, alpha, kappa, u, ode_expression_type);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -424,7 +394,7 @@ int main(int argc, char *argv[])
          }
          std::unique_ptr<ARKStepSolver> arkode(
             new ARKStepSolver(MPI_COMM_WORLD, arkode_solver_type));
-         arkode->Init(&oper);
+         arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 11 || ode_solver_type == 14)
@@ -527,11 +497,10 @@ int main(int argc, char *argv[])
 ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
                                        const real_t alpha, const real_t kappa,
                                        const Vector &u,
-                                       const bool use_mass_form)
-   : TimeDependentOperator(fes.GetTrueVSize(), 0.0),
+                                       const Type &ode_expression_type)
+   : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
      fespace(fes), M(&fespace), alpha(alpha), kappa(kappa),
-     M_solver(fes.GetComm()), T_solver(fes.GetComm()), z(height),
-     use_mass_form(use_mass_form)
+     M_solver(fes.GetComm()), T_solver(fes.GetComm()), z(height)
 {
    // specify a relative tolerance for all solves with MFEM integrators
    const real_t rel_tol = 1e-8;
@@ -559,16 +528,6 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
    SetConductionTensor(u);
 }
 
-int ConductionOperator::ARKSize() const
-{
-   return z.Size();
-}
-
-bool ConductionOperator::ARKInMassForm() const
-{
-   return use_mass_form;
-}
-
 void ConductionOperator::SetConductionTensor(const Vector &u)
 {
    // Compute K(u_n).
@@ -586,27 +545,17 @@ void ConductionOperator::SetConductionTensor(const Vector &u)
    K->FormSystemMatrix(ess_tdof_list, Kmat);
 }
 
-void ConductionOperator::ARKEvaluateRHS(const Vector &u, const real_t t,
-                                        Vector &result) const
+void ConductionOperator::ExplicitMult(const Vector &u, Vector &v) const
 {
-   if (use_mass_form) // compute -K(u_n) u.
-   {
-      Kmat.Mult(u, result);
-      result.Neg();
-   }
-   else // compute -inv(M) K(u_n) u
-   {
-      Kmat.Mult(u, z);
-      z.Neg();
-      M_solver.Mult(z, result);
-   }
+   // Compute - K(u_n) u.
+   Kmat.Mult(u, v);
+   v.Neg();
 }
 
 void ConductionOperator::Mult(const Vector &u, Vector &k) const
 {
    // Compute - inv(M) K(u_n) u.
-   Kmat.Mult(u, z);
-   z.Neg();
+   ExplicitMult(u, z);
    M_solver.Mult(z, k);
 }
 
@@ -614,16 +563,14 @@ void ConductionOperator::ImplicitSolve(const real_t gam, const Vector &u,
                                        Vector &k)
 {
    // Solve for k in M k = - K(u_n) [u + gam*k].
-   Kmat.Mult(u, z);
-   z.Neg();
+   ExplicitMult(u, z);
    T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
    T_solver.SetOperator(*T);
    T_solver.Mult(z, k);
 }
 
-int ConductionOperator::ARKImplicitSetup(const Vector &u, const real_t t,
-                                         const Vector &fu, int jok, int *jcur,
-                                         real_t gam)
+int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
+                                         int jok, int *jcur, real_t gam)
 {
    // Compute T = M + gamma K(u_n).
    T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
@@ -632,23 +579,23 @@ int ConductionOperator::ARKImplicitSetup(const Vector &u, const real_t t,
    return SUN_SUCCESS;
 }
 
-int ConductionOperator::ARKImplicitSolve(const Vector &r, Vector &dk,
+int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
                                          real_t tol)
 {
    // Solve the system [M + gamma K(u_n)] dk = - K(u_n) u - M k.
    // What value r is providing depends on the ODE expression form:
-   //   MFEM form: r = -inv(M) K(u_n) u - k
-   //   mass form: r = -K(u_n) u - M k
+   //   EXPLICIT form: r = -inv(M) K(u_n) u - k
+   //   IMPLICIT form: r = -K(u_n) u - M k
    T_solver.SetRelTol(tol);
-   if (use_mass_form)
-   {
-      T_solver.Mult(r, dk);
-   }
-   else
+   if (isExplicit())
    {
       Mmat.Mult(r, z);
       T_solver.Mult(z, dk);
    }
+   else
+   {
+      T_solver.Mult(r, dk);
+   }
    if (T_solver.GetConverged())
    {
       return SUN_SUCCESS;
@@ -659,13 +606,13 @@ int ConductionOperator::ARKImplicitSolve(const Vector &r, Vector &dk,
    }
 }
 
-int ConductionOperator::ARKMassSetup(const real_t t)
+int ConductionOperator::SUNMassSetup()
 {
    // Do nothing b/c mass solver was setup in constructor.
    return SUN_SUCCESS;
 }
 
-int ConductionOperator::ARKMassSolve(const Vector &b, Vector &x, real_t tol)
+int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
 {
    // Solve the system M x = b.
    M_solver.SetRelTol(tol);
@@ -680,7 +627,7 @@ int ConductionOperator::ARKMassSolve(const Vector &b, Vector &x, real_t tol)
    }
 }
 
-int ConductionOperator::ARKMassMult(const Vector &x, Vector &v)
+int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
 {
    // Compute M x.
    Mmat.Mult(x, v);

examples/sundials/ex9.cpp

@@ -119,7 +119,7 @@ public:
     and advection matrices, and b describes the flow on the boundary. This can
     be written as a general ODE, du/dt = M^{-1} (K u + b), and this class is
     used to evaluate the right-hand side. */
-class FE_Evolution : public TimeDependentOperator, public ARKStepODE
+class FE_Evolution : public TimeDependentOperator
 {
 private:
    BilinearForm &M, &K;
@@ -133,14 +133,9 @@ private:
 public:
    FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);
 
-   // TimeDependentOperator methods for MFEM native and CVODE time integrators
    virtual void Mult(const Vector &x, Vector &y) const;
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
-   // ARKStepODE methods for ARKODE time integrators
-   int ARKSize() const override;
-   void ARKEvaluateRHS(const Vector &u, const real_t t, Vector& result) const override;
-
    virtual ~FE_Evolution();
 };
 
@@ -409,14 +404,14 @@ int main(int argc, char *argv[])
          ode_solver = cvode; break;
       case 8:
          arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(&adv);
+         arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
          arkode->SetOrder(4);
          ode_solver = arkode; break;
       case 9:
          arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(&adv);
+         arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
          arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
@@ -525,19 +520,6 @@ void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
    dg_solver->Mult(z, k);
 }
 
-int FE_Evolution::ARKSize() const
-{
-   return z.Size();
-}
-
-void FE_Evolution::ARKEvaluateRHS(const Vector &u, const real_t t, Vector &result) const
-{
-   // y = M^{-1} (K x + b)
-   K.Mult(u, z);
-   z += b;
-   M_solver.Mult(z, result);
-}
-
 FE_Evolution::~FE_Evolution()
 {
    delete M_prec;

examples/sundials/ex9p.cpp

@@ -206,7 +206,7 @@ public:
     and advection matrices, and b describes the flow on the boundary. This can
     be written as a general ODE, du/dt = M^{-1} (K u + b), and this class is
     used to evaluate the right-hand side. */
-class FE_Evolution : public TimeDependentOperator, public ARKStepODE
+class FE_Evolution : public TimeDependentOperator
 {
 private:
    OperatorHandle M, K;
@@ -221,14 +221,9 @@ public:
    FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_,
                 PrecType prec_type);
 
-   // TimeDependentOperator methods for MFEM native and CVODE time integrators
    virtual void Mult(const Vector &x, Vector &y) const;
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
-   // ARKStepODE methods for ARKODE time integrators
-   int ARKSize() const override;
-   void ARKEvaluateRHS(const Vector &u, const real_t t, Vector& result) const override;
-
    virtual ~FE_Evolution();
 };
 
@@ -580,7 +575,7 @@ int main(int argc, char *argv[])
       case 8:
       case 9:
          arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(&adv);
+         arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 9)
@@ -748,19 +743,6 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
    M_solver.Mult(z, y);
 }
 
-int FE_Evolution::ARKSize() const
-{
-   return z.Size();
-}
-
-void FE_Evolution::ARKEvaluateRHS(const Vector &u, const real_t t, Vector &result) const
-{
-   // y = M^{-1} (K x + b)
-   K->Mult(u, z);
-   z += b;
-   M_solver.Mult(z, result);
-}
-
 FE_Evolution::~FE_Evolution()
 {
    delete M_prec;

linalg/operator.hpp

@@ -578,7 +578,7 @@ public:
 
        Presently, this method is used by SUNDIALS ARKStep integrator, for more
        details, see the ARKode User Guide. */
-   MFEM_DEPRECATED virtual int SUNMassSetup();
+   virtual int SUNMassSetup();
 
    /** @brief Solve the mass matrix linear system  M @a x = @a b, where M is
        defined by the method SUNMassSetup().
@@ -591,7 +591,7 @@ public:
 
        Presently, this method is used by SUNDIALS ARKStep integrator, for more
        details, see the ARKode User Guide. */
-   MFEM_DEPRECATED virtual int SUNMassSolve(const Vector &b, Vector &x, real_t tol);
+   virtual int SUNMassSolve(const Vector &b, Vector &x, real_t tol);
 
    /** @brief Compute the mass matrix-vector product @a v = M @a x, where M is
        defined by the method SUNMassSetup().
@@ -603,7 +603,7 @@ public:
 
        Presently, this method is used by SUNDIALS ARKStep integrator, for more
        details, see the ARKode User Guide. */
-   MFEM_DEPRECATED virtual int SUNMassMult(const Vector &x, Vector &v);
+   virtual int SUNMassMult(const Vector &x, Vector &v);
 
    virtual ~TimeDependentOperator() { }
 };

linalg/sundials.cpp

@@ -1367,84 +1367,6 @@ CVODESSolver::~CVODESSolver()
 // ARKStep interface
 // ---------------------------------------------------------------------------
 
-ARKStepSolver::TimeDependentOperatorWrapper::TimeDependentOperatorWrapper(
-   TimeDependentOperator *f)
-{
-   tdo = f;
-}
-
-int ARKStepSolver::TimeDependentOperatorWrapper::ARKSize() const
-{
-   return tdo->Height();
-}
-
-bool ARKStepSolver::TimeDependentOperatorWrapper::ARKInMassForm() const
-{
-   return (tdo->isExplicit() == false);
-}
-
-void ARKStepSolver::TimeDependentOperatorWrapper::ARKSetEvalMode(
-   const ARKEvalMode new_eval_mode)
-{
-   if (new_eval_mode == NORMAL)
-      tdo->SetEvalMode(tdo->NORMAL);
-   else if (new_eval_mode == ADDITIVE_TERM_1)
-      tdo->SetEvalMode(tdo->ADDITIVE_TERM_1);
-   else if (new_eval_mode == ADDITIVE_TERM_2)
-      tdo->SetEvalMode(tdo->ADDITIVE_TERM_2);
-   else
-      mfem_error("Unrecognized evaluation mode.");
-}
-
-void ARKStepSolver::TimeDependentOperatorWrapper::ARKEvaluateRHS(
-   const Vector &u, const real_t t, Vector &result) const
-{
-   tdo->SetTime(t);
-   if (ARKInMassForm())
-      tdo->Mult(u, result);
-   else
-      tdo->ExplicitMult(u, result);
-}
-
-int ARKStepSolver::TimeDependentOperatorWrapper::ARKImplicitSetup(
-   const Vector &u, const real_t t, const Vector &v, int jok, int *jcur,
-   real_t gamma)
-{
-   tdo->SetTime(t);
-   return tdo->SUNImplicitSetup(u, v, jok, jcur, gamma);
-}
-
-int ARKStepSolver::TimeDependentOperatorWrapper::ARKImplicitSolve(
-   const Vector &r, Vector &dk, real_t tol)
-{
-   return tdo->SUNImplicitSolve(r, dk, tol);
-}
-
-int ARKStepSolver::TimeDependentOperatorWrapper::ARKMassSetup(const real_t t)
-{
-   tdo->SetTime(t);
-   return tdo->SUNMassSetup();
-}
-
-int ARKStepSolver::TimeDependentOperatorWrapper::ARKMassSolve(const Vector &b,
-   Vector &x, real_t tol)
-{
-   return tdo->SUNMassSolve(b, x, tol);
-}
-
-int ARKStepSolver::TimeDependentOperatorWrapper::ARKMassMult(const Vector &x,
-   const real_t t, Vector &v)
-{
-   tdo->SetTime(t);
-   return tdo->SUNMassMult(x, v);
-}
-
-int ARKStepSolver::TimeDependentOperatorWrapper::ARKMassMult(const Vector &x,
-   Vector &v)
-{
-   return tdo->SUNMassMult(x, v);
-}
-
 int ARKStepSolver::RHS1(sunrealtype t, const N_Vector y, N_Vector result,
                         void *user_data)
 {
@@ -1459,11 +1381,19 @@ int ARKStepSolver::RHS1(sunrealtype t, const N_Vector y, N_Vector result,
    // or fe(t, y) in one of
    //   1. y' = fe(t, y) + fi(t, y)
    //   2. M y' = fe(t, y) + fi(t, y)
+   self->f->SetTime(t);
    if (self->rk_type == IMEX)
    {
-      self->f_arkstep->ARKSetEvalMode(ARKStepODE::ADDITIVE_TERM_1);
+      self->f->SetEvalMode(TimeDependentOperator::ADDITIVE_TERM_1);
+   }
+   if (self->f->isExplicit()) // ODE is in form 1
+   {
+      self->f->Mult(mfem_y, mfem_result);
+   }
+   else // ODE is in form 2
+   {
+      self->f->ExplicitMult(mfem_y, mfem_result);
    }
-   self->f_arkstep->ARKEvaluateRHS(mfem_y, t, mfem_result);
 
    // Return success
    return (0);
@@ -1480,8 +1410,16 @@ int ARKStepSolver::RHS2(sunrealtype t, const N_Vector y, N_Vector result,
    // Compute fi(t, y) in one of
    //   1. y' = fe(t, y) + fi(t, y)       (ODE is expressed in EXPLICIT form)
    //   2. M y' = fe(t, y) + fi(y, t)     (ODE is expressed in IMPLICIT form)
-   self->f_arkstep->ARKSetEvalMode(ARKStepODE::ADDITIVE_TERM_2);
-   self->f_arkstep->ARKEvaluateRHS(mfem_y, t, mfem_result);
+   self->f->SetTime(t);
+   self->f->SetEvalMode(TimeDependentOperator::ADDITIVE_TERM_2);
+   if (self->f->isExplicit())
+   {
+      self->f->Mult(mfem_y, mfem_result);
+   }
+   else
+   {
+      self->f->ExplicitMult(mfem_y, mfem_result);
+   }
 
    // Return success
    return (0);
@@ -1498,11 +1436,12 @@ int ARKStepSolver::LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy,
    ARKStepSolver *self = static_cast<ARKStepSolver*>(GET_CONTENT(A));
 
    // Compute the linear system
+   self->f->SetTime(t);
    if (self->rk_type == IMEX)
    {
-      self->f_arkstep->ARKSetEvalMode(ARKStepODE::ADDITIVE_TERM_2);
+      self->f->SetEvalMode(TimeDependentOperator::ADDITIVE_TERM_2);
    }
-   return (self->f_arkstep->ARKImplicitSetup(mfem_y, t, mfem_fy, jok, jcur, gamma));
+   return (self->f->SUNImplicitSetup(mfem_y, mfem_fy, jok, jcur, gamma));
 }
 
 int ARKStepSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
@@ -1515,9 +1454,9 @@ int ARKStepSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
    // Solve the linear system
    if (self->rk_type == IMEX)
    {
-      self->f_arkstep->ARKSetEvalMode(ARKStepODE::ADDITIVE_TERM_2);
+      self->f->SetEvalMode(TimeDependentOperator::ADDITIVE_TERM_2);
    }
-   return (self->f_arkstep->ARKImplicitSolve(mfem_b, mfem_x, tol));
+   return (self->f->SUNImplicitSolve(mfem_b, mfem_x, tol));
 }
 
 int ARKStepSolver::MassSysSetup(sunrealtype t, SUNMatrix M,
@@ -1526,7 +1465,8 @@ int ARKStepSolver::MassSysSetup(sunrealtype t, SUNMatrix M,
    ARKStepSolver *self = static_cast<ARKStepSolver*>(GET_CONTENT(M));
 
    // Compute the mass matrix system
-   return (self->f_arkstep->ARKMassSetup(t));
+   self->f->SetTime(t);
+   return (self->f->SUNMassSetup());
 }
 
 int ARKStepSolver::MassSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
@@ -1537,7 +1477,7 @@ int ARKStepSolver::MassSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
    ARKStepSolver *self = static_cast<ARKStepSolver*>(GET_CONTENT(LS));
 
    // Solve the mass matrix system
-   return (self->f_arkstep->ARKMassSolve(mfem_b, mfem_x, tol));
+   return (self->f->SUNMassSolve(mfem_b, mfem_x, tol));
 }
 
 int ARKStepSolver::MassMult1(SUNMatrix M, N_Vector x, N_Vector v)
@@ -1547,7 +1487,7 @@ int ARKStepSolver::MassMult1(SUNMatrix M, N_Vector x, N_Vector v)
    ARKStepSolver *self = static_cast<ARKStepSolver*>(GET_CONTENT(M));
 
    // Compute the mass matrix-vector product
-   return (self->f_arkstep->ARKMassMult(mfem_x, mfem_v));
+   return (self->f->SUNMassMult(mfem_x, mfem_v));
 }
 
 int ARKStepSolver::MassMult2(N_Vector x, N_Vector v, sunrealtype t,
@@ -1558,7 +1498,8 @@ int ARKStepSolver::MassMult2(N_Vector x, N_Vector v, sunrealtype t,
    ARKStepSolver *self = static_cast<ARKStepSolver*>(mtimes_data);
 
    // Compute the mass matrix-vector product
-   return (self->f_arkstep->ARKMassMult(mfem_x, t, mfem_v));
+   self->f->SetTime(t);
+   return (self->f->SUNMassMult(mfem_x, mfem_v));
 }
 
 ARKStepSolver::ARKStepSolver(Type type)
@@ -1577,12 +1518,13 @@ ARKStepSolver::ARKStepSolver(MPI_Comm comm, Type type)
 }
 #endif
 
-void ARKStepSolver::Init(ARKStepODE *f_ark_)
+void ARKStepSolver::Init(TimeDependentOperator &f_)
 {
-   f_arkstep = f_ark_;
+   // Initialize the base class
+   ODESolver::Init(f_);
 
    // Get the vector length
-   long local_size = f_arkstep->ARKSize();
+   long local_size = f_.Height();
 #ifdef MFEM_USE_MPI
    long global_size;
 #endif
@@ -1596,7 +1538,7 @@ void ARKStepSolver::Init(ARKStepODE *f_ark_)
    }
 
    // Get current time
-   double t = f ? f->GetTime() : 0.0;
+   double t = f_.GetTime();
 
    if (sundials_mem)
    {
@@ -1675,12 +1617,6 @@ void ARKStepSolver::Init(ARKStepODE *f_ark_)
    reinit = true;
 }
 
-void ARKStepSolver::Init(TimeDependentOperator &f_)
-{
-   f_tdo = std::make_unique<TimeDependentOperatorWrapper>(&f_);
-   Init(f_tdo.get());
-}
-
 void ARKStepSolver::Step(Vector &x, real_t &t, real_t &dt)
 {
    Y->MakeRef(x, 0, x.Size());
@@ -1773,9 +1709,6 @@ void ARKStepSolver::UseSundialsLinearSolver()
 
 void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
 {
-   // Check that the ODE is expressed in mass form
-   MFEM_VERIFY(f_arkstep->ARKInMassForm(), "ODE operator is not in mass form.")
-
    // Free any existing matrix and linear solver
    if (M != NULL)   { SUNMatDestroy(M); M = NULL; }
    if (LSM != NULL) { SUNLinSolFree(LSM); LSM = NULL; }
@@ -1806,13 +1739,13 @@ void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
    flag = MFEM_ARKode(SetMassFn)(sundials_mem, ARKStepSolver::MassSysSetup);
    MFEM_VERIFY(flag == ARK_SUCCESS,
                "error in " STR(MFEM_ARKode(SetMassFn)) "()");
+
+   // Check that the ODE is not expressed in EXPLICIT form
+   MFEM_VERIFY(!f->isExplicit(), "ODE operator is expressed in EXPLICIT form")
 }
 
 void ARKStepSolver::UseSundialsMassLinearSolver(int tdep)
 {
-   // Check that the ODE is expressed in mass form
-   MFEM_VERIFY(f_arkstep->ARKInMassForm(), "ODE operator is not in mass form.")
-
    // Free any existing matrix and linear solver
    if (M != NULL)   { SUNMatDestroy(A); M = NULL; }
    if (LSM != NULL) { SUNLinSolFree(LSM); LSM = NULL; }
@@ -1831,6 +1764,9 @@ void ARKStepSolver::UseSundialsMassLinearSolver(int tdep)
                                     ARKStepSolver::MassMult2, this);
    MFEM_VERIFY(flag == ARK_SUCCESS,
                "error in " STR(MFEM_ARKode(SetMassTimes)) "()");
+
+   // Check that the ODE is not expressed in EXPLICIT form
+   MFEM_VERIFY(!f->isExplicit(), "ODE operator is expressed in EXPLICIT form")
 }
 
 void ARKStepSolver::SetStepMode(int itask)

linalg/sundials.hpp

@@ -706,130 +706,9 @@ public:
 // Interface to ARKode's ARKStep module -- Additive Runge-Kutta methods
 // ---------------------------------------------------------------------------
 
-// Interface for defining ODE systems to be evolved using ARKStepSolver:
-//
-//  1)   du/dt = inv(M) f(u,t) ("MFEM" form)
-//  2) M dy/dt = f(u,t)        ("mass" form)
-//
-// where f(u,t) might be additively split, i.e., f(u,t) = f1(u,t) + f2(u,t)
-class ARKStepODE
-{
-
-public:
-
-   // the size of the ODE system
-   virtual int ARKSize() const = 0;
-
-   // return if the ODE system is of the form M du/dt = f(u,t), note the MFEM
-   // default is to use the form du/dt = int(M) f(u,t)
-   virtual bool ARKInMassForm() const { return false; };
-
-   // these flags are used by ARKStepSolver for switching between RK and ARK methods
-   enum ARKEvalMode
-   {  NORMAL,          // evaluate f(u,t)
-      ADDITIVE_TERM_1, // evaluate f1(u,t)
-      ADDITIVE_TERM_2  // evaluate f2(u,t)
-   };
-   virtual void ARKSetEvalMode(const ARKEvalMode new_eval_mode) {}
-
-   // evaluate either f(u,t) (mass form) or inv(M(t)) f(u,t) (MFEM form),
-   // which is necessary for solving ODEs with ERK or IMEX
-   virtual void ARKEvaluateRHS(const Vector &u, const real_t t, Vector &result) const
-   {
-      mfem_error("This function must be specified for ERK or IMEX methods.");
-   }
-
-  /** setup linear system for solving [M(t) - gamma Jf(u)] dk = f(u) - M(t) k,
-      which is necessary for solving ODEs with DIRK or IMEX methods
-      @param[in]  u     The state at which A(@a u,t) should be evaluated.
-      @param[in]  t     The time at which A(u,@a t) should be evaluated.
-      @param[in]  v     The value of inv(M) f(u,t) or f(u,t) for depending on form.
-      @param[in]  jok   Flag indicating if the Jacobian should be updated.
-      @param[out] jcur  Flag to signal if the Jacobian was updated.
-      @param[in]  gamma The scaled time step value.    */
-   virtual int ARKImplicitSetup(const Vector &u, const real_t t, const Vector &v,
-                                int jok, int *jcur, real_t gamma)
-   {
-      mfem_error("This function must be specified for DIRK or IMEX methods.");
-   }
-
-  /** solve for dk in [M - gamma Jf(u)] dk = r, where r is either
-        inv(M) f(u,t) - k       (MFEM form)
-        f(u,t) - M k f(u) - M k (mass form)
-      when using DIRK or IMEX methods
-      @param[in]      r   inv(M) f(u,t) - k or f(u,t) - M k, depending on form.
-      @param[in,out]  dk  On input, the initial guess. On output, the solution.
-      @param[in]      tol Linear solve tolerance. */
-   virtual int ARKImplicitSolve(const Vector &r, Vector &dk, real_t tol)
-   {
-      mfem_error("This function must be specified for DIRK or IMEX methods.");
-   }
-
-  /** for mass form ODEs using an MFEM mass solver, setup the mass linear
-      system M(t) x = b
-      @param[in]  t  The time at which M(@a t) should be evaluated. */
-   virtual int ARKMassSetup(const real_t t)
-   {
-      mfem_error("This function must be specified to use MFEM mass solvers for mass form ODEs.");
-   }
-
-  /** for mass form ODEs using an MFEM mass solver, solve for x in M(t) x = b
-      @param[in]      b   The linear system right-hand side.
-      @param[in,out]  x   On input, the initial guess. On output, the solution.
-      @param[in]      tol Linear solve tolerance. */
-   virtual int ARKMassSolve(const Vector &b, Vector &x, real_t tol)
-   {
-      mfem_error("This function must be specified to use MFEM mass solver for mass form ODEs.");
-   }
-
-  /**  for mass form ODEs using an MFEM mass solver, evaluate M(t) x
-       @param[in]   x The vector to multiply.
-       @param[out]  v The result of the matrix-vector product. */
-   virtual int ARKMassMult(const Vector &x, Vector &v)
-   {
-      mfem_error("This function must be specified to use MFEM mass solver for mass form ODEs.");
-   }
-
-  /**  for mass form ODEs using a SUNDIALS mass solver, evaluate M(t) x
-       @param[in]   x The vector to multiply.
-       @param[in]   t The time at which M(@a t) should be evaluated.
-       @param[out]  v The result of the matrix-vector product. */
-   virtual int ARKMassMult(const Vector &x, const real_t t, Vector &v)
-   {
-      mfem_error("This function must be specified to use SUNDIALS mass solver for mass form ODEs.");
-   }
-
-};
-
 /// Interface to ARKode's ARKStep module -- additive Runge-Kutta methods.
 class ARKStepSolver : public ODESolver, public SundialsSolver
 {
-
-   // Wrapper class to provide backwards compatability with user code that
-   // derives from TimeDependentOperator instead of ARKStepODE
-   class TimeDependentOperatorWrapper : public ARKStepODE
-   {
-      TimeDependentOperator *tdo;
-
-   public:
-
-      TimeDependentOperatorWrapper(TimeDependentOperator *f);
-
-      int ARKSize() const override;
-      bool ARKInMassForm() const override;
-      void ARKSetEvalMode(const ARKEvalMode new_eval_mode) override;
-      void ARKEvaluateRHS(const Vector &u, const real_t t, Vector &result) const override;
-      int ARKImplicitSetup(const Vector &u, const real_t t, const Vector &v,
-         int jok, int *jcur, real_t gamma) override;
-      int ARKImplicitSolve(const Vector &r, Vector &dk, real_t tol) override;
-      int ARKMassSetup(const real_t t) override;
-      int ARKMassSolve(const Vector &b, Vector &x, real_t tol) override;
-      int ARKMassMult(const Vector &x, Vector &v) override;
-      int ARKMassMult(const Vector &x, const real_t t, Vector &v) override;
-
-   };
-
-
 public:
    /// Types of ARKODE solvers.
    enum Type
@@ -843,8 +722,6 @@ protected:
    Type rk_type;      ///< Runge-Kutta type.
    int step_mode;     ///< ARKStep step mode (ARK_NORMAL or ARK_ONE_STEP).
    bool use_implicit; ///< True for implicit or imex integration.
-   ARKStepODE* f_arkstep;
-   std::unique_ptr<TimeDependentOperatorWrapper> f_tdo; // for backwards compatibility
 
    /** @name Wrappers to compute the ODE RHS functions.
        RHS1 is explicit RHS and RHS2 the implicit RHS for IMEX integration. When
@@ -907,19 +784,14 @@ public:
        then ARKStepReInit() will be called in the next call to Step(). If the
        problem size has changed, the ARKStep memory is freed and realloced
        for the new problem size. */
-   /** @param[in] f_ The ARKStepODE that defines the ODE system
+   /** @param[in] f_ The TimeDependentOperator that defines the ODE system
 
        @note All other methods must be called after Init().
 
        @note If this method is called a second time with a different problem
        size, then any non-default user-set options will be lost and will need
        to be set again. */
-   void Init(ARKStepODE *f_ark_);
-
-   // This method is provided for backwards compatibility with classes that
-   // derive TimeDependentOperator instead of ARKStepODE; however, those classes
-   // should be migrated.
-   MFEM_DEPRECATED void Init(TimeDependentOperator &f_) override;
+   void Init(TimeDependentOperator &f_) override;
 
    /// Integrate the ODE with ARKode using the specified step mode.
    /**

linalg/vector_operator.cpp

@@ -0,0 +1,80 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "vector_operator.hpp"
+
+namespace mfem
+{
+
+#ifdef MFEM_USE_MPI
+
+ParVectorOperator::ParVectorOperator(MPI_Comm comm,
+                                     int myid,
+                                     int local_vec_size,
+                                     int num_vecs)
+   : Operator((myid == 0) ? num_vecs : 0, local_vec_size),
+     comm(comm),
+     myid(myid),
+     vecs(num_vecs),
+     coefs(num_vecs),
+     owns(num_vecs)
+{
+   vecs = NULL;
+   coefs = 1.0;
+   owns = false;
+}
+
+ParVectorOperator::~ParVectorOperator()
+{
+   for (int i=0; i < vecs.Size(); i++)
+   {
+      if (owns[i]) { delete vecs[i]; }
+      vecs[i] = NULL;
+   }
+}
+
+void ParVectorOperator::SetVector(int idx, Vector *vec,
+                                  double c, bool own_vec)
+{
+   MFEM_VERIFY(idx >= 0 && idx < vecs.Size(),
+               "ParVectorOperator: Index out of range");
+
+   vecs[idx] = vec;
+   coefs[idx] = c;
+   owns[idx] = own_vec;
+}
+
+void ParVectorOperator::Mult(const Vector &x, Vector &y) const
+{
+   for (int i=0; i<vecs.Size(); i++)
+   {
+      double vo = coefs[i] * (*vecs[i] * x);
+      double vi = 0.0;
+      MPI_Reduce(&vo, &vi, 1, MPI_DOUBLE, MPI_SUM, 0, comm);
+      if (myid == 0) { y[i] = vi; }
+   }
+}
+
+/// Action of the transpose operator: `y=A^t(x)`.
+void ParVectorOperator::MultTranspose(const Vector &x, Vector &y) const
+{
+   y = 0.0;
+   for (int i=0; i<vecs.Size(); i++)
+   {
+      double xi = (myid == 0) ? x[i] : 0.0;
+      MPI_Bcast(&xi, 1, MPI_DOUBLE, 0, comm);
+      y.Add(xi * coefs[i], *vecs[i]);
+   }
+}
+
+}
+
+#endif // MFEM_USE_MPI

linalg/vector_operator.hpp

@@ -0,0 +1,55 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_VECTOR_OPERATOR
+#define MFEM_VECTOR_OPERATOR
+
+#include "operator.hpp"
+#include "vector.hpp"
+
+namespace mfem
+{
+
+#ifdef MFEM_USE_MPI
+
+class ParVectorOperator : public Operator
+{
+private:
+   MPI_Comm comm;
+   int myid;
+
+   Array<Vector*> vecs;
+   Array<double> coefs;
+   Array<bool> owns;
+
+public:
+   ParVectorOperator(MPI_Comm comm,
+                     int myid,
+                     int local_vec_size,
+                     int num_vecs);
+
+   ~ParVectorOperator();
+
+   void SetVector(int idx, Vector *vec,
+                  double c = 1.0, bool own_vec = false);
+
+   /// Operator application: `y=A(x)`.
+   void Mult(const Vector &x, Vector &y) const;
+
+   /// Action of the transpose operator: `y=A^t(x)`.
+   void MultTranspose(const Vector &x, Vector &y) const;
+};
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem
+
+#endif // MFEM_VECTOR_OPERATOR