Merge branch 'gpu_em' into gpu-maxwell

Just use the regular Maxwell miniapp for full assembly.

fem/lor/lor_dg_impl.hpp

@@ -258,8 +258,8 @@ void BatchedLOR_DG::Assemble2D()
                   const int w_idx = (n_idx == 0) ? iy : ix;
                   const int x_idx = (n_idx == 0) ? i_0 : j_0;
 
-                  const real_t J1 = J(ix, iy, n_idx, !n_idx, iel_ho);
-                  const real_t J2 = J(ix, iy, !n_idx, !n_idx, iel_ho);
+                  const real_t J1 = J(ix, iy, n_idx, (int)!n_idx, iel_ho);
+                  const real_t J2 = J(ix, iy, (int)!n_idx, (int)!n_idx, iel_ho);
                   const real_t Jh = (J1*J1 + J2*J2) / detJ(ix, iy, iel_ho);
 
                   V(v_idx, ix, iy, iel_ho) =

fem/qinterp/eval_hdiv.hpp

@@ -136,8 +136,8 @@ inline void EvalHDiv2D(const int NE,
                {
                   qq += QD(qx,dy,vd) * By(dy,qy);
                }
-               if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                            QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+               if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                             QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
                {
                   QQ(qx,qy,vd) = qq;
                }
@@ -153,8 +153,8 @@ inline void EvalHDiv2D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                   QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+      if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                    QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
       {
          if (tidz == 0)
          {
@@ -178,7 +178,7 @@ inline void EvalHDiv2D(const int NE,
                   const real_t detJ = kernels::Det<DIM>(J_loc);
                   kernels::Mult(DIM, DIM, J_loc, u_ref, u_phys);
                   kernels::Set(DIM, 1, 1_r/detJ, u_phys, u_phys);
-                  if (FLAGS & QuadratureInterpolator::PHYSICAL_VALUES)
+                  if ((FLAGS & QuadratureInterpolator::PHYSICAL_VALUES) != 0)
                   {
                      MFEM_UNROLL(DIM)
                      for (int sd = 0; sd < DIM; sd++)
@@ -193,7 +193,8 @@ inline void EvalHDiv2D(const int NE,
                         }
                      }
                   }
-                  else if (FLAGS & QuadratureInterpolator::PHYSICAL_MAGNITUDES)
+                  else if ((FLAGS &
+                            QuadratureInterpolator::PHYSICAL_MAGNITUDES) != 0)
                   {
                      y(qx,qy,0,e) = kernels::Norml2(DIM, u_phys);
                   }
@@ -372,8 +373,9 @@ inline void EvalHDiv3D(const int NE,
                MFEM_UNROLL(MQ1)
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                               QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+                  if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                                QuadratureInterpolator::PHYSICAL_MAGNITUDES)) !=
+                      0)
                   {
                      QQQ(qx,qy,qz,vd) = u[qz];
                   }
@@ -390,8 +392,8 @@ inline void EvalHDiv3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                   QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+      if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                    QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
       {
          MFEM_FOREACH_THREAD(qz,z,Q1D)
          {
@@ -415,7 +417,7 @@ inline void EvalHDiv3D(const int NE,
                   const real_t detJ = kernels::Det<DIM>(J_loc);
                   kernels::Mult(DIM, DIM, J_loc, u_ref, u_phys);
                   kernels::Set(DIM, 1, 1_r/detJ, u_phys, u_phys);
-                  if (FLAGS & QuadratureInterpolator::PHYSICAL_VALUES)
+                  if ((FLAGS & QuadratureInterpolator::PHYSICAL_VALUES) != 0)
                   {
                      MFEM_UNROLL(DIM)
                      for (int sd = 0; sd < DIM; sd++)
@@ -430,7 +432,8 @@ inline void EvalHDiv3D(const int NE,
                         }
                      }
                   }
-                  else if (FLAGS & QuadratureInterpolator::PHYSICAL_MAGNITUDES)
+                  else if ((FLAGS &
+                            QuadratureInterpolator::PHYSICAL_MAGNITUDES) != 0)
                   {
                      y(qx,qy,qz,0,e) = kernels::Norml2(DIM, u_phys);
                   }

general/optparser.cpp

@@ -148,9 +148,26 @@ void parseVector(char * str, Vector & var)
    }
 }
 
+bool parseEnumOption(const char *str, size_t &var,
+                     const std::vector<std::string> &options)
+{
+   for (size_t i = 0; i < options.size(); ++i)
+   {
+      auto &v = options[i];
+      if (v == str)
+      {
+         var = i;
+         // success
+         return true;
+      }
+   }
+   // failure
+   return false;
+}
+
 void OptionsParser::Parse()
 {
-   option_check.SetSize(options.Size());
+   option_check.SetSize(options.size());
    option_check = 0;
    for (int i = 1; i < argc; )
    {
@@ -161,9 +178,9 @@ void OptionsParser::Parse()
          return;
       }
 
-      for (int j = 0; true; j++)
+      for (size_t j = 0; true; j++)
       {
-         if (j >= options.Size())
+         if (j >= options.size())
          {
             // unrecognized option
             error_type = 2;
@@ -171,8 +188,8 @@ void OptionsParser::Parse()
             return;
          }
 
-         if (strcmp(argv[i], options[j].short_name) == 0 ||
-             strcmp(argv[i], options[j].long_name) == 0)
+         if (strcmp(argv[i], options[j].short_name.c_str()) == 0 ||
+             strcmp(argv[i], options[j].long_name.c_str()) == 0)
          {
             OptionType type = options[j].type;
 
@@ -224,6 +241,11 @@ void OptionsParser::Parse()
                case VECTOR:
                   parseVector(argv[i++], *(Vector*)(options[j].var_ptr) );
                   break;
+               case ENUM_OPTION:
+                  isValid =
+                     parseEnumOption(argv[i++], *(size_t *)(options[j].var_ptr),
+                                     options[j].options);
+                  break;
             }
 
             if (!isValid)
@@ -239,7 +261,7 @@ void OptionsParser::Parse()
    }
 
    // check for missing required options
-   for (int i = 0; i < options.Size(); i++)
+   for (size_t i = 0; i < options.size(); i++)
       if (options[i].required &&
           (option_check[i] == 0 ||
            (options[i].type == ENABLE && option_check[++i] == 0)))
@@ -323,6 +345,12 @@ void OptionsParser::WriteValue(const Option &opt, std::ostream &os)
          break;
       }
 
+      case ENUM_OPTION:
+      {
+         os << opt.options.at(*((size_t *)opt.var_ptr));
+         break;
+      }
+
       default: // provide a default to suppress warning
          break;
    }
@@ -333,7 +361,7 @@ void OptionsParser::PrintOptions(ostream &os) const
    static const char *indent = "   ";
 
    os << "Options used:\n";
-   for (int j = 0; j < options.Size(); j++)
+   for (size_t j = 0; j < options.size(); j++)
    {
       OptionType type = options[j].type;
 
@@ -355,7 +383,7 @@ void OptionsParser::PrintOptions(ostream &os) const
          os << options[j].long_name << " ";
          WriteValue(options[j], os);
       }
-      os << '\n';
+      os << std::endl;
    }
 }
 
@@ -410,12 +438,13 @@ void OptionsParser::PrintHelp(ostream &os) const
    static const char *line_sep = "";
    static const char *types[] = { " <int>", " <double>", " <string>",
                                   " <string>", "", "", " '<int>...'",
-                                  " '<double>...'"
+                                  " '<double>...'",
+                                  " <string>",
                                 };
 
    os << indent << "-h" << seprtr << "--help" << descr_sep
       << "Print this help message and exit.\n" << line_sep;
-   for (int j = 0; j < options.Size(); j++)
+   for (size_t j = 0; j < options.size(); j++)
    {
       OptionType type = options[j].type;
 
@@ -451,9 +480,22 @@ void OptionsParser::PrintHelp(ostream &os) const
       }
       os << descr_sep;
 
-      if (options[j].description)
+      if (options[j].description.size())
       {
-         os << options[j].description << '\n';
+         os << options[j].description << std::endl;
+      }
+      if (options[j].type == ENUM_OPTION)
+      {
+         os << "\tavailable choices: ";
+         for (size_t i = 0; i < options[j].options.size(); ++i)
+         {
+            os << '"' << options[j].options[i] << '"';
+            if (i + 1 < options[j].options.size())
+            {
+               os << ", ";
+            }
+         }
+         os << std::endl;
       }
       os << line_sep;
    }

general/optparser.hpp

@@ -15,6 +15,9 @@
 #include "../config/config.hpp"
 #include "array.hpp"
 
+#include <string>
+#include <vector>
+
 namespace mfem
 {
 
@@ -31,17 +34,18 @@ class Vector;
 class OptionsParser
 {
 public:
-   enum OptionType { INT, DOUBLE, STRING, STD_STRING, ENABLE, DISABLE, ARRAY, VECTOR };
+   enum OptionType { INT, DOUBLE, STRING, STD_STRING, ENABLE, DISABLE, ARRAY, VECTOR, ENUM_OPTION };
 
 private:
    struct Option
    {
       OptionType type;
       void *var_ptr;
-      const char *short_name;
-      const char *long_name;
-      const char *description;
+      std::string short_name;
+      std::string long_name;
+      std::string description;
       bool required;
+      std::vector<std::string> options;
 
       Option() = default;
 
@@ -49,11 +53,19 @@ private:
              const char *long_name_, const char *description_, bool req)
          : type(type_), var_ptr(var_ptr_), short_name(short_name_),
            long_name(long_name_), description(description_), required(req) { }
+
+      Option(size_t *var_ptr_, const char *short_name_, const char *long_name_,
+             const char *description_, std::vector<std::string> &&options_,
+             bool req)
+         : type(ENUM_OPTION), var_ptr(var_ptr_), short_name(short_name_),
+           long_name(long_name_), description(description_),
+           required(req), options(std::move(options_))
+      {}
    };
 
    int argc;
    char **argv;
-   Array<Option> options;
+   std::vector<Option> options;
    Array<int> option_check;
    // error_type can be:
    //  0 - no error
@@ -84,26 +96,26 @@ public:
                   const char *disable_long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(ENABLE, var, enable_short_name, enable_long_name,
-                            description, required));
-      options.Append(Option(DISABLE, var, disable_short_name, disable_long_name,
-                            description, required));
+      options.emplace_back(ENABLE, var, enable_short_name, enable_long_name,
+                           description, required);
+      options.emplace_back(DISABLE, var, disable_short_name, disable_long_name,
+                           description, required);
    }
 
    /// Add an integer option and set 'var' to receive the value.
    void AddOption(int *var, const char *short_name, const char *long_name,
                   const char *description, bool required = false)
    {
-      options.Append(Option(INT, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(INT, var, short_name, long_name, description,
+                           required);
    }
 
    /// Add a double option and set 'var' to receive the value.
    void AddOption(real_t *var, const char *short_name, const char *long_name,
                   const char *description, bool required = false)
    {
-      options.Append(Option(DOUBLE, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(DOUBLE, var, short_name, long_name, description,
+                           required);
    }
 
    /// Add a string (char*) option and set 'var' to receive the value.
@@ -111,8 +123,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(STRING, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(STRING, var, short_name, long_name, description,
+                           required);
    }
 
    /// Add a string (std::string) option and set 'var' to receive the value.
@@ -120,8 +132,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(STD_STRING, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(STD_STRING, var, short_name, long_name, description,
+                           required);
    }
 
    /** Add an integer array (separated by spaces) option and set 'var' to
@@ -130,8 +142,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(ARRAY, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(ARRAY, var, short_name, long_name, description,
+                           required);
    }
 
    /** Add a vector (doubles separated by spaces) option and set 'var' to
@@ -140,8 +152,20 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(VECTOR, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(VECTOR, var, short_name, long_name, description,
+                           required);
+   }
+
+   /**
+    * Add an option which must be one of a given list of options
+    */
+   void AddOptionChoice(size_t *index, const char *short_name,
+                        const char *long_name, const char *description,
+                        std::vector<std::string> options_,
+                        bool required = false)
+   {
+      options.emplace_back(index, short_name, long_name, description,
+                           std::move(options_), required);
    }
 
    /** @brief Parse the command-line options.

miniapps/electromagnetics/CMakeLists.txt

@@ -34,6 +34,11 @@ if (MFEM_USE_MPI)
    EXTRA_HEADERS maxwell_solver.hpp ${MFEM_MINIAPPS_COMMON_HEADERS}
    LIBRARIES mfem-common)
 
+ add_mfem_miniapp(maxwell-gpu
+   MAIN maxwell_gpu.cpp
+   EXTRA_HEADERS ${MFEM_MINIAPPS_COMMON_HEADERS}
+   LIBRARIES mfem-common)
+
  if (MFEM_USE_GSLIB)
   add_mfem_miniapp(lorentz
     MAIN lorentz.cpp

miniapps/electromagnetics/electromagnetics.hpp

@@ -12,6 +12,8 @@
 #ifndef MFEM_ELECTROMAGNETICS_HPP
 #define MFEM_ELECTROMAGNETICS_HPP
 
+#include <cmath>
+
 namespace mfem
 {
 

miniapps/electromagnetics/maxwell_solver.cpp

@@ -308,7 +308,7 @@ MaxwellSolver::~MaxwellSolver()
    {
       int i = mit1->first;
       delete pcg_[i];
-      delete diagScale_[i];
+      // delete diagScale_[i];
       delete A1_[i];
       delete a1_[i];
    }
@@ -377,7 +377,7 @@ MaxwellSolver::Mult(const Vector &B, Vector &dEdt) const
 void
 MaxwellSolver::ImplicitSolve(real_t dt, const Vector &B, Vector &dEdt)
 {
-   implicitSolve(dt, B, dEdt);
+   const_cast<const MaxwellSolver *>(this)->implicitSolve(dt, B, dEdt);
 }
 
 void
@@ -420,12 +420,12 @@ MaxwellSolver::setupSolver(const int idt, const real_t dt) const
       a1_[idt]->Finalize();
       A1_[idt] = a1_[idt]->ParallelAssemble();
 
-      diagScale_[idt] = new HypreDiagScale(*A1_[idt]);
+      // diagScale_[idt] = new HypreDiagScale(*A1_[idt]);
       pcg_[idt] = new HyprePCG(*A1_[idt]);
       pcg_[idt]->SetTol(1.0e-12);
       pcg_[idt]->SetMaxIter(200);
       pcg_[idt]->SetPrintLevel(0);
-      pcg_[idt]->SetPreconditioner(*diagScale_[idt]);
+      // pcg_[idt]->SetPreconditioner(*diagScale_[idt]);
    }
 }