more temporary memory in tmop

fem/bilininteg_convection_pa.cpp

@@ -900,11 +900,18 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    ne = fes.GetNE();
    const DofToQuad::Mode mode = DofToQuad::TENSOR;
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
+#ifdef MFEM_USE_UMPIRE
+   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
+      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
+#else
+   const MemoryType temp_type = Device::GetDeviceMemoryType();
+#endif
+
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode, temp_type);
    maps = &el.GetDofToQuad(*ir, mode);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
+   pa_data.SetSize(symmDims * nq * ne, temp_type);
    Vector vel;
    if (VectorConstantCoefficient *cQ =
           dynamic_cast<VectorConstantCoefficient*>(Q))
@@ -915,7 +922,7 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
                dynamic_cast<VectorGridFunctionCoefficient*>(Q))
    {
       Vector xe;
-      vel.SetSize(dim * nq * ne);
+      vel.SetSize(dim * nq * ne, temp_type);
 
       const GridFunction *gf = vgfQ->GetGridFunction();
       const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;

fem/bilininteg_mass_pa.cpp

@@ -49,7 +49,13 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    const DofToQuad::Mode mode = DofToQuad::TENSOR;
    const int flags = GeometricFactors::JACOBIANS |
                      GeometricFactors::COORDINATES;
-   geom = mesh->GetGeometricFactors(*ir, flags, mode);
+#ifdef MFEM_USE_UMPIRE
+   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
+      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
+#else
+   const MemoryType temp_type = Device::GetDeviceMemoryType();
+#endif
+   geom = mesh->GetGeometricFactors(*ir, flags, mode, temp_type);
    maps = &el.GetDofToQuad(*ir, mode);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;

fem/pfespace.cpp

@@ -3147,7 +3147,7 @@ static void SetSubVector(const int N,
                          const Array<int> &indices,
                          const Vector &in, Vector &out)
 {
-   auto y = out.Write();
+   auto y = out.ReadWrite();
    const auto x = in.Read();
    const auto I = indices.Read();
    MFEM_FORALL(i, N, y[I[i]] = x[i];);
@@ -3234,7 +3234,7 @@ static void AddSubVector(const int num_unique_dst_indices,
                          const Vector &src,
                          Vector &dst)
 {
-   auto y = dst.Write();
+   auto y = dst.ReadWrite();
    const auto x = src.Read();
    const auto DST_I = unique_dst_indices.Read();
    const auto SRC_O = unique_to_src_offsets.Read();

fem/tmop.cpp

@@ -1872,6 +1872,17 @@ void AdaptivityEvaluator::SetParMetaInfo(const ParMesh &m,
 }
 #endif
 
+void AdaptivityEvaluator::ClearGeometricFactors()
+{
+#ifdef MFEM_USE_MPI
+   if (pmesh) pmesh->DeleteGeometricFactors();
+   if (pfes) pfes->GetParMesh()->DeleteGeometricFactors();
+#else
+   if (mesh) mesh->DeleteGeometricFactors();
+   if (fes) fes->GetMesh()->DeleteGeometricFactors();
+#endif
+}
+
 AdaptivityEvaluator::~AdaptivityEvaluator()
 {
    delete fes;

fem/tmop.hpp

@@ -606,6 +606,8 @@ public:
 
    virtual void ComputeAtNewPosition(const Vector &new_nodes,
                                      Vector &new_field) = 0;
+
+   void ClearGeometricFactors();
 };
 
 /** @brief Base class representing target-matrix construction algorithms for
@@ -1216,6 +1218,8 @@ public:
       {
          PA.H.GetMemory().DeleteDevice();
          PA.H0.GetMemory().DeleteDevice();
+         //PA.Jtr.GetMemory().DeleteDevice();
+         //PA.setup_Jtr = false;
       }
    }
 };

fem/tmop_pa.cpp

@@ -238,7 +238,7 @@ void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
    PA.O = 1.0;
 
    // TargetConstructor TargetType setup
-   PA.Jtr.SetSize(dim, dim, PA.ne*PA.nq);
+   PA.Jtr.SetSize(dim, dim, PA.ne*PA.nq);//, temp_type);
    ComputeElementTargetsPA();
 
    // Coeff0 PA.C0

general/mem_manager.cpp

@@ -1319,8 +1319,10 @@ void MemoryManager::EraseDevice(void *h_ptr)
    if (!h_ptr) { return; }
    auto mem_map_iter = maps->memories.find(h_ptr);
    if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
-   if (maps->aliases.find(h_ptr) != maps->aliases.end())
+   auto it = maps->aliases.find(h_ptr);
+   if (it != maps->aliases.end())
    {
+      fprintf(stderr, "count = %lu\n", it->second.counter);
       mfem_error("cannot delete aliased obj!");
    }
    internal::Memory &mem = mem_map_iter->second;