phynexis-deps/mfem
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
f7b59cd1815d979aa430b3ca7ea62045bb12f3fa
mfem/general/mem_manager.cpp
T
Tzanio 32d7e036e7 Renamed 
MemoryType GetSuitableMemoryType(MemoryClass mc);

to

  MemoryType GetMemoryType(MemoryClass mc);
2019-05-24 15:54:24 -07:00

713 lines
21 KiB
C++
Raw Blame History

// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
// reserved. See file COPYRIGHT for details.
//
// This file is part of the MFEM library. For more information and source code
// availability see http://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the GNU Lesser General Public License (as published by the Free
// Software Foundation) version 2.1 dated February 1999.
#include "../general/forall.hpp"
#include <cstring> // std::memcpy
#include <list>
#include <unordered_map>
#include <algorithm> // std::max
namespace mfem
{
MemoryType GetMemoryType(MemoryClass mc)
{
switch (mc)
{
case MemoryClass::HOST: return MemoryType::HOST;
case MemoryClass::HOST_32: return MemoryType::HOST_32;
case MemoryClass::HOST_64: return MemoryType::HOST_64;
case MemoryClass::CUDA: return MemoryType::CUDA;
case MemoryClass::CUDA_UVM: return MemoryType::CUDA_UVM;
}
return MemoryType::HOST;
}
MemoryClass operator*(MemoryClass mc1, MemoryClass mc2)
{
// | HOST HOST_32 HOST_64 CUDA CUDA_UVM
// ---------+--------------------------------------------------
// HOST | HOST HOST_32 HOST_64 CUDA CUDA_UVM
// HOST_32 | HOST_32 HOST_32 HOST_64 CUDA CUDA_UVM
// HOST_64 | HOST_64 HOST_64 HOST_64 CUDA CUDA_UVM
// CUDA | CUDA CUDA CUDA CUDA CUDA_UVM
// CUDA_UVM | CUDA_UVM CUDA_UVM CUDA_UVM CUDA_UVM CUDA_UVM
// Using the enumeration ordering:
// HOST < HOST_32 < HOST_64 < CUDA < CUDA_UVM,
// the above table is simply: a*b = max(a,b).
return std::max(mc1, mc2);
}
namespace internal
{
/// Forward declaration of the Alias structure
struct Alias;
/// Memory class that holds:
/// - a boolean telling which memory space is being used
/// - the size in bytes of this memory region,
/// - the host and the device pointer.
struct Memory
{
bool host;
const std::size_t bytes;
void *const h_ptr;
void *d_ptr;
Memory(void* const h, const std::size_t size):
host(true), bytes(size), h_ptr(h), d_ptr(nullptr) {}
};
/// Alias class that holds the base memory region and the offset
struct Alias
{
Memory *const mem;
const long offset;
unsigned long counter;
};
typedef std::unordered_map<const void*, Memory> MemoryMap;
// TODO: use 'Alias' or 'const Alias' as the mapped type in the AliasMap instead
// of 'Alias*'
typedef std::unordered_map<const void*, Alias*> AliasMap;
struct Ledger
{
MemoryMap memories;
AliasMap aliases;
};
} // namespace mfem::internal
static internal::Ledger *maps;
MemoryManager::MemoryManager()
{
exists = true;
maps = new internal::Ledger();
}
MemoryManager::~MemoryManager()
{
if (exists) { Destroy(); }
}
void MemoryManager::Destroy()
{
MFEM_VERIFY(exists, "MemoryManager has been destroyed already!");
for (auto& n : maps->memories)
{
internal::Memory &mem = n.second;
if (mem.d_ptr) { CuMemFree(mem.d_ptr); }
}
for (auto& n : maps->aliases)
{
delete n.second;
}
delete maps;
exists = false;
}
void* MemoryManager::Insert(void *ptr, const std::size_t bytes)
{
if (ptr == NULL)
{
MFEM_VERIFY(bytes == 0, "Trying to add NULL with size " << bytes);
return NULL;
}
auto res = maps->memories.emplace(ptr, internal::Memory(ptr, bytes));
if (res.second == false)
{
mfem_error("Trying to add an already present address!");
}
return ptr;
}
void MemoryManager::InsertDevice(void *ptr, void *h_ptr, size_t bytes)
{
MFEM_VERIFY(ptr != NULL, "cannot register NULL device pointer");
MFEM_VERIFY(h_ptr != NULL, "internal error");
auto res = maps->memories.emplace(h_ptr, internal::Memory(h_ptr, bytes));
if (res.second == false)
{
mfem_error("Trying to add an already present address!");
}
res.first->second.d_ptr = ptr;
}
void *MemoryManager::Erase(void *ptr, bool free_dev_ptr)
{
if (!ptr) { return ptr; }
auto mem_map_iter = maps->memories.find(ptr);
if (mem_map_iter == maps->memories.end())
{
mfem_error("Trying to erase an unknown pointer!");
}
internal::Memory &mem = mem_map_iter->second;
if (mem.d_ptr && free_dev_ptr) { CuMemFree(mem.d_ptr); }
maps->memories.erase(mem_map_iter);
return ptr;
}
bool MemoryManager::IsKnown(const void *ptr)
{
return maps->memories.find(ptr) != maps->memories.end();
}
void *MemoryManager::GetDevicePtr(const void *ptr, size_t bytes, bool copy_data)
{
if (!ptr)
{
MFEM_VERIFY(bytes == 0, "Trying to access NULL with size " << bytes);
return NULL;
}
internal::Memory &base = maps->memories.at(ptr);
if (!base.d_ptr)
{
CuMemAlloc(&base.d_ptr, base.bytes);
}
if (copy_data)
{
MFEM_ASSERT(bytes <= base.bytes, "invalid copy size");
CuMemcpyHtoD(base.d_ptr, ptr, bytes);
base.host = false;
}
return base.d_ptr;
}
void MemoryManager::InsertAlias(const void *base_ptr, void *alias_ptr,
bool base_is_alias)
{
long offset = static_cast<const char*>(alias_ptr) -
static_cast<const char*>(base_ptr);
if (!base_ptr)
{
MFEM_VERIFY(offset == 0,
"Trying to add alias to NULL at offset " << offset);
return;
}
if (base_is_alias)
{
const internal::Alias *alias = maps->aliases.at(base_ptr);
base_ptr = alias->mem->h_ptr;
offset += alias->offset;
}
internal::Memory &mem = maps->memories.at(base_ptr);
auto res = maps->aliases.emplace(alias_ptr, nullptr);
if (res.second == false) // alias_ptr was already in the map
{
if (res.first->second->mem != &mem || res.first->second->offset != offset)
{
mfem_error("alias already exists with different base/offset!");
}
else
{
res.first->second->counter++;
}
}
else
{
res.first->second = new internal::Alias{&mem, offset, 1};
}
}
void MemoryManager::EraseAlias(void *alias_ptr)
{
if (!alias_ptr) { return; }
auto alias_map_iter = maps->aliases.find(alias_ptr);
if (alias_map_iter == maps->aliases.end())
{
mfem_error("alias not found");
}
internal::Alias *alias = alias_map_iter->second;
if (--alias->counter) { return; }
// erase the alias from the alias map:
maps->aliases.erase(alias_map_iter);
delete alias;
}
void *MemoryManager::GetAliasDevicePtr(const void *alias_ptr, size_t bytes,
bool copy_data)
{
if (!alias_ptr)
{
MFEM_VERIFY(bytes == 0, "Trying to access NULL with size " << bytes);
return NULL;
}
auto &alias_map = maps->aliases;
auto alias_map_iter = alias_map.find(alias_ptr);
if (alias_map_iter == alias_map.end())
{
mfem_error("alias not found");
}
const internal::Alias *alias = alias_map_iter->second;
internal::Memory &base = *alias->mem;
MFEM_ASSERT((char*)base.h_ptr + alias->offset == alias_ptr,
"internal error");
if (!base.d_ptr)
{
CuMemAlloc(&base.d_ptr, base.bytes);
}
if (copy_data)
{
CuMemcpyHtoD((char*)base.d_ptr + alias->offset, alias_ptr, bytes);
base.host = false;
}
return (char*)base.d_ptr + alias->offset;
}
static void PullKnown(internal::Ledger *maps,
const void *ptr, const std::size_t bytes, bool copy_data)
{
internal::Memory &base = maps->memories.at(ptr);
MFEM_ASSERT(base.h_ptr == ptr, "internal error");
// There are cases where it is OK if base.d_ptr is not allocated yet:
// for example, when requesting read-write access on host to memory created
// as device memory.
if (copy_data && base.d_ptr)
{
CuMemcpyDtoH(base.h_ptr, base.d_ptr, bytes);
base.host = true;
}
}
static void PullAlias(const internal::Ledger *maps,
const void *ptr, const std::size_t bytes, bool copy_data)
{
const internal::Alias *alias = maps->aliases.at(ptr);
MFEM_ASSERT((char*)alias->mem->h_ptr + alias->offset == ptr,
"internal error");
// There are cases where it is OK if alias->mem->d_ptr is not allocated yet:
// for example, when requesting read-write access on host to memory created
// as device memory.
if (copy_data && alias->mem->d_ptr)
{
CuMemcpyDtoH(const_cast<void*>(ptr),
static_cast<char*>(alias->mem->d_ptr) + alias->offset,
bytes);
}
}
void MemoryManager::RegisterCheck(void *ptr)
{
if (ptr != NULL)
{
if (!IsKnown(ptr))
{
mfem_error("Pointer is not registered!");
}
}
}
void MemoryManager::PrintPtrs(void)
{
for (const auto& n : maps->memories)
{
const internal::Memory &mem = n.second;
mfem::out << std::endl
<< "key " << n.first << ", "
<< "host " << mem.host << ", "
<< "h_ptr " << mem.h_ptr << ", "
<< "d_ptr " << mem.d_ptr;
}
mfem::out << std::endl;
}
// Static private MemoryManager methods used by class Memory
void *MemoryManager::New_(void *h_ptr, std::size_t size, MemoryType mt,
unsigned &flags)
{
// TODO: save the types of the pointers ...
flags = Mem::REGISTERED | Mem::OWNS_INTERNAL;
switch (mt)
{
case MemoryType::HOST: return nullptr; // case is handled outside
case MemoryType::HOST_32:
case MemoryType::HOST_64:
mfem_error("New_(): aligned host types are not implemented yet");
return nullptr;
case MemoryType::CUDA:
mm.Insert(h_ptr, size);
flags = flags | Mem::OWNS_HOST | Mem::OWNS_DEVICE | Mem::VALID_DEVICE;
return h_ptr;
case MemoryType::CUDA_UVM:
mfem_error("New_(): CUDA UVM allocation is not implemented yet");
return nullptr;
}
return nullptr;
}
void *MemoryManager::Register_(void *ptr, void *h_ptr, std::size_t capacity,
MemoryType mt, bool own, bool alias,
unsigned &flags)
{
// TODO: save the type of the registered pointer ...
MFEM_VERIFY(alias == false, "cannot register an alias!");
flags = flags | (Mem::REGISTERED | Mem::OWNS_INTERNAL);
if (IsHostMemory(mt))
{
mm.Insert(ptr, capacity);
flags = (own ? flags | Mem::OWNS_HOST : flags & ~Mem::OWNS_HOST) |
Mem::OWNS_DEVICE | Mem::VALID_HOST;
return ptr;
}
MFEM_VERIFY(mt == MemoryType::CUDA, "Only CUDA pointers are supported");
mm.InsertDevice(ptr, h_ptr, capacity);
flags = (own ? flags | Mem::OWNS_DEVICE : flags & ~Mem::OWNS_DEVICE) |
Mem::OWNS_HOST | Mem::VALID_DEVICE;
return h_ptr;
}
void MemoryManager::Alias_(void *base_h_ptr, std::size_t offset,
std::size_t size, unsigned base_flags,
unsigned &flags)
{
// TODO: store the 'size' in the MemoryManager?
mm.InsertAlias(base_h_ptr, (char*)base_h_ptr + offset,
base_flags & Mem::ALIAS);
flags = (base_flags | Mem::ALIAS | Mem::OWNS_INTERNAL) &
~(Mem::OWNS_HOST | Mem::OWNS_DEVICE);
}
MemoryType MemoryManager::Delete_(void *h_ptr, unsigned flags)
{
// TODO: this logic needs to be updated when support for HOST_32 and HOST_64
// memory types is added.
MFEM_ASSERT(!(flags & Mem::OWNS_DEVICE) || (flags & Mem::OWNS_INTERNAL),
"invalid Memory state");
if (mm.exists && (flags & Mem::OWNS_INTERNAL))
{
if (flags & Mem::ALIAS)
{
mm.EraseAlias(h_ptr);
}
else
{
mm.Erase(h_ptr, flags & Mem::OWNS_DEVICE);
}
}
return MemoryType::HOST;
}
void *MemoryManager::ReadWrite_(void *h_ptr, MemoryClass mc,
std::size_t size, unsigned &flags)
{
switch (mc)
{
case MemoryClass::HOST:
if (!(flags & Mem::VALID_HOST))
{
if (flags & Mem::ALIAS) { PullAlias(maps, h_ptr, size, true); }
else { PullKnown(maps, h_ptr, size, true); }
}
flags = (flags | Mem::VALID_HOST) & ~Mem::VALID_DEVICE;
return h_ptr;
case MemoryClass::HOST_32:
// TODO: check that the host pointer is MemoryType::HOST_32 or
// MemoryType::HOST_64
return h_ptr;
case MemoryClass::HOST_64:
// TODO: check that the host pointer is MemoryType::HOST_64
return h_ptr;
case MemoryClass::CUDA:
{
// TODO: check that the device pointer is MemoryType::CUDA or
// MemoryType::CUDA_UVM
const bool need_copy = !(flags & Mem::VALID_DEVICE);
flags = (flags | Mem::VALID_DEVICE) & ~Mem::VALID_HOST;
// TODO: add support for UVM
if (flags & Mem::ALIAS)
{
return mm.GetAliasDevicePtr(h_ptr, size, need_copy);
}
return mm.GetDevicePtr(h_ptr, size, need_copy);
}
case MemoryClass::CUDA_UVM:
// TODO: check that the host+device pointers are MemoryType::CUDA_UVM
// Do we need to update the validity flags?
return h_ptr; // the host and device pointers are the same
}
return nullptr;
}
const void *MemoryManager::Read_(void *h_ptr, MemoryClass mc,
std::size_t size, unsigned &flags)
{
switch (mc)
{
case MemoryClass::HOST:
if (!(flags & Mem::VALID_HOST))
{
if (flags & Mem::ALIAS) { PullAlias(maps, h_ptr, size, true); }
else { PullKnown(maps, h_ptr, size, true); }
}
flags = flags | Mem::VALID_HOST;
return h_ptr;
case MemoryClass::HOST_32:
// TODO: check that the host pointer is MemoryType::HOST_32 or
// MemoryType::HOST_64
return h_ptr;
case MemoryClass::HOST_64:
// TODO: check that the host pointer is MemoryType::HOST_64
return h_ptr;
case MemoryClass::CUDA:
{
// TODO: check that the device pointer is MemoryType::CUDA or
// MemoryType::CUDA_UVM
const bool need_copy = !(flags & Mem::VALID_DEVICE);
flags = flags | Mem::VALID_DEVICE;
// TODO: add support for UVM
if (flags & Mem::ALIAS)
{
return mm.GetAliasDevicePtr(h_ptr, size, need_copy);
}
return mm.GetDevicePtr(h_ptr, size, need_copy);
}
case MemoryClass::CUDA_UVM:
// TODO: check that the host+device pointers are MemoryType::CUDA_UVM
// Do we need to update the validity flags?
return h_ptr; // the host and device pointers are the same
}
return nullptr;
}
void *MemoryManager::Write_(void *h_ptr, MemoryClass mc, std::size_t size,
unsigned &flags)
{
switch (mc)
{
case MemoryClass::HOST:
flags = (flags | Mem::VALID_HOST) & ~Mem::VALID_DEVICE;
return h_ptr;
case MemoryClass::HOST_32:
// TODO: check that the host pointer is MemoryType::HOST_32 or
// MemoryType::HOST_64
flags = (flags | Mem::VALID_HOST) & ~Mem::VALID_DEVICE;
return h_ptr;
case MemoryClass::HOST_64:
// TODO: check that the host pointer is MemoryType::HOST_64
flags = (flags | Mem::VALID_HOST) & ~Mem::VALID_DEVICE;
return h_ptr;
case MemoryClass::CUDA:
// TODO: check that the device pointer is MemoryType::CUDA or
// MemoryType::CUDA_UVM
flags = (flags | Mem::VALID_DEVICE) & ~Mem::VALID_HOST;
// TODO: add support for UVM
if (flags & Mem::ALIAS)
{
return mm.GetAliasDevicePtr(h_ptr, size, false);
}
return mm.GetDevicePtr(h_ptr, size, false);
case MemoryClass::CUDA_UVM:
// TODO: check that the host+device pointers are MemoryType::CUDA_UVM
// Do we need to update the validity flags?
return h_ptr; // the host and device pointers are the same
}
return nullptr;
}
void MemoryManager::SyncAlias_(const void *base_h_ptr, void *alias_h_ptr,
size_t alias_size, unsigned base_flags,
unsigned &alias_flags)
{
// This is called only when (base_flags & Mem::REGISTERED) is true.
// Note that (alias_flags & REGISTERED) may not be true.
MFEM_ASSERT(alias_flags & Mem::ALIAS, "not an alias");
if ((base_flags & Mem::VALID_HOST) && !(alias_flags & Mem::VALID_HOST))
{
PullAlias(maps, alias_h_ptr, alias_size, true);
}
if ((base_flags & Mem::VALID_DEVICE) && !(alias_flags & Mem::VALID_DEVICE))
{
if (!(alias_flags & Mem::REGISTERED))
{
mm.InsertAlias(base_h_ptr, alias_h_ptr, base_flags & Mem::ALIAS);
alias_flags = (alias_flags | Mem::REGISTERED | Mem::OWNS_INTERNAL) &
~(Mem::OWNS_HOST | Mem::OWNS_DEVICE);
}
mm.GetAliasDevicePtr(alias_h_ptr, alias_size, true);
}
alias_flags = (alias_flags & ~(Mem::VALID_HOST | Mem::VALID_DEVICE)) |
(base_flags & (Mem::VALID_HOST | Mem::VALID_DEVICE));
}
MemoryType MemoryManager::GetMemoryType_(void *h_ptr, unsigned flags)
{
// TODO: support other memory types
if (flags & Mem::VALID_DEVICE) { return MemoryType::CUDA; }
return MemoryType::HOST;
}
void MemoryManager::Copy_(void *dest_h_ptr, const void *src_h_ptr,
std::size_t size, unsigned src_flags,
unsigned &dest_flags)
{
// Type of copy to use based on the src and dest validity flags:
// | src
// | h | d | hd
// -----------+-----+-----+------
// h | h2h d2h h2h
// dest d | h2d d2d d2d
// hd | h2h d2d d2d
const bool src_on_host =
(src_flags & Mem::VALID_HOST) &&
(!(src_flags & Mem::VALID_DEVICE) ||
((dest_flags & Mem::VALID_HOST) && !(dest_flags & Mem::VALID_DEVICE)));
const bool dest_on_host =
(dest_flags & Mem::VALID_HOST) &&
(!(dest_flags & Mem::VALID_DEVICE) ||
((src_flags & Mem::VALID_HOST) && !(src_flags & Mem::VALID_DEVICE)));
const void *src_d_ptr = src_on_host ? NULL :
((src_flags & Mem::ALIAS) ?
mm.GetAliasDevicePtr(src_h_ptr, size, false) :
mm.GetDevicePtr(src_h_ptr, size, false));
if (dest_on_host)
{
if (src_on_host)
{
if (dest_h_ptr != src_h_ptr && size != 0)
{
MFEM_ASSERT((char*)dest_h_ptr + size <= src_h_ptr ||
(char*)src_h_ptr + size <= dest_h_ptr,
"data overlaps!");
std::memcpy(dest_h_ptr, src_h_ptr, size);
}
}
else
{
CuMemcpyDtoH(dest_h_ptr, src_d_ptr, size);
}
}
else
{
void *dest_d_ptr = (dest_flags & Mem::ALIAS) ?
mm.GetAliasDevicePtr(dest_h_ptr, size, false) :
mm.GetDevicePtr(dest_h_ptr, size, false);
if (src_on_host)
{
CuMemcpyHtoD(dest_d_ptr, src_h_ptr, size);
}
else
{
CuMemcpyDtoD(dest_d_ptr, src_d_ptr, size);
}
}
dest_flags = dest_flags &
~(dest_on_host ? Mem::VALID_DEVICE : Mem::VALID_HOST);
}
void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
std::size_t size, unsigned src_flags)
{
const bool src_on_host = src_flags & Mem::VALID_HOST;
if (src_on_host)
{
if (dest_h_ptr != src_h_ptr && size != 0)
{
MFEM_ASSERT((char*)dest_h_ptr + size <= src_h_ptr ||
(char*)src_h_ptr + size <= dest_h_ptr,
"data overlaps!");
std::memcpy(dest_h_ptr, src_h_ptr, size);
}
}
else
{
const void *src_d_ptr = (src_flags & Mem::ALIAS) ?
mm.GetAliasDevicePtr(src_h_ptr, size, false) :
mm.GetDevicePtr(src_h_ptr, size, false);
CuMemcpyDtoH(dest_h_ptr, src_d_ptr, size);
}
}
void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
std::size_t size, unsigned &dest_flags)
{
const bool dest_on_host = dest_flags & Mem::VALID_HOST;
if (dest_on_host)
{
if (dest_h_ptr != src_h_ptr && size != 0)
{
MFEM_ASSERT((char*)dest_h_ptr + size <= src_h_ptr ||
(char*)src_h_ptr + size <= dest_h_ptr,
"data overlaps!");
std::memcpy(dest_h_ptr, src_h_ptr, size);
}
}
else
{
void *dest_d_ptr = (dest_flags & Mem::ALIAS) ?
mm.GetAliasDevicePtr(dest_h_ptr, size, false) :
mm.GetDevicePtr(dest_h_ptr, size, false);
CuMemcpyHtoD(dest_d_ptr, src_h_ptr, size);
}
dest_flags = dest_flags &
~(dest_on_host ? Mem::VALID_DEVICE : Mem::VALID_HOST);
}
void MemoryPrintFlags(unsigned flags)
{
typedef Memory<int> Mem;
mfem::out
<< " registered = " << bool(flags & Mem::REGISTERED)
<< "\n owns host = " << bool(flags & Mem::OWNS_HOST)
<< "\n owns device = " << bool(flags & Mem::OWNS_DEVICE)
<< "\n owns internal = " << bool(flags & Mem::OWNS_INTERNAL)
<< "\n valid host = " << bool(flags & Mem::VALID_HOST)
<< "\n valid device = " << bool(flags & Mem::VALID_DEVICE)
<< "\n alias = " << bool(flags & Mem::ALIAS)
<< "\n device flag = " << bool(flags & Mem::USE_DEVICE)
<< std::endl;
}
MemoryManager mm;
bool MemoryManager::exists = false;
} // namespace mfem
Reference in New Issue View Git Blame Copy Permalink