apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2011 from rcurtin/ensmallen-2-fixes

Browse Source 
ensmallen 2 compatibility fixes
This commit is contained in:
Ryan Curtin
2019-09-10 17:29:14 -04:00
committed by GitHub
parent 65878a16f4 8f3d166bb6
commit cf71ecef94
7 changed files with 562 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1 -1
View File
@@ -41,7 +41,7 @@ matrix:
- brew install openblas armadillo || brew install openblas armadillo
install:
- mkdir build && cd build && cmake $CMAKE_OPTIONS .. && travis_wait 60 make -j2
- mkdir build && cd build && cmake $CMAKE_OPTIONS .. && travis_wait 75 make -j2
script:
- CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2
src/mlpack/methods/reinforcement_learning/q_learning.hpp
+8
View File
@@ -84,6 +84,11 @@ class QLearning
UpdaterType updater = UpdaterType(),
EnvironmentType environment = EnvironmentType());
/**
* Clean memory.
*/
~QLearning();
/**
* Execute a step in an episode.
* @return Reward for the step.
@@ -140,6 +145,9 @@ class QLearning
//! Locally-stored updater.
UpdaterType updater;
#if ENS_VERSION_MAJOR >= 2
typename UpdaterType::template Policy<arma::mat, arma::mat>* updatePolicy;
#endif
//! Locally-stored behavior policy.
PolicyType policy;
src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp
+39 -4
View File
@@ -39,6 +39,9 @@ QLearning<
config(std::move(config)),
learningNetwork(std::move(network)),
updater(std::move(updater)),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
policy(std::move(policy)),
replayMethod(std::move(replayMethod)),
environment(std::move(environment)),
@@ -49,11 +52,39 @@ QLearning<
if (learningNetwork.Parameters().is_empty())
learningNetwork.ResetParameters();
#if ENS_VERSION_MAJOR == 1
this->updater.Initialize(learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
learningNetwork.Parameters().n_cols);
#else
this->updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(this->updater,
learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
#endif
targetNetwork = learningNetwork;
}
template <
typename EnvironmentType,
typename NetworkType,
typename UpdaterType,
typename PolicyType,
typename ReplayType
>
QLearning<
EnvironmentType,
NetworkType,
UpdaterType,
PolicyType,
ReplayType
>::~QLearning()
{
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
}
template <
typename EnvironmentType,
typename NetworkType,
@@ -166,14 +197,18 @@ double QLearning<
}
}
// Learn form experience.
// Learn from experience.
arma::mat gradients;
learningNetwork.Backward(target, gradients);
replayMethod.Update(target, sampledActions,
nextActionValues, gradients);
replayMethod.Update(target, sampledActions, nextActionValues, gradients);
#if ENS_VERSION_MAJOR == 1
updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients);
#else
updatePolicy->Update(learningNetwork.Parameters(), config.StepSize(),
gradients);
#endif
return reward;
}
src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp
+167 -2
View File
@@ -54,20 +54,177 @@ class NStepQLearningWorker
const TrainingConfig& config,
bool deterministic):
updater(updater),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(environment),
config(config),
deterministic(deterministic),
pending(config.UpdateInterval())
{ Reset(); }
/**
* Copy another NStepQLearningWorker.
*
* @param other NStepQLearningWorker to copy.
*/
NStepQLearningWorker(const NStepQLearningWorker& other) :
updater(other.updater),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(other.environment),
config(other.config),
deterministic(other.deterministic),
steps(other.steps),
episodeReturn(other.episodeReturn),
pending(other.pending),
pendingIndex(other.pendingIndex),
network(other.network),
state(other.state)
{
#if ENS_VERSION_MAJOR >= 2
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
Reset();
}
/**
* Take ownership of another NStepQLearningWorker.
*
* @param other NStepQLearningWorker to take ownership of.
*/
NStepQLearningWorker(NStepQLearningWorker&& other) :
updater(std::move(other.updater)),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(std::move(other.environment)),
config(std::move(other.config)),
deterministic(std::move(other.deterministic)),
steps(std::move(other.steps)),
episodeReturn(std::move(other.episodeReturn)),
pending(std::move(other.pending)),
pendingIndex(std::move(other.pendingIndex)),
network(std::move(other.network)),
state(std::move(other.state))
{
#if ENS_VERSION_MAJOR >= 2
other.updatePolicy = NULL;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
}
/**
* Copy another NStepQLearningWorker.
*
* @param other NStepQLearningWorker to copy.
*/
NStepQLearningWorker& operator=(const NStepQLearningWorker& other)
{
if (&other == this)
return *this;
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
updater = other.updater;
environment = other.environment;
config = other.config;
deterministic = other.deterministic;
steps = other.steps;
episodeReturn = other.episodeReturn;
pending = other.pending;
pendingIndex = other.pendingIndex;
network = other.network;
state = other.state;
#if ENS_VERSION_MAJOR >= 2
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
Reset();
return *this;
}
/**
* Take ownership of another NStepQLearningWorker.
*
* @param other NStepQLearningWorker to take ownership of.
*/
NStepQLearningWorker& operator=(NStepQLearningWorker&& other)
{
if (&other == this)
return *this;
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
updater = std::move(other.updater);
environment = std::move(other.environment);
config = std::move(other.config);
deterministic = std::move(other.deterministic);
steps = std::move(other.steps);
episodeReturn = std::move(other.episodeReturn);
pending = std::move(other.pending);
pendingIndex = std::move(other.pendingIndex);
network = std::move(other.network);
state = std::move(other.state);
#if ENS_VERSION_MAJOR >= 2
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
other.updatePolicy = NULL;
#endif
return *this;
}
/**
* Clean memory.
*/
~NStepQLearningWorker()
{
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
}
/**
* Initialize the worker.
* @param learningNetwork The shared network.
*/
void Initialize(NetworkType& learningNetwork)
{
#if ENS_VERSION_MAJOR == 1
updater.Initialize(learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
learningNetwork.Parameters().n_cols);
#else
delete updatePolicy;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
#endif
// Build local network.
network = learningNetwork;
}
@@ -162,8 +319,13 @@ class NStepQLearningWorker
config.GradientLimit()); });
// Perform async update of the global network.
updater.Update(learningNetwork.Parameters(),
#if ENS_VERSION_MAJOR == 1
updater.Update(learningNetwork.Parameters(), config.StepSize(),
totalGradients);
#else
updatePolicy->Update(learningNetwork.Parameters(),
config.StepSize(), totalGradients);
#endif
// Sync the local network with the global network.
network = learningNetwork;
@@ -204,6 +366,9 @@ class NStepQLearningWorker
//! Locally-stored optimizer.
UpdaterType updater;
#if ENS_VERSION_MAJOR >= 2
typename UpdaterType::template Policy<arma::mat, arma::mat>* updatePolicy;
#endif
//! Locally-stored task.
EnvironmentType environment;
src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp
+167 -2
View File
@@ -54,20 +54,177 @@ class OneStepQLearningWorker
const TrainingConfig& config,
bool deterministic):
updater(updater),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(environment),
config(config),
deterministic(deterministic),
pending(config.UpdateInterval())
{ Reset(); }
/**
* Copy another OneStepQLearningWorker.
*
* @param other OneStepQLearningWorker to copy.
*/
OneStepQLearningWorker(const OneStepQLearningWorker& other) :
updater(other.updater),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(other.environment),
config(other.config),
deterministic(other.deterministic),
steps(other.steps),
episodeReturn(other.episodeReturn),
pending(other.pending),
pendingIndex(other.pendingIndex),
network(other.network),
state(other.state)
{
#if ENS_VERSION_MAJOR >= 2
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
Reset();
}
/**
* Take ownership of another OneStepQLearningWorker.
*
* @param other OneStepQLearningWorker to take ownership of.
*/
OneStepQLearningWorker(OneStepQLearningWorker&& other) :
updater(std::move(other.updater)),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(std::move(other.environment)),
config(std::move(other.config)),
deterministic(std::move(other.deterministic)),
steps(std::move(other.steps)),
episodeReturn(std::move(other.episodeReturn)),
pending(std::move(other.pending)),
pendingIndex(std::move(other.pendingIndex)),
network(std::move(other.network)),
state(std::move(other.state))
{
#if ENS_VERSION_MAJOR >= 2
other.updatePolicy = NULL;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
}
/**
* Copy another OneStepQLearningWorker.
*
* @param other OneStepQLearningWorker to copy.
*/
OneStepQLearningWorker& operator=(const OneStepQLearningWorker& other)
{
if (&other == this)
return *this;
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
updater = other.updater;
environment = other.environment;
config = other.config;
deterministic = other.deterministic;
steps = other.steps;
episodeReturn = other.episodeReturn;
pending = other.pending;
pendingIndex = other.pendingIndex;
network = other.network;
state = other.state;
#if ENS_VERSION_MAJOR >= 2
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
Reset();
return *this;
}
/**
* Take ownership of another OneStepQLearningWorker.
*
* @param other OneStepQLearningWorker to take ownership of.
*/
OneStepQLearningWorker& operator=(OneStepQLearningWorker&& other)
{
if (&other == this)
return *this;
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
updater = std::move(other.updater);
environment = std::move(other.environment);
config = std::move(other.config);
deterministic = std::move(other.deterministic);
steps = std::move(other.steps);
episodeReturn = std::move(other.episodeReturn);
pending = std::move(other.pending);
pendingIndex = std::move(other.pendingIndex);
network = std::move(other.network);
state = std::move(other.state);
#if ENS_VERSION_MAJOR >= 2
other.updatePolicy = NULL;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
return *this;
}
/**
* Clean memory.
*/
~OneStepQLearningWorker()
{
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
}
/**
* Initialize the worker.
* @param learningNetwork The shared network.
*/
void Initialize(NetworkType& learningNetwork)
{
#if ENS_VERSION_MAJOR == 1
updater.Initialize(learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
learningNetwork.Parameters().n_cols);
#else
delete updatePolicy;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
#endif
// Build local network.
network = learningNetwork;
}
@@ -162,8 +319,13 @@ class OneStepQLearningWorker
config.GradientLimit()); });
// Perform async update of the global network.
updater.Update(learningNetwork.Parameters(),
#if ENS_VERSION_MAJOR == 1
updater.Update(learningNetwork.Parameters(), config.StepSize(),
totalGradients);
#else
updatePolicy->Update(learningNetwork.Parameters(),
config.StepSize(), totalGradients);
#endif
// Sync the local network with the global network.
network = learningNetwork;
@@ -204,6 +366,9 @@ class OneStepQLearningWorker
//! Locally-stored optimizer.
UpdaterType updater;
#if ENS_VERSION_MAJOR >= 2
typename UpdaterType::template Policy<arma::mat, arma::mat>* updatePolicy;
#endif
//! Locally-stored task.
EnvironmentType environment;
src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp
+171 -2
View File
@@ -55,20 +55,181 @@ class OneStepSarsaWorker
const TrainingConfig& config,
bool deterministic):
updater(updater),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(environment),
config(config),
deterministic(deterministic),
pending(config.UpdateInterval())
{ Reset(); }
/**
* Copy another OneStepSarsaWorker.
*
* @param other OneStepSarsaWorker to copy.
*/
OneStepSarsaWorker(const OneStepSarsaWorker& other) :
updater(other.updater),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(other.environment),
config(other.config),
deterministic(other.deterministic),
steps(other.steps),
episodeReturn(other.episodeReturn),
pending(other.pending),
pendingIndex(other.pendingIndex),
network(other.network),
state(other.state),
action(other.action)
{
Reset();
#if ENS_VERSION_MAJOR >= 2
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
}
/**
* Take ownership of another OneStepSarsaWorker.
*
* @param other OneStepSarsaWorker to take ownership of.
*/
OneStepSarsaWorker(OneStepSarsaWorker&& other) :
updater(std::move(other.updater)),
#if ENS_VERSION_MAJOR >= 2
updatePolicy(NULL),
#endif
environment(std::move(other.environment)),
config(std::move(other.config)),
deterministic(std::move(other.deterministic)),
steps(std::move(other.steps)),
episodeReturn(std::move(other.episodeReturn)),
pending(std::move(other.pending)),
pendingIndex(std::move(other.pendingIndex)),
network(std::move(other.network)),
state(std::move(other.state)),
action(std::move(other.action))
{
#if ENS_VERSION_MAJOR >= 2
other.updatePolicy = NULL;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
}
/**
* Copy another OneStepSarsaWorker.
*
* @param other OneStepSarsaWorker to copy.
*/
OneStepSarsaWorker& operator=(const OneStepSarsaWorker& other)
{
if (&other == this)
return *this;
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
updater = other.updater;
environment = other.environment;
config = other.config;
deterministic = other.deterministic;
steps = other.steps;
episodeReturn = other.episodeReturn;
pending = other.pending;
pendingIndex = other.pendingIndex;
network = other.network;
state = other.state;
action = other.action;
#if ENS_VERSION_MAJOR >= 2
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
Reset();
return *this;
}
/**
* Take ownership of another OneStepSarsaWorker.
*
* @param other OneStepSarsaWorker to take ownership of.
*/
OneStepSarsaWorker& operator=(OneStepSarsaWorker&& other)
{
if (&other == this)
return *this;
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
updater = std::move(other.updater);
environment = std::move(other.environment);
config = std::move(other.config);
deterministic = std::move(other.deterministic);
steps = std::move(other.steps);
episodeReturn = std::move(other.episodeReturn);
pending = std::move(other.pending);
pendingIndex = std::move(other.pendingIndex);
network = std::move(other.network);
state = std::move(other.state);
action = std::move(other.action);
#if ENS_VERSION_MAJOR >= 2
other.updatePolicy = NULL;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
network.Parameters().n_rows,
network.Parameters().n_cols);
#endif
return *this;
}
/**
* Clean memory.
*/
~OneStepSarsaWorker()
{
#if ENS_VERSION_MAJOR >= 2
delete updatePolicy;
#endif
}
/**
* Initialize the worker.
* @param learningNetwork The shared network.
*/
void Initialize(NetworkType& learningNetwork)
{
#if ENS_VERSION_MAJOR == 1
updater.Initialize(learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
learningNetwork.Parameters().n_cols);
#else
delete updatePolicy;
updatePolicy = new typename UpdaterType::template
Policy<arma::mat, arma::mat>(updater,
learningNetwork.Parameters().n_rows,
learningNetwork.Parameters().n_cols);
#endif
// Build local network.
network = learningNetwork;
}
@@ -171,8 +332,13 @@ class OneStepSarsaWorker
config.GradientLimit()); });
// Perform async update of the global network.
updater.Update(learningNetwork.Parameters(),
#if ENS_VERSION_MAJOR == 1
updater.Update(learningNetwork.Parameters(), config.StepSize(),
totalGradients);
#else
updatePolicy->Update(learningNetwork.Parameters(),
config.StepSize(), totalGradients);
#endif
// Sync the local network with the global network.
network = learningNetwork;
@@ -215,6 +381,9 @@ class OneStepSarsaWorker
//! Locally-stored optimizer.
UpdaterType updater;
#if ENS_VERSION_MAJOR >= 2
typename UpdaterType::template Policy<arma::mat, arma::mat>* updatePolicy;
#endif
//! Locally-stored task.
EnvironmentType environment;
src/mlpack/tests/feedforward_network_test.cpp
+9
View File
@@ -159,7 +159,12 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest)
ens::VanillaUpdate opt;
model.ResetParameters();
#if ENS_VERSION_MAJOR == 1
opt.Initialize(model.Parameters().n_rows, model.Parameters().n_cols);
#else
ens::VanillaUpdate::Policy<arma::mat, arma::mat> optPolicy(opt,
model.Parameters().n_rows, model.Parameters().n_cols);
#endif
double stepSize = 0.01;
size_t batchSize = 10;
@@ -179,7 +184,11 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest)
model.Forward(currentData, currentResuls);
arma::mat gradients;
model.Backward(currentLabels, gradients);
#if ENS_VERSION_MAJOR == 1
opt.Update(model.Parameters(), stepSize, gradients);
#else
optPolicy.Update(model.Parameters(), stepSize, gradients);
#endif
batchStart = batchEnd;
arma::mat prediction = arma::zeros<arma::mat>(1, currentResuls.n_cols);