Implement API changes
This commit is contained in:
Shikhar Jaiswal
@@ -3,10 +3,6 @@
|
||||
set(SOURCES
|
||||
ffn.hpp
|
||||
ffn_impl.hpp
|
||||
gan.hpp
|
||||
gan_impl.hpp
|
||||
rbm.hpp
|
||||
rbm_impl.hpp
|
||||
rnn.hpp
|
||||
rnn_impl.hpp
|
||||
)
|
||||
|
||||
@@ -14,8 +14,6 @@ set(SOURCES
|
||||
batch_norm_impl.hpp
|
||||
bilinear_interpolation.hpp
|
||||
bilinear_interpolation_impl.hpp
|
||||
binary_rbm.hpp
|
||||
binary_rbm_impl.hpp
|
||||
concat.hpp
|
||||
concat_impl.hpp
|
||||
concat_performance.hpp
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
/**
|
||||
* @file rbm.hpp
|
||||
* @author Kris Singh
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_HPP
|
||||
|
||||
#include <mlpack/core.hpp>
|
||||
|
||||
#include <mlpack/methods/ann/activation_functions/softplus_function.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial neural networks. */ {
|
||||
/*
|
||||
* RBM class
|
||||
*
|
||||
* @tparam IntialiserType rule to intialise the parameters of the network
|
||||
* @tparam RBMPolicy type of rbm
|
||||
*/
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
class RBM
|
||||
{
|
||||
public:
|
||||
using NetworkType = RBM<InitializationRuleType, RBMPolicy>;
|
||||
typedef typename RBMPolicy::ElemType ElemType;
|
||||
|
||||
/*
|
||||
* Intalise all the parameters of the network
|
||||
* using the intialise rule.
|
||||
*
|
||||
* @tparam RbmPolicy Class of RBM to use(ssRBM / BinaryRBM).
|
||||
* @param predictors Training data to used.
|
||||
* @param numSteps Number of gibbs steps sampling.
|
||||
* @param negSteps Number of negative samples to average negative gradient.
|
||||
* @param useMonitoringCost Indicates whic Evaluation type to use.
|
||||
* @param persistence Indicates whether to use persistent CD or not.
|
||||
*/
|
||||
RBM(arma::Mat<ElemType> predictors,
|
||||
InitializationRuleType initializeRule,
|
||||
RBMPolicy rbmPolicy,
|
||||
const size_t numSteps = 1,
|
||||
const size_t negSteps = 1,
|
||||
const bool useMonitoringCost = true,
|
||||
const bool persistence = false);
|
||||
|
||||
// Reset the network
|
||||
void Reset();
|
||||
|
||||
/*
|
||||
* Train the feedforward network on the given input data.
|
||||
*
|
||||
* This will use the existing model parameters as a starting point for the
|
||||
* optimization. If this is not what you want, then you should access the
|
||||
* parameters vector directly with Parameters() and modify it as desired.
|
||||
*
|
||||
* @param predictors Data points / Traing Data.
|
||||
* @param optimizer Optimizer type.
|
||||
*/
|
||||
template<typename OptimizerType>
|
||||
void Train(const arma::Mat<ElemType>& predictors, OptimizerType& optimizer);
|
||||
|
||||
/**
|
||||
* Evaluate the rbm network with the given parameters.
|
||||
* The function is needed for monitoring the progress of the network.
|
||||
*
|
||||
* @param parameters Matrix model parameters.
|
||||
* @param i Index of point to use for objective function evaluation.
|
||||
*/
|
||||
double Evaluate(const arma::Mat<ElemType>& parameters, const size_t i);
|
||||
|
||||
/**
|
||||
* This function calculates the free energy of the model.
|
||||
*
|
||||
* @param Input data point.
|
||||
*/
|
||||
double FreeEnergy(arma::Mat<ElemType>&& input);
|
||||
|
||||
/*
|
||||
* This functions samples the hidden layer given the visible layer.
|
||||
*
|
||||
* @param input Visible layer input.
|
||||
* @param output The sampled hidden layer.
|
||||
*/
|
||||
void SampleHidden(arma::Mat<ElemType>&& input, arma::Mat<ElemType>&& output);
|
||||
|
||||
/*
|
||||
* This functions samples the visible layer given the hidden layer.
|
||||
*
|
||||
* @param input Hidden layer of the network.
|
||||
* @param output The sampled visible layer.
|
||||
*/
|
||||
void SampleVisible(arma::Mat<ElemType>&& input, arma::Mat<ElemType>&& output);
|
||||
|
||||
/*
|
||||
* This function does the k-step gibbs sampling.
|
||||
*
|
||||
* @param input Input to the gibbs function.
|
||||
* @param output Used for storing the negative sample.
|
||||
* @param steps Number of gibbs sampling steps taken.
|
||||
*/
|
||||
void Gibbs(arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output,
|
||||
size_t steps = SIZE_MAX);
|
||||
|
||||
/*
|
||||
* Calculates the gradients for the rbm network.
|
||||
*
|
||||
* @param parameters The current parameters of the network.
|
||||
* @param input Index of the visible layer/data point.
|
||||
* @param output Used for storing the gradients.
|
||||
*/
|
||||
void Gradient(arma::Mat<ElemType>& parameters,
|
||||
const size_t input,
|
||||
arma::Mat<ElemType>& output);
|
||||
|
||||
//! Return the number of separable functions (the number of predictor points).
|
||||
size_t NumFunctions() const { return numFunctions; }
|
||||
|
||||
//! Return the number of stes of gibbs sampling.
|
||||
size_t NumSteps() const { return numSteps; }
|
||||
|
||||
//! Return the parameters of the network.
|
||||
const arma::Mat<ElemType>& Parameters() const { return parameter; }
|
||||
//! Modify the parameters of the network.
|
||||
arma::Mat<ElemType>& Parameters() { return parameter; }
|
||||
|
||||
//! Retutrn the rbm policy for the network.
|
||||
const RBMPolicy& Policy() const { return rbmPolicy; }
|
||||
//! Modify the rbm policy for the network.
|
||||
RBMPolicy& Policy() { return rbmPolicy; }
|
||||
|
||||
//! Serialize the model.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
|
||||
private:
|
||||
//! Locally stored parameters of the network.
|
||||
arma::Mat<ElemType> parameter;
|
||||
//! Policy type of RBM.
|
||||
RBMPolicy rbmPolicy;
|
||||
//! The matrix of data points (predictors).
|
||||
arma::Mat<ElemType> predictors;
|
||||
// Intialiser for intializing the weights of the network.
|
||||
InitializationRuleType initializeRule;
|
||||
//! Locally-stored state of the persistent cdk.
|
||||
arma::Mat<ElemType> state;
|
||||
//! Locally-stored number of data points.
|
||||
size_t numFunctions;
|
||||
//! Locally-stored number of steps in gibbs sampling.
|
||||
size_t numSteps;
|
||||
//! Locally-stored number of negative samples.
|
||||
size_t negSteps;
|
||||
//! Locally-stored monitoring cost.
|
||||
bool useMonitoringCost;
|
||||
//! Locally-stored persistent cd-k or not.
|
||||
bool persistence;
|
||||
//! Locally-stored reset variable.
|
||||
bool reset;
|
||||
|
||||
//! Locally-stored reconstructed output from hidden layer.
|
||||
arma::Mat<ElemType> hiddenReconstruction;
|
||||
//! Locally-stored reconstructed output from visible layer.
|
||||
arma::Mat<ElemType> visibleReconstruction;
|
||||
|
||||
//! Locally-stored negative samples from gibbs distribution.
|
||||
arma::Mat<ElemType> negativeSamples;
|
||||
//! Locally-stored gradients from the negative phase.
|
||||
arma::Mat<ElemType> negativeGradient;
|
||||
//! Locally-stored temproray negative gradient used for negative phase.
|
||||
arma::Mat<ElemType> tempNegativeGradient;
|
||||
//! Locally-stored gradient for positive phase.
|
||||
arma::Mat<ElemType> positiveGradient;
|
||||
//! Locally-stored temporary output of gibbs chain.
|
||||
arma::Mat<ElemType> gibbsTemporary;
|
||||
//! Locally-stored output of the preActivation function used in FreeEnergy.
|
||||
arma::Mat<ElemType> preActivation;
|
||||
};
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#include "rbm_impl.hpp"
|
||||
|
||||
#endif // MLPACK_METHODS_ANN_RBM_HPP
|
||||
@@ -1,10 +1,10 @@
|
||||
# Define the files we need to compile
|
||||
# Anything not in this list will not be compiled into mlpack.
|
||||
# Define the files we need to compile.
|
||||
# Any file not in this list will not be compiled into mlpack.
|
||||
set(SOURCES
|
||||
binary_rbm_policy.hpp
|
||||
binary_rbm_policy_impl.hpp
|
||||
spike_slab_rbm_policy.hpp
|
||||
spike_slab_rbm_policy_impl.hpp
|
||||
rbm.hpp
|
||||
rbm_impl.hpp
|
||||
rbm_policies.hpp
|
||||
spike_slab_rbm_impl.hpp
|
||||
)
|
||||
|
||||
# Add directory name to sources.
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
/**
|
||||
* @file binary_rbm_policy.hpp
|
||||
* @author Kris Singh
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_HPP
|
||||
|
||||
#include <mlpack/core.hpp>
|
||||
|
||||
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
|
||||
|
||||
namespace mlpack{
|
||||
namespace ann{
|
||||
/**
|
||||
* The BinaryRBMPolicy class
|
||||
*
|
||||
* @tparam DataType the type of matrix to be used.
|
||||
*/
|
||||
template <typename DataType = arma::mat>
|
||||
class BinaryRBMPolicy
|
||||
{
|
||||
public:
|
||||
typedef typename DataType::elem_type ElemType;
|
||||
/**
|
||||
* Intialise the visible and hidden layer of the network.
|
||||
* @param visibelSize Number of visible neurons.
|
||||
* @param hiddenSize Number of hidden neurons.
|
||||
*/
|
||||
BinaryRBMPolicy(const size_t visibleSize, const size_t hiddenSize);
|
||||
|
||||
// Reset function
|
||||
void Reset();
|
||||
|
||||
/**
|
||||
* This function calculates the Free Energy of the binary RBM.
|
||||
* The free energy is given by
|
||||
* $-b^Tv - \sum_{i=1}^M log(1 + e^{c_j+v^TW_j})$.
|
||||
*
|
||||
* @param input The visible neurons.
|
||||
*/
|
||||
ElemType FreeEnergy(DataType&& input);
|
||||
|
||||
/**
|
||||
* Evaluate function computes the perfomance of the RBM at the given.
|
||||
* input in the case persistent = true
|
||||
*
|
||||
* @param predictors Training data of the network.
|
||||
* @param i The idx of the current input.
|
||||
*/
|
||||
ElemType Evaluate(DataType& predictors, size_t i);
|
||||
|
||||
/**
|
||||
* Calculates the Gradient of the RBM network on the
|
||||
* visible input obtained from the training data.
|
||||
*
|
||||
* @param input The visible layer type.
|
||||
* @param gradient Stores the gradient of the rbm network.
|
||||
*/
|
||||
void PositivePhase(DataType&& input, DataType&& gradient);
|
||||
|
||||
/**
|
||||
* Calculate the Gradient of the RBM network on the sampled
|
||||
* visible input obtained from gibbs sampling.
|
||||
*
|
||||
* @param input The negative samples sampled from gibbs distribution.
|
||||
* @param gradient Stores the gradient of the rbm network.
|
||||
*/
|
||||
void NegativePhase(DataType&& negativeSamples, DataType&& gradient);
|
||||
|
||||
/**
|
||||
* The function calculates the mean for the visible layer.
|
||||
*
|
||||
* @param input Hidden neurons from the hidden layer of the network.
|
||||
* @param output Visible neuron activations.
|
||||
*/
|
||||
void VisibleMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* The function calcultes the mean for the hidden layer.
|
||||
*
|
||||
* @param input Visible neurons.
|
||||
* @param output Hidden neuron activations.
|
||||
*/
|
||||
void HiddenMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* SampleVisible function samples the visible layer using bernoulli function.
|
||||
*
|
||||
* @param input Hidden neurons.
|
||||
* @param output Sampled visible neurons.
|
||||
*/
|
||||
void SampleVisible(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* SampleHidden function samples the hidden layer using bernoulli function.
|
||||
*
|
||||
* @param input Visible neurons.
|
||||
* @param output Sampled hidden neurons.
|
||||
*/
|
||||
void SampleHidden(DataType&& input, DataType&& output);
|
||||
|
||||
//! Serialize the model.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
|
||||
//! Return the parameters of the network.
|
||||
const DataType& Parameters() const { return parameter; }
|
||||
//! Modify the parameters of the network.
|
||||
DataType& Parameters() { return parameter; }
|
||||
//! Return the weights of the network.
|
||||
const DataType& Weight() const { return weight; }
|
||||
//! Modify the weight of the network.
|
||||
DataType& Weight() { return weight; }
|
||||
|
||||
//! Return the visible bias of the network.
|
||||
const DataType& VisibleBias() const { return visibleBias; }
|
||||
//! Modify the visible bias of the network.
|
||||
DataType& VisibleBias() { return visibleBias; }
|
||||
|
||||
//! Return the hidden bias of the network.
|
||||
const DataType& HiddenBias() const { return hiddenBias; }
|
||||
//! Modify the hidden bias of the network.
|
||||
DataType& HiddenBias() { return hiddenBias; }
|
||||
|
||||
//! Get the visible size.
|
||||
size_t const& VisibleSize() const { return visibleSize; }
|
||||
//! Get the hidden size.
|
||||
size_t const& HiddenSize() const { return hiddenSize; }
|
||||
|
||||
private:
|
||||
/**
|
||||
* VisiblePreAction function calculates the pre activation
|
||||
* values given the hidden input units.
|
||||
*
|
||||
* @param input Hidden neurons.
|
||||
* @param ouput Visible unit pre-activation values.
|
||||
*/
|
||||
void VisiblePreActivation(DataType&& input, DataType&& output);
|
||||
/**
|
||||
* HiddenPreActivation function calculates the pre activation
|
||||
* values given the visible input units.
|
||||
*
|
||||
* @param input Visible unit neuron.
|
||||
* @param ouput Hidden unit pre-activation values.
|
||||
*/
|
||||
void HiddenPreActivation(DataType&& input, DataType&& output);
|
||||
|
||||
private:
|
||||
//! Locally stored number of visible neurons.
|
||||
size_t visibleSize;
|
||||
//! Locally stored number of hidden neurons
|
||||
size_t hiddenSize;
|
||||
//! Locally stored Parameters of the network.
|
||||
DataType parameter;
|
||||
//! Locally stored weight of the network.
|
||||
DataType weight;
|
||||
//! Locally stored biases of the visible layer.
|
||||
DataType visibleBias;
|
||||
//! Locally stored biases of hidden layer.
|
||||
DataType hiddenBias;
|
||||
//! Locally-stored output of the preActivation function used in FreeEnergy.
|
||||
DataType preActivation;
|
||||
//! Locally-stored corrupInput used for Pseudo-Likelihood.
|
||||
DataType corruptInput;
|
||||
};
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#include "binary_rbm_policy_impl.hpp"
|
||||
|
||||
#endif // MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_HPP
|
||||
@@ -1,167 +0,0 @@
|
||||
/**
|
||||
* @file binary_rbm.hpp
|
||||
* @author Kris Singh
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_IMPL_HPP
|
||||
|
||||
#include <mlpack/core.hpp>
|
||||
|
||||
#include "binary_rbm_policy.hpp"
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann {
|
||||
|
||||
template<typename DataType>
|
||||
BinaryRBMPolicy<DataType>
|
||||
::BinaryRBMPolicy(const size_t visibleSize, const size_t hiddenSize) :
|
||||
visibleSize(visibleSize),
|
||||
hiddenSize(hiddenSize)
|
||||
{
|
||||
parameter.set_size((visibleSize * hiddenSize) + visibleSize + hiddenSize, 1);
|
||||
}
|
||||
|
||||
// Reset function
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::Reset()
|
||||
{
|
||||
weight = DataType(parameter.memptr(), hiddenSize, visibleSize, false, false);
|
||||
hiddenBias = DataType(parameter.memptr() + weight.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
visibleBias = DataType(parameter.memptr() + weight.n_elem +
|
||||
hiddenBias.n_elem , visibleSize, 1, false, false);
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
typename BinaryRBMPolicy<DataType>::ElemType BinaryRBMPolicy<DataType>
|
||||
::FreeEnergy(DataType&& input)
|
||||
{
|
||||
HiddenPreActivation(std::move(input), std::move(preActivation));
|
||||
preActivation = arma::log(1 + arma::trunc_exp(preActivation));
|
||||
return -(arma::accu(preActivation) + arma::dot(input, visibleBias));
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
typename BinaryRBMPolicy<DataType>::ElemType BinaryRBMPolicy<DataType>
|
||||
::Evaluate(DataType& predictors, size_t i)
|
||||
{
|
||||
size_t idx = RandInt(0, predictors.n_rows);
|
||||
DataType temp = arma::round(predictors.col(i));
|
||||
corruptInput = temp;
|
||||
corruptInput.row(idx) = 1 - corruptInput.row(idx);
|
||||
return std::log(LogisticFunction::Fn(FreeEnergy(std::move(corruptInput)) -
|
||||
FreeEnergy(std::move(temp)))) * predictors.n_rows;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::PositivePhase(
|
||||
DataType&& input,
|
||||
DataType&& gradient)
|
||||
{
|
||||
DataType weightGrad = DataType(gradient.memptr(),
|
||||
hiddenSize, visibleSize, false, false);
|
||||
|
||||
DataType hiddenBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
|
||||
DataType visibleBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem +
|
||||
hiddenBiasGrad.n_elem, visibleSize, 1, false, false);
|
||||
|
||||
HiddenMean(std::move(input), std::move(hiddenBiasGrad));
|
||||
weightGrad = hiddenBiasGrad * input.t();
|
||||
visibleBiasGrad = input;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::NegativePhase(
|
||||
DataType&& negativeSamples,
|
||||
DataType&& gradient)
|
||||
{
|
||||
DataType weightGrad = DataType(gradient.memptr(),
|
||||
hiddenSize, visibleSize, false, false);
|
||||
|
||||
DataType hiddenBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
|
||||
DataType visibleBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem +
|
||||
hiddenBiasGrad.n_elem, visibleSize, 1, false, false);
|
||||
|
||||
HiddenMean(std::move(negativeSamples), std::move(hiddenBiasGrad));
|
||||
weightGrad = hiddenBiasGrad * negativeSamples.t();
|
||||
visibleBiasGrad = negativeSamples;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::VisibleMean(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
VisiblePreActivation(std::move(input), std::move(output));
|
||||
LogisticFunction::Fn(output, output);
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::HiddenMean(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
HiddenPreActivation(std::move(input), std::move(output));
|
||||
LogisticFunction::Fn(output, output);
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::SampleVisible(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
VisibleMean(std::move(input), std::move(output));
|
||||
|
||||
for (size_t i = 0; i < output.n_elem; i++)
|
||||
output(i) = math::RandBernoulli(output(i));
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::SampleHidden(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
HiddenMean(std::move(input), std::move(output));
|
||||
|
||||
for (size_t i = 0; i < output.n_elem; i++)
|
||||
output(i) = math::RandBernoulli(output(i));
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::VisiblePreActivation(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
output = weight.t() * input + visibleBias;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void BinaryRBMPolicy<DataType>::HiddenPreActivation(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
output = weight * input + hiddenBias;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
template<typename Archive>
|
||||
void BinaryRBMPolicy<DataType>::serialize(
|
||||
Archive& ar,
|
||||
const unsigned int /* version */)
|
||||
{
|
||||
ar & BOOST_SERIALIZATION_NVP(visibleSize);
|
||||
ar & BOOST_SERIALIZATION_NVP(hiddenSize);
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
#endif // MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_IMPL_HPP
|
||||
@@ -0,0 +1,447 @@
|
||||
/**
|
||||
* @file rbm.hpp
|
||||
* @author Kris Singh
|
||||
* @author Shikhar Jaiswal
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_RBM_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_RBM_HPP
|
||||
|
||||
#include <mlpack/core.hpp>
|
||||
#include <mlpack/methods/ann/rbm/rbm_policies.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
/**
|
||||
* The RBM class.
|
||||
*
|
||||
* @tparam InitializationRuleType Rule used to initialize the network.
|
||||
* @tparam DataType The type of matrix to be used.
|
||||
* @tparam PolicyType The RBM variant to be used (BinaryRBM or SpikeSlabRBM).
|
||||
*/
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType = arma::mat,
|
||||
typename PolicyType = BinaryRBM
|
||||
>
|
||||
class RBM
|
||||
{
|
||||
public:
|
||||
using NetworkType = RBM<InitializationRuleType, DataType, PolicyType>;
|
||||
typedef typename DataType::elem_type ElemType;
|
||||
|
||||
/**
|
||||
* Initialize all the parameters of the network using initializeRule.
|
||||
*
|
||||
* @tparam initializeRule InitializationRule object for
|
||||
* initializing the network parameter.
|
||||
* @param predictors Training data to be used.
|
||||
* @param visibleSize Number of visible neurons.
|
||||
* @param hiddenSize Number of hidden neurons.
|
||||
* @param batchSize Batch size to be used for training.
|
||||
* @param numSteps Number of Gibbs Sampling steps.
|
||||
* @param negSteps Number of negative samples to average negative gradient.
|
||||
* @param persistence Indicates whether to use Persistent CD or not.
|
||||
*/
|
||||
RBM(arma::Mat<ElemType> predictors,
|
||||
InitializationRuleType initializeRule,
|
||||
const size_t visibleSize,
|
||||
const size_t hiddenSize,
|
||||
const size_t batchSize = 1,
|
||||
const size_t numSteps = 1,
|
||||
const size_t negSteps = 1,
|
||||
const size_t poolSize = 2,
|
||||
const ElemType slabPenalty = 8,
|
||||
const ElemType radius = 1,
|
||||
const bool persistence = false);
|
||||
|
||||
// Reset the network.
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
Reset();
|
||||
|
||||
// Reset the network.
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
Reset();
|
||||
|
||||
/**
|
||||
* Train the feed-forward network on the given input data.
|
||||
*
|
||||
* This will use the existing model parameters as a starting point for the
|
||||
* optimization. If this is not what you want, then you should access the
|
||||
* parameters vector directly with Parameters() and modify it as desired.
|
||||
*
|
||||
* @param optimizer Optimizer type.
|
||||
*/
|
||||
template<typename OptimizerType>
|
||||
void Train(OptimizerType& optimizer);
|
||||
|
||||
/**
|
||||
* Evaluate the RBM network with the given parameters.
|
||||
* The function is needed for monitoring the progress of the network.
|
||||
*
|
||||
* @param parameters Matrix model parameters.
|
||||
* @param i Index of the data point.
|
||||
* @param batchSize Variable to store the present number of inputs.
|
||||
*/
|
||||
double Evaluate(const arma::Mat<ElemType>& parameters,
|
||||
const size_t i,
|
||||
const size_t batchSize);
|
||||
|
||||
/**
|
||||
* This function calculates the free energy of the BinaryRBM.
|
||||
* The free energy is given by:
|
||||
* $-b^Tv - \sum_{i=1}^M log(1 + e^{c_j+v^TW_j})$.
|
||||
*
|
||||
* @param input The visible neurons.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, double>::type
|
||||
FreeEnergy(arma::Mat<ElemType>&& input);
|
||||
|
||||
/**
|
||||
* This function calculates the free energy of the SpikeSlabRBM.
|
||||
* The free energy is given by:
|
||||
* $v^t$$\Delta$v - $\sum_{i=1}^N$
|
||||
* $\log{ \sqrt{\frac{(-2\pi)^K}{\prod_{m=1}^{K}(\alpha_i)_m}}}$ -
|
||||
* $\sum_{i=1}^N \log(1+\exp( b_i +
|
||||
* \sum_{m=1}^k \frac{(v(w_i)_m^t)^2}{2(\alpha_i)_m})$
|
||||
*
|
||||
* @param input The visible layer neurons.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value,
|
||||
double>::type
|
||||
FreeEnergy(arma::Mat<ElemType>&& input);
|
||||
|
||||
/**
|
||||
* Calculates the gradient of the RBM network on the provided input.
|
||||
*
|
||||
* @param input The provided input data.
|
||||
* @param gradient Stores the gradient of the RBM network.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
Phase(DataType&& input, DataType&& gradient);
|
||||
|
||||
/**
|
||||
* Calculates the gradient of the RBM network on the provided input.
|
||||
*
|
||||
* @param input The provided input data.
|
||||
* @param gradient Stores the gradient of the RBM network.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
Phase(DataType&& input, DataType&& gradient);
|
||||
|
||||
/**
|
||||
* This function samples the hidden layer given the visible layer using
|
||||
* Bernoulli function.
|
||||
*
|
||||
* @param input Visible layer input.
|
||||
* @param output The sampled hidden layer.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
SampleHidden(arma::Mat<ElemType>&& input, arma::Mat<ElemType>&& output);
|
||||
|
||||
/**
|
||||
* This function samples the slab outputs from the Normal distribution with
|
||||
* mean given by:
|
||||
* $h_i*\alpha^{-1}*W_i^T*v$
|
||||
* and variance:
|
||||
* $\alpha&{-1}$
|
||||
*
|
||||
* @param input Consists of both visible and spike variables.
|
||||
* @param output Sampled slab neurons.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
SampleHidden(arma::Mat<ElemType>&& input, arma::Mat<ElemType>&& output);
|
||||
|
||||
/**
|
||||
* This function samples the visible layer given the hidden layer using
|
||||
* Bernoulli function.
|
||||
*
|
||||
* @param input Hidden layer of the network.
|
||||
* @param output The sampled visible layer.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
SampleVisible(arma::Mat<ElemType>&& input, arma::Mat<ElemType>&& output);
|
||||
|
||||
/**
|
||||
* Sample Hidden function samples the slab outputs from the Normal
|
||||
* distribution with mean given by:
|
||||
* $h_i*\alpha^{-1}*W_i^T*v$
|
||||
* and variance:
|
||||
* $\alpha&{-1}$
|
||||
*
|
||||
* @param input Hidden layer of the network.
|
||||
* @param output The sampled visible layer.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
SampleVisible(arma::Mat<ElemType>&& input, arma::Mat<ElemType>&& output);
|
||||
|
||||
/**
|
||||
* The function calculates the mean for the visible layer.
|
||||
*
|
||||
* @param input Hidden neurons from the hidden layer of the network.
|
||||
* @param output Visible neuron activations.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
VisibleMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* The function calculates the mean of the Normal distribution of P(v|s, h).
|
||||
* The mean is given by:
|
||||
* $\Lambda^{-1} \sum_{i=1}^N W_i * s_i * h_i$
|
||||
*
|
||||
* @param input Consists of both the spike and slab variables.
|
||||
* @param output Mean of the of the Normal distribution.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
VisibleMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* The function calculates the mean for the hidden layer.
|
||||
*
|
||||
* @param input Visible neurons.
|
||||
* @param output Hidden neuron activations.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
HiddenMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* The function calculates the mean of the Normal distribution of P(s|v, h).
|
||||
* The mean is given by:
|
||||
* $h_i*\alpha^{-1}*W_i^T*v$
|
||||
* The variance is given by:
|
||||
* $\alpha^{-1}$
|
||||
*
|
||||
* @param input Visible layer neurons.
|
||||
* @param output Consists of both the spike samples and slab samples.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
HiddenMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* The function calculates the mean of the distribution P(h|v),
|
||||
* where mean is given by:
|
||||
* $sigm(v^T*W_i*\alpha_i^{-1}*W_i^T*v + b_i)$
|
||||
*
|
||||
* @param visible The visible layer neurons.
|
||||
* @param spikeMean Indicates P(h|v).
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
SpikeMean(DataType&& visible, DataType&& spikeMean);
|
||||
|
||||
/**
|
||||
* The function samples the spike function using Bernoulli distribution.
|
||||
* @param spikeMean Indicates P(h|v).
|
||||
* @param spike Sampled binary spike variables.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
SampleSpike(DataType&& spikeMean, DataType&& spike);
|
||||
|
||||
/**
|
||||
* The function calculates the mean of Normal distribution of P(s|v, h),
|
||||
* where the mean is given by:
|
||||
* $h_i*\alpha^{-1}*W_i^T*v$
|
||||
*
|
||||
* @param visible The visible layer neurons.
|
||||
* @param spike The spike variables from hidden layer.
|
||||
* @param slabMean The mean of the Normal distribution of slab neurons.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
SlabMean(DataType&& visible, DataType&& spike, DataType&& slabMean);
|
||||
|
||||
/**
|
||||
* The function samples from the Normal distribution of P(s|v, h),
|
||||
* where the mean is given by:
|
||||
* $h_i*\alpha^{-1}*W_i^T*v$
|
||||
* and variance is given by:
|
||||
* $\alpha^{-1}$
|
||||
*
|
||||
* @param slabMean Mean of the Normal distribution of the slab neurons.
|
||||
* @param slab Sampled slab variable from the Normal distribution.
|
||||
*/
|
||||
template<typename Policy = PolicyType>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
SampleSlab(DataType&& slabMean, DataType&& slab);
|
||||
|
||||
/**
|
||||
* This function does the k-step Gibbs Sampling.
|
||||
*
|
||||
* @param input Input to the Gibbs function.
|
||||
* @param output Used for storing the negative sample.
|
||||
* @param steps Number of Gibbs Sampling steps taken.
|
||||
*/
|
||||
void Gibbs(arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output,
|
||||
size_t steps = SIZE_MAX);
|
||||
|
||||
/**
|
||||
* Calculates the gradients for the RBM network.
|
||||
*
|
||||
* @param parameters The current parameters of the network.
|
||||
* @param i Index of the data point.
|
||||
* @param gradient Variable to store the present gradient.
|
||||
* @param batchSize Variable to store the present number of inputs.
|
||||
*/
|
||||
void Gradient(const arma::Mat<ElemType>& parameters,
|
||||
const size_t i,
|
||||
arma::Mat<ElemType>& gradient,
|
||||
const size_t batchSize);
|
||||
|
||||
/**
|
||||
* Shuffle the order of function visitation. This may be called by the
|
||||
* optimizer.
|
||||
*/
|
||||
void Shuffle();
|
||||
|
||||
//! Return the number of separable functions (the number of predictor points).
|
||||
size_t NumFunctions() const { return numFunctions; }
|
||||
|
||||
//! Return the number of steps of Gibbs Sampling.
|
||||
size_t NumSteps() const { return numSteps; }
|
||||
|
||||
//! Return the parameters of the network.
|
||||
const arma::Mat<ElemType>& Parameters() const { return parameter; }
|
||||
//! Modify the parameters of the network.
|
||||
arma::Mat<ElemType>& Parameters() { return parameter; }
|
||||
|
||||
//! Return the weights of the network.
|
||||
const DataType& Weight() const { return weight; }
|
||||
//! Modify the weights of the network.
|
||||
DataType& Weight() { return weight; }
|
||||
//! Get the weight of the network.
|
||||
arma::cube const& WeightCube() const { return weightCube; }
|
||||
//! Modify the weights of the network.
|
||||
arma::cube& WeightCube() { return weightCube; }
|
||||
|
||||
//! Return the visible bias of the network.
|
||||
DataType const& VisibleBias() const { return visibleBias; }
|
||||
//! Modify the visible bias of the network.
|
||||
DataType& VisibleBias() { return visibleBias; }
|
||||
|
||||
//! Return the hidden bias of the network.
|
||||
DataType const& HiddenBias() const { return hiddenBias; }
|
||||
//! Modify the hidden bias of the network.
|
||||
DataType& HiddenBias() { return hiddenBias; }
|
||||
|
||||
//! Get the regularizer associated with spike variables.
|
||||
DataType const& SpikeBias() const { return spikeBias; }
|
||||
//! Modify the regularizer associated with spike variables.
|
||||
DataType& SpikeBias() { return spikeBias; }
|
||||
|
||||
//! Get the regularizer associated with slab variables.
|
||||
ElemType const& SlabPenalty() const { return 1.0 / slabPenalty; }
|
||||
|
||||
//! Get the regularizer associated with visible variables.
|
||||
DataType const& VisiblePenalty() const { return visiblePenalty; }
|
||||
//! Modify the regularizer associated with visible variables.
|
||||
DataType& VisiblePenalty() { return visiblePenalty; }
|
||||
|
||||
//! Get the visible size.
|
||||
size_t const& VisibleSize() const { return visibleSize; }
|
||||
//! Get the hidden size.
|
||||
size_t const& HiddenSize() const { return hiddenSize; }
|
||||
//! Get the pool size.
|
||||
size_t const& PoolSize() const { return poolSize; }
|
||||
|
||||
//! Serialize the model.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
|
||||
private:
|
||||
//! Locally stored parameters of the network.
|
||||
arma::Mat<ElemType> parameter;
|
||||
//! The matrix of data points (predictors).
|
||||
arma::Mat<ElemType> predictors;
|
||||
// Initializer for initializing the weights of the network.
|
||||
InitializationRuleType initializeRule;
|
||||
//! Locally-stored state of the persistent CD-k.
|
||||
arma::Mat<ElemType> state;
|
||||
//! Locally-stored number of data points.
|
||||
size_t numFunctions;
|
||||
//! Locally stored number of visible neurons.
|
||||
size_t visibleSize;
|
||||
//! Locally stored number of hidden neurons.
|
||||
size_t hiddenSize;
|
||||
//! Locally stored batch size parameter.
|
||||
size_t batchSize;
|
||||
//! Locally-stored number of steps in Gibbs Sampling.
|
||||
size_t numSteps;
|
||||
//! Locally-stored number of negative samples.
|
||||
size_t negSteps;
|
||||
//! Locally stored variable poolSize.
|
||||
size_t poolSize;
|
||||
//! Locally stored weight of the network.
|
||||
DataType weight;
|
||||
//! Locally stored biases of the visible layer.
|
||||
DataType visibleBias;
|
||||
//! Locally stored biases of the hidden layer.
|
||||
DataType hiddenBias;
|
||||
//! Locally-stored output of the preActivation function used in FreeEnergy.
|
||||
DataType preActivation;
|
||||
//! Locally stored weight of the network
|
||||
//! (visibleSize * poolSize * hiddenSize).
|
||||
arma::Cube<ElemType> weightCube;
|
||||
//! Locally stored spikeBias (hiddenSize * 1).
|
||||
DataType spikeBias;
|
||||
//! Locally stored visible Penalty (1 * 1).
|
||||
DataType visiblePenalty;
|
||||
//! Locally stored mean of the P(v|s, h).
|
||||
DataType visibleMean;
|
||||
//! Locally stored mean of the P(v|h).
|
||||
DataType spikeMean;
|
||||
//! Locally stored spike variables.
|
||||
DataType spikeSamples;
|
||||
//! Locally stored mean of the P(s|v, h).
|
||||
DataType slabMean;
|
||||
//! Locally stored slabPenalty.
|
||||
ElemType slabPenalty;
|
||||
//! Locally stored radius used for rejection sampling.
|
||||
ElemType radius;
|
||||
//! Locally-stored reconstructed output from hidden layer.
|
||||
arma::Mat<ElemType> hiddenReconstruction;
|
||||
//! Locally-stored reconstructed output from visible layer.
|
||||
arma::Mat<ElemType> visibleReconstruction;
|
||||
//! Locally-stored negative samples from Gibbs distribution.
|
||||
arma::Mat<ElemType> negativeSamples;
|
||||
//! Locally-stored gradients from the negative phase.
|
||||
arma::Mat<ElemType> negativeGradient;
|
||||
//! Locally-stored temporary negative gradient used for negative phase.
|
||||
arma::Mat<ElemType> tempNegativeGradient;
|
||||
//! Locally-stored gradient for positive phase.
|
||||
arma::Mat<ElemType> positiveGradient;
|
||||
//! Locally-stored temporary output of Gibbs chain.
|
||||
arma::Mat<ElemType> gibbsTemporary;
|
||||
//! Locally-stored persistent CD-k boolean flag.
|
||||
bool persistence;
|
||||
//! Locally-stored reset variable.
|
||||
bool reset;
|
||||
};
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#include "rbm_impl.hpp"
|
||||
#include "spike_slab_rbm_impl.hpp"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,343 @@
|
||||
/**
|
||||
* @file rbm_impl.hpp
|
||||
* @author Kris Singh
|
||||
* @author Shikhar Jaiswal
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_RBM_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_RBM_IMPL_HPP
|
||||
|
||||
// In case it hasn't been included yet.
|
||||
#include "rbm.hpp"
|
||||
|
||||
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial neural networks. */ {
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::RBM(
|
||||
arma::Mat<ElemType> predictors,
|
||||
InitializationRuleType initializeRule,
|
||||
const size_t visibleSize,
|
||||
const size_t hiddenSize,
|
||||
const size_t batchSize,
|
||||
const size_t numSteps,
|
||||
const size_t negSteps,
|
||||
const size_t poolSize,
|
||||
const ElemType slabPenalty,
|
||||
const ElemType radius,
|
||||
const bool persistence):
|
||||
predictors(std::move(predictors)),
|
||||
initializeRule(initializeRule),
|
||||
visibleSize(visibleSize),
|
||||
hiddenSize(hiddenSize),
|
||||
batchSize(batchSize),
|
||||
numSteps(numSteps),
|
||||
negSteps(negSteps),
|
||||
poolSize(poolSize),
|
||||
slabPenalty(slabPenalty),
|
||||
radius(2 * radius),
|
||||
persistence(persistence),
|
||||
reset(false)
|
||||
{
|
||||
numFunctions = this->predictors.n_cols;
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::Reset()
|
||||
{
|
||||
size_t shape = (visibleSize * hiddenSize) + visibleSize + hiddenSize;
|
||||
|
||||
parameter.set_size(shape, 1);
|
||||
positiveGradient.set_size(shape, 1);
|
||||
negativeGradient.set_size(shape, 1);
|
||||
tempNegativeGradient.set_size(shape, 1);
|
||||
negativeSamples.set_size(visibleSize, batchSize);
|
||||
|
||||
weight = DataType(parameter.memptr(), hiddenSize, visibleSize, false, false);
|
||||
hiddenBias = DataType(parameter.memptr() + weight.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
visibleBias = DataType(parameter.memptr() + weight.n_elem +
|
||||
hiddenBias.n_elem, visibleSize, 1, false, false);
|
||||
|
||||
parameter.zeros();
|
||||
positiveGradient.zeros();
|
||||
negativeGradient.zeros();
|
||||
tempNegativeGradient.zeros();
|
||||
initializeRule.Initialize(parameter, parameter.n_elem, 1);
|
||||
|
||||
reset = true;
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename OptimizerType>
|
||||
void RBM<InitializationRuleType, DataType, PolicyType>::Train(
|
||||
OptimizerType& optimizer)
|
||||
{
|
||||
if (!reset)
|
||||
{
|
||||
Reset();
|
||||
}
|
||||
|
||||
optimizer.Optimize(*this, parameter);
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, double>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::FreeEnergy(
|
||||
arma::Mat<ElemType>&& input)
|
||||
{
|
||||
preActivation = arma::log(1 + arma::trunc_exp((weight * input) + hiddenBias));
|
||||
return -(arma::accu(preActivation) + arma::dot(input, visibleBias));
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::Phase(
|
||||
DataType&& input,
|
||||
DataType&& gradient)
|
||||
{
|
||||
DataType weightGrad = DataType(gradient.memptr(),
|
||||
hiddenSize, visibleSize, false, false);
|
||||
|
||||
DataType hiddenBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
|
||||
DataType visibleBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem +
|
||||
hiddenBiasGrad.n_elem, visibleSize, 1, false, false);
|
||||
|
||||
HiddenMean(std::move(input), std::move(hiddenBiasGrad));
|
||||
weightGrad = hiddenBiasGrad * input.t();
|
||||
visibleBiasGrad = input;
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
double RBM<InitializationRuleType, DataType, PolicyType>::Evaluate(
|
||||
const arma::Mat<ElemType>& /* parameters*/,
|
||||
const size_t i,
|
||||
const size_t batchSize)
|
||||
{
|
||||
Gibbs(std::move(predictors.cols(i, i + batchSize - 1)),
|
||||
std::move(negativeSamples));
|
||||
return std::fabs(FreeEnergy(std::move(predictors.cols(i,
|
||||
i + batchSize - 1))) - FreeEnergy(std::move(negativeSamples)));
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SampleHidden(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output)
|
||||
{
|
||||
HiddenMean(std::move(input), std::move(output));
|
||||
|
||||
for (size_t i = 0; i < output.n_elem; i++)
|
||||
{
|
||||
output(i) = math::RandBernoulli(output(i));
|
||||
}
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SampleVisible(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output)
|
||||
{
|
||||
VisibleMean(std::move(input), std::move(output));
|
||||
|
||||
for (size_t i = 0; i < output.n_elem; i++)
|
||||
{
|
||||
output(i) = math::RandBernoulli(output(i));
|
||||
}
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::VisibleMean(DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
output = weight.t() * input + visibleBias;
|
||||
LogisticFunction::Fn(output, output);
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, BinaryRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::HiddenMean(DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
output = weight * input + hiddenBias;
|
||||
LogisticFunction::Fn(output, output);
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
void RBM<InitializationRuleType, DataType, PolicyType>::Gibbs(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output,
|
||||
size_t steps)
|
||||
{
|
||||
steps = (steps == SIZE_MAX) ? this->numSteps : steps;
|
||||
|
||||
if (persistence && !state.is_empty())
|
||||
{
|
||||
SampleHidden(std::move(state), std::move(gibbsTemporary));
|
||||
SampleVisible(std::move(gibbsTemporary), std::move(output));
|
||||
}
|
||||
else
|
||||
{
|
||||
SampleHidden(std::move(input), std::move(gibbsTemporary));
|
||||
SampleVisible(std::move(gibbsTemporary), std::move(output));
|
||||
}
|
||||
|
||||
for (size_t j = 1; j < steps; j++)
|
||||
{
|
||||
SampleHidden(std::move(output), std::move(gibbsTemporary));
|
||||
SampleVisible(std::move(gibbsTemporary), std::move(output));
|
||||
}
|
||||
if (persistence)
|
||||
{
|
||||
state = output;
|
||||
}
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
void RBM<InitializationRuleType, DataType, PolicyType>::Gradient(
|
||||
const arma::Mat<ElemType>& /*parameters*/,
|
||||
const size_t i,
|
||||
arma::Mat<ElemType>& gradient,
|
||||
const size_t batchSize)
|
||||
{
|
||||
positiveGradient.zeros();
|
||||
negativeGradient.zeros();
|
||||
|
||||
Phase(std::move(predictors.cols(i, i + batchSize - 1)),
|
||||
std::move(positiveGradient));
|
||||
|
||||
for (size_t i = 0; i < negSteps; i++)
|
||||
{
|
||||
Gibbs(std::move(predictors.cols(i, i + batchSize - 1)),
|
||||
std::move(negativeSamples));
|
||||
Phase(std::move(negativeSamples), std::move(tempNegativeGradient));
|
||||
|
||||
negativeGradient += tempNegativeGradient;
|
||||
}
|
||||
gradient = ((negativeGradient / negSteps) - positiveGradient);
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
void RBM<InitializationRuleType, DataType, PolicyType>::Shuffle()
|
||||
{
|
||||
predictors = predictors.cols(arma::shuffle(arma::linspace<arma::uvec>(0,
|
||||
predictors.n_cols - 1, predictors.n_cols)));
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Archive>
|
||||
void RBM<InitializationRuleType, DataType, PolicyType>::serialize(
|
||||
Archive& ar, const unsigned int /* version */)
|
||||
{
|
||||
ar & BOOST_SERIALIZATION_NVP(parameter);
|
||||
ar & BOOST_SERIALIZATION_NVP(visibleSize);
|
||||
ar & BOOST_SERIALIZATION_NVP(hiddenSize);
|
||||
ar & BOOST_SERIALIZATION_NVP(state);
|
||||
ar & BOOST_SERIALIZATION_NVP(numFunctions);
|
||||
ar & BOOST_SERIALIZATION_NVP(numSteps);
|
||||
ar & BOOST_SERIALIZATION_NVP(negSteps);
|
||||
ar & BOOST_SERIALIZATION_NVP(persistence);
|
||||
ar & BOOST_SERIALIZATION_NVP(poolSize);
|
||||
ar & BOOST_SERIALIZATION_NVP(weight);
|
||||
ar & BOOST_SERIALIZATION_NVP(weightCube);
|
||||
ar & BOOST_SERIALIZATION_NVP(spikeBias);
|
||||
ar & BOOST_SERIALIZATION_NVP(slabPenalty);
|
||||
ar & BOOST_SERIALIZATION_NVP(radius);
|
||||
ar & BOOST_SERIALIZATION_NVP(visiblePenalty);
|
||||
|
||||
// If we are loading, we need to initialize the weights.
|
||||
if (Archive::is_loading::value)
|
||||
{
|
||||
size_t shape = parameter.n_elem;
|
||||
positiveGradient.set_size(shape, 1);
|
||||
negativeGradient.set_size(shape, 1);
|
||||
negativeSamples.set_size(visibleSize, batchSize);
|
||||
tempNegativeGradient.set_size(shape, 1);
|
||||
spikeMean.set_size(hiddenSize, 1);
|
||||
spikeSamples.set_size(hiddenSize, 1);
|
||||
slabMean.set_size(poolSize, hiddenSize);
|
||||
positiveGradient.zeros();
|
||||
negativeGradient.zeros();
|
||||
tempNegativeGradient.zeros();
|
||||
reset = true;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
#endif
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* @file rbm_policies.hpp
|
||||
* @author Shikhar Jaiswal
|
||||
*
|
||||
* Implementation of the RBM policy types.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_RBM_POLICIES_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_RBM_POLICIES_HPP
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial Neural Network. */ {
|
||||
|
||||
/**
|
||||
* For more information, see the following paper:
|
||||
*
|
||||
* @code
|
||||
* @article{Hinton10,
|
||||
* author = {Geoffrey Hinton},
|
||||
* title = {A Practical Guide to Training Restricted Boltzmann Machines},
|
||||
* year = {2010},
|
||||
* url = {https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf}
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
class BinaryRBM { /* Nothing to do here */ };
|
||||
|
||||
/**
|
||||
* For more information, see the following paper:
|
||||
*
|
||||
* @code
|
||||
* @article{Courville11,
|
||||
* author = {Aaron Courville, James Bergstra and Yoshua Bengio},
|
||||
* title = {A Spike and Slab Restricted Boltzmann Machine},
|
||||
* year = {2011},
|
||||
* url = {http://proceedings.mlr.press/v15/courville11a/courville11a.pdf}
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
class SpikeSlabRBM { /* Nothing to do here */ };
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,302 @@
|
||||
/**
|
||||
* @file spike_slab_rbm_impl.hpp
|
||||
* @author Kris Singh
|
||||
* @author Shikhar Jaiswal
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_IMPL_HPP
|
||||
|
||||
#include "rbm.hpp"
|
||||
|
||||
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/softplus_function.hpp>
|
||||
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann {
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::Reset()
|
||||
{
|
||||
size_t shape = (visibleSize * hiddenSize * poolSize) + visibleSize + hiddenSize;
|
||||
parameter.set_size(shape, 1);
|
||||
positiveGradient.set_size(shape, 1);
|
||||
negativeGradient.set_size(shape, 1);
|
||||
tempNegativeGradient.set_size(shape, 1);
|
||||
negativeSamples.set_size(visibleSize, batchSize);
|
||||
visibleMean.set_size(visibleSize, 1);
|
||||
spikeMean.set_size(hiddenSize, 1);
|
||||
spikeSamples.set_size(hiddenSize, 1);
|
||||
slabMean.set_size(poolSize, hiddenSize);
|
||||
|
||||
// Weight shape D * K * N
|
||||
weightCube = arma::Cube<ElemType>(parameter.memptr(),
|
||||
visibleSize, poolSize, hiddenSize,
|
||||
false, false);
|
||||
// spike bias shape N * 1
|
||||
spikeBias = DataType(parameter.memptr() + weight.n_elem, hiddenSize, 1,
|
||||
false, false);
|
||||
// visible penalty 1 * 1 => D * D(when used)
|
||||
visiblePenalty = DataType(parameter.memptr() + weight.n_elem +
|
||||
spikeBias.n_elem, 1, 1, false, false);
|
||||
|
||||
parameter.zeros();
|
||||
positiveGradient.zeros();
|
||||
negativeGradient.zeros();
|
||||
tempNegativeGradient.zeros();
|
||||
initializeRule.Initialize(parameter, parameter.n_elem, 1);
|
||||
|
||||
reset = true;
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, double>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::FreeEnergy(
|
||||
arma::Mat<ElemType>&& input)
|
||||
{
|
||||
ElemType freeEnergy = 0.5 * arma::as_scalar(visiblePenalty(0) * input.t() *
|
||||
input);
|
||||
|
||||
freeEnergy -= 0.5 * hiddenSize * poolSize *
|
||||
std::log((2.0 * M_PI) / slabPenalty);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
ElemType sum = 0;
|
||||
sum = arma::accu(arma::square(input.t() * weightCube.slice(i))) /
|
||||
(2.0 * slabPenalty);
|
||||
freeEnergy -= SoftplusFunction::Fn(spikeBias(i) - sum);
|
||||
}
|
||||
|
||||
return freeEnergy;
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::Phase(
|
||||
DataType&& input,
|
||||
DataType&& gradient)
|
||||
{
|
||||
arma::Cube<ElemType> weightGrad = arma::Cube<ElemType>
|
||||
(gradient.memptr(), visibleSize, poolSize, hiddenSize, false, false);
|
||||
|
||||
DataType spikeBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
|
||||
DataType visiblePenaltyGrad = DataType(gradient.memptr() +
|
||||
weightGrad.n_elem + spikeBiasGrad.n_elem, 1, 1, false, false);
|
||||
|
||||
SpikeMean(std::move(input), std::move(spikeMean));
|
||||
SampleSpike(std::move(spikeMean), std::move(spikeSamples));
|
||||
SlabMean(std::move(input), std::move(spikeSamples), std::move(slabMean));
|
||||
|
||||
for (size_t i = 0 ; i < hiddenSize; i++)
|
||||
weightGrad.slice(i) = input * slabMean.col(i).t() * spikeMean(i);
|
||||
|
||||
spikeBiasGrad = spikeMean;
|
||||
|
||||
visiblePenaltyGrad = -0.5 * input.t() * input;
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SampleHidden(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output)
|
||||
{
|
||||
output.set_size(hiddenSize + poolSize * hiddenSize, 1);
|
||||
|
||||
DataType spike(output.memptr(), hiddenSize, 1, false, false);
|
||||
DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false,
|
||||
false);
|
||||
|
||||
SpikeMean(std::move(input), std::move(spike));
|
||||
SampleSpike(std::move(spike), std::move(spike));
|
||||
SlabMean(std::move(input), std::move(spike), std::move(slab));
|
||||
SampleSlab(std::move(slab), std::move(slab));
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SampleVisible(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output)
|
||||
{
|
||||
const size_t numMaxTrials = 10;
|
||||
size_t k = 0;
|
||||
|
||||
VisibleMean(std::move(input), std::move(visibleMean));
|
||||
output.set_size(visibleSize, 1);
|
||||
|
||||
for (k = 0; k < numMaxTrials; k++)
|
||||
{
|
||||
for (size_t i = 0; i < visibleSize; i++)
|
||||
{
|
||||
output(i) = math::RandNormal(visibleMean(i), 1.0 / visiblePenalty(0));
|
||||
}
|
||||
if (arma::norm(output, 2) < radius)
|
||||
break;
|
||||
}
|
||||
|
||||
if (k == numMaxTrials)
|
||||
{
|
||||
Log::Warn << "Outputs are still not in visible unit "
|
||||
<< arma::norm(output, 2)
|
||||
<< " terminating optimization."
|
||||
<< std::endl;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::VisibleMean(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
output.zeros(visibleSize, 1);
|
||||
|
||||
DataType spike(input.memptr(), hiddenSize, 1, false, false);
|
||||
DataType slab(input.memptr() + hiddenSize, poolSize, hiddenSize, false,
|
||||
false);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
output += weightCube.slice(i) * slab.col(i) * spike(i);
|
||||
|
||||
output = ((1.0 / visiblePenalty(0)) * output);
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::HiddenMean(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
output.set_size(hiddenSize + poolSize * hiddenSize, 1);
|
||||
|
||||
DataType spike(output.memptr(), hiddenSize, 1, false, false);
|
||||
DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false,
|
||||
false);
|
||||
|
||||
SpikeMean(std::move(input), std::move(spike));
|
||||
SampleSpike(std::move(spike), std::move(spikeSamples));
|
||||
SlabMean(std::move(input), std::move(spikeSamples), std::move(slab));
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SpikeMean(
|
||||
DataType&& visible,
|
||||
DataType&& spikeMean)
|
||||
{
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
spikeMean(i) = LogisticFunction::Fn(0.5 * (1.0 / slabPenalty) *
|
||||
arma::as_scalar(visible.t() * weightCube.slice(i) *
|
||||
weightCube.slice(i).t() * visible) + spikeBias(i));
|
||||
}
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SampleSpike(
|
||||
DataType&& spikeMean,
|
||||
DataType&& spike)
|
||||
{
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
spike(i) = math::RandBernoulli(spikeMean(i));
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SlabMean(
|
||||
DataType&& visible,
|
||||
DataType&& spike,
|
||||
DataType&& slabMean)
|
||||
{
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
slabMean.col(i) = (1.0 / slabPenalty) * spike(i) *
|
||||
weightCube.slice(i).t() * visible;
|
||||
}
|
||||
}
|
||||
|
||||
template<
|
||||
typename InitializationRuleType,
|
||||
typename DataType,
|
||||
typename PolicyType
|
||||
>
|
||||
template<typename Policy>
|
||||
typename std::enable_if<std::is_same<Policy, SpikeSlabRBM>::value, void>::type
|
||||
RBM<InitializationRuleType, DataType, PolicyType>::SampleSlab(
|
||||
DataType&& slabMean,
|
||||
DataType&& slab)
|
||||
{
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
for (size_t j = 0; j < poolSize; j++)
|
||||
{
|
||||
slab(j, i) = math::RandNormal(slabMean(j, i), 1.0 / slabPenalty);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
@@ -1,235 +0,0 @@
|
||||
/**
|
||||
* @file spike_slab_rbm_policy.hpp
|
||||
* @author Kris Singh
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_HPP
|
||||
|
||||
#include <mlpack/core.hpp>
|
||||
|
||||
#include <mlpack/methods/ann/activation_functions/softplus_function.hpp>
|
||||
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann {
|
||||
/**
|
||||
* The SpikeSlabRBMPolicy class.
|
||||
*
|
||||
* @tparam DataType the type of matrix to be used.
|
||||
*/
|
||||
template<typename DataType = arma::mat>
|
||||
class SpikeSlabRBMPolicy
|
||||
{
|
||||
public:
|
||||
typedef typename DataType::elem_type ElemType;
|
||||
/**
|
||||
* Intialise the visible and hidden layer of the network
|
||||
*
|
||||
* @param visibleSize Number of visible neurons.
|
||||
* @param hiddenSize Number of hidden neurons.
|
||||
* @param poolSize Number of hidden neurons to pool together.
|
||||
* @param slabPenalty Regulariser of slab varaibles.
|
||||
* @param radius Feasible regions for visible layer samples.
|
||||
*/
|
||||
SpikeSlabRBMPolicy(const size_t visibleSize,
|
||||
const size_t hiddenSize,
|
||||
const size_t poolSize,
|
||||
const ElemType slabPenalty,
|
||||
ElemType radius);
|
||||
|
||||
// Reset function
|
||||
void Reset();
|
||||
|
||||
/**
|
||||
* Free energy of the spike and slab variable
|
||||
* the free energy of the ssRBM is given my
|
||||
* $v^t$$\Delta$v - $\sum_{i=1}^N$
|
||||
* $\log{ \sqrt{\frac{(-2\pi)^K}{\prod_{m=1}^{K}(\alpha_i)_m}}}$ -
|
||||
* $\sum_{i=1}^N \log(1+\exp( b_i +
|
||||
* \sum_{m=1}^k \frac{(v(w_i)_m^t)^2}{2(\alpha_i)_m})$
|
||||
*
|
||||
* @param input The visible layer neurons.
|
||||
*/
|
||||
ElemType FreeEnergy(DataType&& input);
|
||||
|
||||
/**
|
||||
* Evaluate function is used by the Optimizer to
|
||||
* find the perfomance of the network on the currentInput
|
||||
*
|
||||
* @param predictors The training data used.
|
||||
* @param i The idx of the current input.
|
||||
*/
|
||||
ElemType Evaluate(DataType& /*predictors*/, size_t /*i*/);
|
||||
|
||||
/**
|
||||
* Calculate the Gradient of the RBM network on the
|
||||
* visible input from the training data.
|
||||
*
|
||||
* @param input The visible layer neurons.
|
||||
* @param gradient Stores the gradient of the rbm network.
|
||||
*/
|
||||
void PositivePhase(DataType&& input, DataType&& gradient);
|
||||
|
||||
/**
|
||||
* Calculate the Gradient of the RBM network on the sampled
|
||||
* visible input from gibbs sampling
|
||||
*
|
||||
* @param input The visible layer neurons.
|
||||
* @param output Stores the computed gradient of the rbm network.
|
||||
*/
|
||||
void NegativePhase(DataType&& negativeSamples, DataType&& gradient);
|
||||
|
||||
/**
|
||||
* Visible Mean function calculates the mean of the
|
||||
* normal distribution of P(v| s,h).
|
||||
* Where the mean is given by \Lambda^{-1} \sum_{i=1}^N W_i * s_i * h_i
|
||||
*
|
||||
* @param input Consists of both the spike and slab variables.
|
||||
* @param output Mean of the of the Normal distribution.
|
||||
*/
|
||||
void VisibleMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* Hidden Mean function calculates the mean of the
|
||||
* normal distribution of P(s|v,h).
|
||||
* Where the mean is given by h_i*\alpha^{-1}*W_i^T*v
|
||||
* variance is givenby \alpha^{-1}
|
||||
*
|
||||
* @param input Visible layer neurons.
|
||||
* @param output Consits of both the spike samples and slab samples.
|
||||
*/
|
||||
void HiddenMean(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* Sample Visible function samples
|
||||
* the visible layer from the normal distribution with
|
||||
* mean \Lambda^{-1} \sum_{i=1}^N W_i * s_i * h_i and
|
||||
* variance \Lambda^{-1}
|
||||
*
|
||||
* @param input Consists of spike and slab variables.
|
||||
* @param output Sampled visible layer neurons.
|
||||
*/
|
||||
void SampleVisible(DataType&& input, DataType&& output);
|
||||
|
||||
/**
|
||||
* Sample Hidden function samples
|
||||
* the slab outputs from the normal distribution with
|
||||
* mean by h_i*\alpha^{-1}*W_i^T*v and
|
||||
* variance \alpha&{-1}
|
||||
*
|
||||
* @param input Consists of both visible and spike variables.
|
||||
* @param output Smapled slab neurons.
|
||||
*/
|
||||
void SampleHidden(DataType&& input, DataType&& output);
|
||||
|
||||
//! Serialize function.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const unsigned int /* version */);
|
||||
|
||||
//! Return the parameters of the network.
|
||||
const DataType& Parameters() const { return parameter; }
|
||||
//! Modify the parameters of the network.
|
||||
DataType& Parameters() { return parameter; }
|
||||
|
||||
//! Get the weight of the network.
|
||||
arma::cube const& Weight() const { return weight; }
|
||||
//! Modify the weights of the network.
|
||||
arma::cube& Weight() { return weight; }
|
||||
|
||||
//! Get the regulaliser associated with spike variables.
|
||||
DataType const& SpikeBias() const { return spikeBias; }
|
||||
//! Modify the regulaliser associated with spike variables.
|
||||
DataType& SpikeBias() { return spikeBias; }
|
||||
|
||||
//! Get the regulaliser associated with slab variables.
|
||||
ElemType const& SlabPenalty() const { return 1.0 / slabPenalty; }
|
||||
|
||||
//! Get the regulaliser associated with visible variables.
|
||||
DataType const& VisiblePenalty() const { return visiblePenalty; }
|
||||
//! Modify the regulaliser associated with visible variables.
|
||||
DataType& VisiblePenalty() { return visiblePenalty; }
|
||||
|
||||
//! Get the visible size.
|
||||
size_t const& VisibleSize() const { return visibleSize; }
|
||||
//! Get the hidden size.
|
||||
size_t const& HiddenSize() const { return hiddenSize; }
|
||||
//! Get the pool size.
|
||||
size_t const& PoolSize() const { return poolSize; }
|
||||
|
||||
private:
|
||||
/**
|
||||
* Spike Mean function calculates the mean of the following
|
||||
* distribution P(h|v) where mean is given by
|
||||
* sigm(v^T*W_i*\alpha_i^{-1}*W_i^T*v + b_i)
|
||||
*
|
||||
* @param visible The visible layer neurons.
|
||||
* @param spikeMean Indicates P(h|v).
|
||||
*/
|
||||
void SpikeMean(DataType&& visible, DataType&& spikeMean);
|
||||
/**
|
||||
* Sample Spike function samples the spike
|
||||
* function using bernoulli distribution
|
||||
* @param spikeMean Indicates P(h|v).
|
||||
* @param spike Sampled binary spike variables.
|
||||
*/
|
||||
void SampleSpike(DataType&& spikeMean, DataType&& spike);
|
||||
|
||||
/**
|
||||
* SlabMean function calculates the mean of
|
||||
* normal distribution of P(s|v,h).
|
||||
* Where the mean is given by h_i*\alpha^{-1}*W_i^T*v
|
||||
*
|
||||
* @param visible The visible layer neurons.
|
||||
* @param spike The spike variables from hidden layer.
|
||||
* @param slabMean The mean of the normal distribution of slab neurons.
|
||||
*/
|
||||
void SlabMean(DataType&& visible, DataType&& spike, DataType&& slabMean);
|
||||
/**
|
||||
* SampleSlab function samples from the
|
||||
* normal distribution P(s|v,h).
|
||||
* Where the mean is given by h_i*\alpha^{-1}*W_i^T*v
|
||||
* variance is givenby \alpha^{-1}
|
||||
*
|
||||
* @slabMean Mean of the normal distribution of the slab neurons.
|
||||
* @slab Sampled slab variable from the normal distribution.
|
||||
*/
|
||||
void SampleSlab(DataType&& slabMean, DataType&& slab);
|
||||
|
||||
//! Locally stored number of visible neurons.
|
||||
size_t visibleSize;
|
||||
//! Locally stored number of hidden neurons.
|
||||
size_t hiddenSize;
|
||||
//! Locally stored variable poolSize.
|
||||
size_t poolSize;
|
||||
//! Locally stored parameters.
|
||||
DataType parameter;
|
||||
//! Locally stored weight of the network(visibleSize * poolSize * hiddenSize).
|
||||
arma::Cube<ElemType> weight;
|
||||
//! Locally stored spikeBias (hiddenSize * 1).
|
||||
DataType spikeBias;
|
||||
//! Locally stored slabPenalty.
|
||||
ElemType slabPenalty;
|
||||
//! Locally stored radius used for rejection sampling.
|
||||
ElemType radius;
|
||||
//! Locally stored visible Penalty(1 * 1).
|
||||
DataType visiblePenalty;
|
||||
//! Locally stored mean of the P(v | s,h).
|
||||
DataType visibleMean;
|
||||
//! Locally stored mean of the P(v | h).
|
||||
DataType spikeMean;
|
||||
//! Locally stored spike variables.
|
||||
DataType spikeSamples;
|
||||
//! Locally stored mean of the P(s | v, h).
|
||||
DataType slabMean;
|
||||
};
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
#include "spike_slab_rbm_policy_impl.hpp"
|
||||
|
||||
#endif // MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_HPP
|
||||
@@ -1,335 +0,0 @@
|
||||
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_IMPL_HPP
|
||||
|
||||
#include "spike_slab_rbm_policy.hpp"
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann {
|
||||
template<typename DataType>
|
||||
SpikeSlabRBMPolicy<DataType>::SpikeSlabRBMPolicy(
|
||||
const size_t visibleSize,
|
||||
const size_t hiddenSize,
|
||||
const size_t poolSize,
|
||||
const ElemType slabPenalty,
|
||||
ElemType radius):
|
||||
visibleSize(visibleSize),
|
||||
hiddenSize(hiddenSize),
|
||||
poolSize(poolSize),
|
||||
slabPenalty(slabPenalty),
|
||||
radius(2 * radius)
|
||||
{
|
||||
parameter.set_size(visibleSize * hiddenSize * poolSize +
|
||||
visibleSize + hiddenSize);
|
||||
|
||||
visibleMean.set_size(visibleSize, 1);
|
||||
spikeMean.set_size(hiddenSize, 1);
|
||||
spikeSamples.set_size(hiddenSize, 1);
|
||||
slabMean.set_size(poolSize, hiddenSize);
|
||||
};
|
||||
|
||||
// Reset function
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::Reset()
|
||||
{
|
||||
// Weight shape D * K * N
|
||||
weight = arma::Cube<ElemType>(parameter.memptr(),
|
||||
visibleSize, poolSize, hiddenSize,
|
||||
false, false);
|
||||
|
||||
// spike bias shape N * 1
|
||||
spikeBias = DataType(parameter.memptr() + weight.n_elem, hiddenSize, 1,
|
||||
false, false);
|
||||
|
||||
// visible penalty 1 * 1 => D * D(when used)
|
||||
visiblePenalty = DataType(parameter.memptr() + weight.n_elem +
|
||||
spikeBias.n_elem, 1, 1, false, false);
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
typename SpikeSlabRBMPolicy<DataType>::ElemType
|
||||
SpikeSlabRBMPolicy<DataType>::FreeEnergy(DataType&& input)
|
||||
{
|
||||
assert(input.n_rows == visibleSize);
|
||||
assert(input.n_cols == 1);
|
||||
|
||||
|
||||
ElemType freeEnergy = 0.5 * arma::as_scalar(visiblePenalty(0) * input.t() *
|
||||
input);
|
||||
|
||||
freeEnergy -= 0.5 * hiddenSize * poolSize *
|
||||
std::log((2.0 * M_PI) / slabPenalty);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
ElemType sum = 0;
|
||||
sum = arma::accu(arma::square(input.t() * weight.slice(i))) /
|
||||
(2.0 * slabPenalty);
|
||||
freeEnergy -= SoftplusFunction::Fn(spikeBias(i) - sum);
|
||||
}
|
||||
|
||||
return freeEnergy;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
typename SpikeSlabRBMPolicy<DataType>::ElemType
|
||||
SpikeSlabRBMPolicy<DataType>::Evaluate(DataType& /*predictors*/,
|
||||
size_t /*i*/)
|
||||
{
|
||||
// Return 0 here since we don't have evaluate in case of persistence
|
||||
return 0;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::PositivePhase(
|
||||
DataType&& input,
|
||||
DataType&& gradient)
|
||||
{
|
||||
assert(input.n_rows == visibleSize);
|
||||
assert(input.n_cols == 1);
|
||||
arma::Cube<ElemType> weightGrad = arma::Cube<ElemType>
|
||||
(gradient.memptr(), visibleSize, poolSize, hiddenSize, false, false);
|
||||
|
||||
DataType spikeBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
|
||||
DataType visiblePenaltyGrad = DataType(gradient.memptr() +
|
||||
weightGrad.n_elem + spikeBiasGrad.n_elem, 1, 1, false, false);
|
||||
|
||||
SpikeMean(std::move(input), std::move(spikeMean));
|
||||
SampleSpike(std::move(spikeMean), std::move(spikeSamples));
|
||||
SlabMean(std::move(input), std::move(spikeSamples), std::move(slabMean));
|
||||
|
||||
for (size_t i = 0 ; i < hiddenSize; i++)
|
||||
weightGrad.slice(i) = input * slabMean.col(i).t() * spikeMean(i);
|
||||
|
||||
spikeBiasGrad = spikeMean;
|
||||
|
||||
visiblePenaltyGrad = -0.5 * input.t() * input;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::NegativePhase(
|
||||
DataType&& negativeSamples,
|
||||
DataType&& gradient)
|
||||
{
|
||||
assert(negativeSamples.n_rows == visibleSize);
|
||||
assert(negativeSamples.n_cols == 1);
|
||||
arma::Cube<ElemType> weightGrad = arma::Cube<ElemType>
|
||||
(gradient.memptr(), visibleSize, poolSize, hiddenSize, false, false);
|
||||
|
||||
DataType spikeBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem,
|
||||
hiddenSize, 1, false, false);
|
||||
|
||||
DataType visiblePenaltyGrad = DataType(gradient.memptr() +
|
||||
weightGrad.n_elem + spikeBiasGrad.n_elem, 1, 1, false, false);
|
||||
|
||||
SpikeMean(std::move(negativeSamples), std::move(spikeMean));
|
||||
SampleSpike(std::move(spikeMean), std::move(spikeSamples));
|
||||
SlabMean(std::move(negativeSamples), std::move(spikeSamples),
|
||||
std::move(slabMean));
|
||||
|
||||
for (size_t i = 0 ; i < hiddenSize; i++)
|
||||
weightGrad.slice(i) = negativeSamples * slabMean.col(i).t() * spikeMean(i);
|
||||
|
||||
spikeBiasGrad = spikeMean;
|
||||
|
||||
visiblePenaltyGrad = -0.5 * negativeSamples.t() * negativeSamples;
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::SpikeMean(
|
||||
DataType&& visible,
|
||||
DataType&& spikeMean)
|
||||
{
|
||||
assert(visible.n_rows == visibleSize);
|
||||
assert(visible.n_cols == 1);
|
||||
|
||||
assert(spikeMean.n_rows == hiddenSize);
|
||||
assert(spikeMean.n_cols == 1);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
spikeMean(i) = LogisticFunction::Fn(0.5 * (1.0 / slabPenalty) *
|
||||
arma::as_scalar(visible.t() * weight.slice(i) * weight.slice(i).t() *
|
||||
visible) + spikeBias(i));
|
||||
}
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::SampleSpike(
|
||||
DataType&& spikeMean,
|
||||
DataType&& spike)
|
||||
{
|
||||
assert(spikeMean.n_rows == hiddenSize);
|
||||
assert(spikeMean.n_cols == 1);
|
||||
|
||||
assert(spike.n_rows == hiddenSize);
|
||||
assert(spike.n_cols == 1);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
spike(i) = math::RandBernoulli(spikeMean(i));
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::SlabMean(
|
||||
DataType&& visible,
|
||||
DataType&& spike,
|
||||
DataType&& slabMean)
|
||||
{
|
||||
assert(visible.n_rows == visibleSize);
|
||||
assert(visible.n_cols == 1);
|
||||
|
||||
assert(spike.n_rows == hiddenSize);
|
||||
assert(spike.n_cols == 1);
|
||||
|
||||
assert(slabMean.n_rows == poolSize);
|
||||
assert(slabMean.n_cols == hiddenSize);
|
||||
|
||||
assert(weight.n_rows == visibleSize);
|
||||
assert(weight.n_cols == poolSize);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
slabMean.col(i) = (1.0 / slabPenalty) * spike(i) *
|
||||
weight.slice(i).t() * visible;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::SampleSlab(
|
||||
DataType&& slabMean,
|
||||
DataType&& slab)
|
||||
{
|
||||
assert(slabMean.n_rows == poolSize);
|
||||
assert(slabMean.n_cols == hiddenSize);
|
||||
|
||||
assert(slab.n_rows == poolSize);
|
||||
assert(slab.n_cols == hiddenSize);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
{
|
||||
for (size_t j = 0; j < poolSize; j++)
|
||||
{
|
||||
slab(j, i) = math::RandNormal(slabMean(j, i), 1.0 / slabPenalty);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::VisibleMean(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
assert(input.n_elem == hiddenSize + poolSize * hiddenSize);
|
||||
output.zeros(visibleSize, 1);
|
||||
|
||||
DataType spike(input.memptr(), hiddenSize, 1, false, false);
|
||||
DataType slab(input.memptr() + hiddenSize, poolSize, hiddenSize, false,
|
||||
false);
|
||||
|
||||
for (size_t i = 0; i < hiddenSize; i++)
|
||||
output += weight.slice(i) * slab.col(i) * spike(i);
|
||||
|
||||
output = ((1.0 / visiblePenalty(0)) * output);
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::HiddenMean(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
assert(input.n_elem == visibleSize);
|
||||
output.set_size(hiddenSize + poolSize * hiddenSize, 1);
|
||||
|
||||
DataType spike(output.memptr(), hiddenSize, 1, false, false);
|
||||
DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false,
|
||||
false);
|
||||
|
||||
SpikeMean(std::move(input), std::move(spike));
|
||||
SampleSpike(std::move(spike), std::move(spikeSamples));
|
||||
SlabMean(std::move(input), std::move(spikeSamples), std::move(slab));
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::SampleVisible(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
const size_t numMaxTrials = 10;
|
||||
size_t k = 0;
|
||||
|
||||
VisibleMean(std::move(input), std::move(visibleMean));
|
||||
|
||||
output.set_size(visibleSize, 1);
|
||||
|
||||
assert(visiblePenalty(0) > 0);
|
||||
|
||||
for (k = 0; k < numMaxTrials; k++)
|
||||
{
|
||||
for (size_t i = 0; i < visibleSize; i++)
|
||||
{
|
||||
output(i) = math::RandNormal(visibleMean(i), 1.0 / visiblePenalty(0));
|
||||
}
|
||||
if (arma::norm(output, 2) < radius)
|
||||
break;
|
||||
}
|
||||
|
||||
if (k == numMaxTrials)
|
||||
{
|
||||
Log::Warn << "Outputs are still not in visible unit "
|
||||
<< arma::norm(output, 2)
|
||||
<< " terminating optimization."
|
||||
<< std::endl;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
void SpikeSlabRBMPolicy<DataType>::SampleHidden(
|
||||
DataType&& input,
|
||||
DataType&& output)
|
||||
{
|
||||
assert(input.n_elem == visibleSize);
|
||||
output.set_size(hiddenSize + poolSize * hiddenSize, 1);
|
||||
|
||||
DataType spike(output.memptr(), hiddenSize, 1, false, false);
|
||||
DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false,
|
||||
false);
|
||||
|
||||
SpikeMean(std::move(input), std::move(spike));
|
||||
SampleSpike(std::move(spike), std::move(spike));
|
||||
SlabMean(std::move(input), std::move(spike), std::move(slab));
|
||||
SampleSlab(std::move(slab), std::move(slab));
|
||||
}
|
||||
|
||||
template<typename DataType>
|
||||
template<typename Archive>
|
||||
void SpikeSlabRBMPolicy<DataType>::serialize(
|
||||
Archive& ar,
|
||||
const unsigned int /* version */)
|
||||
{
|
||||
ar & BOOST_SERIALIZATION_NVP(visibleSize);
|
||||
ar & BOOST_SERIALIZATION_NVP(hiddenSize);
|
||||
ar & BOOST_SERIALIZATION_NVP(poolSize);
|
||||
ar & BOOST_SERIALIZATION_NVP(parameter);
|
||||
ar & BOOST_SERIALIZATION_NVP(weight);
|
||||
ar & BOOST_SERIALIZATION_NVP(spikeBias);
|
||||
ar & BOOST_SERIALIZATION_NVP(slabPenalty);
|
||||
ar & BOOST_SERIALIZATION_NVP(radius);
|
||||
ar & BOOST_SERIALIZATION_NVP(visiblePenalty);
|
||||
|
||||
if (Archive::is_loading::value)
|
||||
{
|
||||
spikeMean.set_size(hiddenSize, 1);
|
||||
spikeSamples.set_size(hiddenSize, 1);
|
||||
slabMean.set_size(poolSize, hiddenSize);
|
||||
Reset();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
|
||||
|
||||
#endif // MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_IMPL_HPP
|
||||
@@ -1,228 +0,0 @@
|
||||
/**
|
||||
* @file rbm_impl.hpp
|
||||
* @author Kris Singh
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
|
||||
#ifndef MLPACK_METHODS_ANN_RBM_IMPL_HPP
|
||||
#define MLPACK_METHODS_ANN_RBM_IMPL_HPP
|
||||
|
||||
// In case it hasn't been included yet.
|
||||
#include "rbm.hpp"
|
||||
|
||||
#include <mlpack/core.hpp>
|
||||
|
||||
#include <mlpack/methods/ann/activation_functions/softplus_function.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace ann /** Artificial neural networks. */ {
|
||||
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
RBM<InitializationRuleType, RBMPolicy>::RBM(
|
||||
arma::Mat<ElemType> predictors,
|
||||
InitializationRuleType initializeRule,
|
||||
RBMPolicy rbmPolicy,
|
||||
const size_t numSteps,
|
||||
const size_t negSteps,
|
||||
const bool useMonitoringCost,
|
||||
const bool persistence):
|
||||
rbmPolicy(rbmPolicy),
|
||||
initializeRule(initializeRule),
|
||||
numSteps(numSteps),
|
||||
negSteps(negSteps),
|
||||
useMonitoringCost(useMonitoringCost),
|
||||
persistence(persistence),
|
||||
reset(false)
|
||||
{
|
||||
numFunctions = predictors.n_cols;
|
||||
this->predictors = std::move(predictors);
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
void RBM<InitializationRuleType, RBMPolicy>::Reset()
|
||||
{
|
||||
size_t weight = rbmPolicy.Parameters().n_elem;
|
||||
|
||||
positiveGradient.set_size(weight, 1);
|
||||
negativeGradient.set_size(weight, 1);
|
||||
negativeSamples.set_size(rbmPolicy.VisibleSize(), 1);
|
||||
tempNegativeGradient.set_size(weight, 1);
|
||||
|
||||
parameter.zeros(weight, 1);
|
||||
positiveGradient.zeros();
|
||||
negativeGradient.zeros();
|
||||
tempNegativeGradient.zeros();
|
||||
initializeRule.Initialize(parameter, parameter.n_elem, 1);
|
||||
|
||||
rbmPolicy.Parameters() = arma::Mat<ElemType>(parameter.memptr(), weight, 1,
|
||||
false, false);
|
||||
rbmPolicy.Reset();
|
||||
|
||||
reset = true;
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
template<typename OptimizerType>
|
||||
void RBM<InitializationRuleType, RBMPolicy>::Train(
|
||||
const arma::Mat<ElemType>& predictors,
|
||||
OptimizerType& optimizer)
|
||||
{
|
||||
numFunctions = predictors.n_cols;
|
||||
this->predictors = std::move(predictors);
|
||||
|
||||
if (!reset)
|
||||
Reset();
|
||||
|
||||
// Train the model.
|
||||
Timer::Start("rbm_optimization");
|
||||
optimizer.Optimize(*this, parameter);
|
||||
Timer::Stop("rbm_optimization");
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
double RBM<InitializationRuleType, RBMPolicy>::
|
||||
FreeEnergy(arma::Mat<ElemType>&& input)
|
||||
{
|
||||
return rbmPolicy.FreeEnergy(std::move(input));
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
double RBM<InitializationRuleType, RBMPolicy>::Evaluate(
|
||||
const arma::Mat<ElemType>& /* parameters*/,
|
||||
const size_t i)
|
||||
{
|
||||
if (!useMonitoringCost)
|
||||
{
|
||||
Gibbs(std::move(predictors.col(i)), std::move(negativeSamples));
|
||||
return std::fabs(FreeEnergy(std::move(predictors.col(i))) -
|
||||
FreeEnergy(std::move(negativeSamples)));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (persistence)
|
||||
{
|
||||
return rbmPolicy.Evaluate(predictors, i);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Mean Squared Error
|
||||
rbmPolicy.SampleHidden(std::move(predictors.col(i)),
|
||||
std::move(hiddenReconstruction));
|
||||
rbmPolicy.SampleVisible(std::move(hiddenReconstruction),
|
||||
std::move(visibleReconstruction));
|
||||
return arma::accu(arma::pow(visibleReconstruction - predictors.col(i),
|
||||
2)) / hiddenReconstruction.n_rows;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
void RBM<InitializationRuleType, RBMPolicy>::SampleHidden(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output)
|
||||
{
|
||||
rbmPolicy.SampleHidden(std::move(input), std::move(output));
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
void RBM<InitializationRuleType, RBMPolicy>::SampleVisible(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output)
|
||||
{
|
||||
rbmPolicy.SampleVisible(std::move(input), std::move(output));
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
void RBM<InitializationRuleType, RBMPolicy>::Gibbs(
|
||||
arma::Mat<ElemType>&& input,
|
||||
arma::Mat<ElemType>&& output,
|
||||
size_t steps)
|
||||
{
|
||||
steps = (steps == SIZE_MAX) ? this-> numSteps: steps;
|
||||
|
||||
if (persistence && !state.is_empty())
|
||||
{
|
||||
SampleHidden(std::move(state), std::move(gibbsTemporary));
|
||||
SampleVisible(std::move(gibbsTemporary), std::move(output));
|
||||
}
|
||||
else
|
||||
{
|
||||
SampleHidden(std::move(input), std::move(gibbsTemporary));
|
||||
SampleVisible(std::move(gibbsTemporary), std::move(output));
|
||||
}
|
||||
|
||||
for (size_t j = 1; j < steps; j++)
|
||||
{
|
||||
SampleHidden(std::move(output), std::move(gibbsTemporary));
|
||||
SampleVisible(std::move(gibbsTemporary), std::move(output));
|
||||
}
|
||||
if (persistence)
|
||||
state = output;
|
||||
}
|
||||
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
void RBM<InitializationRuleType, RBMPolicy>::Gradient(
|
||||
arma::Mat<ElemType>& /*parameters*/,
|
||||
const size_t input,
|
||||
arma::Mat<ElemType>& output)
|
||||
{
|
||||
positiveGradient.zeros();
|
||||
// Collect the negative samples
|
||||
rbmPolicy.PositivePhase(std::move(predictors.col(input)),
|
||||
std::move(positiveGradient));
|
||||
|
||||
negativeGradient.zeros();
|
||||
|
||||
for (size_t i = 0; i < negSteps; i++)
|
||||
{
|
||||
Gibbs(std::move(predictors.col(input)), std::move(negativeSamples));
|
||||
rbmPolicy.NegativePhase(std::move(negativeSamples),
|
||||
std::move(tempNegativeGradient));
|
||||
|
||||
negativeGradient += tempNegativeGradient;
|
||||
}
|
||||
output = ((negativeGradient / negSteps) - positiveGradient);
|
||||
}
|
||||
|
||||
//! Serialize the model.
|
||||
template<typename InitializationRuleType, typename RBMPolicy>
|
||||
template<typename Archive>
|
||||
void RBM<InitializationRuleType, RBMPolicy>::
|
||||
serialize(Archive& ar, const unsigned int /* version */)
|
||||
{
|
||||
ar & BOOST_SERIALIZATION_NVP(parameter);
|
||||
ar & BOOST_SERIALIZATION_NVP(rbmPolicy);
|
||||
ar & BOOST_SERIALIZATION_NVP(state);
|
||||
ar & BOOST_SERIALIZATION_NVP(numFunctions);
|
||||
ar & BOOST_SERIALIZATION_NVP(numSteps);
|
||||
ar & BOOST_SERIALIZATION_NVP(negSteps);
|
||||
ar & BOOST_SERIALIZATION_NVP(useMonitoringCost);
|
||||
ar & BOOST_SERIALIZATION_NVP(persistence);
|
||||
// ar & data::CreateNVP(RBMPolicy::ElemType, "ElemType");
|
||||
|
||||
// If we are loading, we need to initialize the weights.
|
||||
if (Archive::is_loading::value)
|
||||
{
|
||||
size_t weight = rbmPolicy.Parameters().n_elem;
|
||||
positiveGradient.set_size(weight, 1);
|
||||
negativeGradient.set_size(weight, 1);
|
||||
negativeSamples.set_size(rbmPolicy.VisibleSize(), 1);
|
||||
tempNegativeGradient.set_size(weight, 1);
|
||||
positiveGradient.zeros();
|
||||
negativeGradient.zeros();
|
||||
tempNegativeGradient.zeros();
|
||||
rbmPolicy.Parameters() = arma::Mat<ElemType>(
|
||||
parameter.memptr(), weight, 1, false, false);
|
||||
rbmPolicy.Reset();
|
||||
reset = true;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ann
|
||||
} // namespace mlpack
|
||||
#endif // MLPACK_METHODS_ANN_RBM_IMPL_HPP
|
||||
@@ -70,14 +70,6 @@ add_executable(mlpack_test
|
||||
loss_functions_test.cpp
|
||||
lrsdp_test.cpp
|
||||
lsh_test.cpp
|
||||
main_tests/decision_tree_test.cpp
|
||||
main_tests/emst_test.cpp
|
||||
main_tests/linear_regression_test.cpp
|
||||
main_tests/pca_test.cpp
|
||||
main_tests/preprocess_binarize_test.cpp
|
||||
main_tests/preprocess_imputer_test.cpp
|
||||
main_tests/preprocess_split_test.cpp
|
||||
main_tests/test_helper.hpp
|
||||
math_test.cpp
|
||||
matrix_completion_test.cpp
|
||||
maximal_inputs_test.cpp
|
||||
|
||||
@@ -1331,40 +1331,6 @@ BOOST_AUTO_TEST_CASE(SimpleBilinearInterpolationLayerTest)
|
||||
std::move(unzoomedOutput));
|
||||
CheckMatrices(unzoomedOutput - expectedOutput,
|
||||
arma::zeros(input.n_rows), 1e-12);
|
||||
|
||||
/*
|
||||
* Simple add module test.
|
||||
*/
|
||||
BOOST_AUTO_TEST_CASE(SimpleBinaryRbmLayerTest)
|
||||
{
|
||||
arma::mat output, input, outputModule, inputModule;
|
||||
// Network 1
|
||||
BinaryLayer<> module(2, 4);
|
||||
module.Parameters().ones();
|
||||
module.Reset();
|
||||
|
||||
// Network 2
|
||||
LinearNoBias<> linear(2, 4);
|
||||
Add<> add(4);
|
||||
SigmoidLayer<> sigmoid;
|
||||
linear.Parameters().ones();
|
||||
add.Parameters().ones();
|
||||
linear.Reset();
|
||||
|
||||
// Check reset function
|
||||
BOOST_REQUIRE_EQUAL(module.Bias().size(), 2);
|
||||
BOOST_REQUIRE_EQUAL(arma::accu(module.Bias() - arma::ones(2)), 0);
|
||||
|
||||
// Test the Forward function.
|
||||
input = arma::vec("0.5 0.5");
|
||||
inputModule = arma::vec("0.5 0.5");
|
||||
module.Forward(std::move(inputModule), std::move(outputModule));
|
||||
linear.Forward(std::move(input), std::move(output));
|
||||
add.Forward(std::move(output), std::move(output));
|
||||
sigmoid.Forward(std::move(output), std::move(output));
|
||||
|
||||
for (size_t i = 0; i < output.size(); i++)
|
||||
BOOST_REQUIRE_EQUAL(output(i), outputModule(i));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
/**
|
||||
* @file rbm_network_test.cpp
|
||||
* @author Kris Singh
|
||||
* @author Shikhar Jaiswal
|
||||
*
|
||||
* Tests the rbm Network
|
||||
* Tests the RBM Network
|
||||
*
|
||||
* digits dataset source:
|
||||
* @misc{Lichman:2013 ,
|
||||
@@ -14,17 +15,15 @@
|
||||
* Irvine, School of Information and Computer Sciences" }
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#include <mlpack/core.hpp>
|
||||
|
||||
#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
|
||||
#include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
|
||||
#include <mlpack/methods/ann/rbm/binary_rbm_policy.hpp>
|
||||
#include <mlpack/methods/ann/rbm/spike_slab_rbm_policy.hpp>
|
||||
#include <mlpack/methods/ann/rbm.hpp>
|
||||
#include <mlpack/methods/ann/rbm/rbm.hpp>
|
||||
#include <mlpack/methods/softmax_regression/softmax_regression.hpp>
|
||||
#include <mlpack/core/optimizers/sgd/sgd.hpp>
|
||||
#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
|
||||
@@ -37,10 +36,11 @@ using namespace mlpack::ann;
|
||||
using namespace mlpack::optimization;
|
||||
using namespace mlpack::regression;
|
||||
|
||||
BOOST_AUTO_TEST_SUITE(RbmNetworkTest);
|
||||
BOOST_AUTO_TEST_SUITE(RBMNetworkTest);
|
||||
|
||||
BOOST_AUTO_TEST_CASE(ClassificationTest)
|
||||
{
|
||||
// Normalised dataset
|
||||
// Normalised dataset.
|
||||
int hiddenLayerSize = 100;
|
||||
size_t batchSize = 10;
|
||||
size_t numEpoches = 30;
|
||||
@@ -68,31 +68,29 @@ BOOST_AUTO_TEST_CASE(ClassificationTest)
|
||||
XRbm.zeros();
|
||||
YRbm.zeros();
|
||||
|
||||
BinaryRBMPolicy<> binary_rbm(trainData.n_rows, hiddenLayerSize);
|
||||
GaussianInitialization gaussian(0, 0.1);
|
||||
RBM<GaussianInitialization, BinaryRBMPolicy<>> model(trainData,
|
||||
gaussian, binary_rbm, 1, 1, true, false);
|
||||
RBM<GaussianInitialization> model(trainData,
|
||||
gaussian, trainData.n_rows, hiddenLayerSize, batchSize);
|
||||
|
||||
size_t numRBMIterations = trainData.n_cols * numEpoches;
|
||||
numRBMIterations /= batchSize;
|
||||
|
||||
MiniBatchSGD msgd(batchSize, 0.06, numRBMIterations, 0, true);
|
||||
optimization::StandardSGD msgd(0.06, batchSize, numRBMIterations, 0, true);
|
||||
model.Reset();
|
||||
model.Policy().VisibleBias().ones();
|
||||
model.Policy().HiddenBias().ones();
|
||||
// test the reset function
|
||||
model.Train(trainData, msgd);
|
||||
model.VisibleBias().ones();
|
||||
model.HiddenBias().ones();
|
||||
|
||||
// Test the reset function.
|
||||
model.Train(msgd);
|
||||
|
||||
for (size_t i = 0; i < trainData.n_cols; i++)
|
||||
{
|
||||
model.Policy().HiddenMean(std::move(trainData.col(i)),
|
||||
std::move(output));
|
||||
model.HiddenMean(std::move(trainData.col(i)), std::move(output));
|
||||
XRbm.col(i) = output;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < testData.n_cols; i++)
|
||||
{
|
||||
model.Policy().HiddenMean(std::move(testData.col(i)),
|
||||
model.HiddenMean(std::move(testData.col(i)),
|
||||
std::move(output));
|
||||
YRbm.col(i) = output;
|
||||
}
|
||||
@@ -107,14 +105,13 @@ BOOST_AUTO_TEST_CASE(ClassificationTest)
|
||||
|
||||
double classificationAccuray = regressor.ComputeAccuracy(testData,
|
||||
testLabels);
|
||||
std::cout << "Softmax Accuracy = " << classificationAccuray << std::endl;
|
||||
|
||||
L_BFGS rbmOptimizer(numBasis, numIterations);
|
||||
SoftmaxRegression rbmRegressor(XRbm, trainLabels, numClasses,
|
||||
0.001, false, rbmOptimizer);
|
||||
double rbmClassificationAccuracy = rbmRegressor.ComputeAccuracy(YRbm,
|
||||
testLabels);
|
||||
std::cout << "RBM Accuracy = " << rbmClassificationAccuracy << std::endl;
|
||||
|
||||
BOOST_REQUIRE_GE(rbmClassificationAccuracy, classificationAccuray);
|
||||
}
|
||||
|
||||
@@ -164,29 +161,29 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest)
|
||||
YRbm.zeros();
|
||||
double slabPenalty = 8;
|
||||
|
||||
SpikeSlabRBMPolicy<> ss_rbm(trainData.n_rows, hiddenLayerSize, poolSize,
|
||||
RBM<GaussianInitialization, arma::mat, SpikeSlabRBM> modelssRBM(trainData,
|
||||
gaussian, trainData.n_rows, hiddenLayerSize, batchSize, 1, 1, poolSize,
|
||||
slabPenalty, radius);
|
||||
RBM<GaussianInitialization, SpikeSlabRBMPolicy<>> modelssRBM(trainData,
|
||||
gaussian, ss_rbm, 1, 1, true, false);
|
||||
|
||||
size_t numRBMIterations = trainData.n_cols * numEpoches;
|
||||
numRBMIterations /= batchSize;
|
||||
|
||||
MiniBatchSGD msgd(batchSize, 0.02, numRBMIterations, 0, true);
|
||||
optimization::StandardSGD msgd(0.02, batchSize, numRBMIterations, 0, true);
|
||||
modelssRBM.Reset();
|
||||
modelssRBM.Policy().VisiblePenalty().fill(5);
|
||||
modelssRBM.Policy().SpikeBias().fill(1);
|
||||
modelssRBM.Train(trainData, msgd);
|
||||
modelssRBM.VisiblePenalty().fill(5);
|
||||
modelssRBM.SpikeBias().fill(1);
|
||||
|
||||
modelssRBM.Train(msgd);
|
||||
for (size_t i = 0; i < trainData.n_cols; i++)
|
||||
{
|
||||
modelssRBM.Policy().HiddenMean(std::move(trainData.col(i)),
|
||||
modelssRBM.HiddenMean(std::move(trainData.col(i)),
|
||||
std::move(output));
|
||||
XRbm.col(i) = output;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < testData.n_cols; i++)
|
||||
{
|
||||
modelssRBM.Policy().HiddenMean(std::move(testData.col(i)),
|
||||
modelssRBM.HiddenMean(std::move(testData.col(i)),
|
||||
std::move(output));
|
||||
YRbm.col(i) = output;
|
||||
}
|
||||
@@ -199,7 +196,6 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest)
|
||||
0.001, false, ssRbmOptimizer);
|
||||
double ssRbmClassificationAccuracy = ssRbmRegressor.ComputeAccuracy(
|
||||
YRbm, testLabels);
|
||||
std::cout << "ssRBM Accuracy = " << ssRbmClassificationAccuracy << std::endl;
|
||||
|
||||
BOOST_REQUIRE_GE(ssRbmClassificationAccuracy, 76.18);
|
||||
}
|
||||
@@ -209,13 +205,12 @@ void BuildVanillaNetwork(MatType& trainData,
|
||||
const size_t hiddenLayerSize)
|
||||
{
|
||||
MatType output;
|
||||
BinaryRBMPolicy<MatType> binary_rbm(trainData.n_rows, hiddenLayerSize);
|
||||
GaussianInitialization gaussian(0, 0.1);
|
||||
RBM<GaussianInitialization, BinaryRBMPolicy<MatType>> model(trainData,
|
||||
gaussian, binary_rbm, 1, true);
|
||||
RBM<GaussianInitialization, MatType, BinaryRBM> model(trainData, gaussian,
|
||||
trainData.n_rows, hiddenLayerSize, 1, 1, 1, 2, 8, 1, true);
|
||||
|
||||
model.Reset();
|
||||
// Set the parmaeters from a learned rbm sklearn random state 23
|
||||
// Set the parameters from a learned RBM Sklearn random state 23
|
||||
model.Parameters() = MatType(
|
||||
"-0.23224054, -0.23000632, -0.25701271, -0.25122418, -0.20716651,"
|
||||
"-0.20962217, -0.59922456, -0.60003836, -0.6, -0.625, -0.475;");
|
||||
@@ -223,21 +218,21 @@ void BuildVanillaNetwork(MatType& trainData,
|
||||
// Check free energy
|
||||
arma::Mat<float> freeEnergy = MatType(
|
||||
"-0.87523715, 0.50615066, 0.46923476, 1.21509084;");
|
||||
arma::vec calcultedFreeEnergy(4);
|
||||
calcultedFreeEnergy.zeros();
|
||||
arma::vec calculatedFreeEnergy(4);
|
||||
calculatedFreeEnergy.zeros();
|
||||
for (size_t i = 0; i < trainData.n_cols; i++)
|
||||
{
|
||||
calcultedFreeEnergy(i) = model.FreeEnergy(std::move(trainData.col(i)));
|
||||
calculatedFreeEnergy(i) = model.FreeEnergy(std::move(trainData.col(i)));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < freeEnergy.n_elem; i++)
|
||||
BOOST_REQUIRE_CLOSE(calcultedFreeEnergy(i), freeEnergy(i), 1e-3);
|
||||
BOOST_REQUIRE_CLOSE(calculatedFreeEnergy(i), freeEnergy(i), 1e-3);
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(MiscTest)
|
||||
{
|
||||
/**
|
||||
* Train and evaluate a vanilla network with the specified structure.
|
||||
* Train and evaluate a Vanilla network with the specified structure.
|
||||
*/
|
||||
|
||||
arma::Mat<float> X = arma::Mat<float>("0.0, 0.0, 0.0;"
|
||||
@@ -247,4 +242,5 @@ BOOST_AUTO_TEST_CASE(MiscTest)
|
||||
X = X.t();
|
||||
BuildVanillaNetwork<arma::Mat<float>>(X, 2);
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_SUITE_END();
|
||||
|
||||
@@ -34,9 +34,7 @@
|
||||
#include <mlpack/methods/lsh/lsh_search.hpp>
|
||||
#include <mlpack/methods/decision_stump/decision_stump.hpp>
|
||||
#include <mlpack/methods/lars/lars.hpp>
|
||||
#include <mlpack/methods/ann/rbm.hpp>
|
||||
#include <mlpack/methods/ann/rbm/binary_rbm_policy.hpp>
|
||||
#include <mlpack/methods/ann/rbm/spike_slab_rbm_policy.hpp>
|
||||
#include <mlpack/methods/ann/rbm/rbm.hpp>
|
||||
#include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
|
||||
|
||||
using namespace mlpack;
|
||||
@@ -1706,32 +1704,31 @@ BOOST_AUTO_TEST_CASE(BinaryRBMTest)
|
||||
size_t hiddenLayerSize = 5;
|
||||
data.randu(3, 100);
|
||||
|
||||
BinaryRBMPolicy<> binary_rbm(data.n_rows, hiddenLayerSize);
|
||||
GaussianInitialization gaussian(0, 0.1);
|
||||
RBM<GaussianInitialization, BinaryRBMPolicy<> > Rbm(data,
|
||||
gaussian, binary_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization, BinaryRBMPolicy<> > RbmXml(data,
|
||||
gaussian, binary_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization, BinaryRBMPolicy<> > RbmText(data,
|
||||
gaussian, binary_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization, BinaryRBMPolicy<> > RbmBinary(data,
|
||||
gaussian, binary_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization> Rbm(data, gaussian, data.n_rows, hiddenLayerSize,
|
||||
1, 1, 1, 2, 8, 1, true);
|
||||
RBM<GaussianInitialization> RbmXml(data, gaussian, data.n_rows,
|
||||
hiddenLayerSize, 1, 1, 1, 2, 8, 1, true);
|
||||
RBM<GaussianInitialization> RbmText(data, gaussian, data.n_rows,
|
||||
hiddenLayerSize, 1, 1, 1, 2, 8, 1, true);
|
||||
RBM<GaussianInitialization> RbmBinary(data, gaussian, data.n_rows,
|
||||
hiddenLayerSize, 1, 1, 1, 2, 8, 1, true);
|
||||
Rbm.Reset();
|
||||
|
||||
SerializeObjectAll(Rbm, RbmXml, RbmText, RbmBinary);
|
||||
CheckMatrices(Rbm.Parameters(), RbmXml.Parameters(), RbmText.Parameters(),
|
||||
RbmBinary.Parameters());
|
||||
CheckMatrices(Rbm.Policy().VisibleBias(), RbmXml.Policy().VisibleBias());
|
||||
CheckMatrices(Rbm.Policy().VisibleBias(), RbmText.Policy().VisibleBias());
|
||||
CheckMatrices(Rbm.Policy().VisibleBias(), RbmBinary.Policy().VisibleBias());
|
||||
CheckMatrices(Rbm.VisibleBias(), RbmXml.VisibleBias());
|
||||
CheckMatrices(Rbm.VisibleBias(), RbmText.VisibleBias());
|
||||
CheckMatrices(Rbm.VisibleBias(), RbmBinary.VisibleBias());
|
||||
|
||||
CheckMatrices(Rbm.Policy().HiddenBias(), RbmXml.Policy().HiddenBias());
|
||||
CheckMatrices(Rbm.Policy().HiddenBias(), RbmText.Policy().HiddenBias());
|
||||
CheckMatrices(Rbm.Policy().HiddenBias(), RbmBinary.Policy().HiddenBias());
|
||||
CheckMatrices(Rbm.HiddenBias(), RbmXml.HiddenBias());
|
||||
CheckMatrices(Rbm.HiddenBias(), RbmText.HiddenBias());
|
||||
CheckMatrices(Rbm.HiddenBias(), RbmBinary.HiddenBias());
|
||||
|
||||
CheckMatrices(Rbm.Policy().Weight(), RbmXml.Policy().Weight());
|
||||
CheckMatrices(Rbm.Policy().Weight(), RbmText.Policy().Weight());
|
||||
CheckMatrices(Rbm.Policy().Weight(), RbmBinary.Policy().Weight());
|
||||
CheckMatrices(Rbm.Weight(), RbmXml.Weight());
|
||||
CheckMatrices(Rbm.Weight(), RbmText.Weight());
|
||||
CheckMatrices(Rbm.Weight(), RbmBinary.Weight());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1755,45 +1752,38 @@ BOOST_AUTO_TEST_CASE(ssRBMTest)
|
||||
|
||||
size_t poolSize = 1;
|
||||
|
||||
SpikeSlabRBMPolicy<> ss_rbm(data.n_rows, hiddenLayerSize, poolSize,
|
||||
slabPenalty, radius);
|
||||
GaussianInitialization gaussian(0, 0.1);
|
||||
RBM<GaussianInitialization, SpikeSlabRBMPolicy<> > Rbm(data,
|
||||
gaussian, ss_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization, SpikeSlabRBMPolicy<>> RbmXml(data,
|
||||
gaussian, ss_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization, SpikeSlabRBMPolicy<>> RbmText(data,
|
||||
gaussian, ss_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization, SpikeSlabRBMPolicy<>> RbmBinary(data,
|
||||
gaussian, ss_rbm, 1, true, true);
|
||||
RBM<GaussianInitialization, arma::mat, SpikeSlabRBM> Rbm(data, gaussian,
|
||||
data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius,
|
||||
true);
|
||||
RBM<GaussianInitialization, arma::mat, SpikeSlabRBM> RbmXml(data, gaussian,
|
||||
data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius,
|
||||
true);
|
||||
RBM<GaussianInitialization, arma::mat, SpikeSlabRBM> RbmText(data, gaussian,
|
||||
data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius,
|
||||
true);
|
||||
RBM<GaussianInitialization, arma::mat, SpikeSlabRBM> RbmBinary(data, gaussian,
|
||||
data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius,
|
||||
true);
|
||||
Rbm.Reset();
|
||||
Rbm.Policy().VisiblePenalty().fill(15);
|
||||
Rbm.Policy().SpikeBias().ones();
|
||||
Rbm.VisiblePenalty().fill(15);
|
||||
Rbm.SpikeBias().ones();
|
||||
|
||||
SerializeObjectAll(Rbm, RbmXml, RbmText, RbmBinary);
|
||||
CheckMatrices(Rbm.Parameters(), RbmXml.Parameters(), RbmText.Parameters(),
|
||||
RbmBinary.Parameters());
|
||||
|
||||
CheckMatrices(Rbm.Policy().VisiblePenalty(),
|
||||
RbmXml.Policy().VisiblePenalty());
|
||||
CheckMatrices(Rbm.Policy().VisiblePenalty(),
|
||||
RbmText.Policy().VisiblePenalty());
|
||||
CheckMatrices(Rbm.Policy().VisiblePenalty(),
|
||||
RbmBinary.Policy().VisiblePenalty());
|
||||
CheckMatrices(Rbm.VisiblePenalty(), RbmXml.VisiblePenalty());
|
||||
CheckMatrices(Rbm.VisiblePenalty(), RbmText.VisiblePenalty());
|
||||
CheckMatrices(Rbm.VisiblePenalty(), RbmBinary.VisiblePenalty());
|
||||
|
||||
CheckMatrices(Rbm.Policy().SpikeBias(),
|
||||
RbmXml.Policy().SpikeBias());
|
||||
CheckMatrices(Rbm.Policy().SpikeBias(),
|
||||
RbmText.Policy().SpikeBias());
|
||||
CheckMatrices(Rbm.Policy().SpikeBias(),
|
||||
RbmBinary.Policy().SpikeBias());
|
||||
CheckMatrices(Rbm.SpikeBias(), RbmXml.SpikeBias());
|
||||
CheckMatrices(Rbm.SpikeBias(), RbmText.SpikeBias());
|
||||
CheckMatrices(Rbm.SpikeBias(), RbmBinary.SpikeBias());
|
||||
|
||||
CheckMatrices(Rbm.Policy().Weight(),
|
||||
RbmXml.Policy().Weight());
|
||||
CheckMatrices(Rbm.Policy().Weight(),
|
||||
RbmText.Policy().Weight());
|
||||
CheckMatrices(Rbm.Policy().Weight(),
|
||||
RbmBinary.Policy().Weight());
|
||||
CheckMatrices(Rbm.WeightCube(), RbmXml.WeightCube());
|
||||
CheckMatrices(Rbm.WeightCube(), RbmText.WeightCube());
|
||||
CheckMatrices(Rbm.WeightCube(), RbmBinary.WeightCube());
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_SUITE_END();
|
||||
|
||||
Reference in New Issue
Block a user