From 9c57ab50fa94b668ec7c080b264d7325f5997efb Mon Sep 17 00:00:00 2001 From: Shikhar Jaiswal Date: Tue, 23 Jan 2018 22:52:43 +0530 Subject: [PATCH] Implement API changes --- src/mlpack/methods/ann/CMakeLists.txt | 4 - src/mlpack/methods/ann/layer/CMakeLists.txt | 2 - src/mlpack/methods/ann/rbm.hpp | 189 -------- src/mlpack/methods/ann/rbm/CMakeLists.txt | 12 +- .../methods/ann/rbm/binary_rbm_policy.hpp | 176 ------- .../ann/rbm/binary_rbm_policy_impl.hpp | 167 ------- src/mlpack/methods/ann/rbm/rbm.hpp | 447 ++++++++++++++++++ src/mlpack/methods/ann/rbm/rbm_impl.hpp | 343 ++++++++++++++ src/mlpack/methods/ann/rbm/rbm_policies.hpp | 49 ++ .../methods/ann/rbm/spike_slab_rbm_impl.hpp | 302 ++++++++++++ .../methods/ann/rbm/spike_slab_rbm_policy.hpp | 235 --------- .../ann/rbm/spike_slab_rbm_policy_impl.hpp | 335 ------------- src/mlpack/methods/ann/rbm_impl.hpp | 228 --------- src/mlpack/tests/CMakeLists.txt | 8 - src/mlpack/tests/ann_layer_test.cpp | 34 -- src/mlpack/tests/rbm_network_test.cpp | 78 ++- src/mlpack/tests/serialization_test.cpp | 92 ++-- 17 files changed, 1225 insertions(+), 1476 deletions(-) delete mode 100644 src/mlpack/methods/ann/rbm.hpp delete mode 100644 src/mlpack/methods/ann/rbm/binary_rbm_policy.hpp delete mode 100644 src/mlpack/methods/ann/rbm/binary_rbm_policy_impl.hpp create mode 100644 src/mlpack/methods/ann/rbm/rbm.hpp create mode 100644 src/mlpack/methods/ann/rbm/rbm_impl.hpp create mode 100644 src/mlpack/methods/ann/rbm/rbm_policies.hpp create mode 100644 src/mlpack/methods/ann/rbm/spike_slab_rbm_impl.hpp delete mode 100644 src/mlpack/methods/ann/rbm/spike_slab_rbm_policy.hpp delete mode 100644 src/mlpack/methods/ann/rbm/spike_slab_rbm_policy_impl.hpp delete mode 100644 src/mlpack/methods/ann/rbm_impl.hpp diff --git a/src/mlpack/methods/ann/CMakeLists.txt b/src/mlpack/methods/ann/CMakeLists.txt index 1d02f48dac..c7e03e056b 100644 --- a/src/mlpack/methods/ann/CMakeLists.txt +++ b/src/mlpack/methods/ann/CMakeLists.txt @@ -3,10 +3,6 @@ set(SOURCES ffn.hpp ffn_impl.hpp - gan.hpp - gan_impl.hpp - rbm.hpp - rbm_impl.hpp rnn.hpp rnn_impl.hpp ) diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 53f644f4e0..1cf544f3f4 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -14,8 +14,6 @@ set(SOURCES batch_norm_impl.hpp bilinear_interpolation.hpp bilinear_interpolation_impl.hpp - binary_rbm.hpp - binary_rbm_impl.hpp concat.hpp concat_impl.hpp concat_performance.hpp diff --git a/src/mlpack/methods/ann/rbm.hpp b/src/mlpack/methods/ann/rbm.hpp deleted file mode 100644 index b129f446b9..0000000000 --- a/src/mlpack/methods/ann/rbm.hpp +++ /dev/null @@ -1,189 +0,0 @@ -/** - * @file rbm.hpp - * @author Kris Singh - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_RBM_HPP -#define MLPACK_METHODS_ANN_RBM_HPP - -#include - -#include - -namespace mlpack { -namespace ann /** Artificial neural networks. */ { -/* - * RBM class - * - * @tparam IntialiserType rule to intialise the parameters of the network - * @tparam RBMPolicy type of rbm - */ - -template -class RBM -{ - public: - using NetworkType = RBM; - typedef typename RBMPolicy::ElemType ElemType; - - /* - * Intalise all the parameters of the network - * using the intialise rule. - * - * @tparam RbmPolicy Class of RBM to use(ssRBM / BinaryRBM). - * @param predictors Training data to used. - * @param numSteps Number of gibbs steps sampling. - * @param negSteps Number of negative samples to average negative gradient. - * @param useMonitoringCost Indicates whic Evaluation type to use. - * @param persistence Indicates whether to use persistent CD or not. - */ - RBM(arma::Mat predictors, - InitializationRuleType initializeRule, - RBMPolicy rbmPolicy, - const size_t numSteps = 1, - const size_t negSteps = 1, - const bool useMonitoringCost = true, - const bool persistence = false); - - // Reset the network - void Reset(); - - /* - * Train the feedforward network on the given input data. - * - * This will use the existing model parameters as a starting point for the - * optimization. If this is not what you want, then you should access the - * parameters vector directly with Parameters() and modify it as desired. - * - * @param predictors Data points / Traing Data. - * @param optimizer Optimizer type. - */ - template - void Train(const arma::Mat& predictors, OptimizerType& optimizer); - - /** - * Evaluate the rbm network with the given parameters. - * The function is needed for monitoring the progress of the network. - * - * @param parameters Matrix model parameters. - * @param i Index of point to use for objective function evaluation. - */ - double Evaluate(const arma::Mat& parameters, const size_t i); - - /** - * This function calculates the free energy of the model. - * - * @param Input data point. - */ - double FreeEnergy(arma::Mat&& input); - - /* - * This functions samples the hidden layer given the visible layer. - * - * @param input Visible layer input. - * @param output The sampled hidden layer. - */ - void SampleHidden(arma::Mat&& input, arma::Mat&& output); - - /* - * This functions samples the visible layer given the hidden layer. - * - * @param input Hidden layer of the network. - * @param output The sampled visible layer. - */ - void SampleVisible(arma::Mat&& input, arma::Mat&& output); - - /* - * This function does the k-step gibbs sampling. - * - * @param input Input to the gibbs function. - * @param output Used for storing the negative sample. - * @param steps Number of gibbs sampling steps taken. - */ - void Gibbs(arma::Mat&& input, - arma::Mat&& output, - size_t steps = SIZE_MAX); - - /* - * Calculates the gradients for the rbm network. - * - * @param parameters The current parameters of the network. - * @param input Index of the visible layer/data point. - * @param output Used for storing the gradients. - */ - void Gradient(arma::Mat& parameters, - const size_t input, - arma::Mat& output); - - //! Return the number of separable functions (the number of predictor points). - size_t NumFunctions() const { return numFunctions; } - - //! Return the number of stes of gibbs sampling. - size_t NumSteps() const { return numSteps; } - - //! Return the parameters of the network. - const arma::Mat& Parameters() const { return parameter; } - //! Modify the parameters of the network. - arma::Mat& Parameters() { return parameter; } - - //! Retutrn the rbm policy for the network. - const RBMPolicy& Policy() const { return rbmPolicy; } - //! Modify the rbm policy for the network. - RBMPolicy& Policy() { return rbmPolicy; } - - //! Serialize the model. - template - void serialize(Archive& ar, const unsigned int /* version */); - - private: - //! Locally stored parameters of the network. - arma::Mat parameter; - //! Policy type of RBM. - RBMPolicy rbmPolicy; - //! The matrix of data points (predictors). - arma::Mat predictors; - // Intialiser for intializing the weights of the network. - InitializationRuleType initializeRule; - //! Locally-stored state of the persistent cdk. - arma::Mat state; - //! Locally-stored number of data points. - size_t numFunctions; - //! Locally-stored number of steps in gibbs sampling. - size_t numSteps; - //! Locally-stored number of negative samples. - size_t negSteps; - //! Locally-stored monitoring cost. - bool useMonitoringCost; - //! Locally-stored persistent cd-k or not. - bool persistence; - //! Locally-stored reset variable. - bool reset; - - //! Locally-stored reconstructed output from hidden layer. - arma::Mat hiddenReconstruction; - //! Locally-stored reconstructed output from visible layer. - arma::Mat visibleReconstruction; - - //! Locally-stored negative samples from gibbs distribution. - arma::Mat negativeSamples; - //! Locally-stored gradients from the negative phase. - arma::Mat negativeGradient; - //! Locally-stored temproray negative gradient used for negative phase. - arma::Mat tempNegativeGradient; - //! Locally-stored gradient for positive phase. - arma::Mat positiveGradient; - //! Locally-stored temporary output of gibbs chain. - arma::Mat gibbsTemporary; - //! Locally-stored output of the preActivation function used in FreeEnergy. - arma::Mat preActivation; -}; -} // namespace ann -} // namespace mlpack - -#include "rbm_impl.hpp" - -#endif // MLPACK_METHODS_ANN_RBM_HPP diff --git a/src/mlpack/methods/ann/rbm/CMakeLists.txt b/src/mlpack/methods/ann/rbm/CMakeLists.txt index 31a42201f2..001b85b752 100644 --- a/src/mlpack/methods/ann/rbm/CMakeLists.txt +++ b/src/mlpack/methods/ann/rbm/CMakeLists.txt @@ -1,10 +1,10 @@ -# Define the files we need to compile -# Anything not in this list will not be compiled into mlpack. +# Define the files we need to compile. +# Any file not in this list will not be compiled into mlpack. set(SOURCES - binary_rbm_policy.hpp - binary_rbm_policy_impl.hpp - spike_slab_rbm_policy.hpp - spike_slab_rbm_policy_impl.hpp + rbm.hpp + rbm_impl.hpp + rbm_policies.hpp + spike_slab_rbm_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/rbm/binary_rbm_policy.hpp b/src/mlpack/methods/ann/rbm/binary_rbm_policy.hpp deleted file mode 100644 index 85216e66fd..0000000000 --- a/src/mlpack/methods/ann/rbm/binary_rbm_policy.hpp +++ /dev/null @@ -1,176 +0,0 @@ -/** - * @file binary_rbm_policy.hpp - * @author Kris Singh - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_HPP -#define MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_HPP - -#include - -#include - -namespace mlpack{ -namespace ann{ -/** - * The BinaryRBMPolicy class - * - * @tparam DataType the type of matrix to be used. - */ -template -class BinaryRBMPolicy -{ - public: - typedef typename DataType::elem_type ElemType; - /** - * Intialise the visible and hidden layer of the network. - * @param visibelSize Number of visible neurons. - * @param hiddenSize Number of hidden neurons. - */ - BinaryRBMPolicy(const size_t visibleSize, const size_t hiddenSize); - - // Reset function - void Reset(); - - /** - * This function calculates the Free Energy of the binary RBM. - * The free energy is given by - * $-b^Tv - \sum_{i=1}^M log(1 + e^{c_j+v^TW_j})$. - * - * @param input The visible neurons. - */ - ElemType FreeEnergy(DataType&& input); - - /** - * Evaluate function computes the perfomance of the RBM at the given. - * input in the case persistent = true - * - * @param predictors Training data of the network. - * @param i The idx of the current input. - */ - ElemType Evaluate(DataType& predictors, size_t i); - - /** - * Calculates the Gradient of the RBM network on the - * visible input obtained from the training data. - * - * @param input The visible layer type. - * @param gradient Stores the gradient of the rbm network. - */ - void PositivePhase(DataType&& input, DataType&& gradient); - - /** - * Calculate the Gradient of the RBM network on the sampled - * visible input obtained from gibbs sampling. - * - * @param input The negative samples sampled from gibbs distribution. - * @param gradient Stores the gradient of the rbm network. - */ - void NegativePhase(DataType&& negativeSamples, DataType&& gradient); - - /** - * The function calculates the mean for the visible layer. - * - * @param input Hidden neurons from the hidden layer of the network. - * @param output Visible neuron activations. - */ - void VisibleMean(DataType&& input, DataType&& output); - - /** - * The function calcultes the mean for the hidden layer. - * - * @param input Visible neurons. - * @param output Hidden neuron activations. - */ - void HiddenMean(DataType&& input, DataType&& output); - - /** - * SampleVisible function samples the visible layer using bernoulli function. - * - * @param input Hidden neurons. - * @param output Sampled visible neurons. - */ - void SampleVisible(DataType&& input, DataType&& output); - - /** - * SampleHidden function samples the hidden layer using bernoulli function. - * - * @param input Visible neurons. - * @param output Sampled hidden neurons. - */ - void SampleHidden(DataType&& input, DataType&& output); - - //! Serialize the model. - template - void serialize(Archive& ar, const unsigned int /* version */); - - //! Return the parameters of the network. - const DataType& Parameters() const { return parameter; } - //! Modify the parameters of the network. - DataType& Parameters() { return parameter; } - //! Return the weights of the network. - const DataType& Weight() const { return weight; } - //! Modify the weight of the network. - DataType& Weight() { return weight; } - - //! Return the visible bias of the network. - const DataType& VisibleBias() const { return visibleBias; } - //! Modify the visible bias of the network. - DataType& VisibleBias() { return visibleBias; } - - //! Return the hidden bias of the network. - const DataType& HiddenBias() const { return hiddenBias; } - //! Modify the hidden bias of the network. - DataType& HiddenBias() { return hiddenBias; } - - //! Get the visible size. - size_t const& VisibleSize() const { return visibleSize; } - //! Get the hidden size. - size_t const& HiddenSize() const { return hiddenSize; } - - private: - /** - * VisiblePreAction function calculates the pre activation - * values given the hidden input units. - * - * @param input Hidden neurons. - * @param ouput Visible unit pre-activation values. - */ - void VisiblePreActivation(DataType&& input, DataType&& output); - /** - * HiddenPreActivation function calculates the pre activation - * values given the visible input units. - * - * @param input Visible unit neuron. - * @param ouput Hidden unit pre-activation values. - */ - void HiddenPreActivation(DataType&& input, DataType&& output); - - private: - //! Locally stored number of visible neurons. - size_t visibleSize; - //! Locally stored number of hidden neurons - size_t hiddenSize; - //! Locally stored Parameters of the network. - DataType parameter; - //! Locally stored weight of the network. - DataType weight; - //! Locally stored biases of the visible layer. - DataType visibleBias; - //! Locally stored biases of hidden layer. - DataType hiddenBias; - //! Locally-stored output of the preActivation function used in FreeEnergy. - DataType preActivation; - //! Locally-stored corrupInput used for Pseudo-Likelihood. - DataType corruptInput; -}; -} // namespace ann -} // namespace mlpack - -#include "binary_rbm_policy_impl.hpp" - -#endif // MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_HPP diff --git a/src/mlpack/methods/ann/rbm/binary_rbm_policy_impl.hpp b/src/mlpack/methods/ann/rbm/binary_rbm_policy_impl.hpp deleted file mode 100644 index 5fe5458013..0000000000 --- a/src/mlpack/methods/ann/rbm/binary_rbm_policy_impl.hpp +++ /dev/null @@ -1,167 +0,0 @@ -/** - * @file binary_rbm.hpp - * @author Kris Singh - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_IMPL_HPP -#define MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_IMPL_HPP - -#include - -#include "binary_rbm_policy.hpp" - -namespace mlpack { -namespace ann { - -template -BinaryRBMPolicy - ::BinaryRBMPolicy(const size_t visibleSize, const size_t hiddenSize) : - visibleSize(visibleSize), - hiddenSize(hiddenSize) -{ - parameter.set_size((visibleSize * hiddenSize) + visibleSize + hiddenSize, 1); -} - -// Reset function -template -void BinaryRBMPolicy::Reset() -{ - weight = DataType(parameter.memptr(), hiddenSize, visibleSize, false, false); - hiddenBias = DataType(parameter.memptr() + weight.n_elem, - hiddenSize, 1, false, false); - visibleBias = DataType(parameter.memptr() + weight.n_elem + - hiddenBias.n_elem , visibleSize, 1, false, false); -} - -template -typename BinaryRBMPolicy::ElemType BinaryRBMPolicy - ::FreeEnergy(DataType&& input) -{ - HiddenPreActivation(std::move(input), std::move(preActivation)); - preActivation = arma::log(1 + arma::trunc_exp(preActivation)); - return -(arma::accu(preActivation) + arma::dot(input, visibleBias)); -} - -template -typename BinaryRBMPolicy::ElemType BinaryRBMPolicy - ::Evaluate(DataType& predictors, size_t i) -{ - size_t idx = RandInt(0, predictors.n_rows); - DataType temp = arma::round(predictors.col(i)); - corruptInput = temp; - corruptInput.row(idx) = 1 - corruptInput.row(idx); - return std::log(LogisticFunction::Fn(FreeEnergy(std::move(corruptInput)) - - FreeEnergy(std::move(temp)))) * predictors.n_rows; -} - -template -void BinaryRBMPolicy::PositivePhase( - DataType&& input, - DataType&& gradient) -{ - DataType weightGrad = DataType(gradient.memptr(), - hiddenSize, visibleSize, false, false); - - DataType hiddenBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem, - hiddenSize, 1, false, false); - - DataType visibleBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem + - hiddenBiasGrad.n_elem, visibleSize, 1, false, false); - - HiddenMean(std::move(input), std::move(hiddenBiasGrad)); - weightGrad = hiddenBiasGrad * input.t(); - visibleBiasGrad = input; -} - -template -void BinaryRBMPolicy::NegativePhase( - DataType&& negativeSamples, - DataType&& gradient) -{ - DataType weightGrad = DataType(gradient.memptr(), - hiddenSize, visibleSize, false, false); - - DataType hiddenBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem, - hiddenSize, 1, false, false); - - DataType visibleBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem + - hiddenBiasGrad.n_elem, visibleSize, 1, false, false); - - HiddenMean(std::move(negativeSamples), std::move(hiddenBiasGrad)); - weightGrad = hiddenBiasGrad * negativeSamples.t(); - visibleBiasGrad = negativeSamples; -} - -template -void BinaryRBMPolicy::VisibleMean( - DataType&& input, - DataType&& output) -{ - VisiblePreActivation(std::move(input), std::move(output)); - LogisticFunction::Fn(output, output); -} - -template -void BinaryRBMPolicy::HiddenMean( - DataType&& input, - DataType&& output) -{ - HiddenPreActivation(std::move(input), std::move(output)); - LogisticFunction::Fn(output, output); -} - -template -void BinaryRBMPolicy::SampleVisible( - DataType&& input, - DataType&& output) -{ - VisibleMean(std::move(input), std::move(output)); - - for (size_t i = 0; i < output.n_elem; i++) - output(i) = math::RandBernoulli(output(i)); -} - -template -void BinaryRBMPolicy::SampleHidden( - DataType&& input, - DataType&& output) -{ - HiddenMean(std::move(input), std::move(output)); - - for (size_t i = 0; i < output.n_elem; i++) - output(i) = math::RandBernoulli(output(i)); -} - -template -void BinaryRBMPolicy::VisiblePreActivation( - DataType&& input, - DataType&& output) -{ - output = weight.t() * input + visibleBias; -} - -template -void BinaryRBMPolicy::HiddenPreActivation( - DataType&& input, - DataType&& output) -{ - output = weight * input + hiddenBias; -} - -template -template -void BinaryRBMPolicy::serialize( - Archive& ar, - const unsigned int /* version */) -{ - ar & BOOST_SERIALIZATION_NVP(visibleSize); - ar & BOOST_SERIALIZATION_NVP(hiddenSize); -} - -} // namespace ann -} // namespace mlpack -#endif // MLPACK_METHODS_ANN_RBM_BINARY_RBM_POLICY_IMPL_HPP diff --git a/src/mlpack/methods/ann/rbm/rbm.hpp b/src/mlpack/methods/ann/rbm/rbm.hpp new file mode 100644 index 0000000000..f170457508 --- /dev/null +++ b/src/mlpack/methods/ann/rbm/rbm.hpp @@ -0,0 +1,447 @@ +/** + * @file rbm.hpp + * @author Kris Singh + * @author Shikhar Jaiswal + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_RBM_RBM_HPP +#define MLPACK_METHODS_ANN_RBM_RBM_HPP + +#include +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The RBM class. + * + * @tparam InitializationRuleType Rule used to initialize the network. + * @tparam DataType The type of matrix to be used. + * @tparam PolicyType The RBM variant to be used (BinaryRBM or SpikeSlabRBM). + */ +template< + typename InitializationRuleType, + typename DataType = arma::mat, + typename PolicyType = BinaryRBM +> +class RBM +{ + public: + using NetworkType = RBM; + typedef typename DataType::elem_type ElemType; + + /** + * Initialize all the parameters of the network using initializeRule. + * + * @tparam initializeRule InitializationRule object for + * initializing the network parameter. + * @param predictors Training data to be used. + * @param visibleSize Number of visible neurons. + * @param hiddenSize Number of hidden neurons. + * @param batchSize Batch size to be used for training. + * @param numSteps Number of Gibbs Sampling steps. + * @param negSteps Number of negative samples to average negative gradient. + * @param persistence Indicates whether to use Persistent CD or not. + */ + RBM(arma::Mat predictors, + InitializationRuleType initializeRule, + const size_t visibleSize, + const size_t hiddenSize, + const size_t batchSize = 1, + const size_t numSteps = 1, + const size_t negSteps = 1, + const size_t poolSize = 2, + const ElemType slabPenalty = 8, + const ElemType radius = 1, + const bool persistence = false); + + // Reset the network. + template + typename std::enable_if::value, void>::type + Reset(); + + // Reset the network. + template + typename std::enable_if::value, void>::type + Reset(); + + /** + * Train the feed-forward network on the given input data. + * + * This will use the existing model parameters as a starting point for the + * optimization. If this is not what you want, then you should access the + * parameters vector directly with Parameters() and modify it as desired. + * + * @param optimizer Optimizer type. + */ + template + void Train(OptimizerType& optimizer); + + /** + * Evaluate the RBM network with the given parameters. + * The function is needed for monitoring the progress of the network. + * + * @param parameters Matrix model parameters. + * @param i Index of the data point. + * @param batchSize Variable to store the present number of inputs. + */ + double Evaluate(const arma::Mat& parameters, + const size_t i, + const size_t batchSize); + + /** + * This function calculates the free energy of the BinaryRBM. + * The free energy is given by: + * $-b^Tv - \sum_{i=1}^M log(1 + e^{c_j+v^TW_j})$. + * + * @param input The visible neurons. + */ + template + typename std::enable_if::value, double>::type + FreeEnergy(arma::Mat&& input); + + /** + * This function calculates the free energy of the SpikeSlabRBM. + * The free energy is given by: + * $v^t$$\Delta$v - $\sum_{i=1}^N$ + * $\log{ \sqrt{\frac{(-2\pi)^K}{\prod_{m=1}^{K}(\alpha_i)_m}}}$ - + * $\sum_{i=1}^N \log(1+\exp( b_i + + * \sum_{m=1}^k \frac{(v(w_i)_m^t)^2}{2(\alpha_i)_m})$ + * + * @param input The visible layer neurons. + */ + template + typename std::enable_if::value, + double>::type + FreeEnergy(arma::Mat&& input); + + /** + * Calculates the gradient of the RBM network on the provided input. + * + * @param input The provided input data. + * @param gradient Stores the gradient of the RBM network. + */ + template + typename std::enable_if::value, void>::type + Phase(DataType&& input, DataType&& gradient); + + /** + * Calculates the gradient of the RBM network on the provided input. + * + * @param input The provided input data. + * @param gradient Stores the gradient of the RBM network. + */ + template + typename std::enable_if::value, void>::type + Phase(DataType&& input, DataType&& gradient); + + /** + * This function samples the hidden layer given the visible layer using + * Bernoulli function. + * + * @param input Visible layer input. + * @param output The sampled hidden layer. + */ + template + typename std::enable_if::value, void>::type + SampleHidden(arma::Mat&& input, arma::Mat&& output); + + /** + * This function samples the slab outputs from the Normal distribution with + * mean given by: + * $h_i*\alpha^{-1}*W_i^T*v$ + * and variance: + * $\alpha&{-1}$ + * + * @param input Consists of both visible and spike variables. + * @param output Sampled slab neurons. + */ + template + typename std::enable_if::value, void>::type + SampleHidden(arma::Mat&& input, arma::Mat&& output); + + /** + * This function samples the visible layer given the hidden layer using + * Bernoulli function. + * + * @param input Hidden layer of the network. + * @param output The sampled visible layer. + */ + template + typename std::enable_if::value, void>::type + SampleVisible(arma::Mat&& input, arma::Mat&& output); + + /** + * Sample Hidden function samples the slab outputs from the Normal + * distribution with mean given by: + * $h_i*\alpha^{-1}*W_i^T*v$ + * and variance: + * $\alpha&{-1}$ + * + * @param input Hidden layer of the network. + * @param output The sampled visible layer. + */ + template + typename std::enable_if::value, void>::type + SampleVisible(arma::Mat&& input, arma::Mat&& output); + + /** + * The function calculates the mean for the visible layer. + * + * @param input Hidden neurons from the hidden layer of the network. + * @param output Visible neuron activations. + */ + template + typename std::enable_if::value, void>::type + VisibleMean(DataType&& input, DataType&& output); + + /** + * The function calculates the mean of the Normal distribution of P(v|s, h). + * The mean is given by: + * $\Lambda^{-1} \sum_{i=1}^N W_i * s_i * h_i$ + * + * @param input Consists of both the spike and slab variables. + * @param output Mean of the of the Normal distribution. + */ + template + typename std::enable_if::value, void>::type + VisibleMean(DataType&& input, DataType&& output); + + /** + * The function calculates the mean for the hidden layer. + * + * @param input Visible neurons. + * @param output Hidden neuron activations. + */ + template + typename std::enable_if::value, void>::type + HiddenMean(DataType&& input, DataType&& output); + + /** + * The function calculates the mean of the Normal distribution of P(s|v, h). + * The mean is given by: + * $h_i*\alpha^{-1}*W_i^T*v$ + * The variance is given by: + * $\alpha^{-1}$ + * + * @param input Visible layer neurons. + * @param output Consists of both the spike samples and slab samples. + */ + template + typename std::enable_if::value, void>::type + HiddenMean(DataType&& input, DataType&& output); + + /** + * The function calculates the mean of the distribution P(h|v), + * where mean is given by: + * $sigm(v^T*W_i*\alpha_i^{-1}*W_i^T*v + b_i)$ + * + * @param visible The visible layer neurons. + * @param spikeMean Indicates P(h|v). + */ + template + typename std::enable_if::value, void>::type + SpikeMean(DataType&& visible, DataType&& spikeMean); + + /** + * The function samples the spike function using Bernoulli distribution. + * @param spikeMean Indicates P(h|v). + * @param spike Sampled binary spike variables. + */ + template + typename std::enable_if::value, void>::type + SampleSpike(DataType&& spikeMean, DataType&& spike); + + /** + * The function calculates the mean of Normal distribution of P(s|v, h), + * where the mean is given by: + * $h_i*\alpha^{-1}*W_i^T*v$ + * + * @param visible The visible layer neurons. + * @param spike The spike variables from hidden layer. + * @param slabMean The mean of the Normal distribution of slab neurons. + */ + template + typename std::enable_if::value, void>::type + SlabMean(DataType&& visible, DataType&& spike, DataType&& slabMean); + + /** + * The function samples from the Normal distribution of P(s|v, h), + * where the mean is given by: + * $h_i*\alpha^{-1}*W_i^T*v$ + * and variance is given by: + * $\alpha^{-1}$ + * + * @param slabMean Mean of the Normal distribution of the slab neurons. + * @param slab Sampled slab variable from the Normal distribution. + */ + template + typename std::enable_if::value, void>::type + SampleSlab(DataType&& slabMean, DataType&& slab); + + /** + * This function does the k-step Gibbs Sampling. + * + * @param input Input to the Gibbs function. + * @param output Used for storing the negative sample. + * @param steps Number of Gibbs Sampling steps taken. + */ + void Gibbs(arma::Mat&& input, + arma::Mat&& output, + size_t steps = SIZE_MAX); + + /** + * Calculates the gradients for the RBM network. + * + * @param parameters The current parameters of the network. + * @param i Index of the data point. + * @param gradient Variable to store the present gradient. + * @param batchSize Variable to store the present number of inputs. + */ + void Gradient(const arma::Mat& parameters, + const size_t i, + arma::Mat& gradient, + const size_t batchSize); + + /** + * Shuffle the order of function visitation. This may be called by the + * optimizer. + */ + void Shuffle(); + + //! Return the number of separable functions (the number of predictor points). + size_t NumFunctions() const { return numFunctions; } + + //! Return the number of steps of Gibbs Sampling. + size_t NumSteps() const { return numSteps; } + + //! Return the parameters of the network. + const arma::Mat& Parameters() const { return parameter; } + //! Modify the parameters of the network. + arma::Mat& Parameters() { return parameter; } + + //! Return the weights of the network. + const DataType& Weight() const { return weight; } + //! Modify the weights of the network. + DataType& Weight() { return weight; } + //! Get the weight of the network. + arma::cube const& WeightCube() const { return weightCube; } + //! Modify the weights of the network. + arma::cube& WeightCube() { return weightCube; } + + //! Return the visible bias of the network. + DataType const& VisibleBias() const { return visibleBias; } + //! Modify the visible bias of the network. + DataType& VisibleBias() { return visibleBias; } + + //! Return the hidden bias of the network. + DataType const& HiddenBias() const { return hiddenBias; } + //! Modify the hidden bias of the network. + DataType& HiddenBias() { return hiddenBias; } + + //! Get the regularizer associated with spike variables. + DataType const& SpikeBias() const { return spikeBias; } + //! Modify the regularizer associated with spike variables. + DataType& SpikeBias() { return spikeBias; } + + //! Get the regularizer associated with slab variables. + ElemType const& SlabPenalty() const { return 1.0 / slabPenalty; } + + //! Get the regularizer associated with visible variables. + DataType const& VisiblePenalty() const { return visiblePenalty; } + //! Modify the regularizer associated with visible variables. + DataType& VisiblePenalty() { return visiblePenalty; } + + //! Get the visible size. + size_t const& VisibleSize() const { return visibleSize; } + //! Get the hidden size. + size_t const& HiddenSize() const { return hiddenSize; } + //! Get the pool size. + size_t const& PoolSize() const { return poolSize; } + + //! Serialize the model. + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally stored parameters of the network. + arma::Mat parameter; + //! The matrix of data points (predictors). + arma::Mat predictors; + // Initializer for initializing the weights of the network. + InitializationRuleType initializeRule; + //! Locally-stored state of the persistent CD-k. + arma::Mat state; + //! Locally-stored number of data points. + size_t numFunctions; + //! Locally stored number of visible neurons. + size_t visibleSize; + //! Locally stored number of hidden neurons. + size_t hiddenSize; + //! Locally stored batch size parameter. + size_t batchSize; + //! Locally-stored number of steps in Gibbs Sampling. + size_t numSteps; + //! Locally-stored number of negative samples. + size_t negSteps; + //! Locally stored variable poolSize. + size_t poolSize; + //! Locally stored weight of the network. + DataType weight; + //! Locally stored biases of the visible layer. + DataType visibleBias; + //! Locally stored biases of the hidden layer. + DataType hiddenBias; + //! Locally-stored output of the preActivation function used in FreeEnergy. + DataType preActivation; + //! Locally stored weight of the network + //! (visibleSize * poolSize * hiddenSize). + arma::Cube weightCube; + //! Locally stored spikeBias (hiddenSize * 1). + DataType spikeBias; + //! Locally stored visible Penalty (1 * 1). + DataType visiblePenalty; + //! Locally stored mean of the P(v|s, h). + DataType visibleMean; + //! Locally stored mean of the P(v|h). + DataType spikeMean; + //! Locally stored spike variables. + DataType spikeSamples; + //! Locally stored mean of the P(s|v, h). + DataType slabMean; + //! Locally stored slabPenalty. + ElemType slabPenalty; + //! Locally stored radius used for rejection sampling. + ElemType radius; + //! Locally-stored reconstructed output from hidden layer. + arma::Mat hiddenReconstruction; + //! Locally-stored reconstructed output from visible layer. + arma::Mat visibleReconstruction; + //! Locally-stored negative samples from Gibbs distribution. + arma::Mat negativeSamples; + //! Locally-stored gradients from the negative phase. + arma::Mat negativeGradient; + //! Locally-stored temporary negative gradient used for negative phase. + arma::Mat tempNegativeGradient; + //! Locally-stored gradient for positive phase. + arma::Mat positiveGradient; + //! Locally-stored temporary output of Gibbs chain. + arma::Mat gibbsTemporary; + //! Locally-stored persistent CD-k boolean flag. + bool persistence; + //! Locally-stored reset variable. + bool reset; +}; + +} // namespace ann +} // namespace mlpack + +#include "rbm_impl.hpp" +#include "spike_slab_rbm_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/rbm/rbm_impl.hpp b/src/mlpack/methods/ann/rbm/rbm_impl.hpp new file mode 100644 index 0000000000..67846f65fc --- /dev/null +++ b/src/mlpack/methods/ann/rbm/rbm_impl.hpp @@ -0,0 +1,343 @@ +/** + * @file rbm_impl.hpp + * @author Kris Singh + * @author Shikhar Jaiswal + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_RBM_RBM_IMPL_HPP +#define MLPACK_METHODS_ANN_RBM_RBM_IMPL_HPP + +// In case it hasn't been included yet. +#include "rbm.hpp" + +#include + +namespace mlpack { +namespace ann /** Artificial neural networks. */ { + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +RBM::RBM( + arma::Mat predictors, + InitializationRuleType initializeRule, + const size_t visibleSize, + const size_t hiddenSize, + const size_t batchSize, + const size_t numSteps, + const size_t negSteps, + const size_t poolSize, + const ElemType slabPenalty, + const ElemType radius, + const bool persistence): + predictors(std::move(predictors)), + initializeRule(initializeRule), + visibleSize(visibleSize), + hiddenSize(hiddenSize), + batchSize(batchSize), + numSteps(numSteps), + negSteps(negSteps), + poolSize(poolSize), + slabPenalty(slabPenalty), + radius(2 * radius), + persistence(persistence), + reset(false) +{ + numFunctions = this->predictors.n_cols; +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::Reset() +{ + size_t shape = (visibleSize * hiddenSize) + visibleSize + hiddenSize; + + parameter.set_size(shape, 1); + positiveGradient.set_size(shape, 1); + negativeGradient.set_size(shape, 1); + tempNegativeGradient.set_size(shape, 1); + negativeSamples.set_size(visibleSize, batchSize); + + weight = DataType(parameter.memptr(), hiddenSize, visibleSize, false, false); + hiddenBias = DataType(parameter.memptr() + weight.n_elem, + hiddenSize, 1, false, false); + visibleBias = DataType(parameter.memptr() + weight.n_elem + + hiddenBias.n_elem, visibleSize, 1, false, false); + + parameter.zeros(); + positiveGradient.zeros(); + negativeGradient.zeros(); + tempNegativeGradient.zeros(); + initializeRule.Initialize(parameter, parameter.n_elem, 1); + + reset = true; +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +void RBM::Train( + OptimizerType& optimizer) +{ + if (!reset) + { + Reset(); + } + + optimizer.Optimize(*this, parameter); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, double>::type +RBM::FreeEnergy( + arma::Mat&& input) +{ + preActivation = arma::log(1 + arma::trunc_exp((weight * input) + hiddenBias)); + return -(arma::accu(preActivation) + arma::dot(input, visibleBias)); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::Phase( + DataType&& input, + DataType&& gradient) +{ + DataType weightGrad = DataType(gradient.memptr(), + hiddenSize, visibleSize, false, false); + + DataType hiddenBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem, + hiddenSize, 1, false, false); + + DataType visibleBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem + + hiddenBiasGrad.n_elem, visibleSize, 1, false, false); + + HiddenMean(std::move(input), std::move(hiddenBiasGrad)); + weightGrad = hiddenBiasGrad * input.t(); + visibleBiasGrad = input; +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +double RBM::Evaluate( + const arma::Mat& /* parameters*/, + const size_t i, + const size_t batchSize) +{ + Gibbs(std::move(predictors.cols(i, i + batchSize - 1)), + std::move(negativeSamples)); + return std::fabs(FreeEnergy(std::move(predictors.cols(i, + i + batchSize - 1))) - FreeEnergy(std::move(negativeSamples))); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SampleHidden( + arma::Mat&& input, + arma::Mat&& output) +{ + HiddenMean(std::move(input), std::move(output)); + + for (size_t i = 0; i < output.n_elem; i++) + { + output(i) = math::RandBernoulli(output(i)); + } +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SampleVisible( + arma::Mat&& input, + arma::Mat&& output) +{ + VisibleMean(std::move(input), std::move(output)); + + for (size_t i = 0; i < output.n_elem; i++) + { + output(i) = math::RandBernoulli(output(i)); + } +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::VisibleMean(DataType&& input, + DataType&& output) +{ + output = weight.t() * input + visibleBias; + LogisticFunction::Fn(output, output); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::HiddenMean(DataType&& input, + DataType&& output) +{ + output = weight * input + hiddenBias; + LogisticFunction::Fn(output, output); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +void RBM::Gibbs( + arma::Mat&& input, + arma::Mat&& output, + size_t steps) +{ + steps = (steps == SIZE_MAX) ? this->numSteps : steps; + + if (persistence && !state.is_empty()) + { + SampleHidden(std::move(state), std::move(gibbsTemporary)); + SampleVisible(std::move(gibbsTemporary), std::move(output)); + } + else + { + SampleHidden(std::move(input), std::move(gibbsTemporary)); + SampleVisible(std::move(gibbsTemporary), std::move(output)); + } + + for (size_t j = 1; j < steps; j++) + { + SampleHidden(std::move(output), std::move(gibbsTemporary)); + SampleVisible(std::move(gibbsTemporary), std::move(output)); + } + if (persistence) + { + state = output; + } +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +void RBM::Gradient( + const arma::Mat& /*parameters*/, + const size_t i, + arma::Mat& gradient, + const size_t batchSize) +{ + positiveGradient.zeros(); + negativeGradient.zeros(); + + Phase(std::move(predictors.cols(i, i + batchSize - 1)), + std::move(positiveGradient)); + + for (size_t i = 0; i < negSteps; i++) + { + Gibbs(std::move(predictors.cols(i, i + batchSize - 1)), + std::move(negativeSamples)); + Phase(std::move(negativeSamples), std::move(tempNegativeGradient)); + + negativeGradient += tempNegativeGradient; + } + gradient = ((negativeGradient / negSteps) - positiveGradient); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +void RBM::Shuffle() +{ + predictors = predictors.cols(arma::shuffle(arma::linspace(0, + predictors.n_cols - 1, predictors.n_cols))); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +void RBM::serialize( + Archive& ar, const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(parameter); + ar & BOOST_SERIALIZATION_NVP(visibleSize); + ar & BOOST_SERIALIZATION_NVP(hiddenSize); + ar & BOOST_SERIALIZATION_NVP(state); + ar & BOOST_SERIALIZATION_NVP(numFunctions); + ar & BOOST_SERIALIZATION_NVP(numSteps); + ar & BOOST_SERIALIZATION_NVP(negSteps); + ar & BOOST_SERIALIZATION_NVP(persistence); + ar & BOOST_SERIALIZATION_NVP(poolSize); + ar & BOOST_SERIALIZATION_NVP(weight); + ar & BOOST_SERIALIZATION_NVP(weightCube); + ar & BOOST_SERIALIZATION_NVP(spikeBias); + ar & BOOST_SERIALIZATION_NVP(slabPenalty); + ar & BOOST_SERIALIZATION_NVP(radius); + ar & BOOST_SERIALIZATION_NVP(visiblePenalty); + + // If we are loading, we need to initialize the weights. + if (Archive::is_loading::value) + { + size_t shape = parameter.n_elem; + positiveGradient.set_size(shape, 1); + negativeGradient.set_size(shape, 1); + negativeSamples.set_size(visibleSize, batchSize); + tempNegativeGradient.set_size(shape, 1); + spikeMean.set_size(hiddenSize, 1); + spikeSamples.set_size(hiddenSize, 1); + slabMean.set_size(poolSize, hiddenSize); + positiveGradient.zeros(); + negativeGradient.zeros(); + tempNegativeGradient.zeros(); + reset = true; + } +} + +} // namespace ann +} // namespace mlpack +#endif diff --git a/src/mlpack/methods/ann/rbm/rbm_policies.hpp b/src/mlpack/methods/ann/rbm/rbm_policies.hpp new file mode 100644 index 0000000000..32c6419b16 --- /dev/null +++ b/src/mlpack/methods/ann/rbm/rbm_policies.hpp @@ -0,0 +1,49 @@ +/** + * @file rbm_policies.hpp + * @author Shikhar Jaiswal + * + * Implementation of the RBM policy types. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_RBM_RBM_POLICIES_HPP +#define MLPACK_METHODS_ANN_RBM_RBM_POLICIES_HPP + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * For more information, see the following paper: + * + * @code + * @article{Hinton10, + * author = {Geoffrey Hinton}, + * title = {A Practical Guide to Training Restricted Boltzmann Machines}, + * year = {2010}, + * url = {https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf} + * } + * @endcode + */ +class BinaryRBM { /* Nothing to do here */ }; + +/** + * For more information, see the following paper: + * + * @code + * @article{Courville11, + * author = {Aaron Courville, James Bergstra and Yoshua Bengio}, + * title = {A Spike and Slab Restricted Boltzmann Machine}, + * year = {2011}, + * url = {http://proceedings.mlr.press/v15/courville11a/courville11a.pdf} + * } + * @endcode + */ +class SpikeSlabRBM { /* Nothing to do here */ }; + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/rbm/spike_slab_rbm_impl.hpp b/src/mlpack/methods/ann/rbm/spike_slab_rbm_impl.hpp new file mode 100644 index 0000000000..0f602c874d --- /dev/null +++ b/src/mlpack/methods/ann/rbm/spike_slab_rbm_impl.hpp @@ -0,0 +1,302 @@ +/** + * @file spike_slab_rbm_impl.hpp + * @author Kris Singh + * @author Shikhar Jaiswal + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_IMPL_HPP +#define MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_IMPL_HPP + +#include "rbm.hpp" + +#include +#include + + +namespace mlpack { +namespace ann { + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::Reset() +{ + size_t shape = (visibleSize * hiddenSize * poolSize) + visibleSize + hiddenSize; + parameter.set_size(shape, 1); + positiveGradient.set_size(shape, 1); + negativeGradient.set_size(shape, 1); + tempNegativeGradient.set_size(shape, 1); + negativeSamples.set_size(visibleSize, batchSize); + visibleMean.set_size(visibleSize, 1); + spikeMean.set_size(hiddenSize, 1); + spikeSamples.set_size(hiddenSize, 1); + slabMean.set_size(poolSize, hiddenSize); + + // Weight shape D * K * N + weightCube = arma::Cube(parameter.memptr(), + visibleSize, poolSize, hiddenSize, + false, false); + // spike bias shape N * 1 + spikeBias = DataType(parameter.memptr() + weight.n_elem, hiddenSize, 1, + false, false); + // visible penalty 1 * 1 => D * D(when used) + visiblePenalty = DataType(parameter.memptr() + weight.n_elem + + spikeBias.n_elem, 1, 1, false, false); + + parameter.zeros(); + positiveGradient.zeros(); + negativeGradient.zeros(); + tempNegativeGradient.zeros(); + initializeRule.Initialize(parameter, parameter.n_elem, 1); + + reset = true; +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, double>::type +RBM::FreeEnergy( + arma::Mat&& input) +{ + ElemType freeEnergy = 0.5 * arma::as_scalar(visiblePenalty(0) * input.t() * + input); + + freeEnergy -= 0.5 * hiddenSize * poolSize * + std::log((2.0 * M_PI) / slabPenalty); + + for (size_t i = 0; i < hiddenSize; i++) + { + ElemType sum = 0; + sum = arma::accu(arma::square(input.t() * weightCube.slice(i))) / + (2.0 * slabPenalty); + freeEnergy -= SoftplusFunction::Fn(spikeBias(i) - sum); + } + + return freeEnergy; +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::Phase( + DataType&& input, + DataType&& gradient) +{ + arma::Cube weightGrad = arma::Cube + (gradient.memptr(), visibleSize, poolSize, hiddenSize, false, false); + + DataType spikeBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem, + hiddenSize, 1, false, false); + + DataType visiblePenaltyGrad = DataType(gradient.memptr() + + weightGrad.n_elem + spikeBiasGrad.n_elem, 1, 1, false, false); + + SpikeMean(std::move(input), std::move(spikeMean)); + SampleSpike(std::move(spikeMean), std::move(spikeSamples)); + SlabMean(std::move(input), std::move(spikeSamples), std::move(slabMean)); + + for (size_t i = 0 ; i < hiddenSize; i++) + weightGrad.slice(i) = input * slabMean.col(i).t() * spikeMean(i); + + spikeBiasGrad = spikeMean; + + visiblePenaltyGrad = -0.5 * input.t() * input; +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SampleHidden( + arma::Mat&& input, + arma::Mat&& output) +{ + output.set_size(hiddenSize + poolSize * hiddenSize, 1); + + DataType spike(output.memptr(), hiddenSize, 1, false, false); + DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false, + false); + + SpikeMean(std::move(input), std::move(spike)); + SampleSpike(std::move(spike), std::move(spike)); + SlabMean(std::move(input), std::move(spike), std::move(slab)); + SampleSlab(std::move(slab), std::move(slab)); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SampleVisible( + arma::Mat&& input, + arma::Mat&& output) +{ + const size_t numMaxTrials = 10; + size_t k = 0; + + VisibleMean(std::move(input), std::move(visibleMean)); + output.set_size(visibleSize, 1); + + for (k = 0; k < numMaxTrials; k++) + { + for (size_t i = 0; i < visibleSize; i++) + { + output(i) = math::RandNormal(visibleMean(i), 1.0 / visiblePenalty(0)); + } + if (arma::norm(output, 2) < radius) + break; + } + + if (k == numMaxTrials) + { + Log::Warn << "Outputs are still not in visible unit " + << arma::norm(output, 2) + << " terminating optimization." + << std::endl; + return; + } +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::VisibleMean( + DataType&& input, + DataType&& output) +{ + output.zeros(visibleSize, 1); + + DataType spike(input.memptr(), hiddenSize, 1, false, false); + DataType slab(input.memptr() + hiddenSize, poolSize, hiddenSize, false, + false); + + for (size_t i = 0; i < hiddenSize; i++) + output += weightCube.slice(i) * slab.col(i) * spike(i); + + output = ((1.0 / visiblePenalty(0)) * output); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::HiddenMean( + DataType&& input, + DataType&& output) +{ + output.set_size(hiddenSize + poolSize * hiddenSize, 1); + + DataType spike(output.memptr(), hiddenSize, 1, false, false); + DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false, + false); + + SpikeMean(std::move(input), std::move(spike)); + SampleSpike(std::move(spike), std::move(spikeSamples)); + SlabMean(std::move(input), std::move(spikeSamples), std::move(slab)); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SpikeMean( + DataType&& visible, + DataType&& spikeMean) +{ + for (size_t i = 0; i < hiddenSize; i++) + { + spikeMean(i) = LogisticFunction::Fn(0.5 * (1.0 / slabPenalty) * + arma::as_scalar(visible.t() * weightCube.slice(i) * + weightCube.slice(i).t() * visible) + spikeBias(i)); + } +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SampleSpike( + DataType&& spikeMean, + DataType&& spike) +{ + for (size_t i = 0; i < hiddenSize; i++) + spike(i) = math::RandBernoulli(spikeMean(i)); +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SlabMean( + DataType&& visible, + DataType&& spike, + DataType&& slabMean) +{ + for (size_t i = 0; i < hiddenSize; i++) + { + slabMean.col(i) = (1.0 / slabPenalty) * spike(i) * + weightCube.slice(i).t() * visible; + } +} + +template< + typename InitializationRuleType, + typename DataType, + typename PolicyType +> +template +typename std::enable_if::value, void>::type +RBM::SampleSlab( + DataType&& slabMean, + DataType&& slab) +{ + for (size_t i = 0; i < hiddenSize; i++) + { + for (size_t j = 0; j < poolSize; j++) + { + slab(j, i) = math::RandNormal(slabMean(j, i), 1.0 / slabPenalty); + } + } +} +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/rbm/spike_slab_rbm_policy.hpp b/src/mlpack/methods/ann/rbm/spike_slab_rbm_policy.hpp deleted file mode 100644 index f2b6091f3e..0000000000 --- a/src/mlpack/methods/ann/rbm/spike_slab_rbm_policy.hpp +++ /dev/null @@ -1,235 +0,0 @@ -/** - * @file spike_slab_rbm_policy.hpp - * @author Kris Singh - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_HPP -#define MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_HPP - -#include - -#include -#include - -namespace mlpack { -namespace ann { -/** - * The SpikeSlabRBMPolicy class. - * - * @tparam DataType the type of matrix to be used. - */ -template -class SpikeSlabRBMPolicy -{ - public: - typedef typename DataType::elem_type ElemType; - /** - * Intialise the visible and hidden layer of the network - * - * @param visibleSize Number of visible neurons. - * @param hiddenSize Number of hidden neurons. - * @param poolSize Number of hidden neurons to pool together. - * @param slabPenalty Regulariser of slab varaibles. - * @param radius Feasible regions for visible layer samples. - */ - SpikeSlabRBMPolicy(const size_t visibleSize, - const size_t hiddenSize, - const size_t poolSize, - const ElemType slabPenalty, - ElemType radius); - - // Reset function - void Reset(); - - /** - * Free energy of the spike and slab variable - * the free energy of the ssRBM is given my - * $v^t$$\Delta$v - $\sum_{i=1}^N$ - * $\log{ \sqrt{\frac{(-2\pi)^K}{\prod_{m=1}^{K}(\alpha_i)_m}}}$ - - * $\sum_{i=1}^N \log(1+\exp( b_i + - * \sum_{m=1}^k \frac{(v(w_i)_m^t)^2}{2(\alpha_i)_m})$ - * - * @param input The visible layer neurons. - */ - ElemType FreeEnergy(DataType&& input); - - /** - * Evaluate function is used by the Optimizer to - * find the perfomance of the network on the currentInput - * - * @param predictors The training data used. - * @param i The idx of the current input. - */ - ElemType Evaluate(DataType& /*predictors*/, size_t /*i*/); - - /** - * Calculate the Gradient of the RBM network on the - * visible input from the training data. - * - * @param input The visible layer neurons. - * @param gradient Stores the gradient of the rbm network. - */ - void PositivePhase(DataType&& input, DataType&& gradient); - - /** - * Calculate the Gradient of the RBM network on the sampled - * visible input from gibbs sampling - * - * @param input The visible layer neurons. - * @param output Stores the computed gradient of the rbm network. - */ - void NegativePhase(DataType&& negativeSamples, DataType&& gradient); - - /** - * Visible Mean function calculates the mean of the - * normal distribution of P(v| s,h). - * Where the mean is given by \Lambda^{-1} \sum_{i=1}^N W_i * s_i * h_i - * - * @param input Consists of both the spike and slab variables. - * @param output Mean of the of the Normal distribution. - */ - void VisibleMean(DataType&& input, DataType&& output); - - /** - * Hidden Mean function calculates the mean of the - * normal distribution of P(s|v,h). - * Where the mean is given by h_i*\alpha^{-1}*W_i^T*v - * variance is givenby \alpha^{-1} - * - * @param input Visible layer neurons. - * @param output Consits of both the spike samples and slab samples. - */ - void HiddenMean(DataType&& input, DataType&& output); - - /** - * Sample Visible function samples - * the visible layer from the normal distribution with - * mean \Lambda^{-1} \sum_{i=1}^N W_i * s_i * h_i and - * variance \Lambda^{-1} - * - * @param input Consists of spike and slab variables. - * @param output Sampled visible layer neurons. - */ - void SampleVisible(DataType&& input, DataType&& output); - - /** - * Sample Hidden function samples - * the slab outputs from the normal distribution with - * mean by h_i*\alpha^{-1}*W_i^T*v and - * variance \alpha&{-1} - * - * @param input Consists of both visible and spike variables. - * @param output Smapled slab neurons. - */ - void SampleHidden(DataType&& input, DataType&& output); - - //! Serialize function. - template - void serialize(Archive& ar, const unsigned int /* version */); - - //! Return the parameters of the network. - const DataType& Parameters() const { return parameter; } - //! Modify the parameters of the network. - DataType& Parameters() { return parameter; } - - //! Get the weight of the network. - arma::cube const& Weight() const { return weight; } - //! Modify the weights of the network. - arma::cube& Weight() { return weight; } - - //! Get the regulaliser associated with spike variables. - DataType const& SpikeBias() const { return spikeBias; } - //! Modify the regulaliser associated with spike variables. - DataType& SpikeBias() { return spikeBias; } - - //! Get the regulaliser associated with slab variables. - ElemType const& SlabPenalty() const { return 1.0 / slabPenalty; } - - //! Get the regulaliser associated with visible variables. - DataType const& VisiblePenalty() const { return visiblePenalty; } - //! Modify the regulaliser associated with visible variables. - DataType& VisiblePenalty() { return visiblePenalty; } - - //! Get the visible size. - size_t const& VisibleSize() const { return visibleSize; } - //! Get the hidden size. - size_t const& HiddenSize() const { return hiddenSize; } - //! Get the pool size. - size_t const& PoolSize() const { return poolSize; } - - private: - /** - * Spike Mean function calculates the mean of the following - * distribution P(h|v) where mean is given by - * sigm(v^T*W_i*\alpha_i^{-1}*W_i^T*v + b_i) - * - * @param visible The visible layer neurons. - * @param spikeMean Indicates P(h|v). - */ - void SpikeMean(DataType&& visible, DataType&& spikeMean); - /** - * Sample Spike function samples the spike - * function using bernoulli distribution - * @param spikeMean Indicates P(h|v). - * @param spike Sampled binary spike variables. - */ - void SampleSpike(DataType&& spikeMean, DataType&& spike); - - /** - * SlabMean function calculates the mean of - * normal distribution of P(s|v,h). - * Where the mean is given by h_i*\alpha^{-1}*W_i^T*v - * - * @param visible The visible layer neurons. - * @param spike The spike variables from hidden layer. - * @param slabMean The mean of the normal distribution of slab neurons. - */ - void SlabMean(DataType&& visible, DataType&& spike, DataType&& slabMean); - /** - * SampleSlab function samples from the - * normal distribution P(s|v,h). - * Where the mean is given by h_i*\alpha^{-1}*W_i^T*v - * variance is givenby \alpha^{-1} - * - * @slabMean Mean of the normal distribution of the slab neurons. - * @slab Sampled slab variable from the normal distribution. - */ - void SampleSlab(DataType&& slabMean, DataType&& slab); - - //! Locally stored number of visible neurons. - size_t visibleSize; - //! Locally stored number of hidden neurons. - size_t hiddenSize; - //! Locally stored variable poolSize. - size_t poolSize; - //! Locally stored parameters. - DataType parameter; - //! Locally stored weight of the network(visibleSize * poolSize * hiddenSize). - arma::Cube weight; - //! Locally stored spikeBias (hiddenSize * 1). - DataType spikeBias; - //! Locally stored slabPenalty. - ElemType slabPenalty; - //! Locally stored radius used for rejection sampling. - ElemType radius; - //! Locally stored visible Penalty(1 * 1). - DataType visiblePenalty; - //! Locally stored mean of the P(v | s,h). - DataType visibleMean; - //! Locally stored mean of the P(v | h). - DataType spikeMean; - //! Locally stored spike variables. - DataType spikeSamples; - //! Locally stored mean of the P(s | v, h). - DataType slabMean; -}; -} // namespace ann -} // namespace mlpack - -#include "spike_slab_rbm_policy_impl.hpp" - -#endif // MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_HPP diff --git a/src/mlpack/methods/ann/rbm/spike_slab_rbm_policy_impl.hpp b/src/mlpack/methods/ann/rbm/spike_slab_rbm_policy_impl.hpp deleted file mode 100644 index b7c9ee2e9f..0000000000 --- a/src/mlpack/methods/ann/rbm/spike_slab_rbm_policy_impl.hpp +++ /dev/null @@ -1,335 +0,0 @@ - -#ifndef MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_IMPL_HPP -#define MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_IMPL_HPP - -#include "spike_slab_rbm_policy.hpp" - -namespace mlpack { -namespace ann { -template -SpikeSlabRBMPolicy::SpikeSlabRBMPolicy( - const size_t visibleSize, - const size_t hiddenSize, - const size_t poolSize, - const ElemType slabPenalty, - ElemType radius): - visibleSize(visibleSize), - hiddenSize(hiddenSize), - poolSize(poolSize), - slabPenalty(slabPenalty), - radius(2 * radius) -{ - parameter.set_size(visibleSize * hiddenSize * poolSize + - visibleSize + hiddenSize); - - visibleMean.set_size(visibleSize, 1); - spikeMean.set_size(hiddenSize, 1); - spikeSamples.set_size(hiddenSize, 1); - slabMean.set_size(poolSize, hiddenSize); -}; - -// Reset function -template -void SpikeSlabRBMPolicy::Reset() -{ - // Weight shape D * K * N - weight = arma::Cube(parameter.memptr(), - visibleSize, poolSize, hiddenSize, - false, false); - - // spike bias shape N * 1 - spikeBias = DataType(parameter.memptr() + weight.n_elem, hiddenSize, 1, - false, false); - - // visible penalty 1 * 1 => D * D(when used) - visiblePenalty = DataType(parameter.memptr() + weight.n_elem + - spikeBias.n_elem, 1, 1, false, false); -} - -template -typename SpikeSlabRBMPolicy::ElemType -SpikeSlabRBMPolicy::FreeEnergy(DataType&& input) -{ - assert(input.n_rows == visibleSize); - assert(input.n_cols == 1); - - - ElemType freeEnergy = 0.5 * arma::as_scalar(visiblePenalty(0) * input.t() * - input); - - freeEnergy -= 0.5 * hiddenSize * poolSize * - std::log((2.0 * M_PI) / slabPenalty); - - for (size_t i = 0; i < hiddenSize; i++) - { - ElemType sum = 0; - sum = arma::accu(arma::square(input.t() * weight.slice(i))) / - (2.0 * slabPenalty); - freeEnergy -= SoftplusFunction::Fn(spikeBias(i) - sum); - } - - return freeEnergy; -} - -template -typename SpikeSlabRBMPolicy::ElemType -SpikeSlabRBMPolicy::Evaluate(DataType& /*predictors*/, - size_t /*i*/) -{ - // Return 0 here since we don't have evaluate in case of persistence - return 0; -} - -template -void SpikeSlabRBMPolicy::PositivePhase( - DataType&& input, - DataType&& gradient) -{ - assert(input.n_rows == visibleSize); - assert(input.n_cols == 1); - arma::Cube weightGrad = arma::Cube - (gradient.memptr(), visibleSize, poolSize, hiddenSize, false, false); - - DataType spikeBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem, - hiddenSize, 1, false, false); - - DataType visiblePenaltyGrad = DataType(gradient.memptr() + - weightGrad.n_elem + spikeBiasGrad.n_elem, 1, 1, false, false); - - SpikeMean(std::move(input), std::move(spikeMean)); - SampleSpike(std::move(spikeMean), std::move(spikeSamples)); - SlabMean(std::move(input), std::move(spikeSamples), std::move(slabMean)); - - for (size_t i = 0 ; i < hiddenSize; i++) - weightGrad.slice(i) = input * slabMean.col(i).t() * spikeMean(i); - - spikeBiasGrad = spikeMean; - - visiblePenaltyGrad = -0.5 * input.t() * input; -} - -template -void SpikeSlabRBMPolicy::NegativePhase( - DataType&& negativeSamples, - DataType&& gradient) -{ - assert(negativeSamples.n_rows == visibleSize); - assert(negativeSamples.n_cols == 1); - arma::Cube weightGrad = arma::Cube - (gradient.memptr(), visibleSize, poolSize, hiddenSize, false, false); - - DataType spikeBiasGrad = DataType(gradient.memptr() + weightGrad.n_elem, - hiddenSize, 1, false, false); - - DataType visiblePenaltyGrad = DataType(gradient.memptr() + - weightGrad.n_elem + spikeBiasGrad.n_elem, 1, 1, false, false); - - SpikeMean(std::move(negativeSamples), std::move(spikeMean)); - SampleSpike(std::move(spikeMean), std::move(spikeSamples)); - SlabMean(std::move(negativeSamples), std::move(spikeSamples), - std::move(slabMean)); - - for (size_t i = 0 ; i < hiddenSize; i++) - weightGrad.slice(i) = negativeSamples * slabMean.col(i).t() * spikeMean(i); - - spikeBiasGrad = spikeMean; - - visiblePenaltyGrad = -0.5 * negativeSamples.t() * negativeSamples; -} - -template -void SpikeSlabRBMPolicy::SpikeMean( - DataType&& visible, - DataType&& spikeMean) -{ - assert(visible.n_rows == visibleSize); - assert(visible.n_cols == 1); - - assert(spikeMean.n_rows == hiddenSize); - assert(spikeMean.n_cols == 1); - - for (size_t i = 0; i < hiddenSize; i++) - { - spikeMean(i) = LogisticFunction::Fn(0.5 * (1.0 / slabPenalty) * - arma::as_scalar(visible.t() * weight.slice(i) * weight.slice(i).t() * - visible) + spikeBias(i)); - } -} - -template -void SpikeSlabRBMPolicy::SampleSpike( - DataType&& spikeMean, - DataType&& spike) -{ - assert(spikeMean.n_rows == hiddenSize); - assert(spikeMean.n_cols == 1); - - assert(spike.n_rows == hiddenSize); - assert(spike.n_cols == 1); - - for (size_t i = 0; i < hiddenSize; i++) - spike(i) = math::RandBernoulli(spikeMean(i)); -} - -template -void SpikeSlabRBMPolicy::SlabMean( - DataType&& visible, - DataType&& spike, - DataType&& slabMean) -{ - assert(visible.n_rows == visibleSize); - assert(visible.n_cols == 1); - - assert(spike.n_rows == hiddenSize); - assert(spike.n_cols == 1); - - assert(slabMean.n_rows == poolSize); - assert(slabMean.n_cols == hiddenSize); - - assert(weight.n_rows == visibleSize); - assert(weight.n_cols == poolSize); - - for (size_t i = 0; i < hiddenSize; i++) - { - slabMean.col(i) = (1.0 / slabPenalty) * spike(i) * - weight.slice(i).t() * visible; - } -} - -template -void SpikeSlabRBMPolicy::SampleSlab( - DataType&& slabMean, - DataType&& slab) -{ - assert(slabMean.n_rows == poolSize); - assert(slabMean.n_cols == hiddenSize); - - assert(slab.n_rows == poolSize); - assert(slab.n_cols == hiddenSize); - - for (size_t i = 0; i < hiddenSize; i++) - { - for (size_t j = 0; j < poolSize; j++) - { - slab(j, i) = math::RandNormal(slabMean(j, i), 1.0 / slabPenalty); - } - } -} - -template -void SpikeSlabRBMPolicy::VisibleMean( - DataType&& input, - DataType&& output) -{ - assert(input.n_elem == hiddenSize + poolSize * hiddenSize); - output.zeros(visibleSize, 1); - - DataType spike(input.memptr(), hiddenSize, 1, false, false); - DataType slab(input.memptr() + hiddenSize, poolSize, hiddenSize, false, - false); - - for (size_t i = 0; i < hiddenSize; i++) - output += weight.slice(i) * slab.col(i) * spike(i); - - output = ((1.0 / visiblePenalty(0)) * output); -} - -template -void SpikeSlabRBMPolicy::HiddenMean( - DataType&& input, - DataType&& output) -{ - assert(input.n_elem == visibleSize); - output.set_size(hiddenSize + poolSize * hiddenSize, 1); - - DataType spike(output.memptr(), hiddenSize, 1, false, false); - DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false, - false); - - SpikeMean(std::move(input), std::move(spike)); - SampleSpike(std::move(spike), std::move(spikeSamples)); - SlabMean(std::move(input), std::move(spikeSamples), std::move(slab)); -} - -template -void SpikeSlabRBMPolicy::SampleVisible( - DataType&& input, - DataType&& output) -{ - const size_t numMaxTrials = 10; - size_t k = 0; - - VisibleMean(std::move(input), std::move(visibleMean)); - - output.set_size(visibleSize, 1); - - assert(visiblePenalty(0) > 0); - - for (k = 0; k < numMaxTrials; k++) - { - for (size_t i = 0; i < visibleSize; i++) - { - output(i) = math::RandNormal(visibleMean(i), 1.0 / visiblePenalty(0)); - } - if (arma::norm(output, 2) < radius) - break; - } - - if (k == numMaxTrials) - { - Log::Warn << "Outputs are still not in visible unit " - << arma::norm(output, 2) - << " terminating optimization." - << std::endl; - return; - } -} - -template -void SpikeSlabRBMPolicy::SampleHidden( - DataType&& input, - DataType&& output) -{ - assert(input.n_elem == visibleSize); - output.set_size(hiddenSize + poolSize * hiddenSize, 1); - - DataType spike(output.memptr(), hiddenSize, 1, false, false); - DataType slab(output.memptr() + hiddenSize, poolSize, hiddenSize, false, - false); - - SpikeMean(std::move(input), std::move(spike)); - SampleSpike(std::move(spike), std::move(spike)); - SlabMean(std::move(input), std::move(spike), std::move(slab)); - SampleSlab(std::move(slab), std::move(slab)); -} - -template -template -void SpikeSlabRBMPolicy::serialize( - Archive& ar, - const unsigned int /* version */) -{ - ar & BOOST_SERIALIZATION_NVP(visibleSize); - ar & BOOST_SERIALIZATION_NVP(hiddenSize); - ar & BOOST_SERIALIZATION_NVP(poolSize); - ar & BOOST_SERIALIZATION_NVP(parameter); - ar & BOOST_SERIALIZATION_NVP(weight); - ar & BOOST_SERIALIZATION_NVP(spikeBias); - ar & BOOST_SERIALIZATION_NVP(slabPenalty); - ar & BOOST_SERIALIZATION_NVP(radius); - ar & BOOST_SERIALIZATION_NVP(visiblePenalty); - - if (Archive::is_loading::value) - { - spikeMean.set_size(hiddenSize, 1); - spikeSamples.set_size(hiddenSize, 1); - slabMean.set_size(poolSize, hiddenSize); - Reset(); - } -} - -} // namespace ann -} // namespace mlpack - - -#endif // MLPACK_METHODS_ANN_RBM_SPIKE_SLAB_RBM_POLICY_IMPL_HPP diff --git a/src/mlpack/methods/ann/rbm_impl.hpp b/src/mlpack/methods/ann/rbm_impl.hpp deleted file mode 100644 index 5de88a1e32..0000000000 --- a/src/mlpack/methods/ann/rbm_impl.hpp +++ /dev/null @@ -1,228 +0,0 @@ -/** - * @file rbm_impl.hpp - * @author Kris Singh - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#ifndef MLPACK_METHODS_ANN_RBM_IMPL_HPP -#define MLPACK_METHODS_ANN_RBM_IMPL_HPP - -// In case it hasn't been included yet. -#include "rbm.hpp" - -#include - -#include - -namespace mlpack { -namespace ann /** Artificial neural networks. */ { - - -template -RBM::RBM( - arma::Mat predictors, - InitializationRuleType initializeRule, - RBMPolicy rbmPolicy, - const size_t numSteps, - const size_t negSteps, - const bool useMonitoringCost, - const bool persistence): - rbmPolicy(rbmPolicy), - initializeRule(initializeRule), - numSteps(numSteps), - negSteps(negSteps), - useMonitoringCost(useMonitoringCost), - persistence(persistence), - reset(false) -{ - numFunctions = predictors.n_cols; - this->predictors = std::move(predictors); -} - -template -void RBM::Reset() -{ - size_t weight = rbmPolicy.Parameters().n_elem; - - positiveGradient.set_size(weight, 1); - negativeGradient.set_size(weight, 1); - negativeSamples.set_size(rbmPolicy.VisibleSize(), 1); - tempNegativeGradient.set_size(weight, 1); - - parameter.zeros(weight, 1); - positiveGradient.zeros(); - negativeGradient.zeros(); - tempNegativeGradient.zeros(); - initializeRule.Initialize(parameter, parameter.n_elem, 1); - - rbmPolicy.Parameters() = arma::Mat(parameter.memptr(), weight, 1, - false, false); - rbmPolicy.Reset(); - - reset = true; -} - -template -template -void RBM::Train( - const arma::Mat& predictors, - OptimizerType& optimizer) -{ - numFunctions = predictors.n_cols; - this->predictors = std::move(predictors); - - if (!reset) - Reset(); - - // Train the model. - Timer::Start("rbm_optimization"); - optimizer.Optimize(*this, parameter); - Timer::Stop("rbm_optimization"); -} - -template -double RBM:: - FreeEnergy(arma::Mat&& input) -{ - return rbmPolicy.FreeEnergy(std::move(input)); -} - -template -double RBM::Evaluate( - const arma::Mat& /* parameters*/, - const size_t i) -{ - if (!useMonitoringCost) - { - Gibbs(std::move(predictors.col(i)), std::move(negativeSamples)); - return std::fabs(FreeEnergy(std::move(predictors.col(i))) - - FreeEnergy(std::move(negativeSamples))); - } - else - { - if (persistence) - { - return rbmPolicy.Evaluate(predictors, i); - } - else - { - // Mean Squared Error - rbmPolicy.SampleHidden(std::move(predictors.col(i)), - std::move(hiddenReconstruction)); - rbmPolicy.SampleVisible(std::move(hiddenReconstruction), - std::move(visibleReconstruction)); - return arma::accu(arma::pow(visibleReconstruction - predictors.col(i), - 2)) / hiddenReconstruction.n_rows; - } - } -} - -template -void RBM::SampleHidden( - arma::Mat&& input, - arma::Mat&& output) -{ - rbmPolicy.SampleHidden(std::move(input), std::move(output)); -} - -template -void RBM::SampleVisible( - arma::Mat&& input, - arma::Mat&& output) -{ - rbmPolicy.SampleVisible(std::move(input), std::move(output)); -} - -template -void RBM::Gibbs( - arma::Mat&& input, - arma::Mat&& output, - size_t steps) -{ - steps = (steps == SIZE_MAX) ? this-> numSteps: steps; - - if (persistence && !state.is_empty()) - { - SampleHidden(std::move(state), std::move(gibbsTemporary)); - SampleVisible(std::move(gibbsTemporary), std::move(output)); - } - else - { - SampleHidden(std::move(input), std::move(gibbsTemporary)); - SampleVisible(std::move(gibbsTemporary), std::move(output)); - } - - for (size_t j = 1; j < steps; j++) - { - SampleHidden(std::move(output), std::move(gibbsTemporary)); - SampleVisible(std::move(gibbsTemporary), std::move(output)); - } - if (persistence) - state = output; -} - -template -void RBM::Gradient( - arma::Mat& /*parameters*/, - const size_t input, - arma::Mat& output) -{ - positiveGradient.zeros(); - // Collect the negative samples - rbmPolicy.PositivePhase(std::move(predictors.col(input)), - std::move(positiveGradient)); - - negativeGradient.zeros(); - - for (size_t i = 0; i < negSteps; i++) - { - Gibbs(std::move(predictors.col(input)), std::move(negativeSamples)); - rbmPolicy.NegativePhase(std::move(negativeSamples), - std::move(tempNegativeGradient)); - - negativeGradient += tempNegativeGradient; - } - output = ((negativeGradient / negSteps) - positiveGradient); -} - -//! Serialize the model. -template -template -void RBM:: - serialize(Archive& ar, const unsigned int /* version */) -{ - ar & BOOST_SERIALIZATION_NVP(parameter); - ar & BOOST_SERIALIZATION_NVP(rbmPolicy); - ar & BOOST_SERIALIZATION_NVP(state); - ar & BOOST_SERIALIZATION_NVP(numFunctions); - ar & BOOST_SERIALIZATION_NVP(numSteps); - ar & BOOST_SERIALIZATION_NVP(negSteps); - ar & BOOST_SERIALIZATION_NVP(useMonitoringCost); - ar & BOOST_SERIALIZATION_NVP(persistence); - // ar & data::CreateNVP(RBMPolicy::ElemType, "ElemType"); - - // If we are loading, we need to initialize the weights. - if (Archive::is_loading::value) - { - size_t weight = rbmPolicy.Parameters().n_elem; - positiveGradient.set_size(weight, 1); - negativeGradient.set_size(weight, 1); - negativeSamples.set_size(rbmPolicy.VisibleSize(), 1); - tempNegativeGradient.set_size(weight, 1); - positiveGradient.zeros(); - negativeGradient.zeros(); - tempNegativeGradient.zeros(); - rbmPolicy.Parameters() = arma::Mat( - parameter.memptr(), weight, 1, false, false); - rbmPolicy.Reset(); - reset = true; - } -} - -} // namespace ann -} // namespace mlpack -#endif // MLPACK_METHODS_ANN_RBM_IMPL_HPP diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 08f400f941..69168afaba 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -70,14 +70,6 @@ add_executable(mlpack_test loss_functions_test.cpp lrsdp_test.cpp lsh_test.cpp - main_tests/decision_tree_test.cpp - main_tests/emst_test.cpp - main_tests/linear_regression_test.cpp - main_tests/pca_test.cpp - main_tests/preprocess_binarize_test.cpp - main_tests/preprocess_imputer_test.cpp - main_tests/preprocess_split_test.cpp - main_tests/test_helper.hpp math_test.cpp matrix_completion_test.cpp maximal_inputs_test.cpp diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 46874f291b..82a9339bab 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -1331,40 +1331,6 @@ BOOST_AUTO_TEST_CASE(SimpleBilinearInterpolationLayerTest) std::move(unzoomedOutput)); CheckMatrices(unzoomedOutput - expectedOutput, arma::zeros(input.n_rows), 1e-12); - -/* - * Simple add module test. - */ -BOOST_AUTO_TEST_CASE(SimpleBinaryRbmLayerTest) -{ - arma::mat output, input, outputModule, inputModule; - // Network 1 - BinaryLayer<> module(2, 4); - module.Parameters().ones(); - module.Reset(); - - // Network 2 - LinearNoBias<> linear(2, 4); - Add<> add(4); - SigmoidLayer<> sigmoid; - linear.Parameters().ones(); - add.Parameters().ones(); - linear.Reset(); - - // Check reset function - BOOST_REQUIRE_EQUAL(module.Bias().size(), 2); - BOOST_REQUIRE_EQUAL(arma::accu(module.Bias() - arma::ones(2)), 0); - - // Test the Forward function. - input = arma::vec("0.5 0.5"); - inputModule = arma::vec("0.5 0.5"); - module.Forward(std::move(inputModule), std::move(outputModule)); - linear.Forward(std::move(input), std::move(output)); - add.Forward(std::move(output), std::move(output)); - sigmoid.Forward(std::move(output), std::move(output)); - - for (size_t i = 0; i < output.size(); i++) - BOOST_REQUIRE_EQUAL(output(i), outputModule(i)); } /** diff --git a/src/mlpack/tests/rbm_network_test.cpp b/src/mlpack/tests/rbm_network_test.cpp index 5c5b9c8836..3944de4c8b 100644 --- a/src/mlpack/tests/rbm_network_test.cpp +++ b/src/mlpack/tests/rbm_network_test.cpp @@ -1,8 +1,9 @@ /** * @file rbm_network_test.cpp * @author Kris Singh + * @author Shikhar Jaiswal * - * Tests the rbm Network + * Tests the RBM Network * * digits dataset source: * @misc{Lichman:2013 , @@ -14,17 +15,15 @@ * Irvine, School of Information and Computer Sciences" } * * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include #include #include -#include -#include -#include +#include #include #include #include @@ -37,10 +36,11 @@ using namespace mlpack::ann; using namespace mlpack::optimization; using namespace mlpack::regression; -BOOST_AUTO_TEST_SUITE(RbmNetworkTest); +BOOST_AUTO_TEST_SUITE(RBMNetworkTest); + BOOST_AUTO_TEST_CASE(ClassificationTest) { - // Normalised dataset + // Normalised dataset. int hiddenLayerSize = 100; size_t batchSize = 10; size_t numEpoches = 30; @@ -68,31 +68,29 @@ BOOST_AUTO_TEST_CASE(ClassificationTest) XRbm.zeros(); YRbm.zeros(); - BinaryRBMPolicy<> binary_rbm(trainData.n_rows, hiddenLayerSize); GaussianInitialization gaussian(0, 0.1); - RBM> model(trainData, - gaussian, binary_rbm, 1, 1, true, false); + RBM model(trainData, + gaussian, trainData.n_rows, hiddenLayerSize, batchSize); size_t numRBMIterations = trainData.n_cols * numEpoches; numRBMIterations /= batchSize; - - MiniBatchSGD msgd(batchSize, 0.06, numRBMIterations, 0, true); + optimization::StandardSGD msgd(0.06, batchSize, numRBMIterations, 0, true); model.Reset(); - model.Policy().VisibleBias().ones(); - model.Policy().HiddenBias().ones(); - // test the reset function - model.Train(trainData, msgd); + model.VisibleBias().ones(); + model.HiddenBias().ones(); + + // Test the reset function. + model.Train(msgd); for (size_t i = 0; i < trainData.n_cols; i++) { - model.Policy().HiddenMean(std::move(trainData.col(i)), - std::move(output)); + model.HiddenMean(std::move(trainData.col(i)), std::move(output)); XRbm.col(i) = output; } for (size_t i = 0; i < testData.n_cols; i++) { - model.Policy().HiddenMean(std::move(testData.col(i)), + model.HiddenMean(std::move(testData.col(i)), std::move(output)); YRbm.col(i) = output; } @@ -107,14 +105,13 @@ BOOST_AUTO_TEST_CASE(ClassificationTest) double classificationAccuray = regressor.ComputeAccuracy(testData, testLabels); - std::cout << "Softmax Accuracy = " << classificationAccuray << std::endl; L_BFGS rbmOptimizer(numBasis, numIterations); SoftmaxRegression rbmRegressor(XRbm, trainLabels, numClasses, 0.001, false, rbmOptimizer); double rbmClassificationAccuracy = rbmRegressor.ComputeAccuracy(YRbm, testLabels); - std::cout << "RBM Accuracy = " << rbmClassificationAccuracy << std::endl; + BOOST_REQUIRE_GE(rbmClassificationAccuracy, classificationAccuray); } @@ -164,29 +161,29 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) YRbm.zeros(); double slabPenalty = 8; - SpikeSlabRBMPolicy<> ss_rbm(trainData.n_rows, hiddenLayerSize, poolSize, + RBM modelssRBM(trainData, + gaussian, trainData.n_rows, hiddenLayerSize, batchSize, 1, 1, poolSize, slabPenalty, radius); - RBM> modelssRBM(trainData, - gaussian, ss_rbm, 1, 1, true, false); size_t numRBMIterations = trainData.n_cols * numEpoches; numRBMIterations /= batchSize; - MiniBatchSGD msgd(batchSize, 0.02, numRBMIterations, 0, true); + optimization::StandardSGD msgd(0.02, batchSize, numRBMIterations, 0, true); modelssRBM.Reset(); - modelssRBM.Policy().VisiblePenalty().fill(5); - modelssRBM.Policy().SpikeBias().fill(1); - modelssRBM.Train(trainData, msgd); + modelssRBM.VisiblePenalty().fill(5); + modelssRBM.SpikeBias().fill(1); + + modelssRBM.Train(msgd); for (size_t i = 0; i < trainData.n_cols; i++) { - modelssRBM.Policy().HiddenMean(std::move(trainData.col(i)), + modelssRBM.HiddenMean(std::move(trainData.col(i)), std::move(output)); XRbm.col(i) = output; } for (size_t i = 0; i < testData.n_cols; i++) { - modelssRBM.Policy().HiddenMean(std::move(testData.col(i)), + modelssRBM.HiddenMean(std::move(testData.col(i)), std::move(output)); YRbm.col(i) = output; } @@ -199,7 +196,6 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) 0.001, false, ssRbmOptimizer); double ssRbmClassificationAccuracy = ssRbmRegressor.ComputeAccuracy( YRbm, testLabels); - std::cout << "ssRBM Accuracy = " << ssRbmClassificationAccuracy << std::endl; BOOST_REQUIRE_GE(ssRbmClassificationAccuracy, 76.18); } @@ -209,13 +205,12 @@ void BuildVanillaNetwork(MatType& trainData, const size_t hiddenLayerSize) { MatType output; - BinaryRBMPolicy binary_rbm(trainData.n_rows, hiddenLayerSize); GaussianInitialization gaussian(0, 0.1); - RBM> model(trainData, - gaussian, binary_rbm, 1, true); + RBM model(trainData, gaussian, + trainData.n_rows, hiddenLayerSize, 1, 1, 1, 2, 8, 1, true); model.Reset(); - // Set the parmaeters from a learned rbm sklearn random state 23 + // Set the parameters from a learned RBM Sklearn random state 23 model.Parameters() = MatType( "-0.23224054, -0.23000632, -0.25701271, -0.25122418, -0.20716651," "-0.20962217, -0.59922456, -0.60003836, -0.6, -0.625, -0.475;"); @@ -223,21 +218,21 @@ void BuildVanillaNetwork(MatType& trainData, // Check free energy arma::Mat freeEnergy = MatType( "-0.87523715, 0.50615066, 0.46923476, 1.21509084;"); - arma::vec calcultedFreeEnergy(4); - calcultedFreeEnergy.zeros(); + arma::vec calculatedFreeEnergy(4); + calculatedFreeEnergy.zeros(); for (size_t i = 0; i < trainData.n_cols; i++) { - calcultedFreeEnergy(i) = model.FreeEnergy(std::move(trainData.col(i))); + calculatedFreeEnergy(i) = model.FreeEnergy(std::move(trainData.col(i))); } for (size_t i = 0; i < freeEnergy.n_elem; i++) - BOOST_REQUIRE_CLOSE(calcultedFreeEnergy(i), freeEnergy(i), 1e-3); + BOOST_REQUIRE_CLOSE(calculatedFreeEnergy(i), freeEnergy(i), 1e-3); } BOOST_AUTO_TEST_CASE(MiscTest) { /** - * Train and evaluate a vanilla network with the specified structure. + * Train and evaluate a Vanilla network with the specified structure. */ arma::Mat X = arma::Mat("0.0, 0.0, 0.0;" @@ -247,4 +242,5 @@ BOOST_AUTO_TEST_CASE(MiscTest) X = X.t(); BuildVanillaNetwork>(X, 2); } + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index ebc59c214b..7454e0a06f 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -34,9 +34,7 @@ #include #include #include -#include -#include -#include +#include #include using namespace mlpack; @@ -1706,32 +1704,31 @@ BOOST_AUTO_TEST_CASE(BinaryRBMTest) size_t hiddenLayerSize = 5; data.randu(3, 100); - BinaryRBMPolicy<> binary_rbm(data.n_rows, hiddenLayerSize); GaussianInitialization gaussian(0, 0.1); - RBM > Rbm(data, - gaussian, binary_rbm, 1, true, true); - RBM > RbmXml(data, - gaussian, binary_rbm, 1, true, true); - RBM > RbmText(data, - gaussian, binary_rbm, 1, true, true); - RBM > RbmBinary(data, - gaussian, binary_rbm, 1, true, true); + RBM Rbm(data, gaussian, data.n_rows, hiddenLayerSize, + 1, 1, 1, 2, 8, 1, true); + RBM RbmXml(data, gaussian, data.n_rows, + hiddenLayerSize, 1, 1, 1, 2, 8, 1, true); + RBM RbmText(data, gaussian, data.n_rows, + hiddenLayerSize, 1, 1, 1, 2, 8, 1, true); + RBM RbmBinary(data, gaussian, data.n_rows, + hiddenLayerSize, 1, 1, 1, 2, 8, 1, true); Rbm.Reset(); SerializeObjectAll(Rbm, RbmXml, RbmText, RbmBinary); CheckMatrices(Rbm.Parameters(), RbmXml.Parameters(), RbmText.Parameters(), RbmBinary.Parameters()); - CheckMatrices(Rbm.Policy().VisibleBias(), RbmXml.Policy().VisibleBias()); - CheckMatrices(Rbm.Policy().VisibleBias(), RbmText.Policy().VisibleBias()); - CheckMatrices(Rbm.Policy().VisibleBias(), RbmBinary.Policy().VisibleBias()); + CheckMatrices(Rbm.VisibleBias(), RbmXml.VisibleBias()); + CheckMatrices(Rbm.VisibleBias(), RbmText.VisibleBias()); + CheckMatrices(Rbm.VisibleBias(), RbmBinary.VisibleBias()); - CheckMatrices(Rbm.Policy().HiddenBias(), RbmXml.Policy().HiddenBias()); - CheckMatrices(Rbm.Policy().HiddenBias(), RbmText.Policy().HiddenBias()); - CheckMatrices(Rbm.Policy().HiddenBias(), RbmBinary.Policy().HiddenBias()); + CheckMatrices(Rbm.HiddenBias(), RbmXml.HiddenBias()); + CheckMatrices(Rbm.HiddenBias(), RbmText.HiddenBias()); + CheckMatrices(Rbm.HiddenBias(), RbmBinary.HiddenBias()); - CheckMatrices(Rbm.Policy().Weight(), RbmXml.Policy().Weight()); - CheckMatrices(Rbm.Policy().Weight(), RbmText.Policy().Weight()); - CheckMatrices(Rbm.Policy().Weight(), RbmBinary.Policy().Weight()); + CheckMatrices(Rbm.Weight(), RbmXml.Weight()); + CheckMatrices(Rbm.Weight(), RbmText.Weight()); + CheckMatrices(Rbm.Weight(), RbmBinary.Weight()); } /** @@ -1755,45 +1752,38 @@ BOOST_AUTO_TEST_CASE(ssRBMTest) size_t poolSize = 1; - SpikeSlabRBMPolicy<> ss_rbm(data.n_rows, hiddenLayerSize, poolSize, - slabPenalty, radius); GaussianInitialization gaussian(0, 0.1); - RBM > Rbm(data, - gaussian, ss_rbm, 1, true, true); - RBM> RbmXml(data, - gaussian, ss_rbm, 1, true, true); - RBM> RbmText(data, - gaussian, ss_rbm, 1, true, true); - RBM> RbmBinary(data, - gaussian, ss_rbm, 1, true, true); + RBM Rbm(data, gaussian, + data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius, + true); + RBM RbmXml(data, gaussian, + data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius, + true); + RBM RbmText(data, gaussian, + data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius, + true); + RBM RbmBinary(data, gaussian, + data.n_rows, hiddenLayerSize, 1, 1, 1, poolSize, slabPenalty, radius, + true); Rbm.Reset(); - Rbm.Policy().VisiblePenalty().fill(15); - Rbm.Policy().SpikeBias().ones(); + Rbm.VisiblePenalty().fill(15); + Rbm.SpikeBias().ones(); SerializeObjectAll(Rbm, RbmXml, RbmText, RbmBinary); CheckMatrices(Rbm.Parameters(), RbmXml.Parameters(), RbmText.Parameters(), RbmBinary.Parameters()); - CheckMatrices(Rbm.Policy().VisiblePenalty(), - RbmXml.Policy().VisiblePenalty()); - CheckMatrices(Rbm.Policy().VisiblePenalty(), - RbmText.Policy().VisiblePenalty()); - CheckMatrices(Rbm.Policy().VisiblePenalty(), - RbmBinary.Policy().VisiblePenalty()); + CheckMatrices(Rbm.VisiblePenalty(), RbmXml.VisiblePenalty()); + CheckMatrices(Rbm.VisiblePenalty(), RbmText.VisiblePenalty()); + CheckMatrices(Rbm.VisiblePenalty(), RbmBinary.VisiblePenalty()); - CheckMatrices(Rbm.Policy().SpikeBias(), - RbmXml.Policy().SpikeBias()); - CheckMatrices(Rbm.Policy().SpikeBias(), - RbmText.Policy().SpikeBias()); - CheckMatrices(Rbm.Policy().SpikeBias(), - RbmBinary.Policy().SpikeBias()); + CheckMatrices(Rbm.SpikeBias(), RbmXml.SpikeBias()); + CheckMatrices(Rbm.SpikeBias(), RbmText.SpikeBias()); + CheckMatrices(Rbm.SpikeBias(), RbmBinary.SpikeBias()); - CheckMatrices(Rbm.Policy().Weight(), - RbmXml.Policy().Weight()); - CheckMatrices(Rbm.Policy().Weight(), - RbmText.Policy().Weight()); - CheckMatrices(Rbm.Policy().Weight(), - RbmBinary.Policy().Weight()); + CheckMatrices(Rbm.WeightCube(), RbmXml.WeightCube()); + CheckMatrices(Rbm.WeightCube(), RbmText.WeightCube()); + CheckMatrices(Rbm.WeightCube(), RbmBinary.WeightCube()); } BOOST_AUTO_TEST_SUITE_END();