diff --git a/src/mlpack/methods/ann/performance_functions/sparse_function.hpp b/src/mlpack/methods/ann/performance_functions/sparse_function.hpp index cf46b04ba6..d8e0118d43 100644 --- a/src/mlpack/methods/ann/performance_functions/sparse_function.hpp +++ b/src/mlpack/methods/ann/performance_functions/sparse_function.hpp @@ -1,5 +1,6 @@ /** * @file sparse_function.hpp + * @author Siddharth Agrawal * @author Tham Ngap Wei * * Definition and implementation of the sparse performance function. diff --git a/src/mlpack/methods/sparse_autoencoder/CMakeLists.txt b/src/mlpack/methods/sparse_autoencoder/CMakeLists.txt index d0d958f66d..2165e4c71d 100644 --- a/src/mlpack/methods/sparse_autoencoder/CMakeLists.txt +++ b/src/mlpack/methods/sparse_autoencoder/CMakeLists.txt @@ -2,9 +2,6 @@ # Anything not in this list will not be compiled into mlpack. set(SOURCES sparse_autoencoder.hpp - sparse_autoencoder_impl.hpp - sparse_autoencoder_function.hpp - sparse_autoencoder_function_impl.hpp maximal_inputs.hpp maximal_inputs.cpp ) diff --git a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp index 8820662229..316b961348 100644 --- a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp +++ b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp @@ -1,17 +1,18 @@ /** - * @file one_hot_layer.hpp - * @author Shangtong Zhang + * @file sparse_autoencoder.hpp + * @author Siddharth Agrawal + * @author Tham Ngap Wei * - * Definition of the OneHotLayer class, which implements a standard network - * layer. + * Definition of the sparse autoencoder class, for automatic learning of + * representative features. */ -#ifndef __MLPACK_METHODS_ANN_SPARSE_AUTOENCODER_HPP -#define __MLPACK_METHODS_ANN_SPARSE_AUTOENCODER_HPP +#ifndef __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP +#define __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP #include -#include #include +#include #include #include @@ -21,37 +22,76 @@ #include #include +#include #include namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template, - typename OutputActivate = HiddenActivate, - typename MatType = arma::mat, - template class Optimize = RMSPROP, - typename HiddenLayer = SparseInputLayer - , - typename OutputLayer = SparseOutputLayer - - > +/** + * A sparse autoencoder is a neural network whose aim to learn compressed + * representations of the data, typically for dimensionality reduction, with a + * constraint on the activity of the neurons in the network. Sparse autoencoders + * can be stacked together to learn a hierarchy of features, which provide a + * better representation of the data for classification. This is a method used + * in the recently developed field of deep learning. More technical details + * about the model can be found on the following webpage: + * + * http://deeplearning.stanford.edu/wiki/index.php/UFLDL_Tutorial + * + * This implementation allows the use of arbitrary mlpack optimizers via the + * Optimizer template parameter. + * + * @tparam MatType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam Optimizer The optimizer to use; by default this is RMSPROP. + * + * @tparam HiddenActivate Activation function used for the hidden layer. + * + * @tparam OutputActivate Activation function used for the output layer. + * + * @tparam HiddenLayer The layer type of the hidden layer, this type must + * provide functions "Forward(const InputType& input, + * OutputType& output)" and "Backward(const DataType& input, + * const DataType& gy, DataType& g)" the InputType, OutpuType, DataType + * must be able to accept arma::mat. + * + * @tparam OutputLayer The layer type of the output, this type must + * provide functions "Forward(const InputType& input, + * OutputType& output)" and "Backward(const DataType& input, + * const DataType& gy, DataType& g)" the InputType, OutpuType, DataType + * must be able to accept arma::mat. + */ +template< + typename HiddenActivate = BaseLayer, + typename OutputActivate = HiddenActivate, + typename MatType = arma::mat, + template class Optimizer = RMSPROP, + typename HiddenLayer = SparseInputLayer< + Optimizer, RandomInitialization, MatType, MatType>, + typename OutputLayer = SparseOutputLayer< + Optimizer, RandomInitialization, MatType, MatType> +> class SparseAutoencoder -{ +{ + // Convenience typedefs for the internal model construction. using BiasLayer = - SparseBiasLayer; + SparseBiasLayer; using Network = std::tuple; + OutputLayer, BiasLayer, OutputActivate>; + + using FFNet = FFN >; - using FFNet = FFN>; public: /** - * Construct sparse autoencoder function - * @param visibleSize Visible size of the input data, it is same as the feature - * size of the training data + * Construct sparse autoencoder function. + * + * @param visibleSize Visible size of the input data, it is same as the + * feature size of the training data. * @param hiddenSize Hidden size of the hidden layer, usually it should be - * smaller than the visible size + * smaller than the visible size. * @param batchSize The batch size used to train the network. * @param lambda L2-regularization parameter. * @param beta KL divergence parameter. @@ -65,30 +105,63 @@ class SparseAutoencoder const double rho = 0.01) : range(std::sqrt(6) / std::sqrt(visibleSize + hiddenSize + 1)), encoder(std::make_tuple( - HiddenLayer(visibleSize, hiddenSize, {-range, range}, - lambda), - BiasLayer(hiddenSize, batchSize), - HiddenActivate(), - OutputLayer(hiddenSize, visibleSize, {-range, range}, - lambda, beta, rho), - BiasLayer(visibleSize, batchSize), - OutputActivate()), - oneHotLayer, - SparseErrorFunction(lambda, beta, rho)) + HiddenLayer(visibleSize, hiddenSize, {-range, range}, lambda), + BiasLayer(hiddenSize, batchSize), + HiddenActivate(), + OutputLayer(hiddenSize, visibleSize, {-range, range}, lambda, beta, rho), + BiasLayer(visibleSize, batchSize), + OutputActivate()), + oneHotLayer, + SparseErrorFunction(lambda, beta, rho)) { + // Nothing to do here. } /** - * Train the sparse autoencoder network + * Construct sparse autoencoder function. + * + * @param data Input data with each column as one example. + * @param visibleSize Visible size of the input data, it is same as the + * feature size of the training data. + * @param hiddenSize Hidden size of the hidden layer, usually it should be + * smaller than the visible size. + * @param batchSize The batch size used to train the network. + * @param lambda L2-regularization parameter. + * @param beta KL divergence parameter. + * @param rho Sparsity parameter. + */ + SparseAutoencoder(const MatType& input, + size_t visibleSize, + size_t hiddenSize, + const double lambda = 0.0001, + const double beta = 3, + const double rho = 0.01) : + range(std::sqrt(6) / std::sqrt(visibleSize + hiddenSize + 1)), + encoder(std::make_tuple( + HiddenLayer(visibleSize, hiddenSize, {-range, range}, lambda), + BiasLayer(hiddenSize, input.n_rows), + HiddenActivate(), + OutputLayer(hiddenSize, visibleSize, {-range, range}, lambda, beta, rho), + BiasLayer(visibleSize, input.n_rows), + OutputActivate()), + oneHotLayer, + SparseErrorFunction(lambda, beta, rho)), + input(input) + { + // Nothing to do here. + } + + /** + * Train the sparse autoencoder network. * * @param input Data used to train the network * @param maxEpochs The number of maximal trained iterations (0 means no - * limit). + * limit). * @param batchSize The batch size used to train the network. * @param tolerance Train the network until it converges against - * the specified threshold. + * the specified threshold. * @param shuffle If true, the order of the training set is shuffled; - * otherwise, each data is visited in linear order. + * otherwise, each data is visited in linear order. */ void Train(MatType const &input, const size_t maxEpochs = 0, @@ -96,89 +169,144 @@ class SparseAutoencoder const bool shuffle = true) { Trainer trainer(encoder, maxEpochs, - std::get<1>(encoder.Network()).BatchSize(), + std::get<1>(encoder.Model()).BatchSize(), tolerance, shuffle); trainer.Train(input, input, input, input); } /** - * Get the weights of decoder + * Evaluates the objective function of the sparse autoencoder model using the + * given parameters. The cost function has terms for the reconstruction + * error, regularization cost and the sparsity cost. The objective function + * takes a low value when the model is able to reconstruct the data well + * using weights which are low in value and when the average activations of + * neurons in the hidden layers agrees well with the sparsity parameter 'rho'. + * + * @param parameters Current values of the model parameters. + */ + double Evaluate(const arma::mat& parameters) + { + const size_t l1 = (parameters.n_rows - 1) / 2; + const size_t l2 = parameters.n_cols - 1; + const size_t l3 = 2 * l1; + + std::get<0>(encoder.Model()).Weights() = parameters.submat( + 0, 0, l1 - 1, l2 - 1); + std::get<1>(encoder.Model()).Weights() = parameters.submat( + 0, l2, l1 - 1, l2); + std::get<3>(encoder.Model()).Weights() = parameters.submat( + l1, 0, l3 - 1, l2 - 1).t(); + std::get<4>(encoder.Model()).Weights() = parameters.submat( + l3, 0, l3, l2 - 1).t(); + + encoder.FeedForward(input, input, error); + return encoder.Error(); + } + + /** + * Get the weights of decoder. + * * @return Weights of decoder */ MatType const& DecoderWeights() const { - return std::get<3>(encoder.Network()).Weights(); + return std::get<3>(encoder.Model()).Weights(); } MatType& DecoderWeights() { - return std::get<3>(encoder.Network()).Weights(); + return std::get<3>(encoder.Model()).Weights(); } /** - * Get the bias of decoder - * @return Bias of decoder + * Get the bias of decoder. + * + * @return Bias of decoder. */ MatType const& DecoderBias() const { - return std::get<4>(encoder.Network()).Weights(); + return std::get<4>(encoder.Model()).Weights(); } /** - * Get the weights of encoder - * @return Weights of encoder + * Get the weights of encoder. + * + * @return Weights of encoder. */ MatType const& EncoderWeights() const { - return std::get<0>(encoder.Network()).Weights(); + return std::get<0>(encoder.Model()).Weights(); } MatType& EncoderWeights() { - return std::get<0>(encoder.Network()).Weights(); + return std::get<0>(encoder.Model()).Weights(); } /** - * Get the bias of encoder - * @return bias of encoder + * Get the bias of encoder. + * + * @return bias of encoder. */ MatType const& EncoderBias() const { - return std::get<1>(encoder.Network()).Weights(); + return std::get<1>(encoder.Model()).Weights(); } /** - * Encode the input, in other words, extract the features - * of the input - * @param input input data - * @param output features extracted from input + * Encode the input, in other words, extract the features of the input. + * + * @param input input data. + * @param output features extracted from input. */ void EncodeInput(MatType const &input, MatType &output) const { - arma::mat const encodedInput = EncoderWeights() * input + - arma::repmat(EncoderBias(), - 1, input.n_cols); - std::get<2>(encoder.Network()).fn(encodedInput, output); - } + arma::mat const encodedInput = EncoderWeights() * input + arma::repmat( + EncoderBias(), 1, input.n_cols); + std::get<2>(encoder.Model()).fn(encodedInput, output); + } - private: + //! Get the constructed model object. + FFNet const& Model() const { return encoder; } + //! Modify the constructed model object. + FFNet& Model() { return encoder; } + + private: + //! Locally-stored parameter that specifies the weight initialization range. const double range; + + //! Locally-stored output layer. OneHotLayer oneHotLayer; + + //! Locally-stored sparse autoencoder object. FFNet encoder; + + //! Locally-stored error parameter. + MatType error; + + //! Locally-stored input parameter. + MatType input; }; -template class Optimize = RMSPROP> +// Convenience typedefs. + +/** + * Standard sparse autoencoder using the logistic activation function. + */ +template< + typename MatType = arma::mat, + template class Optimizer = RMSPROP +> using LogisticSparseAutoencoder = SparseAutoencoder< BaseLayer, BaseLayer, MatType, - Optimize, - SparseInputLayer, - SparseOutputLayer>; + Optimizer, + SparseInputLayer, + SparseOutputLayer >; -}; // namespace ann -}; // namespace mlpack +} // namespace ann +} // namespace mlpack #endif diff --git a/src/mlpack/tests/sparse_autoencoder_test.cpp b/src/mlpack/tests/sparse_autoencoder_test.cpp index 5d912acd73..354cf5a0ee 100644 --- a/src/mlpack/tests/sparse_autoencoder_test.cpp +++ b/src/mlpack/tests/sparse_autoencoder_test.cpp @@ -1,276 +1,206 @@ /** -* @file sparse_autoencoder_test.cpp -* @author Siddharth Agrawal -* -* Test the SparseAutoencoder class. -*/ -#include -#include + * @file sparse_autoencoder_test.cpp + * @author Siddharth Agrawal + * @author Tham Ngap Wei + * + * Test the SparseAutoencoder class. + */ +#include #include -#include #include #include "old_boost_test_definitions.hpp" using namespace mlpack; -using namespace arma; +using namespace mlpack::ann; -using FSigmoidLayer = ann::SigmoidLayer; - -//sparse autoencoder function -using SAEF = nn::SparseAutoencoderFunction; -//sparse autoencoder function greedy -using SAEFG = nn::SparseAutoencoderFunction; - -BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest2); +BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest); BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionEvaluate) { - const size_t vSize = 5; - const size_t hSize = 3; - const size_t r = 2 * hSize + 1; - const size_t c = vSize + 1; + const size_t vSize = 5; + const size_t hSize = 3; + const size_t r = 2 * hSize + 1; + const size_t c = vSize + 1; - // Simple fake dataset. - arma::mat data1("0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5"); - // Transpose of the above dataset. - arma::mat data2 = data1.t(); + // Simple fake dataset. + arma::mat data1("0.1 0.2 0.3 0.4 0.5;" + "0.1 0.2 0.3 0.4 0.5;" + "0.1 0.2 0.3 0.4 0.5;" + "0.1 0.2 0.3 0.4 0.5;" + "0.1 0.2 0.3 0.4 0.5"); - // Create a SparseAutoencoderFunction. Regularization and KL divergence terms - // ignored. - SAEF saf1(data1, vSize, hSize, 0, 0); + // Transpose of the above dataset. + arma::mat data2 = data1.t(); - // Test using first dataset. Values were calculated using Octave. - BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::ones(r, c)), 1.190472606540, 1e-5); - BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5); - BOOST_REQUIRE_CLOSE(saf1.Evaluate(-arma::ones(r, c)), 0.048800332266, 1e-5); + // Create a SparseAutoencoderFunction. Regularization and KL divergence terms + // ignored. + SparseAutoencoder<> saf1(data1, vSize, hSize, 0, 0); - // Create a SparseAutoencoderFunction. Regularization and KL divergence terms - // ignored. - SAEF saf2(data2, vSize, hSize, 0, 0); + // Test using first dataset. Values were calculated using Octave. + BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::ones(r, c)), 1.190472606540, 1e-5); + BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5); + BOOST_REQUIRE_CLOSE(saf1.Evaluate(-arma::ones(r, c)), 0.048800332266, 1e-5); - // Test using second dataset. Values were calculated using Octave. - BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::ones(r, c)), 1.197585812647, 1e-5); - BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5); - BOOST_REQUIRE_CLOSE(saf2.Evaluate(-arma::ones(r, c)), 0.063466617408, 1e-5); + // Create a SparseAutoencoderFunction. Regularization and KL divergence terms + // ignored. + SparseAutoencoder<> saf2(data2, vSize, hSize, 0, 0); + + // Test using second dataset. Values were calculated using Octave. + BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::ones(r, c)), 1.197585812647, 1e-5); + BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5); + BOOST_REQUIRE_CLOSE(saf2.Evaluate(-arma::ones(r, c)), 0.063466617408, 1e-5); } BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate) { - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l1 = hSize; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; + const size_t points = 1000; + const size_t trials = 50; + const size_t vSize = 20; + const size_t hSize = 10; + const size_t l1 = hSize; + const size_t l2 = vSize; + const size_t l3 = 2 * hSize; - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); + // Initialize a random dataset. + arma::mat data; + data.randu(vSize, points); - // Create a SparseAutoencoderFunction. Regularization and KL divergence terms - // ignored. - SAEF saf(data, vSize, hSize, 0, 0); + // Create a SparseAutoencoderFunction. Regularization and KL divergence terms + // ignored. + SparseAutoencoder<> saf(data, vSize, hSize, 0, 0); - // Run a number of trials. - for (size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); + // Run a number of trials. + for (size_t i = 0; i < trials; i++) + { + // Create a random set of parameters. + arma::mat parameters; + parameters.randu(l3 + 1, l2 + 1); - double reconstructionError = 0; + double reconstructionError = 0; - // Compute error for each training example. - for (size_t j = 0; j < points; j++) - { - arma::mat hiddenLayer, outputLayer, diff; + // Compute error for each training example. + for (size_t j = 0; j < points; j++) + { + arma::mat hiddenLayer, outputLayer, diff; - hiddenLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * - data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - outputLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1, l2 - 1).t() - * hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t()))); - diff = outputLayer - data.col(j); + hiddenLayer = 1.0 / + (1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * + data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff)); - } - reconstructionError /= points; + outputLayer = 1.0 / + (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1, l2 - 1).t() + * hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t()))); - // Compare with the value returned by the function. - BOOST_REQUIRE_CLOSE(saf.Evaluate(parameters), reconstructionError, 1e-5); - } + diff = outputLayer - data.col(j); + + reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff)); + } + reconstructionError /= points; + + // Compare with the value returned by the function. + BOOST_REQUIRE_CLOSE(saf.Evaluate(parameters), reconstructionError, 1e-5); + } } BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRegularizationEvaluate) { - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; + const size_t points = 1000; + const size_t trials = 50; + const size_t vSize = 20; + const size_t hSize = 10; + const size_t l2 = vSize; + const size_t l3 = 2 * hSize; - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); + // Initialize a random dataset. + arma::mat data; + data.randu(vSize, points); - // 3 objects for comparing regularization costs. - SAEF safNoReg(data, vSize, hSize, 0, 0); - SAEF safSmallReg(data, vSize, hSize, 0.5, 0); - SAEF safBigReg(data, vSize, hSize, 20, 0); + // 3 objects for comparing regularization costs. + SparseAutoencoder<> safNoReg(data, vSize, hSize, 0, 0); + SparseAutoencoder<> safSmallReg(data, vSize, hSize, 0.5, 0); + SparseAutoencoder<> safBigReg(data, vSize, hSize, 20, 0); - // Run a number of trials. - for (size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); + // Run a number of trials. + for (size_t i = 0; i < trials; i++) + { + // Create a random set of parameters. + arma::mat parameters; + parameters.randu(l3 + 1, l2 + 1); - double wL2SquaredNorm; + double wL2SquaredNorm; - wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) % - parameters.submat(0, 0, l3 - 1, l2 - 1)); + wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) % + parameters.submat(0, 0, l3 - 1, l2 - 1)); - // Calculate regularization terms. - const double smallRegTerm = 0.25 * wL2SquaredNorm; - const double bigRegTerm = 10 * wL2SquaredNorm; + // Calculate regularization terms. + const double smallRegTerm = 0.25 * wL2SquaredNorm; + const double bigRegTerm = 10 * wL2SquaredNorm; - BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + smallRegTerm, - safSmallReg.Evaluate(parameters), 1e-5); - BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + bigRegTerm, - safBigReg.Evaluate(parameters), 1e-5); - } + BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + smallRegTerm, + safSmallReg.Evaluate(parameters), 1e-5); + BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + bigRegTerm, + safBigReg.Evaluate(parameters), 1e-5); + } } BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate) { - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l1 = hSize; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; + const size_t points = 1000; + const size_t trials = 50; + const size_t vSize = 20; + const size_t hSize = 10; + const size_t l1 = hSize; + const size_t l2 = vSize; + const size_t l3 = 2 * hSize; - const double rho = 0.01; + const double rho = 0.01; - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); + // Initialize a random dataset. + arma::mat data; + data.randu(vSize, points); - // 3 objects for comparing divergence costs. - SAEF safNoDiv(data, vSize, hSize, 0, 0, rho); - SAEF safSmallDiv(data, vSize, hSize, 0, 5, rho); - SAEF safBigDiv(data, vSize, hSize, 0, 20, rho); + // 3 objects for comparing divergence costs. + SparseAutoencoder<> safNoDiv(data, vSize, hSize, 0, 0, rho); + SparseAutoencoder<> safSmallDiv(data, vSize, hSize, 0, 5, rho); + SparseAutoencoder<> safBigDiv(data, vSize, hSize, 0, 20, rho); - // Run a number of trials. - for (size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); + // Run a number of trials. + for (size_t i = 0; i < trials; i++) + { + // Create a random set of parameters. + arma::mat parameters; + parameters.randu(l3 + 1, l2 + 1); - arma::mat rhoCap; - rhoCap.zeros(hSize, 1); + arma::mat rhoCap; + rhoCap.zeros(hSize, 1); - // Compute hidden layer activations for each example. - for (size_t j = 0; j < points; j++) - { - arma::mat hiddenLayer; + // Compute hidden layer activations for each example. + for (size_t j = 0; j < points; j++) + { + arma::mat hiddenLayer; - hiddenLayer = 1.0 / (1 + - arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * - data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - rhoCap += hiddenLayer; - } - rhoCap /= points; + hiddenLayer = 1.0 / (1 + + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * + data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - // Calculate divergence terms. - const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) + - (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); - const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) + - (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); + rhoCap += hiddenLayer; + } + rhoCap /= points; - BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + smallDivTerm, - safSmallDiv.Evaluate(parameters), 1e-5); - BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + bigDivTerm, - safBigDiv.Evaluate(parameters), 1e-5); - } -} + // Calculate divergence terms. + const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) + + (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionGradient) -{ - const size_t points = 1000; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; + const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) + + (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); - - // 3 objects for 3 terms in the cost function. Each term contributes towards - // the gradient and thus need to be checked independently. - SAEFG saf1(data, vSize, hSize, 0, 0); - SAEFG saf2(data, vSize, hSize, 20, 0); - SAEFG saf3(data, vSize, hSize, 20, 20); - - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - // Get gradients for the current parameters. - arma::mat gradient1, gradient2, gradient3; - saf1.Gradient(parameters, gradient1); - saf2.Gradient(parameters, gradient2); - saf3.Gradient(parameters, gradient3); - - // Perturbation constant. - const double epsilon = 0.0001; - double costPlus1, costMinus1, numGradient1; - double costPlus2, costMinus2, numGradient2; - double costPlus3, costMinus3, numGradient3; - - // For each parameter. - for (size_t i = 0; i <= l3; i++) - { - for (size_t j = 0; j <= l2; j++) - { - // Perturb parameter with a positive constant and get costs. - parameters(i, j) += epsilon; - costPlus1 = saf1.Evaluate(parameters); - costPlus2 = saf2.Evaluate(parameters); - costPlus3 = saf3.Evaluate(parameters); - - // Perturb parameter with a negative constant and get costs. - parameters(i, j) -= 2 * epsilon; - costMinus1 = saf1.Evaluate(parameters); - costMinus2 = saf2.Evaluate(parameters); - costMinus3 = saf3.Evaluate(parameters); - - // Compute numerical gradients using the costs calculated above. - numGradient1 = (costPlus1 - costMinus1) / (2 * epsilon); - numGradient2 = (costPlus2 - costMinus2) / (2 * epsilon); - numGradient3 = (costPlus3 - costMinus3) / (2 * epsilon); - - // Restore the parameter value. - parameters(i, j) += epsilon; - - // Compare numerical and backpropagation gradient values. - BOOST_REQUIRE_CLOSE(numGradient1, gradient1(i, j), 1e-2); - BOOST_REQUIRE_CLOSE(numGradient2, gradient2(i, j), 1e-2); - BOOST_REQUIRE_CLOSE(numGradient3, gradient3(i, j), 1e-2); - } - } + BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + smallDivTerm, + safSmallDiv.Evaluate(parameters), 1e-5); + BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + bigDivTerm, + safBigDiv.Evaluate(parameters), 1e-5); + } } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/sparse_autoencoder_test_2.cpp b/src/mlpack/tests/sparse_autoencoder_test_2.cpp deleted file mode 100644 index 24fffcddde..0000000000 --- a/src/mlpack/tests/sparse_autoencoder_test_2.cpp +++ /dev/null @@ -1,306 +0,0 @@ -/** - * @file sparse_autoencoder_test.cpp - * @author Siddharth Agrawal - * - * Test the SparseAutoencoder class. - */ -#include -#include -#include - -#include -#include - -#include -#include "old_boost_test_definitions.hpp" - -using namespace mlpack; -using namespace arma; - -using SigmoidLayer = nn::SigmoidLayer; - -//sparse autoencoder function -using SAEF = nn::SparseAutoencoderFunction; -//sparse autoencoder function greedy -using SAEFG = nn::SparseAutoencoderFunction; - -BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest2); - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionEvaluate) -{ - const size_t vSize = 5; - const size_t hSize = 3; - const size_t r = 2 * hSize + 1; - const size_t c = vSize + 1; - - // Simple fake dataset. - arma::mat data1("0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5"); - // Transpose of the above dataset. - arma::mat data2 = data1.t(); - - // Create a SparseAutoencoderFunction. Regularization and KL divergence terms - // ignored. - SAEF saf1(data1, vSize, hSize, 0, 0); - - // Test using first dataset. Values were calculated using Octave. - BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::ones(r, c)), 1.190472606540, 1e-5); - BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5); - BOOST_REQUIRE_CLOSE(saf1.Evaluate(-arma::ones(r, c)), 0.048800332266, 1e-5); - - // Create a SparseAutoencoderFunction. Regularization and KL divergence terms - // ignored. - SAEF saf2(data2, vSize, hSize, 0, 0); - - // Test using second dataset. Values were calculated using Octave. - BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::ones(r, c)), 1.197585812647, 1e-5); - BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5); - BOOST_REQUIRE_CLOSE(saf2.Evaluate(-arma::ones(r, c)), 0.063466617408, 1e-5); -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate) -{ - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l1 = hSize; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; - - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); - - // Create a SparseAutoencoderFunction. Regularization and KL divergence terms - // ignored. - SAEF saf(data, vSize, hSize, 0, 0); - - // Run a number of trials. - for (size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - double reconstructionError = 0; - - // Compute error for each training example. - for (size_t j = 0; j < points; j++) - { - arma::mat hiddenLayer, outputLayer, diff; - - hiddenLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * - data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - outputLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1,l2 - 1).t() - * hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t()))); - diff = outputLayer - data.col(j); - - reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff)); - } - reconstructionError /= points; - - // Compare with the value returned by the function. - BOOST_REQUIRE_CLOSE(saf.Evaluate(parameters), reconstructionError, 1e-5); - } -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRegularizationEvaluate) -{ - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; - - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); - - // 3 objects for comparing regularization costs. - SAEF safNoReg(data, vSize, hSize, 0, 0); - SAEF safSmallReg(data, vSize, hSize, 0.5, 0); - SAEF safBigReg(data, vSize, hSize, 20, 0); - - // Run a number of trials. - for (size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - double wL2SquaredNorm; - - wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) % - parameters.submat(0, 0, l3 - 1, l2 - 1)); - - // Calculate regularization terms. - const double smallRegTerm = 0.25 * wL2SquaredNorm; - const double bigRegTerm = 10 * wL2SquaredNorm; - - BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + smallRegTerm, - safSmallReg.Evaluate(parameters), 1e-5); - BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + bigRegTerm, - safBigReg.Evaluate(parameters), 1e-5); - } -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate) -{ - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l1 = hSize; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; - - const double rho = 0.01; - - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); - - // 3 objects for comparing divergence costs. - SAEF safNoDiv(data, vSize, hSize, 0, 0, rho); - SAEF safSmallDiv(data, vSize, hSize, 0, 5, rho); - SAEF safBigDiv(data, vSize, hSize, 0, 20, rho); - - // Run a number of trials. - for(size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - arma::mat rhoCap; - rhoCap.zeros(hSize, 1); - - // Compute hidden layer activations for each example. - for (size_t j = 0; j < points; j++) - { - arma::mat hiddenLayer; - - hiddenLayer = 1.0 / (1 + - arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * - data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - rhoCap += hiddenLayer; - } - rhoCap /= points; - - // Calculate divergence terms. - const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) + - (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); - const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) + - (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); - - BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + smallDivTerm, - safSmallDiv.Evaluate(parameters), 1e-5); - BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + bigDivTerm, - safBigDiv.Evaluate(parameters), 1e-5); - } -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionGradient) -{ - const size_t points = 1000; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; - - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); - - // 3 objects for 3 terms in the cost function. Each term contributes towards - // the gradient and thus need to be checked independently. - SAEFG saf1(data, vSize, hSize, 0, 0); - SAEFG saf2(data, vSize, hSize, 20, 0); - SAEFG saf3(data, vSize, hSize, 20, 20); - - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - // Get gradients for the current parameters. - arma::mat gradient1, gradient2, gradient3; - saf1.Gradient(parameters, gradient1); - saf2.Gradient(parameters, gradient2); - saf3.Gradient(parameters, gradient3); - - // Perturbation constant. - const double epsilon = 0.0001; - double costPlus1, costMinus1, numGradient1; - double costPlus2, costMinus2, numGradient2; - double costPlus3, costMinus3, numGradient3; - - // For each parameter. - for (size_t i = 0; i <= l3; i++) - { - for (size_t j = 0; j <= l2; j++) - { - // Perturb parameter with a positive constant and get costs. - parameters(i, j) += epsilon; - costPlus1 = saf1.Evaluate(parameters); - costPlus2 = saf2.Evaluate(parameters); - costPlus3 = saf3.Evaluate(parameters); - - // Perturb parameter with a negative constant and get costs. - parameters(i, j) -= 2 * epsilon; - costMinus1 = saf1.Evaluate(parameters); - costMinus2 = saf2.Evaluate(parameters); - costMinus3 = saf3.Evaluate(parameters); - - // Compute numerical gradients using the costs calculated above. - numGradient1 = (costPlus1 - costMinus1) / (2 * epsilon); - numGradient2 = (costPlus2 - costMinus2) / (2 * epsilon); - numGradient3 = (costPlus3 - costMinus3) / (2 * epsilon); - - // Restore the parameter value. - parameters(i, j) += epsilon; - - // Compare numerical and backpropagation gradient values. - BOOST_REQUIRE_CLOSE(numGradient1, gradient1(i, j), 1e-2); - BOOST_REQUIRE_CLOSE(numGradient2, gradient2(i, j), 1e-2); - BOOST_REQUIRE_CLOSE(numGradient3, gradient3(i, j), 1e-2); - } - } -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderMoveTest) -{ - using SAE = mlpack::nn::SparseAutoencoder<>; - - SAE sae1(2, 2); - sae1.Lambda(0.45); - sae1.Beta(0.33); - sae1.Rho(0.11); - const auto Params = sae1.Parameters(); - - SAE sae2(std::move(sae1)); - BOOST_REQUIRE(sae1.Parameters().n_elem == 0); - BOOST_REQUIRE(sae1.VisibleSize() == sae2.VisibleSize()); - BOOST_REQUIRE(sae1.HiddenSize() == sae2.HiddenSize()); - BOOST_REQUIRE_CLOSE(sae1.Lambda(), sae2.Lambda(), 1e-5); - BOOST_REQUIRE_CLOSE(sae1.Beta(), sae2.Beta(), 1e-5); - BOOST_REQUIRE_CLOSE(sae1.Rho(), sae2.Rho(), 1e-5); - - const auto Params2 = sae2.Parameters(); - auto func = [](double lhs, double rhs) - { - BOOST_REQUIRE_CLOSE(lhs, rhs, 1e-5); - return true; - }; - std::equal(std::begin(Params), std::end(Params), - std::begin(Params2), - func); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/sparse_autoencoder_test_3.cpp b/src/mlpack/tests/sparse_autoencoder_test_3.cpp deleted file mode 100644 index b992913809..0000000000 --- a/src/mlpack/tests/sparse_autoencoder_test_3.cpp +++ /dev/null @@ -1,381 +0,0 @@ -/** - * @file sparse_autoencoder_test_3.cpp - * @author Tham Ngap Wei - * - * Test the SparseAutoencoder class. - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include - -#include -#include "old_boost_test_definitions.hpp" - -using namespace mlpack; -using namespace arma; -using namespace mlpack::ann; - -BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest3); - -using Network = std::tuple, SparseBiasLayer<>, - BaseLayer, SparseOutputLayer<>, SparseBiasLayer<>, - BaseLayer >; - -using FFNet = FFN >; - -Network create_network(size_t visibleSize, size_t hiddenSize, - size_t sampleSize, double lambda = 0.0001, - double beta = 3, double rho = 0.01) -{ - const double range = std::sqrt(6) / std::sqrt(visibleSize + hiddenSize + 1); - - SparseInputLayer<> hiddenLayer(visibleSize, hiddenSize, {-range, range}, - lambda); - SparseBiasLayer<> hiddenBiasLayer(hiddenSize, sampleSize); - BaseLayer hiddenBaseLayer; - - SparseOutputLayer<> outputLayer(hiddenSize, visibleSize, {-range, range}, - lambda, beta, rho); - SparseBiasLayer<> outputBiasLayer(visibleSize, sampleSize); - BaseLayer outputBaseLayer; - - return std::make_tuple(std::move(hiddenLayer), std::move(hiddenBiasLayer), - std::move(hiddenBaseLayer), std::move(outputLayer), - std::move(outputBiasLayer), std::move(outputBaseLayer)); -} - -FFNet create_ffn(size_t visibleSize, size_t hiddenSize, - size_t sampleSize, double lambda = 0.0001, - double beta = 3, double rho = 0.01) -{ - auto network = create_network(visibleSize, hiddenSize, - sampleSize, lambda, - beta, rho); - FFNet ffn(std::move(network), OneHotLayer(), - SparseErrorFunction<>(lambda,beta,rho)); - - return ffn; -} - -template -void initWeights(Net &net, UnaryFunc func) -{ - std::get<0>(net).Weights() = func(std::get<0>(net).Weights()); - std::get<1>(net).Weights() = func(std::get<1>(net).Weights()); - std::get<3>(net).Weights() = func(std::get<3>(net).Weights()); - std::get<4>(net).Weights() = func(std::get<4>(net).Weights()); -} - -template -void initWeights(Net &net, arma::mat const ¶meters) -{ - const size_t l1 = (parameters.n_rows - 1) / 2; - const size_t l2 = parameters.n_cols - 1; - const size_t l3 = 2 * l1; - std::get<0>(net).Weights() = parameters.submat(0, 0, l1-1, l2-1); - std::get<1>(net).Weights() = parameters.submat(0, l2, l1-1, l2); - std::get<3>(net).Weights() = parameters.submat(l1, 0, l3-1, l2-1).t(); - std::get<4>(net).Weights() = parameters.submat(l3, 0, l3, l2-1).t(); -} - -arma::mat initGradient(arma::mat const &input, - arma::mat const ¶meters, - FFNet &ffn) -{ - initWeights(ffn.Network(), parameters); - arma::mat error; - ffn.FeedForward(input, input, error); - ffn.FeedBackward(arma::mat(), error); - - arma::mat gradient = arma::zeros(parameters.n_rows, parameters.n_cols); - const size_t l1 = (parameters.n_rows - 1) / 2; - const size_t l2 = parameters.n_cols - 1; - const size_t l3 = 2 * l1; - auto const &net = ffn.Network(); - gradient.submat(0, 0, l1-1, l2-1) = std::get<0>(net).Gradient(); - gradient.submat(0, l2, l1-1, l2) = std::get<1>(net).Gradient(); - gradient.submat(l1, 0, l3-1, l2-1) = std::get<3>(net).Gradient().t(); - gradient.submat(l3, 0, l3, l2-1) = std::get<4>(net).Gradient().t(); - - return gradient; -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionEvaluate) -{ - const size_t vSize = 5; - const size_t hSize = 3; - - // Simple fake dataset. - arma::mat data1("0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5;" - "0.1 0.2 0.3 0.4 0.5"); - - // Create SparseAutoencoder. Regularization and KL divergence terms - // ignored. - auto ffn1 = create_ffn(vSize, hSize, 5, 0, 0); - initWeights(ffn1.Network(), [](arma::mat &v){ - return v.ones(); - }); - arma::mat error; - ffn1.FeedForward(data1, data1, error); - - auto ffn2 = create_ffn(vSize, hSize, 5, 0, 0); - initWeights(ffn2.Network(), [](arma::mat &v){ - return v.zeros(); - }); - ffn2.FeedForward(data1, data1, error); - - auto ffn3 = create_ffn(vSize, hSize, 5, 0, 0); - initWeights(ffn3.Network(), [](arma::mat &v){ - return -v.ones(); - }); - ffn3.FeedForward(data1, data1, error); - - // Test using first dataset. Values were calculated using Octave. - BOOST_REQUIRE_CLOSE(ffn1.Error(), 1.190472606540, 1e-5); - BOOST_REQUIRE_CLOSE(ffn2.Error(), 0.150000000000, 1e-5); - BOOST_REQUIRE_CLOSE(ffn3.Error(), 0.048800332266, 1e-5); - - arma::mat const data2 = data1.t(); - auto ffn4 = create_ffn(vSize, hSize, 5, 0, 0); - initWeights(ffn4.Network(), [](arma::mat &v){ - return v.ones(); - }); - ffn4.FeedForward(data2, data2, error); - - auto ffn5 = create_ffn(vSize, hSize, 5, 0, 0); - initWeights(ffn5.Network(), [](arma::mat &v){ - return v.zeros(); - }); - ffn5.FeedForward(data2, data2, error); - - auto ffn6 = create_ffn(vSize, hSize, 5, 0, 0); - initWeights(ffn6.Network(), [](arma::mat &v){ - return -v.ones(); - }); - ffn6.FeedForward(data2, data2, error); - - // Test using second dataset. Values were calculated using Octave. - BOOST_REQUIRE_CLOSE(ffn4.Error(), 1.197585812647, 1e-5); - BOOST_REQUIRE_CLOSE(ffn5.Error(), 0.150000000000, 1e-5); - BOOST_REQUIRE_CLOSE(ffn6.Error(), 0.063466617408, 1e-5); -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate) -{ - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l1 = hSize; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; - - // Initialize a random dataset. - arma::mat data1; - data1.randu(vSize, points); - - // Run a number of trials. - for(size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - double reconstructionError = 0; - - // Compute error for each training example. - for (size_t j = 0; j < points; j++) - { - arma::mat hiddenLayer, outputLayer, diff; - - hiddenLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * - data1.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - outputLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1,l2 - 1).t() - * hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t()))); - diff = outputLayer - data1.col(j); - - reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff)); - } - reconstructionError /= points; - - // Compare with the value returned by the function. - auto ffn = create_ffn(vSize, hSize, points, 0, 0); - auto &net = ffn.Network(); - initWeights(net, parameters); - arma::mat error; - ffn.FeedForward(data1, data1, error); - BOOST_REQUIRE_CLOSE(ffn.Error(), reconstructionError, 1e-5); - } -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRegularizationEvaluate) -{ - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; - - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); - - // Run a number of trials. - for (size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - double wL2SquaredNorm; - - wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) % - parameters.submat(0, 0, l3 - 1, l2 - 1)); - - // Calculate regularization terms. - const double smallRegTerm = 0.25 * wL2SquaredNorm; - const double bigRegTerm = 10 * wL2SquaredNorm; - - // 3 objects for comparing regularization costs. - auto safNoReg = create_ffn(vSize, hSize, points, 0, 0); - initWeights(safNoReg.Network(), parameters); - arma::mat error; - safNoReg.FeedForward(data, data, error); - - auto safSmallReg = create_ffn(vSize, hSize, points, 0.5, 0); - initWeights(safSmallReg.Network(), parameters); - safSmallReg.FeedForward(data, data, error); - - auto safBigReg = create_ffn(vSize, hSize, points, 20, 0); - initWeights(safBigReg.Network(), parameters); - safBigReg.FeedForward(data, data, error); - - BOOST_REQUIRE_CLOSE(safNoReg.Error() + smallRegTerm, - safSmallReg.Error(), 1e-5); - BOOST_REQUIRE_CLOSE(safNoReg.Error() + bigRegTerm, - safBigReg.Error(), 1e-5); - } -} - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate) -{ - const size_t points = 1000; - const size_t trials = 50; - const size_t vSize = 20; - const size_t hSize = 10; - const size_t l1 = hSize; - const size_t l2 = vSize; - const size_t l3 = 2 * hSize; - - const double rho = 0.01; - - // Initialize a random dataset. - arma::mat data; - data.randu(vSize, points); - - // Run a number of trials. - for(size_t i = 0; i < trials; i++) - { - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(l3 + 1, l2 + 1); - - arma::mat rhoCap; - rhoCap.zeros(hSize, 1); - - // Compute hidden layer activations for each example. - for (size_t j = 0; j < points; j++) - { - arma::mat hiddenLayer; - - hiddenLayer = 1.0 / (1 + - arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * - data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); - rhoCap += hiddenLayer; - } - rhoCap /= static_cast(points); - - // Calculate divergence terms. - const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) + - (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); - const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) + - (1 - rho) * arma::log((1 - rho) / (1 - rhoCap))); - - // 3 objects for comparing divergence costs. - auto safNoDiv = create_ffn(vSize, hSize, points, 0, 0, rho); - initWeights(safNoDiv.Network(), parameters); - arma::mat error; - safNoDiv.FeedForward(data, data, error); - - auto safSmallDiv = create_ffn(vSize, hSize, points, 0, 5, rho); - initWeights(safSmallDiv.Network(), parameters); - safSmallDiv.FeedForward(data, data, error); - - auto safBigDiv = create_ffn(vSize, hSize, points, 0, 20, rho); - initWeights(safBigDiv.Network(), parameters); - safBigDiv.FeedForward(data, data, error); - - BOOST_REQUIRE_CLOSE(safNoDiv.Error() + smallDivTerm, - safSmallDiv.Error(), 1e-5); - BOOST_REQUIRE_CLOSE(safNoDiv.Error() + bigDivTerm, - safBigDiv.Error(), 1e-5); - } -}//*/ - -BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionGradient) -{ - const arma::mat trainData("0.0013 0.5850 0.8228 0.7105;" - "0.1933 0.3503 0.1741 0.3040"); - const size_t visibleSize = trainData.n_rows; - const size_t hiddenSize = trainData.n_rows / 2; - - arma::mat const parameters( "-1.2029 -0.8175 0;" - " 0.0776 -0.1205 0;" - " 0 0 0"); - - auto ffn = create_ffn(visibleSize, hiddenSize, 4); - initGradient(trainData, parameters, ffn); - auto &net = ffn.Network(); - - arma::mat const w1 = std::get<0>(net).Gradient(); - arma::mat const b1 = std::get<1>(net).Gradient(); - arma::mat const w2 = std::get<3>(net).Gradient(); - arma::mat const b2 = std::get<4>(net).Gradient(); - - //these values come from original implementation - //because ffn do not provide ease to use Evalulate - //api to do the gradient checking, I prefer golden - //model(FeedForward api will alter inner value, - //everytime you call it, the error value will change) - BOOST_REQUIRE_CLOSE(w1(0, 0), 0.41743157405, 1e-5); - BOOST_REQUIRE_CLOSE(w1(0, 1), 0.21509458293, 1e-5); - BOOST_REQUIRE_CLOSE(b1(0, 0), 0.85303876677, 1e-5); - BOOST_REQUIRE_CLOSE(w2(0, 0), 0.00520531433, 1e-5); - BOOST_REQUIRE_CLOSE(w2(1, 0), 0.01858690990, 1e-5); - BOOST_REQUIRE_CLOSE(b2(0, 0), -0.00599756012, 1e-5); - BOOST_REQUIRE_CLOSE(b2(1, 0), 0.05881613659, 1e-5); -} - -BOOST_AUTO_TEST_SUITE_END();