apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use the sparse autoencoder class for the sparse autoencoder test.

Browse Source
This commit is contained in:
marcus
2016-01-31 23:54:32 +01:00
parent f34ae33e2c
commit 443ecdcd35
6 changed files with 346 additions and 977 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/performance_functions/sparse_function.hpp
+1
View File
@@ -1,5 +1,6 @@
/**
* @file sparse_function.hpp
* @author Siddharth Agrawal
* @author Tham Ngap Wei
*
* Definition and implementation of the sparse performance function.
src/mlpack/methods/sparse_autoencoder/CMakeLists.txt
-3
View File
@@ -2,9 +2,6 @@
# Anything not in this list will not be compiled into mlpack.
set(SOURCES
sparse_autoencoder.hpp
sparse_autoencoder_impl.hpp
sparse_autoencoder_function.hpp
sparse_autoencoder_function_impl.hpp
maximal_inputs.hpp
maximal_inputs.cpp
)
src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp
+197 -69
View File
@@ -1,17 +1,18 @@
/**
* @file one_hot_layer.hpp
* @author Shangtong Zhang
* @file sparse_autoencoder.hpp
* @author Siddharth Agrawal
* @author Tham Ngap Wei
*
* Definition of the OneHotLayer class, which implements a standard network
* layer.
* Definition of the sparse autoencoder class, for automatic learning of
* representative features.
*/
#ifndef __MLPACK_METHODS_ANN_SPARSE_AUTOENCODER_HPP
#define __MLPACK_METHODS_ANN_SPARSE_AUTOENCODER_HPP
#ifndef __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP
#define __MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP
#include <mlpack/core.hpp>
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
#include <mlpack/methods/ann/ffn.hpp>
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
#include <mlpack/methods/ann/init_rules/random_init.hpp>
#include <mlpack/methods/ann/layer/base_layer.hpp>
@@ -21,37 +22,76 @@
#include <mlpack/methods/ann/layer/sparse_output_layer.hpp>
#include <mlpack/methods/ann/optimizer/rmsprop.hpp>
#include <mlpack/methods/ann/trainer/trainer.hpp>
#include <mlpack/methods/ann/performance_functions/sparse_function.hpp>
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename HiddenActivate = BaseLayer<LogisticFunction>,
typename OutputActivate = HiddenActivate,
typename MatType = arma::mat,
template<typename,typename> class Optimize = RMSPROP,
typename HiddenLayer = SparseInputLayer
<Optimize, RandomInitialization, MatType, MatType>,
typename OutputLayer = SparseOutputLayer
<Optimize, RandomInitialization, MatType, MatType>
>
/**
* A sparse autoencoder is a neural network whose aim to learn compressed
* representations of the data, typically for dimensionality reduction, with a
* constraint on the activity of the neurons in the network. Sparse autoencoders
* can be stacked together to learn a hierarchy of features, which provide a
* better representation of the data for classification. This is a method used
* in the recently developed field of deep learning. More technical details
* about the model can be found on the following webpage:
*
* http://deeplearning.stanford.edu/wiki/index.php/UFLDL_Tutorial
*
* This implementation allows the use of arbitrary mlpack optimizers via the
* Optimizer template parameter.
*
* @tparam MatType Type of the input data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
* @tparam Optimizer The optimizer to use; by default this is RMSPROP.
*
* @tparam HiddenActivate Activation function used for the hidden layer.
*
* @tparam OutputActivate Activation function used for the output layer.
*
* @tparam HiddenLayer The layer type of the hidden layer, this type must
* provide functions "Forward(const InputType& input,
* OutputType& output)" and "Backward(const DataType& input,
* const DataType& gy, DataType& g)" the InputType, OutpuType, DataType
* must be able to accept arma::mat.
*
* @tparam OutputLayer The layer type of the output, this type must
* provide functions "Forward(const InputType& input,
* OutputType& output)" and "Backward(const DataType& input,
* const DataType& gy, DataType& g)" the InputType, OutpuType, DataType
* must be able to accept arma::mat.
*/
template<
typename HiddenActivate = BaseLayer<LogisticFunction>,
typename OutputActivate = HiddenActivate,
typename MatType = arma::mat,
template<typename, typename> class Optimizer = RMSPROP,
typename HiddenLayer = SparseInputLayer<
Optimizer, RandomInitialization, MatType, MatType>,
typename OutputLayer = SparseOutputLayer<
Optimizer, RandomInitialization, MatType, MatType>
>
class SparseAutoencoder
{
{
// Convenience typedefs for the internal model construction.
using BiasLayer =
SparseBiasLayer<Optimize, ZeroInitialization, MatType, MatType>;
SparseBiasLayer<Optimizer, ZeroInitialization, MatType, MatType>;
using Network = std::tuple<HiddenLayer, BiasLayer, HiddenActivate,
OutputLayer, BiasLayer, OutputActivate>;
OutputLayer, BiasLayer, OutputActivate>;
using FFNet = FFN<Network, OneHotLayer, SparseErrorFunction<MatType> >;
using FFNet = FFN<Network, OneHotLayer, SparseErrorFunction<MatType>>;
public:
/**
* Construct sparse autoencoder function
* @param visibleSize Visible size of the input data, it is same as the feature
* size of the training data
* Construct sparse autoencoder function.
*
* @param visibleSize Visible size of the input data, it is same as the
* feature size of the training data.
* @param hiddenSize Hidden size of the hidden layer, usually it should be
* smaller than the visible size
* smaller than the visible size.
* @param batchSize The batch size used to train the network.
* @param lambda L2-regularization parameter.
* @param beta KL divergence parameter.
@@ -65,30 +105,63 @@ class SparseAutoencoder
const double rho = 0.01) :
range(std::sqrt(6) / std::sqrt(visibleSize + hiddenSize + 1)),
encoder(std::make_tuple(
HiddenLayer(visibleSize, hiddenSize, {-range, range},
lambda),
BiasLayer(hiddenSize, batchSize),
HiddenActivate(),
OutputLayer(hiddenSize, visibleSize, {-range, range},
lambda, beta, rho),
BiasLayer(visibleSize, batchSize),
OutputActivate()),
oneHotLayer,
SparseErrorFunction<MatType>(lambda, beta, rho))
HiddenLayer(visibleSize, hiddenSize, {-range, range}, lambda),
BiasLayer(hiddenSize, batchSize),
HiddenActivate(),
OutputLayer(hiddenSize, visibleSize, {-range, range}, lambda, beta, rho),
BiasLayer(visibleSize, batchSize),
OutputActivate()),
oneHotLayer,
SparseErrorFunction<MatType>(lambda, beta, rho))
{
// Nothing to do here.
}
/**
* Train the sparse autoencoder network
* Construct sparse autoencoder function.
*
* @param data Input data with each column as one example.
* @param visibleSize Visible size of the input data, it is same as the
* feature size of the training data.
* @param hiddenSize Hidden size of the hidden layer, usually it should be
* smaller than the visible size.
* @param batchSize The batch size used to train the network.
* @param lambda L2-regularization parameter.
* @param beta KL divergence parameter.
* @param rho Sparsity parameter.
*/
SparseAutoencoder(const MatType& input,
size_t visibleSize,
size_t hiddenSize,
const double lambda = 0.0001,
const double beta = 3,
const double rho = 0.01) :
range(std::sqrt(6) / std::sqrt(visibleSize + hiddenSize + 1)),
encoder(std::make_tuple(
HiddenLayer(visibleSize, hiddenSize, {-range, range}, lambda),
BiasLayer(hiddenSize, input.n_rows),
HiddenActivate(),
OutputLayer(hiddenSize, visibleSize, {-range, range}, lambda, beta, rho),
BiasLayer(visibleSize, input.n_rows),
OutputActivate()),
oneHotLayer,
SparseErrorFunction<MatType>(lambda, beta, rho)),
input(input)
{
// Nothing to do here.
}
/**
* Train the sparse autoencoder network.
*
* @param input Data used to train the network
* @param maxEpochs The number of maximal trained iterations (0 means no
* limit).
* limit).
* @param batchSize The batch size used to train the network.
* @param tolerance Train the network until it converges against
* the specified threshold.
* the specified threshold.
* @param shuffle If true, the order of the training set is shuffled;
* otherwise, each data is visited in linear order.
* otherwise, each data is visited in linear order.
*/
void Train(MatType const &input,
const size_t maxEpochs = 0,
@@ -96,89 +169,144 @@ class SparseAutoencoder
const bool shuffle = true)
{
Trainer<FFNet, MatType> trainer(encoder, maxEpochs,
std::get<1>(encoder.Network()).BatchSize(),
std::get<1>(encoder.Model()).BatchSize(),
tolerance,
shuffle);
trainer.Train(input, input, input, input);
}
/**
* Get the weights of decoder
* Evaluates the objective function of the sparse autoencoder model using the
* given parameters. The cost function has terms for the reconstruction
* error, regularization cost and the sparsity cost. The objective function
* takes a low value when the model is able to reconstruct the data well
* using weights which are low in value and when the average activations of
* neurons in the hidden layers agrees well with the sparsity parameter 'rho'.
*
* @param parameters Current values of the model parameters.
*/
double Evaluate(const arma::mat& parameters)
{
const size_t l1 = (parameters.n_rows - 1) / 2;
const size_t l2 = parameters.n_cols - 1;
const size_t l3 = 2 * l1;
std::get<0>(encoder.Model()).Weights() = parameters.submat(
0, 0, l1 - 1, l2 - 1);
std::get<1>(encoder.Model()).Weights() = parameters.submat(
0, l2, l1 - 1, l2);
std::get<3>(encoder.Model()).Weights() = parameters.submat(
l1, 0, l3 - 1, l2 - 1).t();
std::get<4>(encoder.Model()).Weights() = parameters.submat(
l3, 0, l3, l2 - 1).t();
encoder.FeedForward(input, input, error);
return encoder.Error();
}
/**
* Get the weights of decoder.
*
* @return Weights of decoder
*/
MatType const& DecoderWeights() const
{
return std::get<3>(encoder.Network()).Weights();
return std::get<3>(encoder.Model()).Weights();
}
MatType& DecoderWeights()
{
return std::get<3>(encoder.Network()).Weights();
return std::get<3>(encoder.Model()).Weights();
}
/**
* Get the bias of decoder
* @return Bias of decoder
* Get the bias of decoder.
*
* @return Bias of decoder.
*/
MatType const& DecoderBias() const
{
return std::get<4>(encoder.Network()).Weights();
return std::get<4>(encoder.Model()).Weights();
}
/**
* Get the weights of encoder
* @return Weights of encoder
* Get the weights of encoder.
*
* @return Weights of encoder.
*/
MatType const& EncoderWeights() const
{
return std::get<0>(encoder.Network()).Weights();
return std::get<0>(encoder.Model()).Weights();
}
MatType& EncoderWeights()
{
return std::get<0>(encoder.Network()).Weights();
return std::get<0>(encoder.Model()).Weights();
}
/**
* Get the bias of encoder
* @return bias of encoder
* Get the bias of encoder.
*
* @return bias of encoder.
*/
MatType const& EncoderBias() const
{
return std::get<1>(encoder.Network()).Weights();
return std::get<1>(encoder.Model()).Weights();
}
/**
* Encode the input, in other words, extract the features
* of the input
* @param input input data
* @param output features extracted from input
* Encode the input, in other words, extract the features of the input.
*
* @param input input data.
* @param output features extracted from input.
*/
void EncodeInput(MatType const &input, MatType &output) const
{
arma::mat const encodedInput = EncoderWeights() * input +
arma::repmat(EncoderBias(),
1, input.n_cols);
std::get<2>(encoder.Network()).fn(encodedInput, output);
}
arma::mat const encodedInput = EncoderWeights() * input + arma::repmat(
EncoderBias(), 1, input.n_cols);
std::get<2>(encoder.Model()).fn(encodedInput, output);
}
private:
//! Get the constructed model object.
FFNet const& Model() const { return encoder; }
//! Modify the constructed model object.
FFNet& Model() { return encoder; }
private:
//! Locally-stored parameter that specifies the weight initialization range.
const double range;
//! Locally-stored output layer.
OneHotLayer oneHotLayer;
//! Locally-stored sparse autoencoder object.
FFNet encoder;
//! Locally-stored error parameter.
MatType error;
//! Locally-stored input parameter.
MatType input;
};
template<typename MatType = arma::mat,
template<typename,typename> class Optimize = RMSPROP>
// Convenience typedefs.
/**
* Standard sparse autoencoder using the logistic activation function.
*/
template<
typename MatType = arma::mat,
template<typename, typename> class Optimizer = RMSPROP
>
using LogisticSparseAutoencoder = SparseAutoencoder<
BaseLayer<LogisticFunction>,
BaseLayer<LogisticFunction>,
MatType,
Optimize,
SparseInputLayer<Optimize, RandomInitialization, MatType, MatType>,
SparseOutputLayer<Optimize, RandomInitialization, MatType, MatType>>;
Optimizer,
SparseInputLayer<Optimizer, RandomInitialization, MatType, MatType>,
SparseOutputLayer<Optimizer, RandomInitialization, MatType, MatType> >;
}; // namespace ann
}; // namespace mlpack
} // namespace ann
} // namespace mlpack
#endif
src/mlpack/tests/sparse_autoencoder_test.cpp
+148 -218
View File
@@ -1,276 +1,206 @@
/**
* @file sparse_autoencoder_test.cpp
* @author Siddharth Agrawal
*
* Test the SparseAutoencoder class.
*/
#include <mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.hpp>
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
* @file sparse_autoencoder_test.cpp
* @author Siddharth Agrawal
* @author Tham Ngap Wei
*
* Test the SparseAutoencoder class.
*/
#include <mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp>
#include <mlpack/core.hpp>
#include <mlpack/methods/ann/layer/base_layer.hpp>
#include <boost/test/unit_test.hpp>
#include "old_boost_test_definitions.hpp"
using namespace mlpack;
using namespace arma;
using namespace mlpack::ann;
using FSigmoidLayer = ann::SigmoidLayer<ann::LogisticFunction>;
//sparse autoencoder function
using SAEF = nn::SparseAutoencoderFunction<FSigmoidLayer, FSigmoidLayer>;
//sparse autoencoder function greedy
using SAEFG = nn::SparseAutoencoderFunction<FSigmoidLayer, FSigmoidLayer, std::true_type>;
BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest2);
BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest);
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionEvaluate)
{
const size_t vSize = 5;
const size_t hSize = 3;
const size_t r = 2 * hSize + 1;
const size_t c = vSize + 1;
const size_t vSize = 5;
const size_t hSize = 3;
const size_t r = 2 * hSize + 1;
const size_t c = vSize + 1;
// Simple fake dataset.
arma::mat data1("0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5");
// Transpose of the above dataset.
arma::mat data2 = data1.t();
// Simple fake dataset.
arma::mat data1("0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5");
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SAEF saf1(data1, vSize, hSize, 0, 0);
// Transpose of the above dataset.
arma::mat data2 = data1.t();
// Test using first dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::ones(r, c)), 1.190472606540, 1e-5);
BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(saf1.Evaluate(-arma::ones(r, c)), 0.048800332266, 1e-5);
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SparseAutoencoder<> saf1(data1, vSize, hSize, 0, 0);
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SAEF saf2(data2, vSize, hSize, 0, 0);
// Test using first dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::ones(r, c)), 1.190472606540, 1e-5);
BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(saf1.Evaluate(-arma::ones(r, c)), 0.048800332266, 1e-5);
// Test using second dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::ones(r, c)), 1.197585812647, 1e-5);
BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(saf2.Evaluate(-arma::ones(r, c)), 0.063466617408, 1e-5);
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SparseAutoencoder<> saf2(data2, vSize, hSize, 0, 0);
// Test using second dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::ones(r, c)), 1.197585812647, 1e-5);
BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(saf2.Evaluate(-arma::ones(r, c)), 0.063466617408, 1e-5);
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SAEF saf(data, vSize, hSize, 0, 0);
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SparseAutoencoder<> saf(data, vSize, hSize, 0, 0);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
double reconstructionError = 0;
double reconstructionError = 0;
// Compute error for each training example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer, outputLayer, diff;
// Compute error for each training example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer, outputLayer, diff;
hiddenLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
outputLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1, l2 - 1).t()
* hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t())));
diff = outputLayer - data.col(j);
hiddenLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff));
}
reconstructionError /= points;
outputLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1, l2 - 1).t()
* hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t())));
// Compare with the value returned by the function.
BOOST_REQUIRE_CLOSE(saf.Evaluate(parameters), reconstructionError, 1e-5);
}
diff = outputLayer - data.col(j);
reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff));
}
reconstructionError /= points;
// Compare with the value returned by the function.
BOOST_REQUIRE_CLOSE(saf.Evaluate(parameters), reconstructionError, 1e-5);
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRegularizationEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// 3 objects for comparing regularization costs.
SAEF safNoReg(data, vSize, hSize, 0, 0);
SAEF safSmallReg(data, vSize, hSize, 0.5, 0);
SAEF safBigReg(data, vSize, hSize, 20, 0);
// 3 objects for comparing regularization costs.
SparseAutoencoder<> safNoReg(data, vSize, hSize, 0, 0);
SparseAutoencoder<> safSmallReg(data, vSize, hSize, 0.5, 0);
SparseAutoencoder<> safBigReg(data, vSize, hSize, 20, 0);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
double wL2SquaredNorm;
double wL2SquaredNorm;
wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) %
parameters.submat(0, 0, l3 - 1, l2 - 1));
wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) %
parameters.submat(0, 0, l3 - 1, l2 - 1));
// Calculate regularization terms.
const double smallRegTerm = 0.25 * wL2SquaredNorm;
const double bigRegTerm = 10 * wL2SquaredNorm;
// Calculate regularization terms.
const double smallRegTerm = 0.25 * wL2SquaredNorm;
const double bigRegTerm = 10 * wL2SquaredNorm;
BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + smallRegTerm,
safSmallReg.Evaluate(parameters), 1e-5);
BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + bigRegTerm,
safBigReg.Evaluate(parameters), 1e-5);
}
BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + smallRegTerm,
safSmallReg.Evaluate(parameters), 1e-5);
BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + bigRegTerm,
safBigReg.Evaluate(parameters), 1e-5);
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
const double rho = 0.01;
const double rho = 0.01;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// 3 objects for comparing divergence costs.
SAEF safNoDiv(data, vSize, hSize, 0, 0, rho);
SAEF safSmallDiv(data, vSize, hSize, 0, 5, rho);
SAEF safBigDiv(data, vSize, hSize, 0, 20, rho);
// 3 objects for comparing divergence costs.
SparseAutoencoder<> safNoDiv(data, vSize, hSize, 0, 0, rho);
SparseAutoencoder<> safSmallDiv(data, vSize, hSize, 0, 5, rho);
SparseAutoencoder<> safBigDiv(data, vSize, hSize, 0, 20, rho);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
arma::mat rhoCap;
rhoCap.zeros(hSize, 1);
arma::mat rhoCap;
rhoCap.zeros(hSize, 1);
// Compute hidden layer activations for each example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer;
// Compute hidden layer activations for each example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer;
hiddenLayer = 1.0 / (1 +
arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
rhoCap += hiddenLayer;
}
rhoCap /= points;
hiddenLayer = 1.0 / (1 +
arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
// Calculate divergence terms.
const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
rhoCap += hiddenLayer;
}
rhoCap /= points;
BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + smallDivTerm,
safSmallDiv.Evaluate(parameters), 1e-5);
BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + bigDivTerm,
safBigDiv.Evaluate(parameters), 1e-5);
}
}
// Calculate divergence terms.
const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionGradient)
{
const size_t points = 1000;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// 3 objects for 3 terms in the cost function. Each term contributes towards
// the gradient and thus need to be checked independently.
SAEFG saf1(data, vSize, hSize, 0, 0);
SAEFG saf2(data, vSize, hSize, 20, 0);
SAEFG saf3(data, vSize, hSize, 20, 20);
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
// Get gradients for the current parameters.
arma::mat gradient1, gradient2, gradient3;
saf1.Gradient(parameters, gradient1);
saf2.Gradient(parameters, gradient2);
saf3.Gradient(parameters, gradient3);
// Perturbation constant.
const double epsilon = 0.0001;
double costPlus1, costMinus1, numGradient1;
double costPlus2, costMinus2, numGradient2;
double costPlus3, costMinus3, numGradient3;
// For each parameter.
for (size_t i = 0; i <= l3; i++)
{
for (size_t j = 0; j <= l2; j++)
{
// Perturb parameter with a positive constant and get costs.
parameters(i, j) += epsilon;
costPlus1 = saf1.Evaluate(parameters);
costPlus2 = saf2.Evaluate(parameters);
costPlus3 = saf3.Evaluate(parameters);
// Perturb parameter with a negative constant and get costs.
parameters(i, j) -= 2 * epsilon;
costMinus1 = saf1.Evaluate(parameters);
costMinus2 = saf2.Evaluate(parameters);
costMinus3 = saf3.Evaluate(parameters);
// Compute numerical gradients using the costs calculated above.
numGradient1 = (costPlus1 - costMinus1) / (2 * epsilon);
numGradient2 = (costPlus2 - costMinus2) / (2 * epsilon);
numGradient3 = (costPlus3 - costMinus3) / (2 * epsilon);
// Restore the parameter value.
parameters(i, j) += epsilon;
// Compare numerical and backpropagation gradient values.
BOOST_REQUIRE_CLOSE(numGradient1, gradient1(i, j), 1e-2);
BOOST_REQUIRE_CLOSE(numGradient2, gradient2(i, j), 1e-2);
BOOST_REQUIRE_CLOSE(numGradient3, gradient3(i, j), 1e-2);
}
}
BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + smallDivTerm,
safSmallDiv.Evaluate(parameters), 1e-5);
BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + bigDivTerm,
safBigDiv.Evaluate(parameters), 1e-5);
}
}
BOOST_AUTO_TEST_SUITE_END();
src/mlpack/tests/sparse_autoencoder_test_2.cpp
-306
View File
@@ -1,306 +0,0 @@
/**
* @file sparse_autoencoder_test.cpp
* @author Siddharth Agrawal
*
* Test the SparseAutoencoder class.
*/
#include <mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp>
#include <mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.hpp>
#include <mlpack/methods/sparse_autoencoder/activation_functions/logistic_function.hpp>
#include <mlpack/core.hpp>
#include <mlpack/methods/sparse_autoencoder/layer/base_layer.hpp>
#include <boost/test/unit_test.hpp>
#include "old_boost_test_definitions.hpp"
using namespace mlpack;
using namespace arma;
using SigmoidLayer = nn::SigmoidLayer<nn::LogisticFunction>;
//sparse autoencoder function
using SAEF = nn::SparseAutoencoderFunction<SigmoidLayer, SigmoidLayer>;
//sparse autoencoder function greedy
using SAEFG = nn::SparseAutoencoderFunction<SigmoidLayer, SigmoidLayer, std::true_type>;
BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest2);
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionEvaluate)
{
const size_t vSize = 5;
const size_t hSize = 3;
const size_t r = 2 * hSize + 1;
const size_t c = vSize + 1;
// Simple fake dataset.
arma::mat data1("0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5");
// Transpose of the above dataset.
arma::mat data2 = data1.t();
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SAEF saf1(data1, vSize, hSize, 0, 0);
// Test using first dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::ones(r, c)), 1.190472606540, 1e-5);
BOOST_REQUIRE_CLOSE(saf1.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(saf1.Evaluate(-arma::ones(r, c)), 0.048800332266, 1e-5);
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SAEF saf2(data2, vSize, hSize, 0, 0);
// Test using second dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::ones(r, c)), 1.197585812647, 1e-5);
BOOST_REQUIRE_CLOSE(saf2.Evaluate(arma::zeros(r, c)), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(saf2.Evaluate(-arma::ones(r, c)), 0.063466617408, 1e-5);
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// Create a SparseAutoencoderFunction. Regularization and KL divergence terms
// ignored.
SAEF saf(data, vSize, hSize, 0, 0);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
double reconstructionError = 0;
// Compute error for each training example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer, outputLayer, diff;
hiddenLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
outputLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1,l2 - 1).t()
* hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t())));
diff = outputLayer - data.col(j);
reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff));
}
reconstructionError /= points;
// Compare with the value returned by the function.
BOOST_REQUIRE_CLOSE(saf.Evaluate(parameters), reconstructionError, 1e-5);
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRegularizationEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// 3 objects for comparing regularization costs.
SAEF safNoReg(data, vSize, hSize, 0, 0);
SAEF safSmallReg(data, vSize, hSize, 0.5, 0);
SAEF safBigReg(data, vSize, hSize, 20, 0);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
double wL2SquaredNorm;
wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) %
parameters.submat(0, 0, l3 - 1, l2 - 1));
// Calculate regularization terms.
const double smallRegTerm = 0.25 * wL2SquaredNorm;
const double bigRegTerm = 10 * wL2SquaredNorm;
BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + smallRegTerm,
safSmallReg.Evaluate(parameters), 1e-5);
BOOST_REQUIRE_CLOSE(safNoReg.Evaluate(parameters) + bigRegTerm,
safBigReg.Evaluate(parameters), 1e-5);
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
const double rho = 0.01;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// 3 objects for comparing divergence costs.
SAEF safNoDiv(data, vSize, hSize, 0, 0, rho);
SAEF safSmallDiv(data, vSize, hSize, 0, 5, rho);
SAEF safBigDiv(data, vSize, hSize, 0, 20, rho);
// Run a number of trials.
for(size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
arma::mat rhoCap;
rhoCap.zeros(hSize, 1);
// Compute hidden layer activations for each example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer;
hiddenLayer = 1.0 / (1 +
arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
rhoCap += hiddenLayer;
}
rhoCap /= points;
// Calculate divergence terms.
const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + smallDivTerm,
safSmallDiv.Evaluate(parameters), 1e-5);
BOOST_REQUIRE_CLOSE(safNoDiv.Evaluate(parameters) + bigDivTerm,
safBigDiv.Evaluate(parameters), 1e-5);
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionGradient)
{
const size_t points = 1000;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// 3 objects for 3 terms in the cost function. Each term contributes towards
// the gradient and thus need to be checked independently.
SAEFG saf1(data, vSize, hSize, 0, 0);
SAEFG saf2(data, vSize, hSize, 20, 0);
SAEFG saf3(data, vSize, hSize, 20, 20);
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
// Get gradients for the current parameters.
arma::mat gradient1, gradient2, gradient3;
saf1.Gradient(parameters, gradient1);
saf2.Gradient(parameters, gradient2);
saf3.Gradient(parameters, gradient3);
// Perturbation constant.
const double epsilon = 0.0001;
double costPlus1, costMinus1, numGradient1;
double costPlus2, costMinus2, numGradient2;
double costPlus3, costMinus3, numGradient3;
// For each parameter.
for (size_t i = 0; i <= l3; i++)
{
for (size_t j = 0; j <= l2; j++)
{
// Perturb parameter with a positive constant and get costs.
parameters(i, j) += epsilon;
costPlus1 = saf1.Evaluate(parameters);
costPlus2 = saf2.Evaluate(parameters);
costPlus3 = saf3.Evaluate(parameters);
// Perturb parameter with a negative constant and get costs.
parameters(i, j) -= 2 * epsilon;
costMinus1 = saf1.Evaluate(parameters);
costMinus2 = saf2.Evaluate(parameters);
costMinus3 = saf3.Evaluate(parameters);
// Compute numerical gradients using the costs calculated above.
numGradient1 = (costPlus1 - costMinus1) / (2 * epsilon);
numGradient2 = (costPlus2 - costMinus2) / (2 * epsilon);
numGradient3 = (costPlus3 - costMinus3) / (2 * epsilon);
// Restore the parameter value.
parameters(i, j) += epsilon;
// Compare numerical and backpropagation gradient values.
BOOST_REQUIRE_CLOSE(numGradient1, gradient1(i, j), 1e-2);
BOOST_REQUIRE_CLOSE(numGradient2, gradient2(i, j), 1e-2);
BOOST_REQUIRE_CLOSE(numGradient3, gradient3(i, j), 1e-2);
}
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderMoveTest)
{
using SAE = mlpack::nn::SparseAutoencoder<>;
SAE sae1(2, 2);
sae1.Lambda(0.45);
sae1.Beta(0.33);
sae1.Rho(0.11);
const auto Params = sae1.Parameters();
SAE sae2(std::move(sae1));
BOOST_REQUIRE(sae1.Parameters().n_elem == 0);
BOOST_REQUIRE(sae1.VisibleSize() == sae2.VisibleSize());
BOOST_REQUIRE(sae1.HiddenSize() == sae2.HiddenSize());
BOOST_REQUIRE_CLOSE(sae1.Lambda(), sae2.Lambda(), 1e-5);
BOOST_REQUIRE_CLOSE(sae1.Beta(), sae2.Beta(), 1e-5);
BOOST_REQUIRE_CLOSE(sae1.Rho(), sae2.Rho(), 1e-5);
const auto Params2 = sae2.Parameters();
auto func = [](double lhs, double rhs)
{
BOOST_REQUIRE_CLOSE(lhs, rhs, 1e-5);
return true;
};
std::equal(std::begin(Params), std::end(Params),
std::begin(Params2),
func);
}
BOOST_AUTO_TEST_SUITE_END();
src/mlpack/tests/sparse_autoencoder_test_3.cpp
-381
View File
@@ -1,381 +0,0 @@
/**
* @file sparse_autoencoder_test_3.cpp
* @author Tham Ngap Wei
*
* Test the SparseAutoencoder class.
*/
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
#include <mlpack/methods/ann/init_rules/random_init.hpp>
#include <mlpack/methods/ann/layer/base_layer.hpp>
#include <mlpack/methods/ann/layer/one_hot_layer.hpp>
#include <mlpack/methods/ann/layer/softmax_layer.hpp>
#include <mlpack/methods/ann/layer/sparse_input_layer.hpp>
#include <mlpack/methods/ann/layer/sparse_output_layer.hpp>
#include <mlpack/methods/ann/layer/sparse_bias_layer.hpp>
#include <mlpack/methods/ann/trainer/trainer.hpp>
#include <mlpack/methods/ann/ffn.hpp>
#include <mlpack/methods/ann/performance_functions/mse_function.hpp>
#include <mlpack/methods/ann/performance_functions/sparse_function.hpp>
#include <mlpack/methods/ann/optimizer/rmsprop.hpp>
#include <mlpack/core.hpp>
#include <boost/test/unit_test.hpp>
#include "old_boost_test_definitions.hpp"
using namespace mlpack;
using namespace arma;
using namespace mlpack::ann;
BOOST_AUTO_TEST_SUITE(SparseAutoencoderTest3);
using Network = std::tuple<SparseInputLayer<>, SparseBiasLayer<>,
BaseLayer<LogisticFunction>, SparseOutputLayer<>, SparseBiasLayer<>,
BaseLayer<LogisticFunction> >;
using FFNet = FFN<Network, OneHotLayer, SparseErrorFunction<> >;
Network create_network(size_t visibleSize, size_t hiddenSize,
size_t sampleSize, double lambda = 0.0001,
double beta = 3, double rho = 0.01)
{
const double range = std::sqrt(6) / std::sqrt(visibleSize + hiddenSize + 1);
SparseInputLayer<> hiddenLayer(visibleSize, hiddenSize, {-range, range},
lambda);
SparseBiasLayer<> hiddenBiasLayer(hiddenSize, sampleSize);
BaseLayer<LogisticFunction> hiddenBaseLayer;
SparseOutputLayer<> outputLayer(hiddenSize, visibleSize, {-range, range},
lambda, beta, rho);
SparseBiasLayer<> outputBiasLayer(visibleSize, sampleSize);
BaseLayer<LogisticFunction> outputBaseLayer;
return std::make_tuple(std::move(hiddenLayer), std::move(hiddenBiasLayer),
std::move(hiddenBaseLayer), std::move(outputLayer),
std::move(outputBiasLayer), std::move(outputBaseLayer));
}
FFNet create_ffn(size_t visibleSize, size_t hiddenSize,
size_t sampleSize, double lambda = 0.0001,
double beta = 3, double rho = 0.01)
{
auto network = create_network(visibleSize, hiddenSize,
sampleSize, lambda,
beta, rho);
FFNet ffn(std::move(network), OneHotLayer(),
SparseErrorFunction<>(lambda,beta,rho));
return ffn;
}
template<typename Net, typename UnaryFunc>
void initWeights(Net &net, UnaryFunc func)
{
std::get<0>(net).Weights() = func(std::get<0>(net).Weights());
std::get<1>(net).Weights() = func(std::get<1>(net).Weights());
std::get<3>(net).Weights() = func(std::get<3>(net).Weights());
std::get<4>(net).Weights() = func(std::get<4>(net).Weights());
}
template<typename Net>
void initWeights(Net &net, arma::mat const &parameters)
{
const size_t l1 = (parameters.n_rows - 1) / 2;
const size_t l2 = parameters.n_cols - 1;
const size_t l3 = 2 * l1;
std::get<0>(net).Weights() = parameters.submat(0, 0, l1-1, l2-1);
std::get<1>(net).Weights() = parameters.submat(0, l2, l1-1, l2);
std::get<3>(net).Weights() = parameters.submat(l1, 0, l3-1, l2-1).t();
std::get<4>(net).Weights() = parameters.submat(l3, 0, l3, l2-1).t();
}
arma::mat initGradient(arma::mat const &input,
arma::mat const &parameters,
FFNet &ffn)
{
initWeights(ffn.Network(), parameters);
arma::mat error;
ffn.FeedForward(input, input, error);
ffn.FeedBackward(arma::mat(), error);
arma::mat gradient = arma::zeros(parameters.n_rows, parameters.n_cols);
const size_t l1 = (parameters.n_rows - 1) / 2;
const size_t l2 = parameters.n_cols - 1;
const size_t l3 = 2 * l1;
auto const &net = ffn.Network();
gradient.submat(0, 0, l1-1, l2-1) = std::get<0>(net).Gradient();
gradient.submat(0, l2, l1-1, l2) = std::get<1>(net).Gradient();
gradient.submat(l1, 0, l3-1, l2-1) = std::get<3>(net).Gradient().t();
gradient.submat(l3, 0, l3, l2-1) = std::get<4>(net).Gradient().t();
return gradient;
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionEvaluate)
{
const size_t vSize = 5;
const size_t hSize = 3;
// Simple fake dataset.
arma::mat data1("0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5;"
"0.1 0.2 0.3 0.4 0.5");
// Create SparseAutoencoder. Regularization and KL divergence terms
// ignored.
auto ffn1 = create_ffn(vSize, hSize, 5, 0, 0);
initWeights(ffn1.Network(), [](arma::mat &v){
return v.ones();
});
arma::mat error;
ffn1.FeedForward(data1, data1, error);
auto ffn2 = create_ffn(vSize, hSize, 5, 0, 0);
initWeights(ffn2.Network(), [](arma::mat &v){
return v.zeros();
});
ffn2.FeedForward(data1, data1, error);
auto ffn3 = create_ffn(vSize, hSize, 5, 0, 0);
initWeights(ffn3.Network(), [](arma::mat &v){
return -v.ones();
});
ffn3.FeedForward(data1, data1, error);
// Test using first dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(ffn1.Error(), 1.190472606540, 1e-5);
BOOST_REQUIRE_CLOSE(ffn2.Error(), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(ffn3.Error(), 0.048800332266, 1e-5);
arma::mat const data2 = data1.t();
auto ffn4 = create_ffn(vSize, hSize, 5, 0, 0);
initWeights(ffn4.Network(), [](arma::mat &v){
return v.ones();
});
ffn4.FeedForward(data2, data2, error);
auto ffn5 = create_ffn(vSize, hSize, 5, 0, 0);
initWeights(ffn5.Network(), [](arma::mat &v){
return v.zeros();
});
ffn5.FeedForward(data2, data2, error);
auto ffn6 = create_ffn(vSize, hSize, 5, 0, 0);
initWeights(ffn6.Network(), [](arma::mat &v){
return -v.ones();
});
ffn6.FeedForward(data2, data2, error);
// Test using second dataset. Values were calculated using Octave.
BOOST_REQUIRE_CLOSE(ffn4.Error(), 1.197585812647, 1e-5);
BOOST_REQUIRE_CLOSE(ffn5.Error(), 0.150000000000, 1e-5);
BOOST_REQUIRE_CLOSE(ffn6.Error(), 0.063466617408, 1e-5);
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
// Initialize a random dataset.
arma::mat data1;
data1.randu(vSize, points);
// Run a number of trials.
for(size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
double reconstructionError = 0;
// Compute error for each training example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer, outputLayer, diff;
hiddenLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data1.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
outputLayer = 1.0 /
(1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1,l2 - 1).t()
* hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t())));
diff = outputLayer - data1.col(j);
reconstructionError += 0.5 * arma::sum(arma::sum(diff % diff));
}
reconstructionError /= points;
// Compare with the value returned by the function.
auto ffn = create_ffn(vSize, hSize, points, 0, 0);
auto &net = ffn.Network();
initWeights(net, parameters);
arma::mat error;
ffn.FeedForward(data1, data1, error);
BOOST_REQUIRE_CLOSE(ffn.Error(), reconstructionError, 1e-5);
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRegularizationEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// Run a number of trials.
for (size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
double wL2SquaredNorm;
wL2SquaredNorm = arma::accu(parameters.submat(0, 0, l3 - 1, l2 - 1) %
parameters.submat(0, 0, l3 - 1, l2 - 1));
// Calculate regularization terms.
const double smallRegTerm = 0.25 * wL2SquaredNorm;
const double bigRegTerm = 10 * wL2SquaredNorm;
// 3 objects for comparing regularization costs.
auto safNoReg = create_ffn(vSize, hSize, points, 0, 0);
initWeights(safNoReg.Network(), parameters);
arma::mat error;
safNoReg.FeedForward(data, data, error);
auto safSmallReg = create_ffn(vSize, hSize, points, 0.5, 0);
initWeights(safSmallReg.Network(), parameters);
safSmallReg.FeedForward(data, data, error);
auto safBigReg = create_ffn(vSize, hSize, points, 20, 0);
initWeights(safBigReg.Network(), parameters);
safBigReg.FeedForward(data, data, error);
BOOST_REQUIRE_CLOSE(safNoReg.Error() + smallRegTerm,
safSmallReg.Error(), 1e-5);
BOOST_REQUIRE_CLOSE(safNoReg.Error() + bigRegTerm,
safBigReg.Error(), 1e-5);
}
}
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate)
{
const size_t points = 1000;
const size_t trials = 50;
const size_t vSize = 20;
const size_t hSize = 10;
const size_t l1 = hSize;
const size_t l2 = vSize;
const size_t l3 = 2 * hSize;
const double rho = 0.01;
// Initialize a random dataset.
arma::mat data;
data.randu(vSize, points);
// Run a number of trials.
for(size_t i = 0; i < trials; i++)
{
// Create a random set of parameters.
arma::mat parameters;
parameters.randu(l3 + 1, l2 + 1);
arma::mat rhoCap;
rhoCap.zeros(hSize, 1);
// Compute hidden layer activations for each example.
for (size_t j = 0; j < points; j++)
{
arma::mat hiddenLayer;
hiddenLayer = 1.0 / (1 +
arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) *
data.col(j) + parameters.submat(0, l2, l1 - 1, l2))));
rhoCap += hiddenLayer;
}
rhoCap /= static_cast<double>(points);
// Calculate divergence terms.
const double smallDivTerm = 5 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
const double bigDivTerm = 20 * arma::accu(rho * arma::log(rho / rhoCap) +
(1 - rho) * arma::log((1 - rho) / (1 - rhoCap)));
// 3 objects for comparing divergence costs.
auto safNoDiv = create_ffn(vSize, hSize, points, 0, 0, rho);
initWeights(safNoDiv.Network(), parameters);
arma::mat error;
safNoDiv.FeedForward(data, data, error);
auto safSmallDiv = create_ffn(vSize, hSize, points, 0, 5, rho);
initWeights(safSmallDiv.Network(), parameters);
safSmallDiv.FeedForward(data, data, error);
auto safBigDiv = create_ffn(vSize, hSize, points, 0, 20, rho);
initWeights(safBigDiv.Network(), parameters);
safBigDiv.FeedForward(data, data, error);
BOOST_REQUIRE_CLOSE(safNoDiv.Error() + smallDivTerm,
safSmallDiv.Error(), 1e-5);
BOOST_REQUIRE_CLOSE(safNoDiv.Error() + bigDivTerm,
safBigDiv.Error(), 1e-5);
}
}//*/
BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionGradient)
{
const arma::mat trainData("0.0013 0.5850 0.8228 0.7105;"
"0.1933 0.3503 0.1741 0.3040");
const size_t visibleSize = trainData.n_rows;
const size_t hiddenSize = trainData.n_rows / 2;
arma::mat const parameters( "-1.2029 -0.8175 0;"
" 0.0776 -0.1205 0;"
" 0 0 0");
auto ffn = create_ffn(visibleSize, hiddenSize, 4);
initGradient(trainData, parameters, ffn);
auto &net = ffn.Network();
arma::mat const w1 = std::get<0>(net).Gradient();
arma::mat const b1 = std::get<1>(net).Gradient();
arma::mat const w2 = std::get<3>(net).Gradient();
arma::mat const b2 = std::get<4>(net).Gradient();
//these values come from original implementation
//because ffn do not provide ease to use Evalulate
//api to do the gradient checking, I prefer golden
//model(FeedForward api will alter inner value,
//everytime you call it, the error value will change)
BOOST_REQUIRE_CLOSE(w1(0, 0), 0.41743157405, 1e-5);
BOOST_REQUIRE_CLOSE(w1(0, 1), 0.21509458293, 1e-5);
BOOST_REQUIRE_CLOSE(b1(0, 0), 0.85303876677, 1e-5);
BOOST_REQUIRE_CLOSE(w2(0, 0), 0.00520531433, 1e-5);
BOOST_REQUIRE_CLOSE(w2(1, 0), 0.01858690990, 1e-5);
BOOST_REQUIRE_CLOSE(b2(0, 0), -0.00599756012, 1e-5);
BOOST_REQUIRE_CLOSE(b2(1, 0), 0.05881613659, 1e-5);
}
BOOST_AUTO_TEST_SUITE_END();