apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix some minor compilation issues.

Browse Source
This commit is contained in:
Ryan Curtin
2021-05-28 12:52:21 -04:00
parent f8123469e9
commit 5220de7d14
34 changed files with 405 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/layer/add_merge.hpp
+1 -1
View File
@@ -34,7 +34,7 @@ template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
class AddMerge
class AddMerge : public Layer<InputType, OutputType>
{
public:
/**
src/mlpack/methods/ann/layer/alpha_dropout.hpp
+1 -1
View File
@@ -89,7 +89,7 @@ class AlphaDropout : public Layer<InputType, OutputType>
OutputType& Delta() { return delta; }
//! The value of the deterministic parameter.
bool Deterministic() const { return deterministic; }
const bool& Deterministic() const { return deterministic; }
//! Modify the value of the deterministic parameter.
bool& Deterministic() { return deterministic; }
src/mlpack/methods/ann/layer/atrous_convolution.hpp
+15 -16
View File
@@ -159,7 +159,6 @@ class AtrousConvolution : public Layer<InputType, OutputType>
* @param error The calculated error.
* @param gradient The calculated gradient.
*/
template<typename eT>
void Gradient(const InputType& /* input */,
const OutputType& error,
OutputType& gradient);
@@ -198,65 +197,65 @@ class AtrousConvolution : public Layer<InputType, OutputType>
OutputType& Gradient() { return gradient; }
//! Get the input width.
size_t InputWidth() const { return inputWidth; }
const size_t& InputWidth() const { return inputWidth; }
//! Modify input the width.
size_t& InputWidth() { return inputWidth; }
//! Get the input height.
size_t InputHeight() const { return inputHeight; }
const size_t& InputHeight() const { return inputHeight; }
//! Modify the input height.
size_t& InputHeight() { return inputHeight; }
//! Get the output width.
size_t OutputWidth() const { return outputWidth; }
const size_t& OutputWidth() const { return outputWidth; }
//! Modify the output width.
size_t& OutputWidth() { return outputWidth; }
//! Get the output height.
size_t OutputHeight() const { return outputHeight; }
const size_t& OutputHeight() const { return outputHeight; }
//! Modify the output height.
size_t& OutputHeight() { return outputHeight; }
//! Get the input size.
size_t InputSize() const { return inSize; }
const size_t& InputSize() const { return inSize; }
//! Get the output size.
size_t OutputSize() const { return outSize; }
const size_t& OutputSize() const { return outSize; }
//! Get the kernel width.
size_t KernelWidth() const { return kernelWidth; }
const size_t& KernelWidth() const { return kernelWidth; }
//! Modify the kernel width.
size_t& KernelWidth() { return kernelWidth; }
//! Get the kernel height.
size_t KernelHeight() const { return kernelHeight; }
const size_t& KernelHeight() const { return kernelHeight; }
//! Modify the kernel height.
size_t& KernelHeight() { return kernelHeight; }
//! Get the stride width.
size_t StrideWidth() const { return strideWidth; }
const size_t& StrideWidth() const { return strideWidth; }
//! Modify the stride width.
size_t& StrideWidth() { return strideWidth; }
//! Get the stride height.
size_t StrideHeight() const { return strideHeight; }
const size_t& StrideHeight() const { return strideHeight; }
//! Modify the stride height.
size_t& StrideHeight() { return strideHeight; }
//! Get the dilation rate on the X axis.
size_t DilationWidth() const { return dilationWidth; }
const size_t& DilationWidth() const { return dilationWidth; }
//! Modify the dilation rate on the X axis.
size_t& DilationWidth() { return dilationWidth; }
//! Get the dilation rate on the Y axis.
size_t DilationHeight() const { return dilationHeight; }
const size_t& DilationHeight() const { return dilationHeight; }
//! Modify the dilation rate on the Y axis.
size_t& DilationHeight() { return dilationHeight; }
//! Get the internal Padding layer.
ann::Padding<> const& Padding() const { return padding; }
PaddingType<InputType, OutputType> const& Padding() const { return padding; }
//! Modify the internal Padding layer.
ann::Padding<>& Padding() { return padding; }
PaddingType<InputType, OutputType>& Padding() { return padding; }
//! Get size of the weight matrix.
size_t WeightSize() const
@@ -390,7 +389,7 @@ class AtrousConvolution : public Layer<InputType, OutputType>
arma::Cube<typename OutputType::elem_type> gradientTemp;
//! Locally-stored padding layer.
ann::Padding<> padding;
PaddingType<InputType, OutputType> padding;
//! Locally-stored delta object.
OutputType delta;
src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp
+4 -3
View File
@@ -144,7 +144,8 @@ AtrousConvolution<
InitializeSamePadding(padWLeft, padWRight, padHTop, padHBottom);
}
padding = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom);
padding = PaddingType<InputType, OutputType>(padWLeft, padWRight, padHTop,
padHBottom);
}
template<
@@ -329,7 +330,7 @@ void AtrousConvolution<
((OutputType&) error).memptr(), outputWidth, outputHeight, outSize *
batchSize, false, false);
arma::Cube<typename InputType::elem_type> inputTemp(
const_cast<arma::Mat<eT>&>(input).memptr(), inputWidth, inputHeight,
const_cast<InputType&>(input).memptr(), inputWidth, inputHeight,
inSize * batchSize, false, false);
gradient.set_size(weights.n_elem, 1);
@@ -417,7 +418,7 @@ void AtrousConvolution<
OutputType
>::serialize(Archive& ar, const uint32_t /* version */)
{
ar(cereal::base_class<Layer<InputDataType, OutputDataType>>(this));
ar(cereal::base_class<Layer<InputType, OutputType>>(this));
ar(CEREAL_NVP(inSize));
ar(CEREAL_NVP(outSize));
src/mlpack/methods/ann/layer/base_layer.hpp
+66 -103
View File
@@ -139,160 +139,123 @@ class BaseLayer : public Layer<InputType, OutputType>
/**
* Standard Sigmoid-Layer using the logistic activation function.
*/
// template <
// class ActivationFunction = LogisticFunction,
// typename InputType = arma::mat,
// typename OutputType = arma::mat
// >
// using SigmoidLayer = BaseLayer<ActivationFunction, InputType, OutputType>;
typedef BaseLayer<LogisticFunction, arma::mat, arma::mat> Sigmoid;
typedef BaseLayer<LogisticFunction, arma::mat, arma::mat> SigmoidLayer;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using SigmoidLayer = BaseLayer<LogisticFunction, InputType, OutputType>;
/**
* Standard Identity-Layer using the identity activation function.
*/
// template <
// class ActivationFunction = IdentityFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using IdentityLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<IdentityFunction, arma::mat, arma::mat> IdentityLayer;
typedef BaseLayer<IdentityFunction, arma::mat, arma::mat> Identity;
template <
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using IdentityLayer = BaseLayer<IdentityFunction, InputType, OutputType>;
/**
* Standard rectified linear unit non-linearity layer.
*/
// template <
// class ActivationFunction = RectifierFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using ReLULayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<RectifierFunction, arma::mat, arma::mat> ReLULayer;
typedef BaseLayer<RectifierFunction, arma::mat, arma::mat> ReLU;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using ReLULayer = BaseLayer<RectifierFunction, InputType, OutputType>;
/**
* Standard hyperbolic tangent layer.
*/
// template <
// class ActivationFunction = TanhFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using TanHLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<TanhFunction, arma::mat, arma::mat> TanHLayer;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using TanHLayer = BaseLayer<TanhFunction, InputType, OutputType>;
/**
* Standard Softplus-Layer using the Softplus activation function.
*/
// template <
// class ActivationFunction = SoftplusFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using SoftPlusLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<SoftplusFunction, arma::mat, arma::mat> SoftPlusLayer;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using SoftPlusLayer = BaseLayer<SoftplusFunction, InputType, OutputType>;
/**
* Standard HardSigmoid-Layer using the HardSigmoid activation function.
*/
// template <
// class ActivationFunction = HardSigmoidFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using HardSigmoidLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<HardSigmoidFunction, arma::mat, arma::mat> HardSigmoidLayer;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using HardSigmoidLayer = BaseLayer<HardSigmoidFunction, InputType, OutputType>;
/**
* Standard Swish-Layer using the Swish activation function.
*/
// template <
// class ActivationFunction = SwishFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using SwishFunctionLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<SwishFunction, arma::mat, arma::mat> SwishFunctionLayer;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using SwishFunctionLayer = BaseLayer<SwishFunction, InputType, OutputType>;
/**
* Standard Mish-Layer using the Mish activation function.
*/
// template <
// class ActivationFunction = MishFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using MishFunctionLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<MishFunction, arma::mat, arma::mat> MishFunctionLayer;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using MishFunctionLayer = BaseLayer<MishFunction, InputType, OutputType>;
/**
* Standard LiSHT-Layer using the LiSHT activation function.
*/
// template <
// class ActivationFunction = LiSHTFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using LiSHTFunctionLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<LiSHTFunction, arma::mat, arma::mat> LiSHTFunctionLayer;
template<
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using LiSHTFunctionLayer = BaseLayer<LiSHTFunction, InputType, OutputType>;
/**
* Standard GELU-Layer using the GELU activation function.
*/
// template <
// class ActivationFunction = GELUFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using GELUFunctionLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<GELUFunction, arma::mat, arma::mat> GELUFunctionLayer;
template <
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using GELUFunctionLayer = BaseLayer<GELUFunction, InputType, OutputType>;
/**
* Standard Elliot-Layer using the Elliot activation function.
*/
// template <
// class ActivationFunction = ElliotFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using ElliotFunctionLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<ElliotFunction, arma::mat, arma::mat> ElliotFunctionLayer;
template <
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using ElliotFunctionLayer = BaseLayer<ElliotFunction, InputType, OutputType>;
/**
* Standard ELiSH-Layer using the ELiSH activation function.
*/
// template <
// class ActivationFunction = ElishFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using ElishFunctionLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<ElishFunction, arma::mat, arma::mat> ElishFunctionLayer;
template <
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using ElishFunctionLayer = BaseLayer<ElishFunction, InputType, OutputType>;
/**
* Standard Gaussian-Layer using the Gaussian activation function.
*/
// template <
// class ActivationFunction = GaussianFunction,
// typename InputDataType = arma::mat,
// typename OutputDataType = arma::mat
// >
// using GaussianFunctionLayer = BaseLayer<
// ActivationFunction, InputDataType, OutputDataType>;
typedef BaseLayer<GaussianFunction, arma::mat, arma::mat> GaussianFunctionLayer;
template <
typename InputType = arma::mat,
typename OutputType = arma::mat
>
using GaussianFunctionLayer = BaseLayer<GaussianFunction, InputType,
OutputType>;
} // namespace ann
} // namespace mlpack
src/mlpack/methods/ann/layer/batch_norm.hpp
+1 -1
View File
@@ -131,7 +131,7 @@ class BatchNorm : public Layer<InputType, OutputType>
OutputType& Gradient() { return gradient; }
//! Get the value of deterministic parameter.
bool Deterministic() const { return deterministic; }
const bool& Deterministic() const { return deterministic; }
//! Modify the value of deterministic parameter.
bool& Deterministic() { return deterministic; }
src/mlpack/methods/ann/layer/concat_performance.hpp
+1 -1
View File
@@ -54,7 +54,7 @@ class ConcatPerformance : public Layer<InputType, OutputType>
* @param input Input data used for evaluating the specified function.
* @param output Resulting output activation.
*/
double Forward(const InputType& input, OutputType& target);
void Forward(const InputType& input, OutputType& target);
/**
* Ordinary feed backward pass of a neural network. The negative log
src/mlpack/methods/ann/layer/concat_performance_impl.hpp
+12 -12
View File
@@ -39,23 +39,24 @@ template<
typename InputType,
typename OutputType
>
template<typename eT>
double ConcatPerformance<
void ConcatPerformance<
OutputLayerType,
InputType,
OutputType
>::Forward(const arma::Mat<eT>& input, arma::Mat<eT>& target)
>::Forward(const InputType& input, OutputType& target)
{
const size_t elements = input.n_elem / inSize;
double output = 0;
for (size_t i = 0; i < input.n_elem; i+= elements)
for (size_t i = 0; i < input.n_elem; i += elements)
{
arma::mat subInput = input.submat(i, 0, i + elements - 1, 0);
InputType subInput = input.submat(i, 0, i + elements - 1, 0);
output += outputLayer.Forward(subInput, target);
}
return output;
// TODO: what to do with output?
//return output;
return;
}
template<
@@ -63,20 +64,19 @@ template<
typename InputType,
typename OutputType
>
template<typename eT>
void ConcatPerformance<
OutputLayerType,
InputType,
OutputType
>::Backward(
const arma::Mat<eT>& input,
const arma::Mat<eT>& target,
arma::Mat<eT>& output)
const InputType& input,
const OutputType& target,
OutputType& output)
{
const size_t elements = input.n_elem / inSize;
arma::mat subInput = input.submat(0, 0, elements - 1, 0);
arma::mat subOutput;
InputType subInput = input.submat(0, 0, elements - 1, 0);
OutputType subOutput;
outputLayer.Backward(subInput, target, subOutput);
src/mlpack/methods/ann/layer/concatenate.hpp
+1 -1
View File
@@ -102,7 +102,7 @@ class ConcatenateType : public Layer<InputType, OutputType>
* Serialize the layer.
*/
template<typename Archive>
void serialize(Archive& /* ar */, const uint32_t /* version */)
void serialize(Archive& ar, const uint32_t /* version */)
{
ar(cereal::base_class<Layer<InputType, OutputType>>(this));
}
src/mlpack/methods/ann/layer/constant.hpp
+15
View File
@@ -34,6 +34,11 @@ template<typename InputType = arma::mat, typename OutputType = arma::mat>
class ConstantType : public Layer<InputType, OutputType>
{
public:
/**
* Create an empty Constant layer.
*/
ConstantType();
/**
* Create the Constant object that outputs a given constant scalar value
* given any input value.
@@ -43,8 +48,18 @@ class ConstantType : public Layer<InputType, OutputType>
*/
ConstantType(const size_t outSize, const double scalar = 0);
//! Copy another ConstantType.
ConstantType(const ConstantType& layer);
//! Take ownership of another ConstantType.
ConstantType(ConstantType&& layer);
//! Copy another ConstantType.
ConstantType& operator=(const ConstantType& layer);
//! Take ownership of another ConstantType.
ConstantType& operator=(ConstantType&& layer);
//! Clone the ConstantType object. This handles polymorphism correctly.
ConstantType* Clone() const { return new ConstantType(*this); }
/**
* Ordinary feed forward pass of a neural network. The forward pass fills the
* output with the specified constant parameter.
src/mlpack/methods/ann/layer/constant_impl.hpp
+64
View File
@@ -19,6 +19,14 @@
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputType, typename OutputType>
ConstantType<InputType, OutputType>::ConstantType() :
inSize(0),
outSize(0)
{
// Nothing to do.
}
template<typename InputType, typename OutputType>
ConstantType<InputType, OutputType>::ConstantType(
const size_t outSize,
@@ -30,6 +38,62 @@ ConstantType<InputType, OutputType>::ConstantType(
constantOutput.fill(scalar);
}
template<typename InputType, typename OutputType>
ConstantType<InputType, OutputType>::ConstantType(
const ConstantType<InputType, OutputType>& other) :
inSize(other.inSize),
outSize(other.outSize),
constantOutput(other.constantOutput)
{
// Nothing else to do.
}
template<typename InputType, typename OutputType>
ConstantType<InputType, OutputType>::ConstantType(
ConstantType<InputType, OutputType>&& other) :
inSize(other.inSize),
outSize(other.outSize),
constantOutput(std::move(other.constantOutput))
{
other.inSize = 0;
other.outSize = 1;
other.constantOutput = OutputType(other.outSize, 1);
}
template<typename InputType, typename OutputType>
ConstantType<InputType, OutputType>&
ConstantType<InputType, OutputType>::operator=(
const ConstantType<InputType, OutputType>& other)
{
if (this != &other)
{
inSize = other.inSize;
outSize = other.outSize;
constantOutput = other.constantOutput;
}
return *this;
}
template<typename InputType, typename OutputType>
ConstantType<InputType, OutputType>&
ConstantType<InputType, OutputType>::operator=(
ConstantType<InputType, OutputType>&& other)
{
if (this != *other)
{
inSize = other.inSize;
outSize = other.outSize;
constantOutput = std::move(other.constantOutput);
other.inSize = 0;
other.outSize = 1;
other.constantOutput = OutputType(other.outSize, 1);
}
return *this;
}
template<typename InputType, typename OutputType>
void ConstantType<InputType, OutputType>::Forward(
const InputType& input, OutputType& output)
src/mlpack/methods/ann/layer/dropout.hpp
-2
View File
@@ -138,8 +138,6 @@ typedef DropoutType<arma::mat, arma::mat> Dropout;
} // namespace ann
} // namespace mlpack
CEREAL_REGISTER_TYPE(mlpack::ann::Dropout);
// Include implementation.
#include "dropout_impl.hpp"
src/mlpack/methods/ann/layer/fast_lstm_impl.hpp
+5 -5
View File
@@ -134,14 +134,14 @@ void FastLSTMType<InputType, OutputType>::Reset()
{
// Set the weight parameter for the input to gate layer (linear layer) using
// the overall layer parameter matrix.
input2GateWeight = OutputDataType(weights.memptr(),
input2GateWeight = OutputType(weights.memptr(),
4 * outSize, inSize, false, false);
input2GateBias = OutputDataType(weights.memptr() + input2GateWeight.n_elem,
input2GateBias = OutputType(weights.memptr() + input2GateWeight.n_elem,
4 * outSize, 1, false, false);
// Set the weight parameter for the output to gate layer
// (linear no bias layer) using the overall layer parameter matrix.
output2GateWeight = OutputDataType(weights.memptr() + input2GateWeight.n_elem
output2GateWeight = OutputType(weights.memptr() + input2GateWeight.n_elem
+ input2GateBias.n_elem, 4 * outSize, outSize, false, false);
}
@@ -210,8 +210,8 @@ void FastLSTMType<InputType, OutputType>::Forward(
forwardStep, forwardStep + batchStep);
gate.cols(forwardStep, forwardStep + batchStep).each_col() += input2GateBias;
arma::subview<double> sigmoidOut = gateActivation.cols(forwardStep,
forwardStep + batchStep);
InputType sigmoidOut(gateActivation.colptr(forwardStep),
gateActivation.n_rows, batchStep, false, false);
FastSigmoid(
gate.submat(0, forwardStep, 3 * outSize - 1, forwardStep + batchStep),
sigmoidOut);
src/mlpack/methods/ann/layer/gru.hpp
+7 -11
View File
@@ -31,9 +31,6 @@
#include <mlpack/prereqs.hpp>
#include "../visitor/delta_visitor.hpp"
#include "../visitor/output_parameter_visitor.hpp"
#include "layer_types.hpp"
#include "add_merge.hpp"
#include "sequential.hpp"
@@ -101,10 +98,9 @@ class GRU : public Layer<InputType, OutputType>
* @param error The calculated error.
* @param gradient The calculated gradient.
*/
template<typename eT>
void Gradient(const arma::Mat<eT>& input,
const arma::Mat<eT>& /* error */,
arma::Mat<eT>& /* gradient */);
void Gradient(const InputType& input,
const OutputType& /* error */,
OutputType& /* gradient */);
/*
* Resets the cell to accept a new input. This breaks the BPTT chain starts a
@@ -115,7 +111,7 @@ class GRU : public Layer<InputType, OutputType>
void ResetCell(const size_t size);
//! The value of the deterministic parameter.
bool Deterministic() const { return deterministic; }
const bool& Deterministic() const { return deterministic; }
//! Modify the value of the deterministic parameter.
bool& Deterministic() { return deterministic; }
@@ -212,13 +208,13 @@ class GRU : public Layer<InputType, OutputType>
OutputType allZeros;
//! Iterator pointed to the last output produced by the cell
std::list<OutputType>::iterator prevOutput;
typename std::list<OutputType>::iterator prevOutput;
//! Iterator pointed to the last output processed by backward
std::list<OutputType>::iterator backIterator;
typename std::list<OutputType>::iterator backIterator;
//! Iterator pointed to the last output processed by gradient
std::list<OutputType>::iterator gradIterator;
typename std::list<OutputType>::iterator gradIterator;
//! Locally-stored previous error.
OutputType prevError;
src/mlpack/methods/ann/layer/gru_impl.hpp
+10 -12
View File
@@ -16,10 +16,6 @@
// In case it hasn't yet been included.
#include "gru.hpp"
#include "../visitor/forward_visitor.hpp"
#include "../visitor/backward_visitor.hpp"
#include "../visitor/gradient_visitor.hpp"
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
@@ -44,21 +40,23 @@ GRU<InputType, OutputType>::GRU(
deterministic(false)
{
// Input specific linear layers(for zt, rt, ot).
input2GateModule = new Linear<>(inSize, 3 * outSize);
input2GateModule = new LinearType<InputType, OutputType>(inSize, 3 * outSize);
// Previous output gates (for zt and rt).
output2GateModule = new LinearNoBias<>(outSize, 2 * outSize);
output2GateModule = new LinearNoBiasType<InputType, OutputType>(outSize,
2 * outSize);
// Previous output gate for ot.
outputHidden2GateModule = new LinearNoBias<>(outSize, outSize);
outputHidden2GateModule = new LinearNoBiasType<InputType, OutputType>(outSize,
outSize);
network.push_back(input2GateModule);
network.push_back(output2GateModule);
network.push_back(outputHidden2GateModule);
inputGateModule = new SigmoidLayer<>();
forgetGateModule = new SigmoidLayer<>();
hiddenStateModule = new TanHLayer<>();
inputGateModule = new SigmoidLayer<InputType, OutputType>();
forgetGateModule = new SigmoidLayer<InputType, OutputType>();
hiddenStateModule = new TanHLayer<InputType, OutputType>();
network.push_back(inputGateModule);
network.push_back(hiddenStateModule);
@@ -139,7 +137,7 @@ void GRU<InputType, OutputType>::Forward(
// cmul2 is (1 - input gate) * hidden gate.
output = (inputGateModule->OutputParameter()
% (*prevOutput - hiddenStateModule->OutputParameter())) +
% hiddenStateModule->OutputParameter();
hiddenStateModule->OutputParameter();
forwardStep++;
if (forwardStep == rho)
@@ -225,7 +223,7 @@ void GRU<InputType, OutputType>::Backward(
// Delta ot.
OutputType dOt = gyLocal % (arma::ones<OutputType>(outSize, batchSize) -
inputGateModule->OutputParameter();
inputGateModule->OutputParameter());
// Delta of input gate.
inputGateModule->Backward(inputGateModule->OutputParameter(), dZt,
src/mlpack/methods/ann/layer/highway_impl.hpp
+8
View File
@@ -208,6 +208,14 @@ void HighwayType<InputType, OutputType>::serialize(
ar(CEREAL_NVP(weights));
ar(CEREAL_NVP(model));
ar(CEREAL_VECTOR_POINTER(network));
// Reset the memory.
if (Archive::is_loading::value)
{
networkOwnerships.clear();
networkOwnerships.resize(network.size(), true);
Reset();
}
}
} // namespace ann
src/mlpack/methods/ann/layer/layer.hpp
+6
View File
@@ -12,6 +12,9 @@
#ifndef MLPACK_METHODS_ANN_LAYER_LAYER_HPP
#define MLPACK_METHODS_ANN_LAYER_LAYER_HPP
namespace mlpack {
namespace ann {
/**
* A layer is an abstract class implementing common neural networks operations,
* such as convolution, batch norm, etc. These operations require managing
@@ -292,4 +295,7 @@ class Layer
};
} // namespace ann
} // namespace mlpack
#endif
src/mlpack/methods/ann/layer/layer_types.hpp
+24 -9
View File
@@ -12,6 +12,8 @@
#ifndef MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP
#define MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP
#include <mlpack/methods/ann/layer/layer.hpp>
// Layer modules.
#include <mlpack/methods/ann/layer/adaptive_max_pooling.hpp>
#include <mlpack/methods/ann/layer/adaptive_mean_pooling.hpp>
@@ -44,6 +46,8 @@
#include <mlpack/methods/ann/layer/linear3d.hpp>
#include <mlpack/methods/ann/layer/log_softmax.hpp>
#include <mlpack/methods/ann/layer/lookup.hpp>
#include <mlpack/methods/ann/layer/lstm.hpp>
#include <mlpack/methods/ann/layer/minibatch_discrimination.hpp>
#include <mlpack/methods/ann/layer/multihead_attention.hpp>
#include <mlpack/methods/ann/layer/multiply_constant.hpp>
#include <mlpack/methods/ann/layer/multiply_merge.hpp>
@@ -53,12 +57,15 @@
#include <mlpack/methods/ann/layer/padding.hpp>
#include <mlpack/methods/ann/layer/parametric_relu.hpp>
#include <mlpack/methods/ann/layer/positional_encoding.hpp>
// #include <mlpack/methods/ann/layer/radial_basis_function.hpp>
#include <mlpack/methods/ann/layer/radial_basis_function.hpp>
#include <mlpack/methods/ann/layer/recurrent.hpp>
#include <mlpack/methods/ann/layer/recurrent_attention.hpp>
#include <mlpack/methods/ann/layer/reinforce_normal.hpp>
#include <mlpack/methods/ann/layer/reparametrization.hpp>
#include <mlpack/methods/ann/layer/select.hpp>
#include <mlpack/methods/ann/layer/sequential.hpp>
#include <mlpack/methods/ann/layer/softmax.hpp>
#include <mlpack/methods/ann/layer/softmin.hpp>
#include <mlpack/methods/ann/layer/softshrink.hpp>
#include <mlpack/methods/ann/layer/spatial_dropout.hpp>
#include <mlpack/methods/ann/layer/subview.hpp>
@@ -66,16 +73,24 @@
#include <mlpack/methods/ann/layer/virtual_batch_norm.hpp>
#include <mlpack/methods/ann/layer/weight_norm.hpp>
// // Convolution modules.
// #include <mlpack/methods/ann/convolution_rules/border_modes.hpp>
// #include <mlpack/methods/ann/convolution_rules/fft_convolution.hpp>
// #include <mlpack/methods/ann/convolution_rules/naive_convolution.hpp>
// Depends on PaddingType<>.
#include <mlpack/methods/ann/layer/atrous_convolution.hpp>
// Depends on Linear<> and LinearNoBias<>.
#include <mlpack/methods/ann/layer/gru.hpp>
// // Regularizers.
// #include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
// Convolution modules.
#include <mlpack/methods/ann/convolution_rules/border_modes.hpp>
#include <mlpack/methods/ann/convolution_rules/fft_convolution.hpp>
#include <mlpack/methods/ann/convolution_rules/naive_convolution.hpp>
// // Loss function modules.
// #include <mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp>
// Regularizers.
#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
// Loss function modules.
#include <mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp>
// Depends on NegativeLogLikelihood.
#include <mlpack/methods/ann/layer/concat_performance.hpp>
// Include definitions for polymorphic serialization.
#include <mlpack/methods/ann/layer/serialization.hpp>
src/mlpack/methods/ann/layer/linear.hpp
-2
View File
@@ -179,8 +179,6 @@ typedef LinearType<arma::mat, arma::mat, NoRegularizer> Linear;
} // namespace ann
} // namespace mlpack
CEREAL_REGISTER_TYPE(mlpack::ann::Linear);
// Include implementation.
#include "linear_impl.hpp"
src/mlpack/methods/ann/layer/log_softmax.hpp
-2
View File
@@ -84,8 +84,6 @@ typedef LogSoftMaxType<arma::mat, arma::mat> LogSoftMax;
} // namespace ann
} // namespace mlpack
CEREAL_REGISTER_TYPE(mlpack::ann::LogSoftMax);
// Include implementation.
#include "log_softmax_impl.hpp"
src/mlpack/methods/ann/layer/lstm.hpp
+5 -9
View File
@@ -59,7 +59,7 @@ template <
typename InputType = arma::mat,
typename OutputType = arma::mat
>
class LSTM
class LSTM : public Layer<InputType, OutputType>
{
public:
//! Create the LSTM object.
@@ -95,7 +95,6 @@ class LSTM
* @param input Input data used for evaluating the specified function.
* @param output Resulting output activation.
*/
template<typename InputType, typename OutputType>
void Forward(const InputType& input, OutputType& output);
/**
@@ -107,7 +106,6 @@ class LSTM
* @param cellState Cell state of the LSTM.
* @param useCellState Use the cellState passed in the LSTM cell.
*/
template<typename InputType, typename OutputType>
void Forward(const InputType& input,
OutputType& output,
OutputType& cellState,
@@ -122,10 +120,9 @@ class LSTM
* @param gy The backpropagated error.
* @param g The calculated gradient.
*/
template<typename InputType, typename ErrorType, typename GradientType>
void Backward(const InputType& input,
const ErrorType& gy,
GradientType& g);
const OutputType& gy,
OutputType& g);
/*
* Reset the layer parameter.
@@ -147,10 +144,9 @@ class LSTM
* @param error The calculated error.
* @param gradient The calculated gradient.
*/
template<typename InputType, typename ErrorType, typename GradientType>
void Gradient(const InputType& input,
const ErrorType& error,
GradientType& gradient);
const OutputType& error,
OutputType& gradient);
//! Get the maximum number of steps to backpropagate through time (BPTT).
size_t Rho() const { return rho; }
src/mlpack/methods/ann/layer/lstm_impl.hpp
+17 -21
View File
@@ -26,7 +26,7 @@ LSTM<InputType, OutputType>::LSTM()
template<typename InputType, typename OutputType>
LSTM<InputType, OutputType>::LSTM(
const LSTM& layer) :
const LSTM& layer) :
inSize(layer.inSize),
outSize(layer.outSize),
rho(layer.rho),
@@ -45,7 +45,7 @@ LSTM<InputType, OutputType>::LSTM(
template<typename InputType, typename OutputType>
LSTM<InputType, OutputType>::LSTM(
LSTM&& layer) :
LSTM&& layer) :
inSize(std::move(layer.inSize)),
outSize(std::move(layer.outSize)),
rho(std::move(layer.rho)),
@@ -63,8 +63,8 @@ LSTM<InputType, OutputType>::LSTM(
}
template <typename InputType, typename OutputType>
LSTM<InputType, OutputType>&
LSTM<InputType, OutputType> :: operator=(const LSTM& layer)
LSTM<InputType, OutputType>&
LSTM<InputType, OutputType>::operator=(const LSTM& layer)
{
if (this != &layer)
{
@@ -82,12 +82,12 @@ LSTM<InputType, OutputType> :: operator=(const LSTM& layer)
rhoSize = layer.rho;
bpttSteps = layer.bpttSteps;
}
return *this;
return *this;
}
template <typename InputType, typename OutputType>
LSTM<InputType, OutputType>&
LSTM<InputType, OutputType> :: operator=(LSTM&& layer)
LSTM<InputType, OutputType>&
LSTM<InputType, OutputType>::operator=(LSTM&& layer)
{
if (this != &layer)
{
@@ -105,7 +105,7 @@ LSTM<InputType, OutputType> :: operator=(LSTM&& layer)
rhoSize = std::move(layer.rho);
bpttSteps = std::move(layer.bpttSteps);
}
return *this;
return *this;
}
template <typename InputType, typename OutputType>
@@ -245,7 +245,6 @@ void LSTM<InputType, OutputType>::Reset()
// Forward when cellState is not needed.
template<typename InputType, typename OutputType>
template<typename InputType, typename OutputType>
void LSTM<InputType, OutputType>::Forward(
const InputType& input, OutputType& output)
{
@@ -256,11 +255,10 @@ void LSTM<InputType, OutputType>::Forward(
// Forward when cellState is needed overloaded LSTM::Forward().
template<typename InputType, typename OutputType>
template<typename InputType, typename OutputType>
void LSTM<InputType, OutputType>::Forward(const InputType& input,
OutputType& output,
OutputType& cellState,
bool useCellState)
OutputType& output,
OutputType& cellState,
bool useCellState)
{
// Check if the batch size changed, the number of cols is defines the input
// batch size.
@@ -370,11 +368,10 @@ void LSTM<InputType, OutputType>::Forward(const InputType& input,
}
template<typename InputType, typename OutputType>
template<typename InputType, typename ErrorType, typename GradientType>
void LSTM<InputType, OutputType>::Backward(
const InputType& /* input */, const ErrorType& gy, GradientType& g)
const InputType& /* input */, const OutputType& gy, OutputType& g)
{
ErrorType gyLocal;
OutputType gyLocal;
if (gradientStepIdx > 0)
{
gyLocal = gy + prevError;
@@ -382,8 +379,8 @@ void LSTM<InputType, OutputType>::Backward(
else
{
// Make an alias.
gyLocal = ErrorType(((ErrorType&) gy).memptr(), gy.n_rows, gy.n_cols, false,
false);
gyLocal = OutputType(((OutputType&) gy).memptr(), gy.n_rows, gy.n_cols,
false, false);
}
outputGateError =
@@ -448,11 +445,10 @@ void LSTM<InputType, OutputType>::Backward(
}
template<typename InputType, typename OutputType>
template<typename InputType, typename ErrorType, typename GradientType>
void LSTM<InputType, OutputType>::Gradient(
const InputType& input,
const ErrorType& /* error */,
GradientType& gradient)
const OutputType& /* error */,
OutputType& gradient)
{
// Input2GateOutputWeight and input2GateOutputBias gradients.
gradient.submat(0, 0, input2GateOutputWeight.n_elem - 1, 0) =
src/mlpack/methods/ann/layer/max_pooling.hpp
+4
View File
@@ -70,6 +70,10 @@ class MaxPoolingType : public Layer<InputType, OutputType>
const size_t strideHeight = 1,
const bool floor = true);
// TODO: copy constructor / move constructor
MaxPoolingType* Clone() const { return new MaxPoolingType(*this); }
/**
* Ordinary feed forward pass of a neural network, evaluating the function
* f(x) by propagating the activity forward through f.
src/mlpack/methods/ann/layer/mean_pooling.hpp
+3
View File
@@ -51,6 +51,9 @@ class MeanPoolingType : public Layer<InputType, OutputType>
const size_t strideHeight = 1,
const bool floor = true);
// TODO: copy constructor / move constructor
MeanPoolingType* Clone() const { return new MeanPoolingType(*this); }
/**
* Ordinary feed forward pass of a neural network, evaluating the function
* f(x) by propagating the activity forward through f.
src/mlpack/methods/ann/layer/padding.hpp
+2 -2
View File
@@ -47,8 +47,8 @@ class PaddingType : public Layer<InputType, OutputType>
const size_t padHTop = 0,
const size_t padHBottom = 0);
//! Clone the PaddingType object. This handles polymorphism correctly.
PaddingType* Clone() const { return new PaddingType(*this); }
//! Clone the PaddingType object. This handles polymorphism correctly.
PaddingType* Clone() const { return new PaddingType(*this); }
/**
* Ordinary feed forward pass of a neural network, evaluating the function
src/mlpack/methods/ann/layer/recurrent.hpp
+4 -12
View File
@@ -14,11 +14,6 @@
#include <mlpack/core.hpp>
#include "../visitor/delete_visitor.hpp"
#include "../visitor/delta_visitor.hpp"
#include "../visitor/copy_visitor.hpp"
#include "../visitor/output_parameter_visitor.hpp"
#include "layer_types.hpp"
#include "add_merge.hpp"
#include "sequential.hpp"
@@ -37,7 +32,7 @@ namespace ann /** Artificial Neural Network. */ {
*/
template <
typename InputType = arma::mat,
typename OutputType = arma::mat,
typename OutputType = arma::mat
>
class Recurrent : public Layer<InputType, OutputType>
{
@@ -107,7 +102,7 @@ class Recurrent : public Layer<InputType, OutputType>
std::vector<Layer<InputType, OutputType>*>& Model() { return network; }
//! The value of the deterministic parameter.
bool Deterministic() const { return deterministic; }
const bool& Deterministic() const { return deterministic; }
//! Modify the value of the deterministic parameter.
bool& Deterministic() { return deterministic; }
@@ -141,9 +136,6 @@ class Recurrent : public Layer<InputType, OutputType>
void serialize(Archive& ar, const uint32_t /* version */);
private:
//! Locally-stored delete visitor module object.
DeleteVisitor deleteVisitor;
//! Locally-stored start module.
Layer<InputType, OutputType>* startModule;
@@ -179,10 +171,10 @@ class Recurrent : public Layer<InputType, OutputType>
OutputType parameters;
//! Locally-stored initial module.
Sequential<InputType, OutputType>* initialModule;
SequentialType<InputType, OutputType>* initialModule;
//! Locally-stored recurrent module.
Sequential<InputType, OutputType>* recurrentModule;
SequentialType<InputType, OutputType>* recurrentModule;
//! Locally-stored model modules.
std::vector<Layer<InputType, OutputType>*> network;
src/mlpack/methods/ann/layer/recurrent_attention.hpp
+2 -8
View File
@@ -13,12 +13,6 @@
#define MLPACK_METHODS_ANN_LAYER_RECURRENT_ATTENTION_HPP
#include <mlpack/prereqs.hpp>
#include <boost/ptr_container/ptr_vector.hpp>
#include "../visitor/delta_visitor.hpp"
#include "../visitor/output_parameter_visitor.hpp"
#include "../visitor/reset_visitor.hpp"
#include "../visitor/weight_size_visitor.hpp"
#include "layer_types.hpp"
#include "add_merge.hpp"
@@ -110,10 +104,10 @@ class RecurrentAttention : public Layer<InputType, OutputType>
OutputType& /* gradient */);
//! Get the model modules.
std::vector<LayerTypes<>>& Model() { return network; }
std::vector<Layer<InputType, OutputType>*>& Model() { return network; }
//! The value of the deterministic parameter.
bool Deterministic() const { return deterministic; }
const bool& Deterministic() const { return deterministic; }
//! Modify the value of the deterministic parameter.
bool& Deterministic() { return deterministic; }
src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp
+1 -9
View File
@@ -15,14 +15,6 @@
// In case it hasn't yet been included.
#include "recurrent_attention.hpp"
#include "../visitor/load_output_parameter_visitor.hpp"
#include "../visitor/save_output_parameter_visitor.hpp"
#include "../visitor/backward_visitor.hpp"
#include "../visitor/forward_visitor.hpp"
#include "../visitor/gradient_set_visitor.hpp"
#include "../visitor/gradient_update_visitor.hpp"
#include "../visitor/gradient_visitor.hpp"
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
@@ -97,7 +89,7 @@ void RecurrentAttention<InputType, OutputType>::Forward(
}
}
output = boost::apply_visitor(outputParameterVisitor, rnnModule);
output = rnnModule->OutputParameter();
forwardStep = 0;
backwardStep = 0;
src/mlpack/methods/ann/layer/recurrent_impl.hpp
+16 -24
View File
@@ -15,11 +15,6 @@
// In case it hasn't yet been included.
#include "recurrent.hpp"
#include "../visitor/add_visitor.hpp"
#include "../visitor/backward_visitor.hpp"
#include "../visitor/gradient_visitor.hpp"
#include "../visitor/gradient_zero_visitor.hpp"
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
@@ -60,9 +55,9 @@ Recurrent<InputType, OutputType>::Recurrent(
deterministic(false),
ownsLayer(true)
{
initialModule = new Sequential<>();
mergeModule = new AddMerge<>(false, false, false);
recurrentModule = new Sequential<>(false, false);
initialModule = new SequentialType<InputType, OutputType>();
mergeModule = new AddMerge<InputType, OutputType>(false, false, false);
recurrentModule = new SequentialType<InputType, OutputType>(false, false);
initialModule->Add(inputModule);
initialModule->Add(startModule);
@@ -95,9 +90,9 @@ Recurrent<InputType, OutputType>::Recurrent(
feedbackModule = network.feedbackModule->Clone();
transferModule = network.transferModule->Clone();
initialModule = new Sequential<>();
mergeModule = new AddMerge<>(false, false, false);
recurrentModule = new Sequential<>(false, false);
initialModule = new SequentialType<InputType, OutputType>();
mergeModule = new AddMerge<InputType, OutputType>(false, false, false);
recurrentModule = new SequentialType<InputType, OutputType>(false, false);
initialModule->Add(inputModule);
initialModule->Add(startModule);
@@ -116,7 +111,7 @@ Recurrent<InputType, OutputType>::Recurrent(
}
template<typename InputType, typename OutputType>
void Recurrent<InputType, OutputType, CustomLayers...>::Forward(
void Recurrent<InputType, OutputType>::Forward(
const InputType& input, OutputType& output)
{
if (forwardStep == 0)
@@ -152,9 +147,8 @@ void Recurrent<InputType, OutputType, CustomLayers...>::Forward(
}
}
template<typename InputType, typename OutputType,
typename... CustomLayers>
void Recurrent<InputType, OutputType, CustomLayers...>::Backward(
template<typename InputType, typename OutputType>
void Recurrent<InputType, OutputType>::Backward(
const InputType& /* input */, const OutputType& gy, OutputType& g)
{
if (!recurrentError.is_empty())
@@ -185,9 +179,8 @@ void Recurrent<InputType, OutputType, CustomLayers...>::Backward(
backwardStep++;
}
template<typename InputType, typename OutputType,
typename... CustomLayers>
void Recurrent<InputType, OutputType, CustomLayers...>::Gradient(
template<typename InputType, typename OutputType>
void Recurrent<InputType, OutputType>::Gradient(
const InputType& input,
const OutputType& error,
OutputType& /* gradient */)
@@ -219,10 +212,9 @@ void Recurrent<InputType, OutputType, CustomLayers...>::Gradient(
}
}
template<typename InputType, typename OutputType,
typename... CustomLayers>
template<typename InputType, typename OutputType>
template<typename Archive>
void Recurrent<InputType, OutputType, CustomLayers...>::serialize(
void Recurrent<InputType, OutputType>::serialize(
Archive& ar, const uint32_t /* version */)
{
ar(cereal::base_class<Layer<InputType, OutputType>>(this));
@@ -246,9 +238,9 @@ void Recurrent<InputType, OutputType, CustomLayers...>::serialize(
// Set up the network.
if (cereal::is_loading<Archive>())
{
initialModule = new Sequential<>();
mergeModule = new AddMerge<>(false, false, false);
recurrentModule = new Sequential<>(false, false);
initialModule = new SequentialType<InputType, OutputType>();
mergeModule = new AddMerge<InputType, OutputType>(false, false, false);
recurrentModule = new SequentialType<InputType, OutputType>(false, false);
initialModule->Add(inputModule);
initialModule->Add(startModule);
src/mlpack/methods/ann/layer/serialization.hpp
+16 -32
View File
@@ -21,37 +21,19 @@
mlpack::ann::NaiveConvolution<mlpack::ann::FullConvolution>, \
mlpack::ann::NaiveConvolution<mlpack::ann::ValidConvolution>, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::LogisticFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::IdentityFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer< \
mlpack::ann::RectifierFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::TanhFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::SoftplusFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer< \
mlpack::ann::HardSigmoidFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer< \
mlpack::ann::RectifierFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::SwishFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::MishFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::LiSHTFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::GELUFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::ElliotFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::ElishFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BaseLayer<mlpack::ann::GaussianFunction, \
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::SigmoidLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::IdentityLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::ReLULayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::TanHLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::SoftPlusLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::HardSigmoidLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::SwishFunctionLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::MishFunctionLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::LiSHTFunctionLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::GELUFunctionLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::ElliotFunctionLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::ElishFunctionLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::GaussianFunctionLayer<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BatchNorm<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::BilinearInterpolationType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::CELUType<__VA_ARGS__>); \
@@ -97,7 +79,7 @@
CEREAL_REGISTER_TYPE(mlpack::ann::PositionalEncodingType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::RBF<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::RecurrentAttention<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::RecurrentType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::Recurrent<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::ReinforceNormalType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::ReparametrizationType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::SelectType<__VA_ARGS__>); \
@@ -117,6 +99,8 @@
// TODO: continue...
CEREAL_REGISTER_MLPACK_LAYERS(arma::mat, arma::mat);
// TODO: I think this below is not needed.
/**
* Register an mlpack layer with the given INPUT_TYPE and OUTPUT_TYPE.
src/mlpack/methods/ann/layer/spatial_dropout.hpp
+1 -1
View File
@@ -86,7 +86,7 @@ class SpatialDropoutType : public Layer<InputType, OutputType>
size_t& Size() { return size; }
//! Get the value of the deterministic parameter.
bool Deterministic() const { return deterministic; }
const bool& Deterministic() const { return deterministic; }
//! Modify the value of the deterministic parameter.
bool& Deterministic() { return deterministic; }
src/mlpack/methods/ann/layer/weight_norm.hpp
+14
View File
@@ -52,6 +52,11 @@ template <
class WeightNormType : public Layer<InputType, OutputType>
{
public:
/**
* Create an empty WeightNorm layer.
*/
WeightNormType();
/**
* Create the WeightNorm layer object.
*
@@ -62,6 +67,15 @@ class WeightNormType : public Layer<InputType, OutputType>
//! Destructor to release allocated memory.
~WeightNormType();
//! Create a WeightNorm layer by copying the given layer.
WeightNormType(const WeightNormType& other);
//! Create a WeightNorm layer by taking ownership of the other layer.
WeightNormType(WeightNormType&& other);
//! Copy the given layer.
WeightNormType& operator=(const WeightNormType& other);
//! Take ownership of the data in the given layer.
WeightNormType& operator=(WeightNormType&& other);
//! Clone the WeightNormType object. This handles polymorphism correctly.
WeightNormType* Clone() const { return new WeightNormType(*this); }
src/mlpack/methods/ann/layer/weight_norm_impl.hpp
+73
View File
@@ -19,6 +19,17 @@
namespace mlpack {
namespace ann { /** Artificial Neural Network. */
template<typename InputType, typename OutputType>
WeightNormType<InputType, OutputType>::
WeightNormType() : wrappedLayer(new LinearType<InputType, OutputType>())
{
layerWeightSize = wrappedLayer->WeightSize();
weights.set_size(layerWeightSize + 1, 1);
layerWeights.set_size(layerWeightSize, 1);
layerGradients.set_size(layerWeightSize, 1);
}
template<typename InputType, typename OutputType>
WeightNormType<InputType, OutputType>::
WeightNormType(Layer<InputType, OutputType>* layer) : wrappedLayer(layer)
@@ -30,6 +41,68 @@ WeightNormType(Layer<InputType, OutputType>* layer) : wrappedLayer(layer)
layerGradients.set_size(layerWeightSize, 1);
}
template<typename InputType, typename OutputType>
WeightNormType<InputType, OutputType>::WeightNormType(
const WeightNormType<InputType, OutputType>& other) :
wrappedLayer(other.wrappedLayer->Clone()),
layerWeightSize(other.layerWeightSize),
weights(other.weights),
layerGradients(other.layerGradients),
layerWeights(other.layerWeights)
{
// Nothing else to do.
}
template<typename InputType, typename OutputType>
WeightNormType<InputType, OutputType>::WeightNormType(
WeightNormType<InputType, OutputType>&& other) :
wrappedLayer(std::move(other.wrappedLayer)),
layerWeightSize(other.layerWeightSize),
weights(std::move(other.weights)),
layerGradients(std::move(other.layerGradients)),
layerWeights(std::move(other.layerWeights))
{
// Reset the other layer.
other = WeightNormType<InputType, OutputType>();
}
template<typename InputType, typename OutputType>
WeightNormType<InputType, OutputType>&
WeightNormType<InputType, OutputType>::operator=(
const WeightNormType<InputType, OutputType>& other)
{
if (this != &other)
{
wrappedLayer = other.wrappedLayer->Clone();
layerWeightSize = other.layerWeightSize;
weights = other.weights;
layerWeights = other.layerWeights;
layerGradients = other.layerGradients;
}
return *this;
}
template<typename InputType, typename OutputType>
WeightNormType<InputType, OutputType>&
WeightNormType<InputType, OutputType>::operator=(
WeightNormType<InputType, OutputType>&& other)
{
if (this != &other)
{
wrappedLayer = std::move(other.wrappedLayer);
layerWeightSize = other.layerWeightSize;
weights = std::move(other.weights);
layerWeights = std::move(other.layerWeights);
layerGradients = std::move(other.layerGradients);
// Reset the other layer.
other = WeightNormType<InputType, OutputType>();
}
return *this;
}
template<typename InputType, typename OutputType>
WeightNormType<InputType, OutputType>::~WeightNormType()
{
src/mlpack/tests/loss_functions_test.cpp
+6 -6
View File
@@ -424,9 +424,9 @@ TEST_CASE("GradientMeanSquaredErrorTest", "[LossFunctionsTest]")
model = new FFN<MeanSquaredError<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
model->Add<IdentityLayer>();
model->Add<Linear>(10, 2);
model->Add<SigmoidLayer>();
model->Add<IdentityLayer<>>();
model->Add<LinearType<>>(10, 2);
model->Add<SigmoidLayer<>>();
}
~GradientFunction()
@@ -467,9 +467,9 @@ TEST_CASE("GradientReconstructionLossTest", "[LossFunctionsTest]")
model = new FFN<ReconstructionLoss<>, NguyenWidrowInitialization>();
model->Predictors() = input;
model->Responses() = target;
model->Add<IdentityLayer>();
model->Add<Linear>(10, 2);
model->Add<SigmoidLayer>();
model->Add<IdentityLayer<>>();
model->Add<LinearType<>>(10, 2);
model->Add<SigmoidLayer<>>();
}
~GradientFunction()