diff --git a/.ci/macos-steps.yaml b/.ci/macos-steps.yaml index 7439d0b2b2..bdc11fe3f7 100644 --- a/.ci/macos-steps.yaml +++ b/.ci/macos-steps.yaml @@ -15,6 +15,7 @@ steps: sudo xcode-select --switch /Applications/Xcode_10.1.app/Contents/Developer unset BOOST_ROOT pip install cython numpy pandas zipp + brew update brew install openblas armadillo boost if [ "a$(julia.version)" != "a" ]; then diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index abf573a352..9b2e3df868 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -125,6 +125,8 @@ Copyright: Copyright 2019, Rohit Kartik Copyright 2019, Aditya Viki Copyright 2019, Kartik Dutt + Copyright 2020, Sriram S K + Copyright 2020, Manoranjan Kumar Bharti ( Nakul Bharti ) License: BSD-3-clause All rights reserved. diff --git a/HISTORY.md b/HISTORY.md index 98f39f236a..c34a395540 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,7 +2,9 @@ ###### ????-??-?? * Added `mean squared logarithmic error` loss function for neural networks (#2210). - + + * Added `mean bias loss function` for neural networks (#2210). + * The DecisionStump class has been marked deprecated; use the `DecisionTree` class with `NoRecursion=true` or use `ID3DecisionStump` instead (#2099). @@ -33,10 +35,17 @@ * Add Mish activation function (#2158). + * Update `init_rules` in AMF to allow users to merge two initialization + rules (#2151). + * Add GELU activation function (#2183). * Better error handling of eigendecompositions and Cholesky decompositions (#2088, #1840). + + * Add LiSHT activation function (#2182). + + * Add Valid and Same Padding for Transposed Convolution layer (#2163). ### mlpack 3.2.2 ###### 2019-11-26 diff --git a/doc/guide/sample_ml_app.hpp b/doc/guide/sample_ml_app.hpp index eb8bade6f0..6e42a19c07 100644 --- a/doc/guide/sample_ml_app.hpp +++ b/doc/guide/sample_ml_app.hpp @@ -27,20 +27,20 @@ mlpack and dependencies in Release Mode). - Right click on the project and select Properties, select the x64 Debug profile - Under C/C++ > General > Additional Include Directories add: @code - - C:\boost\boost_1_66_0 - - C:\mlpack\armadillo-8.500.1\include - - C:\mlpack\mlpack-3.2.1\build\include + - C:\boost\boost_1_71_0\lib\native\include + - C:\mlpack\armadillo-9.800.3\include + - C:\mlpack\mlpack-3.2.2\build\include @endcode - Under Linker > Input > Additional Dependencies add: @code - - C:\mlpack\mlpack-3.2.1\build\Debug\mlpack.lib - - C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_serialization-vc141-mt-gd-x64-1_66.lib - - C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_program_options-vc141-mt-gd-x64-1_66.lib + - C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.lib + - C:\boost\boost_1_71_0\lib64-msvc-14.2\libboost_serialization-vc142-mt-gd-x64-1_71.lib + - C:\boost\boost_1_71_0\lib64-msvc-14.2\libboost_program_options-vc142-mt-gd-x64-1_71.lib @endcode - Under Build Events > Post-Build Event > Command Line add: @code - - xcopy /y "C:\mlpack\mlpack-3.2.1\build\Debug\mlpack.dll" $(OutDir) - - xcopy /y "C:\mlpack\mlpack-3.2.1\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.2.2\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) @endcode @note Recent versions of Visual Studio set "Conformance Mode" enabled by default. This causes some issues with diff --git a/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp b/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp index 32486902e9..12b744d697 100644 --- a/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp +++ b/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp @@ -273,7 +273,7 @@ class BinarySpaceTree * Create a binary space tree by copying the other tree. Be careful! This * can take a long time and use a lot of memory. * - * @param other Tree to be replicated. + * @param other Tree to be copied. */ BinarySpaceTree(const BinarySpaceTree& other); @@ -283,6 +283,20 @@ class BinarySpaceTree */ BinarySpaceTree(BinarySpaceTree&& other); + /** + * Copy the given BinarySaceTree. + * + * @param other The tree to be copied. + */ + BinarySpaceTree& operator=(const BinarySpaceTree& other); + + /** + * Take ownership of the given BinarySpaceTree. + * + * @param other The tree to take ownership of. + */ + BinarySpaceTree& operator=(BinarySpaceTree&& other); + /** * Initialize the tree from a boost::serialization archive. * diff --git a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp index a046d8e4db..7c2f4c08b9 100644 --- a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp +++ b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp @@ -341,6 +341,7 @@ BinarySpaceTree( stat(other.stat), parentDistance(other.parentDistance), furthestDescendantDistance(other.furthestDescendantDistance), + minimumBoundDistance(other.minimumBoundDistance), // Copy matrix, but only if we are the root. dataset((other.parent == NULL) ? new MatType(*other.dataset) : NULL) { @@ -379,6 +380,126 @@ BinarySpaceTree( } } +/** + * Copy assignment operator: copy the given other tree. + */ +template class BoundType, + template + class SplitType> +BinarySpaceTree& +BinarySpaceTree:: +operator=(const BinarySpaceTree& other) +{ + // Return if it's the same tree. + if (this == &other) + return *this; + + // Freeing memory that will not be used anymore. + delete dataset; + delete left; + delete right; + + left = NULL; + right = NULL; + parent = other.Parent(); + begin = other.Begin(); + count = other.Count(); + bound = other.bound; + stat = other.stat; + parentDistance = other.ParentDistance(); + furthestDescendantDistance = other.FurthestDescendantDistance(); + minimumBoundDistance = other.MinimumBoundDistance(); + // Copy matrix, but only if we are the root. + dataset = ((other.parent == NULL) ? new MatType(*other.dataset) : NULL); + + // Create left and right children (if any). + if (other.Left()) + { + left = new BinarySpaceTree(*other.Left()); + left->Parent() = this; // Set parent to this, not other tree. + } + + if (other.Right()) + { + right = new BinarySpaceTree(*other.Right()); + right->Parent() = this; // Set parent to this, not other tree. + } + + // Propagate matrix, but only if we are the root. + if (parent == NULL) + { + std::queue queue; + if (left) + queue.push(left); + if (right) + queue.push(right); + while (!queue.empty()) + { + BinarySpaceTree* node = queue.front(); + queue.pop(); + + node->dataset = dataset; + if (node->left) + queue.push(node->left); + if (node->right) + queue.push(node->right); + } + } + + return *this; +} + +/** + * Move assignment operator: take ownership of the given tree. + */ +template class BoundType, + template + class SplitType> +BinarySpaceTree& +BinarySpaceTree:: +operator=(BinarySpaceTree&& other) +{ + // Return if it's the same tree. + if (this == &other) + return *this; + + // Freeing memory that will not be used anymore. + delete dataset; + delete left; + delete right; + + parent = other.Parent(); + left = other.Left(); + right = other.Right(); + begin = other.Begin(); + count = other.Count(); + bound = std::move(other.bound); + stat = std::move(other.stat); + parentDistance = other.ParentDistance(); + furthestDescendantDistance = other.FurthestDescendantDistance(); + minimumBoundDistance = other.MinimumBoundDistance(); + dataset = other.dataset; + + other.left = NULL; + other.right = NULL; + other.parent = NULL; + other.begin = 0; + other.count = 0; + other.parentDistance = 0.0; + other.furthestDescendantDistance = 0.0; + other.minimumBoundDistance = 0.0; + other.dataset = NULL; + + return *this; +} + + /** * Move constructor. */ @@ -406,6 +527,7 @@ BinarySpaceTree(BinarySpaceTree&& other) : // tree's contents, so it doesn't delete anything when it is destructed. other.left = NULL; other.right = NULL; + other.parent = NULL; other.begin = 0; other.count = 0; other.parentDistance = 0.0; diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp index 40a747c875..c7947e14ed 100644 --- a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp +++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp @@ -18,11 +18,12 @@ namespace mlpack { namespace tree { CosineTree::CosineTree(const arma::mat& dataset) : - dataset(dataset), + dataset(&dataset), parent(NULL), left(NULL), right(NULL), - numColumns(dataset.n_cols) + numColumns(dataset.n_cols), + localDataset(false) { // Initialize sizes of column indices and l2 norms. indices.resize(numColumns); @@ -47,11 +48,12 @@ CosineTree::CosineTree(const arma::mat& dataset) : CosineTree::CosineTree(CosineTree& parentNode, const std::vector& subIndices) : - dataset(parentNode.GetDataset()), + dataset(&parentNode.GetDataset()), parent(&parentNode), left(NULL), right(NULL), - numColumns(subIndices.size()) + numColumns(subIndices.size()), + localDataset(false) { // Initialize sizes of column indices and l2 norms. indices.resize(numColumns); @@ -76,10 +78,11 @@ CosineTree::CosineTree(CosineTree& parentNode, CosineTree::CosineTree(const arma::mat& dataset, const double epsilon, const double delta) : - dataset(dataset), + dataset(&dataset), delta(delta), left(NULL), - right(NULL) + right(NULL), + localDataset(false) { // Declare the cosine tree priority queue. CosineNodeQueue treeQueue; @@ -150,8 +153,214 @@ CosineTree::CosineTree(const arma::mat& dataset, ConstructBasis(treeQueue); } +//! Copy the given tree. +CosineTree::CosineTree(const CosineTree& other) : + // Copy matrix, but only if we are the root. + dataset((other.parent == NULL) ? new arma::mat(*other.dataset) : NULL), + delta(other.delta), + parent(NULL), + left(NULL), + right(NULL), + indices(other.indices), + l2NormsSquared(other.l2NormsSquared), + centroid(other.centroid), + basisVector(other.basisVector), + splitPointIndex(other.SplitPointIndex()), + numColumns(other.NumColumns()), + l2Error(other.L2Error()), + frobNormSquared(other.FrobNormSquared()), + localDataset(other.parent == NULL) +{ + // Create left and right children (if any). + if (other.Left()) + { + left = new CosineTree(*other.Left()); + left->Parent() = this; // Set parent to this, not other tree. + } + + if (other.Right()) + { + right = new CosineTree(*other.Right()); + right->Parent() = this; // Set parent to this, not other tree. + } + + // Propagate matrix, but only if we are the root. + if (parent == NULL && localDataset) + { + std::queue queue; + if (left) + queue.push(left); + if (right) + queue.push(right); + while (!queue.empty()) + { + CosineTree* node = queue.front(); + queue.pop(); + + node->dataset = dataset; + if (node->left) + queue.push(node->left); + if (node->right) + queue.push(node->right); + } + } +} + +//! Copy assignment operator: copy the given other tree. +CosineTree& CosineTree::operator=(const CosineTree& other) +{ + // Return if it's the same tree. + if (this == &other) + return *this; + + // Freeing memory that will not be used anymore. + if (localDataset) + delete dataset; + + delete left; + delete right; + + // Performing a deep copy of the dataset. + dataset = (other.parent == NULL) ? new arma::mat(*other.dataset) : NULL; + + delta = other.delta; + parent = other.Parent(); + left = other.Left(); + right = other.Right(); + indices = other.indices; + l2NormsSquared = other.l2NormsSquared; + centroid = other.centroid; + basisVector = other.basisVector; + splitPointIndex = other.SplitPointIndex(); + numColumns = other.NumColumns(); + l2Error = other.L2Error(); + localDataset = (other.parent == NULL) ? true : false; + frobNormSquared = other.FrobNormSquared(); + + // Create left and right children (if any). + if (other.Left()) + { + left = new CosineTree(*other.Left()); + left->Parent() = this; // Set parent to this, not other tree. + } + + if (other.Right()) + { + right = new CosineTree(*other.Right()); + right->Parent() = this; // Set parent to this, not other tree. + } + + // Propagate matrix, but only if we are the root. + if (parent == NULL && localDataset) + { + std::queue queue; + if (left) + queue.push(left); + if (right) + queue.push(right); + while (!queue.empty()) + { + CosineTree* node = queue.front(); + queue.pop(); + + node->dataset = dataset; + if (node->left) + queue.push(node->left); + if (node->right) + queue.push(node->right); + } + } + + return *this; +} + +//! Move the given tree. +CosineTree::CosineTree(CosineTree&& other) : + dataset(other.dataset), + delta(std::move(other.delta)), + parent(other.parent), + left(other.left), + right(other.right), + indices(std::move(other.indices)), + l2NormsSquared(std::move(other.l2NormsSquared)), + centroid(std::move(other.centroid)), + basisVector(std::move(other.basisVector)), + splitPointIndex(other.splitPointIndex), + numColumns(other.numColumns), + l2Error(other.l2Error), + frobNormSquared(other.frobNormSquared), + localDataset(other.localDataset) +{ + // Now we are a clone of the other tree. But we must also clear the other + // tree's contents, so it doesn't delete anything when it is destructed. + other.dataset = NULL; + other.parent = NULL; + other.left = NULL; + other.right = NULL; + other.splitPointIndex = 0; + other.numColumns = 0; + other.l2Error = -1; + other.localDataset = false; + other.frobNormSquared = 0; + // Set new parent. + if (left) + left->parent = this; + if (right) + right->parent = this; +} + +//! Move assignment operator: take ownership of the given tree. +CosineTree& CosineTree::operator=(CosineTree&& other) +{ + // Return if it's the same tree. + if (this == &other) + return *this; + + // Freeing memory that will not be used anymore. + if (localDataset) + delete dataset; + delete left; + delete right; + + dataset = other.dataset; + delta = std::move(other.delta); + parent = other.Parent(); + left = other.Left(); + right = other.Right(); + indices = std::move(other.indices); + l2NormsSquared = std::move(other.l2NormsSquared); + centroid = std::move(other.centroid); + basisVector = std::move(other.basisVector); + splitPointIndex = other.SplitPointIndex(); + numColumns = other.NumColumns(); + l2Error = other.L2Error(); + localDataset = other.localDataset; + frobNormSquared = other.FrobNormSquared(); + + // Now we are a clone of the other tree. But we must also clear the other + // tree's contents, so it doesn't delete anything when it is destructed. + other.dataset = NULL; + other.parent = NULL; + other.left = NULL; + other.right = NULL; + other.splitPointIndex = 0; + other.numColumns = 0; + other.l2Error = -1; + other.localDataset = false; + other.frobNormSquared = 0; + // Set new parent. + if (left) + left->parent = this; + if (right) + right->parent = this; + + return *this; +} + CosineTree::~CosineTree() { + if (localDataset) + delete dataset; if (left) delete left; if (right) @@ -206,7 +415,7 @@ double CosineTree::MonteCarloError(CosineTree* node, node->ColumnSamplesLS(sampledIndices, probabilities, numSamples); // Get pointer to the original dataset. - arma::mat dataset = node->GetDataset(); + const arma::mat& dataset = node->GetDataset(); // Initialize weighted projection magnitudes as zeros. arma::vec weightedMagnitudes; @@ -280,7 +489,7 @@ double CosineTree::MonteCarloError(CosineTree* node, void CosineTree::ConstructBasis(CosineNodeQueue& treeQueue) { // Initialize basis as matrix of zeros. - basis.zeros(dataset.n_rows, treeQueue.size()); + basis.zeros(dataset->n_rows, treeQueue.size()); // Variables for iterating through the priority queue. CosineTree *currentNode; @@ -435,8 +644,8 @@ void CosineTree::CalculateCosines(arma::vec& cosines) else { cosines(i) = - std::abs(arma::norm_dot(dataset.col(indices[splitPointIndex]), - dataset.col(indices[i]))); + std::abs(arma::norm_dot(dataset->col(indices[splitPointIndex]), + dataset->col(indices[i]))); } } } @@ -444,12 +653,12 @@ void CosineTree::CalculateCosines(arma::vec& cosines) void CosineTree::CalculateCentroid() { // Initialize centroid as vector of zeros. - centroid.zeros(dataset.n_rows); + centroid.zeros(dataset->n_rows); // Calculate centroid of columns in the node. for (size_t i = 0; i < numColumns; i++) { - centroid += dataset.col(indices[i]); + centroid += dataset->col(indices[i]); } centroid /= numColumns; } diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp index a94562afba..b2244619aa 100644 --- a/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp +++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp @@ -68,6 +68,35 @@ class CosineTree const double epsilon, const double delta); + /** + * Copy the given tree. Be careful! This may use a lot of memory. + * + * @param other Tree to copy from. + */ + CosineTree(const CosineTree& other); + + /** + * Move the given tree. The tree passed as a parameter will be emptied and + * will not be usable after this call. + * + * @param other Tree to move. + */ + CosineTree(CosineTree&& other); + + /** + * Copy the given Cosine Tree. + * + * @param other The tree to be copied. + */ + CosineTree& operator=(const CosineTree& other); + + /** + * Take ownership of the given Cosine Tree. + * + * @param other The tree to take ownership of. + */ + CosineTree& operator=(CosineTree&& other); + /** * Clean up the CosineTree: release allocated memory (including children). */ @@ -169,7 +198,7 @@ class CosineTree void GetFinalBasis(arma::mat& finalBasis) { finalBasis = basis; } //! Get pointer to the dataset matrix. - const arma::mat& GetDataset() const { return dataset; } + const arma::mat& GetDataset() const { return *dataset; } //! Get the indices of columns in the node. std::vector& VectorIndices() { return indices; } @@ -214,7 +243,7 @@ class CosineTree private: //! Matrix for which cosine tree is constructed. - const arma::mat& dataset; + const arma::mat* dataset; //! Cumulative probability for Monte Carlo error lower bound. double delta; //! Subspace basis of the input dataset. @@ -241,6 +270,8 @@ class CosineTree double l2Error; //! Frobenius norm squared of columns in the node. double frobNormSquared; + //! If true, we own the dataset and need to destroy it in the destructor. + bool localDataset; }; class CompareCosineNode diff --git a/src/mlpack/core/tree/cover_tree/cover_tree_impl.hpp b/src/mlpack/core/tree/cover_tree/cover_tree_impl.hpp index ce56ce7300..49e95ca866 100644 --- a/src/mlpack/core/tree/cover_tree/cover_tree_impl.hpp +++ b/src/mlpack/core/tree/cover_tree/cover_tree_impl.hpp @@ -548,7 +548,7 @@ CoverTree::CoverTree( } } -// Copy Assignment. +// Copy assignment operator: copy the given other tree. template< typename MetricType, typename StatisticType, @@ -658,7 +658,7 @@ CoverTree::CoverTree( other.metric = NULL; } -// Move Assignment. +// Move assignment operator: take ownership of the given tree. template< typename MetricType, typename StatisticType, diff --git a/src/mlpack/core/tree/octree/octree.hpp b/src/mlpack/core/tree/octree/octree.hpp index 78184d30f7..d24338730c 100644 --- a/src/mlpack/core/tree/octree/octree.hpp +++ b/src/mlpack/core/tree/octree/octree.hpp @@ -220,6 +220,20 @@ class Octree */ Octree(Octree&& other); + /** + * Copy the given Octree. + * + * @param other The tree to be copied. + */ + Octree& operator=(const Octree& other); + + /** + * Take ownership of the given Octree. + * + * @param other The tree to take ownership of. + */ + Octree& operator=(Octree&& other); + /** * Initialize the tree from a boost::serialization archive. * diff --git a/src/mlpack/core/tree/octree/octree_impl.hpp b/src/mlpack/core/tree/octree/octree_impl.hpp index 7477d1e4ba..017034c54e 100644 --- a/src/mlpack/core/tree/octree/octree_impl.hpp +++ b/src/mlpack/core/tree/octree/octree_impl.hpp @@ -363,6 +363,43 @@ Octree::Octree(const Octree& other) : } } +//! Copy assignment operator: copy the given other tree. +template +Octree& +Octree:: +operator=(const Octree& other) +{ + // Return if it's the same tree. + if (this == &other) + return *this; + + // Freeing memory that will not be used anymore. + delete dataset; + for (size_t i = 0; i < children.size(); ++i) + delete children[i]; + children.clear(); + + begin = other.Begin(); + count = other.Count(); + bound = other.bound; + dataset = ((other.parent == NULL) ? new MatType(*other.dataset) : NULL); + parent = NULL; + stat = other.stat; + parentDistance = other.ParentDistance(); + furthestDescendantDistance = other.FurthestDescendantDistance(); + metric = other.metric; + + // If we have any children, we need to create them, and then ensure that their + // parent links are set right. + for (size_t i = 0; i < other.NumChildren(); ++i) + { + children.push_back(new Octree(other.Child(i))); + children[i]->parent = this; + children[i]->dataset = this->dataset; + } + return *this; +} + //! Move the given tree. template Octree::Octree(Octree&& other) : @@ -389,6 +426,48 @@ Octree::Octree(Octree&& other) : other.parent = NULL; } +//! Move assignment operator: take ownership of the given tree. +template +Octree& +Octree:: +operator=(Octree&& other) +{ + // Return if it's the same tree. + if (this == &other) + return *this; + + // Freeing memory that will not be used anymore. + delete dataset; + for (size_t i = 0; i < children.size(); ++i) + delete children[i]; + children.clear(); + + children = std::move(other.children); + begin = other.Begin(); + count = other.Count(); + bound = std::move(other.bound); + dataset = other.dataset; + parent = other.Parent(); + stat = std::move(other.stat); + parentDistance = other.ParentDistance(); + furthestDescendantDistance = other.furthestDescendantDistance(); + metric = std::move(other.metric); + + // Update the parent pointers of the direct children. + for (size_t i = 0; i < children.size(); ++i) + children[i]->parent = this; + + other.begin = 0; + other.count = 0; + other.dataset = new MatType(); + other.parentDistance = 0.0; + other.numDescendants = 0; + other.furthestDescendantDistance = 0.0; + other.parent = NULL; + + return *this; +} + template Octree::Octree() : begin(0), diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp index de08e4e1ee..9d56dc69a0 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp @@ -183,6 +183,7 @@ RectangleTree( maxLeafSize(other.MaxLeafSize()), minLeafSize(other.MinLeafSize()), bound(other.bound), + stat(other.stat), parentDistance(other.ParentDistance()), dataset(deepCopy ? (parent ? parent->dataset : new MatType(*other.dataset)) : @@ -203,6 +204,9 @@ RectangleTree( children = other.children; } +/** + * Move constructor. + */ templateparent = this; } + // Now we are a clone of the other tree. But we must also clear the other + // tree's contents, so it doesn't delete anything when it is destructed. other.maxNumChildren = 0; other.minNumChildren = 0; other.numChildren = 0; @@ -256,6 +263,9 @@ RectangleTree(RectangleTree&& other) : other.ownsDataset = false; } +/** + * Copy assignment operator: copy the given other tree. + */ template + inline static void InitializeOne(const MatType& V, + const size_t r, + arma::mat& M, + const bool whichMatrix = true) + { + const size_t n = V.n_rows; + const size_t m = V.n_cols; + + double avgV = 0; + double min = DBL_MAX; + + // Iterate over all elements in the matrix (for sparse matrices, this only + // iterates over nonzeros). + for (typename MatType::const_row_col_iterator it = V.begin(); + it != V.end(); ++it) + { + avgV += *it; + // Track the minimum value. + if (*it < min) + min = *it; + } + if (whichMatrix) + { + // Initialize W to random values + M.randu(n, r); + } + else + { + // Initialize H to random values + M.randu(r, m); + } + M += sqrt(((avgV / (n * m)) - min) / r); } //! Serialize the object (in this case, there is nothing to do). diff --git a/src/mlpack/methods/amf/init_rules/given_init.hpp b/src/mlpack/methods/amf/init_rules/given_init.hpp index 8da4736786..faf4d5c333 100644 --- a/src/mlpack/methods/amf/init_rules/given_init.hpp +++ b/src/mlpack/methods/amf/init_rules/given_init.hpp @@ -1,5 +1,5 @@ /** - * @file given_initialization.hpp + * @file given_init.hpp * @author Ryan Curtin * * Initialization rule for alternating matrix factorization (AMF). This simple @@ -28,25 +28,62 @@ class GivenInitialization { public: // Empty constructor required for the InitializeRule template. - GivenInitialization() { } + GivenInitialization() : wIsGiven(false), hIsGiven(false) { } // Initialize the GivenInitialization object with the given matrices. - GivenInitialization(const arma::mat& w, const arma::mat& h) : w(w), h(h) { } + GivenInitialization(const arma::mat& w, const arma::mat& h) : + w(w), h(h), wIsGiven(true), hIsGiven(true) { } // Initialize the GivenInitialization object, taking control of the given // matrices. GivenInitialization(const arma::mat&& w, const arma::mat&& h) : w(std::move(w)), - h(std::move(h)) + h(std::move(h)), + wIsGiven(true), + hIsGiven(true) { } + // Initialize either H or W with the given matrix. + GivenInitialization(const arma::mat& m, const bool whichMatrix = true) + { + if (whichMatrix) + { + w = m; + wIsGiven = true; + hIsGiven = false; + } + else + { + h = m; + wIsGiven = false; + hIsGiven = true; + } + } + + // Initialize either H or W, taking control of the given matrix. + GivenInitialization(const arma::mat&& m, const bool whichMatrix = true) + { + if (whichMatrix) + { + w = std::move(m); + wIsGiven = true; + hIsGiven = false; + } + else + { + h = std::move(m); + wIsGiven = false; + hIsGiven = true; + } + } + /** - * Fill W and H with random uniform noise. + * Fill W and H with given matrices. * * @param V Input matrix. * @param r Rank of decomposition. - * @param W W matrix, to be filled with random noise. - * @param H H matrix, to be filled with random noise. + * @param W W matrix, to be initialized to given matrix. + * @param H H matrix, to be initialized to given matrix. */ template inline void Initialize(const MatType& V, @@ -54,6 +91,16 @@ class GivenInitialization arma::mat& W, arma::mat& H) { + // Make sure the initial W, H matrices are given + if (!wIsGiven) + { + Log::Fatal << "Initial W matrix is not given!" << std::endl; + } + if (!hIsGiven) + { + Log::Fatal << "Initial H matrix is not given!" << std::endl; + } + // Make sure the initial W, H matrices have correct size. if (w.n_rows != V.n_rows) { @@ -85,6 +132,72 @@ class GivenInitialization H = h; } + /** + * Fill W or H with given matrix. + * + * @param V Input matrix. + * @param r Rank of decomposition. + * @param M W or H matrix, to be initialized to given matrix. + * @param whichMatrix If true, initialize W. Otherwise, initialize H. + */ + template + inline void InitializeOne(const MatType& V, + const size_t r, + arma::mat& M, + const bool whichMatrix = true) + { + if (whichMatrix) + { + // Make sure the initial W matrix is given. + if (!wIsGiven) + { + Log::Fatal << "Initial W matrix is not given!" << std::endl; + } + + // Make sure the initial W matrix has correct size. + if (w.n_rows != V.n_rows) + { + Log::Fatal << "The number of rows in given W (" << w.n_rows + << ") doesn't equal the number of rows in V (" << V.n_rows + << ") !" << std::endl; + } + if (w.n_cols != r) + { + Log::Fatal << "The number of columns in given W (" << w.n_cols + << ") doesn't equal the rank of factorization (" << r + << ") !" << std::endl; + } + + // Initialize W to the given matrix. + M = w; + } + else + { + // Make sure the initial H matrix is given. + if (!hIsGiven) + { + Log::Fatal << "Initial H matrix is not given!" << std::endl; + } + + // Make sure the initial H matrix has correct size. + if (h.n_cols != V.n_cols) + { + Log::Fatal << "The number of columns in given H (" << h.n_cols + << ") doesn't equal the number of columns in V (" << V.n_cols + << ") !" << std::endl; + } + if (h.n_rows != r) + { + Log::Fatal << "The number of rows in given H (" << h.n_rows + << ") doesn't equal the rank of factorization (" << r + << ") !"<< std::endl; + } + + // Initialize H to the given matrix. + M = h; + } + } + //! Serialize the object (in this case, there is nothing to serialize). template void serialize(Archive& ar, const unsigned int /* version */) @@ -98,6 +211,10 @@ class GivenInitialization arma::mat w; //! The H matrix for initialization. arma::mat h; + //! Whether initial W is given. + bool wIsGiven; + //! Whether initial H is given. + bool hIsGiven; }; } // namespace amf diff --git a/src/mlpack/methods/amf/init_rules/merge_init.hpp b/src/mlpack/methods/amf/init_rules/merge_init.hpp new file mode 100644 index 0000000000..8abe5e1d8a --- /dev/null +++ b/src/mlpack/methods/amf/init_rules/merge_init.hpp @@ -0,0 +1,69 @@ +/** + * @file merge_init.hpp + * @author Ziyang Jiang + * + * Initialization rule for alternating matrix factorization (AMF). This simple + * initialization is performed by assigning a given matrix to W or H and a + * random matrix to another one. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_AMF_MERGE_INIT_HPP +#define MLPACK_METHODS_AMF_MERGE_INIT_HPP + +#include + +namespace mlpack { +namespace amf { + +/** + * This initialization rule for AMF simply takes in two initialization rules, + * and initialize W with the first rule and H with the second rule. + */ +template +class MergeInitialization +{ + public: + // Empty constructor required for the InitializeRule template + MergeInitialization() { } + + // Initialize the MergeInitialization object with existing initialization + // rules. + MergeInitialization(const WInitializationRuleType& wInitRule, + const HInitializationRuleType& hInitRule) : + wInitializationRule(wInitRule), + hInitializationRule(hInitRule) + { } + + /** + * Initialize W and H with the corresponding initialization rules. + * + * @param V Input matrix. + * @param r Rank of decomposition. + * @param W W matrix, to be initialized to given matrix. + * @param H H matrix, to be initialized to given matrix. + */ + template + inline void Initialize(const MatType& V, + const size_t r, + arma::mat& W, + arma::mat& H) + { + wInitializationRule.InitializeOne(V, r, W); + hInitializationRule.InitializeOne(V, r, H, false); + } + + private: + // Initialization rule for W matrix + WInitializationRuleType wInitializationRule; + // Initialization rule for H matrix + HInitializationRuleType hInitializationRule; +}; + +} // namespace amf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/amf/init_rules/random_init.hpp b/src/mlpack/methods/amf/init_rules/random_init.hpp index 1f775a12c4..8bc2a8087e 100644 --- a/src/mlpack/methods/amf/init_rules/random_init.hpp +++ b/src/mlpack/methods/amf/init_rules/random_init.hpp @@ -51,6 +51,35 @@ class RandomInitialization H.randu(r, m); } + /** + * Fill W or H with random uniform noise. + * + * @param V Input matrix. + * @param r Rank of decomposition. + * @param M W or H matrix, to be filled with random noise. + * @param whichMatrix If true, initialize W. Otherwise, initialize H. + */ + template + inline void InitializeOne(const MatType& V, + const size_t r, + arma::mat& M, + const bool whichMatrix = true) + { + // Simple implementation (left in the header file due to its simplicity). + const size_t n = V.n_rows; + const size_t m = V.n_cols; + + // Initialize W or H to random values + if (whichMatrix) + { + M.randu(n, r); + } + else + { + M.randu(r, m); + } + } + //! Serialize the object (in this case, there is nothing to serialize). template void serialize(Archive& /* ar */, const unsigned int /* version */) { } diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt index 4cf9e40295..ad995034c7 100644 --- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt @@ -9,6 +9,7 @@ set(SOURCES softplus_function.hpp swish_function.hpp mish_function.hpp + lisht_function.hpp gelu_function.hpp ) diff --git a/src/mlpack/methods/ann/activation_functions/lisht_function.hpp b/src/mlpack/methods/ann/activation_functions/lisht_function.hpp new file mode 100644 index 0000000000..b403f5977e --- /dev/null +++ b/src/mlpack/methods/ann/activation_functions/lisht_function.hpp @@ -0,0 +1,95 @@ +/** + * @file lisht_function.hpp + * @author Kartik Dutt + * + * Definition and implementation of the LiSHT function as described by + * Swalpa K. Roy, Suvojit Manna, Shiv Ram Dubey and Bidyut B. Chaudhuri. + * + * For more information, see the following paper. + * + * @code + * @misc{ + * author = {Swalpa K. Roy, Suvojit Manna, Shiv R. Dubey and + * Bidyut B. Chaudhuri}, + * title = {LiSHT: Non-Parametric Linearly Scaled Hyperbolic Tangent + * Activation Function for Neural Networks}, + * year = {2019} + * } + * @endcode + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_LISHT_FUNCTION_HPP +#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_LISHT_FUNCTION_HPP + +#include +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The LiSHT function, defined by + * + * @f{eqnarray*}{ + * f(x) = x * tanh(x) + * f'(x) = tanh(x) + x * (1 - tanh^{2}(x)) + * @f} + */ +class LiSHTFunction +{ + public: + /** + * Computes the LiSHT function. + * + * @param x Input data. + * @return f(x). + */ + static double Fn(const double x) + { + return x * std::tanh(x); + } + + /** + * Computes the LiSHT function. + * + * @param x Input data. + * @param y The resulting output activation. + */ + template + static void Fn(const InputVecType &x, OutputVecType &y) + { + y = x % arma::tanh(x); + } + + /** + * Computes the first derivative of the LiSHT function. + * + * @param y Input data. + * @return f'(x) + */ + static double Deriv(const double y) + { + return std::tanh(y) + y * (1 - std::pow(std::tanh(y), 2)); + } + + /** + * Computes the first derivatives of the LiSHT function. + * + * @param y Input activations. + * @param x The resulting derivatives. + */ + template + static void Deriv(const InputVecType &y, OutputVecType &x) + { + x = arma::tanh(y) + y % (1 - arma::pow(arma::tanh(y), 2)); + } +}; // class LishtFunction + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp index f18c4477cc..36d4f9618b 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -85,7 +85,7 @@ class AtrousConvolution const size_t inputHeight = 0, const size_t dilationWidth = 1, const size_t dilationHeight = 1, - const std::string paddingType = "None"); + const std::string& paddingType = "None"); /** * Create the AtrousConvolution object using the specified number of @@ -116,13 +116,13 @@ class AtrousConvolution const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, - const std::tuple padW, - const std::tuple padH, + const std::tuple& padW, + const std::tuple& padH, const size_t inputWidth = 0, const size_t inputHeight = 0, const size_t dilationWidth = 1, const size_t dilationHeight = 1, - const std::string paddingType = "None"); + const std::string& paddingType = "None"); /* * Set the weight and bias term. diff --git a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index 52f2a78b5c..32362f71a2 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -63,45 +63,23 @@ AtrousConvolution< const size_t inputHeight, const size_t dilationWidth, const size_t dilationHeight, - const std::string paddingType) : - inSize(inSize), - outSize(outSize), - kernelWidth(kernelWidth), - kernelHeight(kernelHeight), - strideWidth(strideWidth), - strideHeight(strideHeight), - inputWidth(inputWidth), - inputHeight(inputHeight), - outputWidth(0), - outputHeight(0), - dilationWidth(dilationWidth), - dilationHeight(dilationHeight) + const std::string& paddingType) : + AtrousConvolution( + inSize, + outSize, + kernelWidth, + kernelHeight, + strideWidth, + strideHeight, + std::tuple(padW, padW), + std::tuple(padH, padH), + inputWidth, + inputHeight, + dilationWidth, + dilationHeight, + paddingType) { - weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, - 1); - - // Transform paddingType to lowercase. - std::string paddingTypeLow = paddingType; - std::transform(paddingType.begin(), paddingType.end(), paddingTypeLow.begin(), - [](unsigned char c){ return std::tolower(c); }); - - size_t padWLeft = padW; - size_t padWRight = padW; - size_t padHBottom = padH; - size_t padHTop = padH; - if (paddingTypeLow == "valid") - { - padWLeft = 0; - padWRight = 0; - padHTop = 0; - padHBottom = 0; - } - else if (paddingTypeLow == "same") - { - InitializeSamePadding(padWLeft, padWRight, padHTop, padHBottom); - } - - padding = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom); + // Nothing to do here. } template< @@ -124,13 +102,13 @@ AtrousConvolution< const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, - const std::tuple padW, - const std::tuple padH, + const std::tuple& padW, + const std::tuple& padH, const size_t inputWidth, const size_t inputHeight, const size_t dilationWidth, const size_t dilationHeight, - const std::string paddingType) : + const std::string& paddingType) : inSize(inSize), outSize(outSize), kernelWidth(kernelWidth), diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index 841d797416..c08185488b 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace mlpack { @@ -209,6 +210,17 @@ template < using MishFunctionLayer = BaseLayer< ActivationFunction, InputDataType, OutputDataType>; +/** + * Standard LiSHT-Layer using the LiSHT activation function. + */ +template < + class ActivationFunction = LiSHTFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using LiSHTFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; + /** * Standard GELU-Layer using the GELU activation function. */ diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index f94633b0f9..11352ec371 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -76,7 +76,7 @@ class Convolution const size_t padH = 0, const size_t inputWidth = 0, const size_t inputHeight = 0, - const std::string paddingType = "None"); + const std::string& paddingType = "None"); /** * Create the Convolution object using the specified number of input maps, @@ -104,11 +104,11 @@ class Convolution const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, - const std::tuple padW, - const std::tuple padH, + const std::tuple& padW, + const std::tuple& padH, const size_t inputWidth = 0, const size_t inputHeight = 0, - const std::string paddingType = "None"); + const std::string& paddingType = "None"); /* * Set the weight and bias term. diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index f5b3c4dafc..5f44836a3c 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -60,43 +60,20 @@ Convolution< const size_t padH, const size_t inputWidth, const size_t inputHeight, - const std::string paddingType) : - inSize(inSize), - outSize(outSize), - kernelWidth(kernelWidth), - kernelHeight(kernelHeight), - strideWidth(strideWidth), - strideHeight(strideHeight), - padWLeft(padW), - padWRight(padW), - padHBottom(padH), - padHTop(padH), - inputWidth(inputWidth), - inputHeight(inputHeight), - outputWidth(0), - outputHeight(0) + const std::string& paddingType) : + Convolution( + inSize, + outSize, + kernelWidth, + kernelHeight, + strideWidth, + strideHeight, + std::tuple(padW, padW), + std::tuple(padH, padH), + inputWidth, + inputHeight) { - weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, - 1); - - // Transform paddingType to lowercase. - std::string paddingTypeLow = paddingType; - std::transform(paddingType.begin(), paddingType.end(), paddingTypeLow.begin(), - [](unsigned char c){ return std::tolower(c); }); - - if (paddingTypeLow == "valid") - { - padWLeft = 0; - padWRight = 0; - padHTop = 0; - padHBottom = 0; - } - else if (paddingTypeLow == "same") - { - InitializeSamePadding(); - } - - padding = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom); + // Nothing to do here. } template< @@ -119,11 +96,11 @@ Convolution< const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, - const std::tuple padW, - const std::tuple padH, + const std::tuple& padW, + const std::tuple& padH, const size_t inputWidth, const size_t inputHeight, - const std::string paddingType) : + const std::string& paddingType) : inSize(inSize), outSize(outSize), kernelWidth(kernelWidth), diff --git a/src/mlpack/methods/ann/layer/transposed_convolution.hpp b/src/mlpack/methods/ann/layer/transposed_convolution.hpp index 1e90e9e945..2ce2fb3b72 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution.hpp @@ -67,12 +67,13 @@ class TransposedConvolution * @param kernelHeight Height of the filter/kernel. * @param strideWidth Stride of filter application in the x direction. * @param strideHeight Stride of filter application in the y direction. - * @param padWidth Padding width of the input. - * @param padHeight Padding height of the input. + * @param padW Padding width of the input. + * @param padH Padding height of the input. * @param inputWidth The width of the input data. * @param inputHeight The height of the input data. * @param outputWidth The width of the output data. * @param outputHeight The height of the output data. + * @param paddingType The type of padding (Valid or Same). Defaults to None. */ TransposedConvolution(const size_t inSize, const size_t outSize, @@ -80,12 +81,55 @@ class TransposedConvolution const size_t kernelHeight, const size_t strideWidth = 1, const size_t strideHeight = 1, - const size_t padWidth = 0, - const size_t padHeight = 0, + const size_t padW = 0, + const size_t padH = 0, const size_t inputWidth = 0, const size_t inputHeight = 0, const size_t outputWidth = 0, - const size_t outputHeight = 0); + const size_t outputHeight = 0, + const std::string& paddingType = "None"); + + /** + * Create the Transposed Convolution object using the specified number of + * input maps, output maps, filter size, stride and padding parameter. + * + * Note: The equivalent stride of a transposed convolution operation is always + * equal to 1. In this implementation, stride of filter represents the stride + * of the associated convolution operation. + * Note: Padding of input represents padding of associated convolution + * operation. + * + * @param inSize The number of input maps. + * @param outSize The number of output maps. + * @param kernelWidth Width of the filter/kernel. + * @param kernelHeight Height of the filter/kernel. + * @param strideWidth Stride of filter application in the x direction. + * @param strideHeight Stride of filter application in the y direction. + * @param padW A two-value tuple indicating padding widths of the input. + * First value is padding at left side. Second value is padding on + * right side. + * @param padH A two-value tuple indicating padding heights of the input. + * First value is padding at top. Second value is padding on + * bottom. + * @param inputWidth The width of the input data. + * @param inputHeight The height of the input data. + * @param outputWidth The width of the output data. + * @param outputHeight The height of the output data. + * @param paddingType The type of padding (Valid or Same). Defaults to None. + */ + TransposedConvolution(const size_t inSize, + const size_t outSize, + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth, + const size_t strideHeight, + const std::tuple& padW, + const std::tuple& padH, + const size_t inputWidth = 0, + const size_t inputHeight = 0, + const size_t outputWidth = 0, + const size_t outputHeight = 0, + const std::string& paddingType = "None"); /* * Set the weight and bias term. @@ -199,15 +243,25 @@ class TransposedConvolution //! Modify the stride height. size_t& StrideHeight() { return strideHeight; } - //! Get the padding width. - size_t PadWidth() const { return padWidth; } - //! Modify the padding width. - size_t& PadWidth() { return padWidth; } + //! Get the top padding height. + size_t PadHTop() const { return padHTop; } + //! Modify the top padding height. + size_t& PadHTop() { return padHTop; } - //! Get the padding height. - size_t PadHeight() const { return padHeight; } - //! Modify the padding height. - size_t& PadHeight() { return padHeight; } + //! Get the bottom padding height. + size_t PadHBottom() const { return padHBottom; } + //! Modify the bottom padding height. + size_t& PadHBottom() { return padHBottom; } + + //! Get the left padding width. + size_t PadWLeft() const { return padWLeft; } + //! Modify the left padding width. + size_t& PadWLeft() { return padWLeft; } + + //! Get the right padding width. + size_t PadWRight() const { return padWRight; } + //! Modify the right padding width. + size_t& PadWRight() { return padWRight; } //! Modify the bias weights of the layer. arma::mat& Bias() { return bias; } @@ -235,6 +289,11 @@ class TransposedConvolution output.slice(s) = arma::fliplr(arma::flipud(input.slice(s))); } + /* + * Function to assign padding such that output size is same as input size. + */ + void InitializeSamePadding(); + /* * Rotates a dense matrix counterclockwise by 180 degrees. * @@ -328,11 +387,17 @@ class TransposedConvolution //! Locally-stored stride of the filter in y-direction. size_t strideHeight; - //! Locally-stored padding width. - size_t padWidth; + //! Locally-stored left-side padding width. + size_t padWLeft; - //! Locally-stored padding height. - size_t padHeight; + //! Locally-stored right-side padding width. + size_t padWRight; + + //! Locally-stored bottom padding height. + size_t padHBottom; + + //! Locally-stored top padding height. + size_t padHTop; //! Locally-stored number of zeros added to the right of input. size_t aW; diff --git a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp index ec710409ca..c0ec126337 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp @@ -57,18 +57,68 @@ TransposedConvolution< const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, - const size_t padWidth, - const size_t padHeight, + const size_t padW, + const size_t padH, const size_t inputWidth, const size_t inputHeight, const size_t outputWidth, - const size_t outputHeight) : + const size_t outputHeight, + const std::string& paddingType) : + TransposedConvolution( + inSize, + outSize, + kernelWidth, + kernelHeight, + strideWidth, + strideHeight, + std::tuple(padW, padW), + std::tuple(padH, padH), + inputWidth, + inputHeight, + outputWidth, + outputHeight, + paddingType) +{ + // Nothing to do here. +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +TransposedConvolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>::TransposedConvolution( + const size_t inSize, + const size_t outSize, + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth, + const size_t strideHeight, + const std::tuple& padW, + const std::tuple& padH, + const size_t inputWidth, + const size_t inputHeight, + const size_t outputWidth, + const size_t outputHeight, + const std::string& paddingType) : inSize(inSize), outSize(outSize), kernelWidth(kernelWidth), kernelHeight(kernelHeight), strideWidth(strideWidth), strideHeight(strideHeight), + padWLeft(std::get<0>(padW)), + padWRight(std::get<1>(padW)), + padHBottom(std::get<1>(padH)), + padHTop(std::get<0>(padH)), inputWidth(inputWidth), inputHeight(inputHeight), outputWidth(outputWidth), @@ -76,26 +126,49 @@ TransposedConvolution< { weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, 1); + // Transform paddingType to lowercase. + std::string paddingTypeLow = paddingType; + std::transform(paddingType.begin(), paddingType.end(), paddingTypeLow.begin(), + [](unsigned char c){ return std::tolower(c); }); - aW = (outputWidth + 2 * padWidth - kernelWidth) % strideWidth; - aH = (outputHeight + 2 * padHeight - kernelHeight) % strideHeight; + if (paddingTypeLow == "valid") + { + // Set Padding to 0. + padWLeft = 0; + padWRight = 0; + padHTop = 0; + padHBottom = 0; + } + else if (paddingTypeLow == "same") + { + InitializeSamePadding(); + } - const size_t padWidthForward = kernelWidth - padWidth - 1; - const size_t padHeightForward = kernelHeight - padHeight - 1; + const size_t totalPadWidth = padWLeft + padWRight; + const size_t totalPadHeight = padHTop + padHBottom; - paddingForward = ann::Padding<>(padWidthForward, padWidthForward + aW, - padHeightForward, padHeightForward + aH); - paddingBackward = ann::Padding<>(padWidth, padWidth, padHeight, padHeight); + aW = (outputWidth + totalPadWidth - kernelWidth) % strideWidth; + aH = (outputHeight + totalPadHeight - kernelHeight) % strideHeight; + + const size_t padWidthLeftForward = kernelWidth - padWLeft - 1; + const size_t padHeightTopForward = kernelHeight - padHTop - 1; + const size_t padWidthRightForward = kernelWidth - padWRight - 1; + const size_t padHeightBottomtForward = kernelHeight - padHBottom - 1; + + paddingForward = ann::Padding<>(padWidthLeftForward, + padWidthRightForward + aW, padHeightTopForward, + padHeightBottomtForward + aH); + paddingBackward = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom); // Check if the output height and width are possible given the other // parameters of the layer. if (outputWidth != strideWidth * (inputWidth - 1) + - aW + kernelWidth - 2 * padWidth || + aW + kernelWidth - totalPadWidth || outputHeight != strideHeight * (inputHeight - 1) + - aH + kernelHeight - 2 * padHeight) + aH + kernelHeight - totalPadHeight) { Log::Fatal << "The output width / output height is not possible given " - << "the other parameters of the layer." << std::endl; + << "the other parameters of the layer." << std::endl; } } @@ -144,12 +217,13 @@ void TransposedConvolution< { InsertZeros(inputTemp, strideWidth, strideHeight, inputExpandedTemp); - if (paddingForward.PadWLeft() != 0 || paddingForward.PadHTop() != 0 || - aW != 0 || aH != 0) + if (paddingForward.PadWLeft() != 0 || paddingForward.PadWRight() != 0 || + paddingForward.PadHTop() != 0 || paddingForward.PadHBottom() != 0) { inputPaddedTemp.set_size(inputExpandedTemp.n_rows + - paddingForward.PadWLeft() * 2 + aW, inputExpandedTemp.n_cols + - paddingForward.PadHTop() * 2 + aH, inputExpandedTemp.n_slices); + paddingForward.PadWLeft() + paddingForward.PadWRight(), + inputExpandedTemp.n_cols + paddingForward.PadHTop() + + paddingForward.PadHBottom(), inputExpandedTemp.n_slices); for (size_t i = 0; i < inputExpandedTemp.n_slices; ++i) { @@ -165,12 +239,13 @@ void TransposedConvolution< } } else if (paddingForward.PadWLeft() != 0 || + paddingForward.PadWRight() != 0 || paddingForward.PadHTop() != 0 || - aW != 0 || - aH != 0) + paddingForward.PadHBottom() != 0) { - inputPaddedTemp.set_size(inputTemp.n_rows + paddingForward.PadWLeft() * 2 + - aW, inputTemp.n_cols + paddingForward.PadHTop() * 2 + aH, + inputPaddedTemp.set_size(inputTemp.n_rows + paddingForward.PadWLeft() + + paddingForward.PadWRight(), inputTemp.n_cols + + paddingForward.PadHTop() + paddingForward.PadHBottom(), inputTemp.n_slices); for (size_t i = 0; i < inputTemp.n_slices; ++i) @@ -202,9 +277,9 @@ void TransposedConvolution< if (strideWidth > 1 || strideHeight > 1 || paddingForward.PadWLeft() != 0 || + paddingForward.PadWRight() != 0 || paddingForward.PadHTop() != 0 || - aW != 0 || - aH != 0) + paddingForward.PadHBottom() != 0) { ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap + batchCount * inSize), rotatedFilter, convOutput, 1, 1); @@ -242,11 +317,13 @@ void TransposedConvolution< arma::Cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize * batchSize, false, false); arma::Cube mappedErrorPadded; - if (paddingBackward.PadWLeft() != 0 || paddingBackward.PadHTop() != 0) + if (paddingBackward.PadWLeft() != 0 || paddingBackward.PadWRight() != 0 || + paddingBackward.PadHTop() != 0 || paddingBackward.PadHBottom() != 0) { mappedErrorPadded.set_size(mappedError.n_rows + - paddingBackward.PadWLeft() * 2, mappedError.n_cols + - paddingBackward.PadHTop() * 2, mappedError.n_slices); + paddingBackward.PadWLeft() + paddingBackward.PadWRight(), + mappedError.n_cols + paddingBackward.PadHTop() + + paddingBackward.PadHBottom(), mappedError.n_slices); for (size_t i = 0; i < mappedError.n_slices; ++i) { @@ -273,7 +350,8 @@ void TransposedConvolution< { arma::Mat output; - if (paddingBackward.PadWLeft() != 0 || paddingBackward.PadHTop() != 0) + if (paddingBackward.PadWLeft() != 0 || paddingBackward.PadWRight() != 0 || + paddingBackward.PadHTop() != 0 || paddingBackward.PadHBottom() != 0) { BackwardConvolutionRule::Convolution(mappedErrorPadded.slice(outMap), weight.slice(outMapIdx), output, strideWidth, strideHeight); @@ -334,9 +412,9 @@ void TransposedConvolution< if (strideWidth > 1 || strideHeight > 1 || paddingForward.PadWLeft() != 0 || + paddingForward.PadWRight() != 0 || paddingForward.PadHTop() != 0 || - aW != 0 || - aH != 0) + paddingForward.PadHBottom() != 0) { inputSlice = inputPaddedTemp.slice(inMap + batchCount * inSize); } @@ -387,6 +465,10 @@ void TransposedConvolution< ar & BOOST_SERIALIZATION_NVP(padWidth); ar & BOOST_SERIALIZATION_NVP(padHeight); } + ar &BOOST_SERIALIZATION_NVP(padWLeft); + ar &BOOST_SERIALIZATION_NVP(padWRight); + ar &BOOST_SERIALIZATION_NVP(padHBottom); + ar &BOOST_SERIALIZATION_NVP(padHTop); ar & BOOST_SERIALIZATION_NVP(inputWidth); ar & BOOST_SERIALIZATION_NVP(inputHeight); ar & BOOST_SERIALIZATION_NVP(outputWidth); @@ -402,9 +484,49 @@ void TransposedConvolution< { weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, 1); + size_t totalPadWidth = padWLeft + padWRight; + size_t totalPadHeight = padHTop + padHBottom; + aW = (outputWidth + kernelWidth - totalPadWidth - 2) % strideWidth; + aH = (outputHeight + kernelHeight - totalPadHeight - 2) % strideHeight; + } +} +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +void TransposedConvolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>::InitializeSamePadding(){ + /** + * Using O=s*(I-1) + K -2P + A + * where + * s=stride + * I=Input Shape + * K=Kernel Size + * P=Padding + */ + const size_t totalHorizontalPadding = (strideWidth - 1) * inputWidth + + kernelWidth - strideWidth; + const size_t totalVerticalPadding = (strideHeight - 1) * inputHeight + + kernelHeight - strideHeight; - aW = (outputWidth + kernelWidth - 2 * padWidth - 2) % strideWidth; - aH = (outputHeight + kernelHeight - 2 * padHeight - 2) % strideHeight; + padWLeft = totalVerticalPadding / 2; + padWRight = totalVerticalPadding - totalVerticalPadding / 2; + padHTop = totalHorizontalPadding / 2; + padHBottom = totalHorizontalPadding - totalHorizontalPadding / 2; + + // If Padding is negative throw a fatal error. + if (totalHorizontalPadding < 0 || totalVerticalPadding < 0) + { + Log::Fatal << "The output width / output height is not possible given " + << "same padding for the layer." << std::endl; } } diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index ce804c0809..5a74ca4a2e 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -9,6 +9,8 @@ set(SOURCES earth_mover_distance_impl.hpp kl_divergence.hpp kl_divergence_impl.hpp + mean_bias_error.hpp + mean_bias_error_impl.hpp mean_squared_error.hpp mean_squared_error_impl.hpp mean_squared_logarithmic_error.hpp diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp new file mode 100644 index 0000000000..1ec50e67b5 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -0,0 +1,84 @@ +/** + * @file mean_bias_error.hpp + * @author Saksham Rastogi + * + * Definition of the mean bias error performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_MEAN_BIAS_ERROR_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_MEAN_BIAS_ERROR_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The mean bias error performance function measures the network's + * performance according to the mean of errors. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MeanBiasError +{ + public: + /** + * Create the MeanBiasError object. + */ + MeanBiasError(); + + /** + * Computes the mean bias error function. + * + * @param input Input data used for evaluating the specified function. + * @param target The target vector. + */ + template + double Forward(const InputType&& input, const TargetType&& target); + + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const InputType&& input, + const TargetType&& target, + OutputType&& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class MeanBiasError + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "mean_bias_error_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp new file mode 100644 index 0000000000..4a7a2114d2 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -0,0 +1,59 @@ +/** + * @file mean_bias_error_impl.hpp + * @author Saksham Rastogi + * + * Implementation of the mean bias error performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_MEAN_BIAS_ERROR_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_MEAN_BIAS_ERROR_IMPL_HPP + + +// In case it hasn't yet been included. +#include "mean_bias_error.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +MeanBiasError::MeanBiasError() +{ + // Nothing to do here. +} + +template +template +double MeanBiasError::Forward( + const InputType&& input, const TargetType&& target) +{ + return arma::accu(target - input) / target.n_cols; +} + +template +template +void MeanBiasError::Backward( + const InputType&& input, + const TargetType&& target, + OutputType&& output) +{ + output.set_size(arma::size(input)); + output.fill(-1.0); +} + +template +template +void MeanBiasError::serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/nmf/nmf_main.cpp b/src/mlpack/methods/nmf/nmf_main.cpp index 07fb75d5bb..2021f9d03c 100644 --- a/src/mlpack/methods/nmf/nmf_main.cpp +++ b/src/mlpack/methods/nmf/nmf_main.cpp @@ -14,9 +14,9 @@ #include #include - #include #include +#include #include #include #include @@ -130,6 +130,68 @@ void SaveWH(const bool bindingTransposed, arma::mat&& w, arma::mat&& h) } } +template +void ApplyFactorization(const arma::mat& V, + const size_t r, + arma::mat& W, + arma::mat& H) +{ + const size_t maxIterations = CLI::GetParam("max_iterations"); + const double minResidue = CLI::GetParam("min_residue"); + + SimpleResidueTermination srt(minResidue, maxIterations); + + // Load input dataset. We know if the data is transposed based on the + // BINDING_MATRIX_TRANSPOSED macro, which will be 'true' or 'false'. + arma::mat initialW, initialH; + LoadInitialWH(BINDING_MATRIX_TRANSPOSED, initialW, initialH); + if (CLI::HasParam("initial_w") && CLI::HasParam("initial_h")) + { + // Initialize W and H with given matrices + GivenInitialization ginit = GivenInitialization(initialW, initialH); + AMF amf(srt, ginit); + amf.Apply(V, r, W, H); + } + else if (CLI::HasParam("initial_w")) + { + // Merge GivenInitialization and RandomInitialization rules + // to initialize W with the given matrix, and H with random noise + GivenInitialization ginit = GivenInitialization(initialW); + RandomInitialization rinit = RandomInitialization(); + MergeInitialization minit = + MergeInitialization + (ginit, rinit); + AMF, + UpdateRuleType> amf(srt, minit); + amf.Apply(V, r, W, H); + } + else if (CLI::HasParam("initial_h")) + { + // Merge GivenInitialization and RandomInitialization rules + // to initialize H with the given matrix, and W with random noise + GivenInitialization ginit = GivenInitialization(initialH, false); + RandomInitialization rinit = RandomInitialization(); + MergeInitialization minit = + MergeInitialization + (rinit, ginit); + AMF, + UpdateRuleType> amf(srt, minit); + amf.Apply(V, r, W, H); + } + else + { + // Use random initialization + AMF amf(srt); + amf.Apply(V, r, W, H); + } +} + static void mlpackMain() { // Initialize random seed. @@ -140,8 +202,6 @@ static void mlpackMain() // Gather parameters. const size_t r = CLI::GetParam("rank"); - const size_t maxIterations = CLI::GetParam("max_iterations"); - const double minResidue = CLI::GetParam("min_residue"); const string updateRules = CLI::GetParam("update_rules"); // Validate parameters. @@ -153,10 +213,7 @@ static void mlpackMain() true, "max_iterations must be non-negative"); RequireAtLeastOnePassed({ "h", "w" }, false, "no output will be saved"); - RequireNoneOrAllPassed({"initial_w", "initial_h"}, true); - // Load input dataset. We know if the data is transposed based on the - // BINDING_MATRIX_TRANSPOSED macro, which will be 'true' or 'false'. arma::mat V = std::move(CLI::GetParam("input")); arma::mat W; @@ -167,76 +224,19 @@ static void mlpackMain() { Log::Info << "Performing NMF with multiplicative distance-based update " << "rules." << std::endl; - - SimpleResidueTermination srt(minResidue, maxIterations); - if (CLI::HasParam("initial_w")) - { - // Initialization with given W, H matrices. - arma::mat initialW, initialH; - LoadInitialWH(BINDING_MATRIX_TRANSPOSED, initialW, initialH); - GivenInitialization ginit = GivenInitialization(initialW, initialH); - - AMF amf(srt, ginit); - amf.Apply(V, r, W, H); - } - else - { - AMF<> amf(srt); - amf.Apply(V, r, W, H); - } + ApplyFactorization(V, r, W, H); } else if (updateRules == "multdiv") { Log::Info << "Performing NMF with multiplicative divergence-based update " << "rules." << std::endl; - - SimpleResidueTermination srt(minResidue, maxIterations); - if (CLI::HasParam("initial_w")) - { - // Initialization with given W, H matrices. - arma::mat initialW, initialH; - LoadInitialWH(BINDING_MATRIX_TRANSPOSED, initialW, initialH); - GivenInitialization ginit = GivenInitialization(initialW, initialH); - - AMF amf(srt, ginit); - amf.Apply(V, r, W, H); - } - else - { - AMF amf(srt); - amf.Apply(V, r, W, H); - } + ApplyFactorization(V, r, W, H); } else if (updateRules == "als") { Log::Info << "Performing NMF with alternating least squared update rules." << std::endl; - - SimpleResidueTermination srt(minResidue, maxIterations); - if (CLI::HasParam("initial_w")) - { - // Initialization with given W, H matrices. - arma::mat initialW, initialH; - LoadInitialWH(BINDING_MATRIX_TRANSPOSED, initialW, initialH); - GivenInitialization ginit = GivenInitialization(initialW, initialH); - - AMF amf(srt, ginit); - amf.Apply(V, r, W, H); - } - else - { - AMF amf(srt); - amf.Apply(V, r, W, H); - } + ApplyFactorization(V, r, W, H); } // Save results. Remember from our discussion in the comments earlier that we diff --git a/src/mlpack/methods/softmax_regression/softmax_regression.hpp b/src/mlpack/methods/softmax_regression/softmax_regression.hpp index 28a2a2de6c..e956ee512d 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression.hpp @@ -71,7 +71,6 @@ class SoftmaxRegression SoftmaxRegression(const size_t inputSize = 0, const size_t numClasses = 0, const bool fitIntercept = false); - /** * Construct the SoftmaxRegression class with the provided data and labels. * This will train the model. Optionally, the parameter 'lambda' can be @@ -94,23 +93,45 @@ class SoftmaxRegression const double lambda = 0.0001, const bool fitIntercept = false, OptimizerType optimizer = OptimizerType()); - + /** + * Construct the SoftmaxRegression class with the provided data and labels. + * This will train the model. Optionally, the parameter 'lambda' can be + * passed, which controls the amount of L2-regularization in the objective + * function. By default, the model takes a small value. + * + * @tparam OptimizerType Desired optimizer type. + * @tparam CallbackTypes Types of Callback Functions. + * @param data Input training features. Each column associate with one sample + * @param labels Labels associated with the feature data. + * @param inputSize Size of the input feature vector. + * @param numClasses Number of classes for classification. + * @param lambda L2-regularization constant. + * @param fitIntercept add intercept term or not. + * @param optimizer Desired optimizer. + * @param callbacks Callback function for ensmallen optimizer `OptimizerType`. + * See https://www.ensmallen.org/docs.html#callback-documentation. + */ + template + SoftmaxRegression(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const bool fitIntercept, + OptimizerType optimizer, + CallbackTypes&&... callbacks); /** * Classify the given points, returning the predicted labels for each point. * The function calculates the probabilities for every class, given a data * point. It then chooses the class which has the highest probability among * all. - * * @param dataset Set of points to classify. * @param labels Predicted labels for each point. */ void Classify(const arma::mat& dataset, arma::Row& labels) const; - /** * Classify the given point. The predicted class label is returned. * The function calculates the probabilites for every class, given the point. * It then chooses the class which has the highest probability among all. - * * @param point Point to be classified. * @return Predicted class label of the point. */ @@ -151,7 +172,6 @@ class SoftmaxRegression */ double ComputeAccuracy(const arma::mat& testData, const arma::Row& labels) const; - /** * Train the softmax regression with the given training data. * @@ -167,6 +187,25 @@ class SoftmaxRegression const arma::Row& labels, const size_t numClasses, OptimizerType optimizer = OptimizerType()); + /** + * Train the softmax regression with the given training data. + * + * @tparam OptimizerType Desired optimizer type. + * @tparam CallbackTypes Types of Callback Functions. + * @param data Input data with each column as one example. + * @param labels Labels associated with the feature data. + * @param numClasses Number of classes for classification. + * @param optimizer Desired optimizer. + * @param callbacks Callback function for ensmallen optimizer `OptimizerType`. + * See https://www.ensmallen.org/docs.html#callback-documentation. + * @return Objective value of the final point. + */ + template + double Train(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + CallbackTypes&&... callbacks); //! Sets the number of classes. size_t& NumClasses() { return numClasses; } @@ -188,7 +227,7 @@ class SoftmaxRegression //! Gets the features size of the training data size_t FeatureSize() const - { return fitIntercept ? parameters.n_cols - 1 : + { return fitIntercept ? parameters.n_cols - 1: parameters.n_cols; } /** diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_function.hpp b/src/mlpack/methods/softmax_regression/softmax_regression_function.hpp index ed091609db..1d6e4cb36a 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_function.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_function.hpp @@ -175,6 +175,11 @@ class SoftmaxRegressionFunction { return initialPoint.n_cols; } + /** + * Return the number of separable functions + (the number of predictor points). + */ + size_t NumFunctions() const { return data.n_cols; } //! Sets the regularization parameter. double& Lambda() { return lambda; } diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp index 8caceb3919..30fc341cf1 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp @@ -33,6 +33,22 @@ SoftmaxRegression::SoftmaxRegression( Train(data, labels, numClasses, optimizer); } +template +SoftmaxRegression::SoftmaxRegression( + const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const bool fitIntercept, + OptimizerType optimizer, + CallbackTypes&&... callbacks) : + numClasses(numClasses), + lambda(lambda), + fitIntercept(fitIntercept) +{ + Train(data, labels, numClasses, optimizer, callbacks...); +} + template size_t SoftmaxRegression::Classify(const VecType& point) const { @@ -47,8 +63,8 @@ double SoftmaxRegression::Train(const arma::mat& data, const size_t numClasses, OptimizerType optimizer) { - SoftmaxRegressionFunction regressor(data, labels, numClasses, - lambda, fitIntercept); + SoftmaxRegressionFunction regressor(data, labels, numClasses, lambda, + fitIntercept); if (parameters.is_empty()) parameters = regressor.GetInitialPoint(); @@ -63,6 +79,29 @@ double SoftmaxRegression::Train(const arma::mat& data, return out; } +template +double SoftmaxRegression::Train(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + CallbackTypes&&... callbacks) +{ + SoftmaxRegressionFunction regressor(data, labels, numClasses, lambda, + fitIntercept); + if (parameters.is_empty()) + parameters = regressor.GetInitialPoint(); + + // Train the model. + Timer::Start("softmax_regression_optimization"); + const double out = optimizer.Optimize(regressor, parameters, callbacks...); + Timer::Stop("softmax_regression_optimization"); + + Log::Info << "SoftmaxRegression::SoftmaxRegression(): final objective of " + << "trained model is " << out << "." << std::endl; + + return out; +} + } // namespace regression } // namespace mlpack diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp index ed04aaa7dc..bbda39bafb 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp @@ -273,6 +273,5 @@ Model* TrainSoftmax(const size_t maxIterations) sm = new Model(trainData, trainLabels, numClasses, CLI::GetParam("lambda"), intercept, std::move(optimizer)); } - return sm; } diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 143f037c89..43a00b85f3 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -635,6 +636,7 @@ BOOST_AUTO_TEST_CASE(HardSigmoidFunctionTest) CheckDerivativeCorrect(desiredActivations, desiredDerivatives); } + /** * Basic test of the Mish function. */ @@ -657,6 +659,31 @@ BOOST_AUTO_TEST_CASE(MishFunctionTest) desiredDerivatives); } +/** + * Basic test of the LiSHT function. + */ +BOOST_AUTO_TEST_CASE(LiSHTFunctionTest) +{ + // Calculated using tfa.activations.LiSHT(). + // where tfa is tensorflow_addons. + const arma::colvec desiredActivations("1.928055 3.189384 \ + 4.4988894 100.2 0.7615942 \ + 0.7615942 1.9280552 0"); + + const arma::colvec desiredDerivatives("1.1150033 1.0181904 \ + 1.001978 1.0 \ + 1.0896928 1.0896928 \ + 1.1150033 0.0"); + + CheckActivationCorrect(activationData, + desiredActivations); + CheckDerivativeCorrect(desiredActivations, + desiredDerivatives); +} + +/** + * Basic test of the GELU function. + */ BOOST_AUTO_TEST_CASE(GELUFunctionTest) { // Calculated using torch.nn.gelu(). @@ -682,4 +709,5 @@ BOOST_AUTO_TEST_CASE(GELUFunctionTest) CheckDerivativeCorrect(desiredActivations, desiredDerivatives); } + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index e66c4ca748..136460c2f1 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -2931,4 +2931,105 @@ BOOST_AUTO_TEST_CASE(ConvolutionLayerPaddingTest) module2.Backward(std::move(input), std::move(output), std::move(delta)); } +/** + * Test that the padding options in Transposed Convolution layer. + */ +BOOST_AUTO_TEST_CASE(TransposedConvolutionLayerPaddingTest) +{ + arma::mat output, input, delta; + + TransposedConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6, "VALID"); + // Test the forward function. + // Valid Should give the same result. + input = arma::linspace(0, 15, 16); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Reset(); + module1.Forward(std::move(input), std::move(output)); + // Value calculated using tensorflow.nn.conv2d_transpose(). + BOOST_REQUIRE_EQUAL(arma::accu(output), 0.0); + + // Test the Backward Function. + module1.Backward(std::move(input), std::move(output), std::move(delta)); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); + + // Test Valid for non zero padding. + TransposedConvolution<> module2(1, 1, 3, 3, 2, 2, + std::tuple(0, 0), std::tuple(0, 0), + 2, 2, 5, 5, "VALID"); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module2.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module2.Parameters()(2) = 8.0; + module2.Parameters()(4) = 6.0; + module2.Parameters()(6) = 4.0; + module2.Parameters()(8) = 2.0; + module2.Reset(); + module2.Forward(std::move(input), std::move(output)); + // Value calculated using torch.nn.functional.conv_transpose2d(). + BOOST_REQUIRE_EQUAL(arma::accu(output), 120.0); + + // Test the Backward Function. + module2.Backward(std::move(input), std::move(output), std::move(delta)); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 960.0); + + // Test for same padding type. + TransposedConvolution<> module3(1, 1, 3, 3, 2, 2, 0, 0, 3, 3, 3, 3, "SAME"); + // Test the forward function. + input = arma::linspace(0, 8, 9); + module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module3.Reset(); + module3.Forward(std::move(input), std::move(output)); + BOOST_REQUIRE_EQUAL(arma::accu(output), 0); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + + // Test the Backward Function. + module3.Backward(std::move(input), std::move(output), std::move(delta)); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); + + // Output shape should equal input. + TransposedConvolution<> module4(1, 1, 3, 3, 1, 1, + std::tuple(2, 2), std::tuple(2, 2), + 5, 5, 5, 5, "SAME"); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module4.Reset(); + module4.Forward(std::move(input), std::move(output)); + BOOST_REQUIRE_EQUAL(arma::accu(output), 0); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + + // Test the Backward Function. + module4.Backward(std::move(input), std::move(output), std::move(delta)); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); + + TransposedConvolution<> module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); + // Test the forward function. + input = arma::linspace(0, 3, 4); + module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); + module5.Reset(); + module5.Forward(std::move(input), std::move(output)); + BOOST_REQUIRE_EQUAL(arma::accu(output), 0); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + + // Test the Backward Function. + module5.Backward(std::move(input), std::move(output), std::move(delta)); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); + + TransposedConvolution<> module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); + // Test the forward function. + input = arma::linspace(0, 24, 25); + module6.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); + module6.Reset(); + module6.Forward(std::move(input), std::move(output)); + BOOST_REQUIRE_EQUAL(arma::accu(output), 0); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + + // Test the Backward Function. + module6.Backward(std::move(input), std::move(output), std::move(delta)); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); +} BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/callback_test.cpp b/src/mlpack/tests/callback_test.cpp index 7e0dd154c3..6bd3db2147 100644 --- a/src/mlpack/tests/callback_test.cpp +++ b/src/mlpack/tests/callback_test.cpp @@ -19,8 +19,9 @@ #include #include #include +#include +#include #include - #include using namespace mlpack; @@ -29,6 +30,7 @@ using namespace mlpack::regression; using namespace mlpack::lmnn; using namespace mlpack::metric; using namespace mlpack::nca; +using namespace mlpack::distribution; BOOST_AUTO_TEST_SUITE(CallbackTest); @@ -94,12 +96,12 @@ BOOST_AUTO_TEST_CASE(RNNCallbackTest) // Create model with user defined rho parameter. RNN, RandomInitialization> model( rho, false, NegativeLogLikelihood<>(), init); - model.Add >(); - model.Add >(1, 10); + model.Add>(); + model.Add>(1, 10); // Use LSTM layer with rho. - model.Add >(10, 3, rho); - model.Add >(); + model.Add>(10, 3, rho); + model.Add>(); std::stringstream stream; model.Train(input, target, ens::PrintLoss(stream)); @@ -120,12 +122,12 @@ BOOST_AUTO_TEST_CASE(RNNWithOptimizerCallbackTest) // Create model with user defined rho parameter. RNN, RandomInitialization> model( rho, false, NegativeLogLikelihood<>(), init); - model.Add >(); - model.Add >(1, 10); + model.Add>(); + model.Add>(1, 10); // Use LSTM layer with rho. - model.Add >(10, 3, rho); - model.Add >(); + model.Add>(10, 3, rho); + model.Add>(); std::stringstream stream; ens::StandardSGD opt(0.1, 1, 5); @@ -139,17 +141,17 @@ BOOST_AUTO_TEST_CASE(RNNWithOptimizerCallbackTest) */ BOOST_AUTO_TEST_CASE(LRWithOptimizerCallback) { - arma::mat data("1 2 3;" - "1 2 3"); - arma::Row responses("1 1 0"); + arma::mat data("1 2 3;" + "1 2 3"); + arma::Row responses("1 1 0"); - ens::StandardSGD sgd(0.1, 1, 5); - LogisticRegression<> logisticRegression(data, responses, sgd, 0.001); - std::stringstream stream; - logisticRegression.Train(data, responses, sgd, - ens::PrintLoss(stream)); + ens::StandardSGD sgd(0.1, 1, 5); + LogisticRegression<> logisticRegression(data, responses, sgd, 0.001); + std::stringstream stream; + logisticRegression.Train(data, responses, sgd, + ens::PrintLoss(stream)); - BOOST_REQUIRE_GT(stream.str().length(), 0); + BOOST_REQUIRE_GT(stream.str().length(), 0); } /** @@ -158,8 +160,8 @@ BOOST_AUTO_TEST_CASE(LRWithOptimizerCallback) BOOST_AUTO_TEST_CASE(LMNNWithOptimizerCallback) { // Useful but simple dataset with six points and two classes. - arma::mat dataset = "-0.1 -0.1 -0.1 0.1 0.1 0.1;" - " 1.0 0.0 -1.0 1.0 0.0 -1.0 "; + arma::mat dataset = "-0.1 -0.1 -0.1 0.1 0.1 0.1;" + " 1.0 0.0 -1.0 1.0 0.0 -1.0 "; arma::Row labels = " 0 0 0 1 1 1 "; LMNN<> lmnn(dataset, labels, 1); @@ -177,8 +179,8 @@ BOOST_AUTO_TEST_CASE(LMNNWithOptimizerCallback) BOOST_AUTO_TEST_CASE(NCAWithOptimizerCallback) { // Useful but simple dataset with six points and two classes. - arma::mat data = "-0.1 -0.1 -0.1 0.1 0.1 0.1;" - " 1.0 0.0 -1.0 1.0 0.0 -1.0 "; + arma::mat data = "-0.1 -0.1 -0.1 0.1 0.1 0.1;" + " 1.0 0.0 -1.0 1.0 0.0 -1.0 "; arma::Row labels = " 0 0 0 1 1 1 "; NCA nca(data, labels); @@ -190,6 +192,41 @@ BOOST_AUTO_TEST_CASE(NCAWithOptimizerCallback) BOOST_REQUIRE_GT(stream.str().length(), 0); } +/** + * Test softmax_regression implementation with PrintLoss callback. + */ +BOOST_AUTO_TEST_CASE(SRWithOptimizerCallback) +{ + const size_t points = 1000; + const size_t inputSize = 3; + const size_t numClasses = 3; + const double lambda = 0.5; + + // Generate two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 9.0 1.0"), arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("4.0 3.0 4.0"), arma::eye(3, 3)); + + arma::mat data(inputSize, points); + arma::Row labels(points); + + for (size_t i = 0; i < points / 2; i++) + { + data.col(i) = g1.Random(); + labels(i) = 0; + } + for (size_t i = points / 2; i < points; i++) + { + data.col(i) = g2.Random(); + labels(i) = 1; + } + ens::StandardSGD sgd(0.1, 1, 5); + std::stringstream stream; + // Train softmax regression object. + SoftmaxRegression sr(data, labels, numClasses, lambda); + sr.Train(data, labels, numClasses, sgd, ens::ProgressBar(70, stream)); + + BOOST_REQUIRE_GT(stream.str().length(), 0); +} /* * Tests the RBM Implementation with PrintLoss callback. @@ -205,7 +242,10 @@ BOOST_AUTO_TEST_CASE(RBMCallbackTest) GaussianInitialization gaussian(0, 0.1); RBM model(trainData, - gaussian, trainData.n_rows, hiddenLayerSize, batchSize); + gaussian, + trainData.n_rows, + hiddenLayerSize, + batchSize); size_t numRBMIterations = 10; ens::StandardSGD msgd(0.03, batchSize, numRBMIterations, 0, true); diff --git a/src/mlpack/tests/cosine_tree_test.cpp b/src/mlpack/tests/cosine_tree_test.cpp index 56fe3ba7f1..1c299d6e16 100644 --- a/src/mlpack/tests/cosine_tree_test.cpp +++ b/src/mlpack/tests/cosine_tree_test.cpp @@ -222,4 +222,218 @@ BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt) } } +/** + * Test the copy constructor & copy assignment using Cosine trees. + */ +BOOST_AUTO_TEST_CASE(CopyConstructorAndOperatorCosineTreeTest) +{ + // Initialize constants required for the test. + const size_t numRows = 10; + const size_t numCols = 15; + + // Vectors to hold depth-first traversal + // of the number of columns in each node. + std::vector v1, v2, v3; + + // Make a random dataset. + arma::mat* data = new arma::mat(numRows, numCols, arma::fill::randu); + + // Make a cosine tree, with the generated dataset. + CosineTree* ctree1 = new CosineTree(*data); + + // Stacks for depth first search of the tree. + std::vector nodeStack1, nodeStack2, nodeStack3; + nodeStack1.push_back(ctree1); + + // While stack is not empty. + while (nodeStack1.size()) + { + // Pop a node from the stack and split it. + CosineTree *currentNode1, *currentLeft1, *currentRight1; + + currentNode1 = nodeStack1.back(); + currentNode1->CosineNodeSplit(); + nodeStack1.pop_back(); + + // Obtain pointers to the children of the node. + currentLeft1 = currentNode1->Left(); + currentRight1 = currentNode1->Right(); + + // If children exist. + if (currentLeft1 && currentRight1) + { + // Push the child nodes on to the stack. + nodeStack1.push_back(currentLeft1); + nodeStack1.push_back(currentRight1); + + v1.push_back(currentNode1->NumColumns()); + } + } + + // Copy constructor and operator. + CosineTree ctree2(*ctree1); + CosineTree ctree3 = *ctree1; + + delete ctree1; + delete data; + + nodeStack2.push_back(&ctree2); + nodeStack3.push_back(&ctree3); + + // While stacks are not empty. + while (nodeStack2.size() && nodeStack3.size()) + { + // Pop a node from the stack and split it. + CosineTree *currentNode2, *currentLeft2, *currentRight2; + CosineTree *currentNode3, *currentLeft3, *currentRight3; + + currentNode2 = nodeStack2.back(); + nodeStack2.pop_back(); + + currentNode3 = nodeStack3.back(); + nodeStack3.pop_back(); + + // Obtain pointers to the children of the node. + currentLeft2 = currentNode2->Left(); + currentRight2 = currentNode2->Right(); + + currentLeft3 = currentNode3->Left(); + currentRight3 = currentNode3->Right(); + + // If children exist. + if (currentLeft2 && currentRight2 && currentLeft3 && currentRight3) + { + // Push the child nodes on to the stack. + nodeStack2.push_back(currentLeft2); + nodeStack2.push_back(currentRight2); + + v2.push_back(currentNode2->NumColumns()); + + nodeStack3.push_back(currentLeft3); + nodeStack3.push_back(currentRight3); + + v3.push_back(currentNode3->NumColumns()); + } + } + + for (size_t i = 0; i < v1.size(); i++) + { + BOOST_REQUIRE_EQUAL(v1.at(i), v2.at(i)); + BOOST_REQUIRE_EQUAL(v1.at(i), v3.at(i)); + } +} + +/** + * Test the move constructor & move assignment using Cosine trees. + */ +BOOST_AUTO_TEST_CASE(MoveConstructorAndOperatorCosineTreeTest) +{ + // Initialize constants required for the test. + const size_t numRows = 10; + const size_t numCols = 15; + + // Vectors to hold depth-first traversal + // of the number of columns in each node. + std::vector v1, v2, v3; + + // Make a random dataset. + arma::mat data = arma::randu(numRows, numCols); + + // Make a cosine tree, with the generated dataset. + CosineTree ctree1(data); + + // Stacks for depth first search of the tree. + std::vector nodeStack1, nodeStack2, nodeStack3; + nodeStack1.push_back(&ctree1); + + // While stack is not empty. + while (nodeStack1.size()) + { + // Pop a node from the stack and split it. + CosineTree *currentNode1, *currentLeft1, *currentRight1; + + currentNode1 = nodeStack1.back(); + currentNode1->CosineNodeSplit(); + nodeStack1.pop_back(); + + // Obtain pointers to the children of the node. + currentLeft1 = currentNode1->Left(); + currentRight1 = currentNode1->Right(); + + // If children exist. + if (currentLeft1 && currentRight1) + { + // Push the child nodes on to the stack. + nodeStack1.push_back(currentLeft1); + nodeStack1.push_back(currentRight1); + + v1.push_back(currentNode1->NumColumns()); + } + } + + // Move constructor. + CosineTree ctree2(std::move(ctree1)); + + nodeStack2.push_back(&ctree2); + + // While stacks are not empty. + while (nodeStack2.size()) + { + // Pop a node from the stack and split it. + CosineTree *currentNode2, *currentLeft2, *currentRight2; + + currentNode2 = nodeStack2.back(); + nodeStack2.pop_back(); + + // Obtain pointers to the children of the node. + currentLeft2 = currentNode2->Left(); + currentRight2 = currentNode2->Right(); + + // If children exist. + if (currentLeft2 && currentRight2) + { + // Push the child nodes on to the stack. + nodeStack2.push_back(currentLeft2); + nodeStack2.push_back(currentRight2); + + v2.push_back(currentNode2->NumColumns()); + } + } + + // Move operator. + CosineTree ctree3 = std::move(ctree2); + + nodeStack3.push_back(&ctree3); + + // While stacks are not empty. + while (nodeStack3.size()) + { + // Pop a node from the stack and split it. + CosineTree *currentNode3, *currentLeft3, *currentRight3; + + currentNode3 = nodeStack3.back(); + nodeStack3.pop_back(); + + // Obtain pointers to the children of the node. + currentLeft3 = currentNode3->Left(); + currentRight3 = currentNode3->Right(); + + // If children exist. + if (currentLeft3 && currentRight3) + { + // Push the child nodes on to the stack. + nodeStack3.push_back(currentLeft3); + nodeStack3.push_back(currentRight3); + + v3.push_back(currentNode3->NumColumns()); + } + } + + for (size_t i = 0; i < v1.size(); i++) + { + BOOST_REQUIRE_EQUAL(v1.at(i), v2.at(i)); + BOOST_REQUIRE_EQUAL(v1.at(i), v3.at(i)); + } +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/knn_test.cpp b/src/mlpack/tests/knn_test.cpp index da7b97e9ca..92d921d5c8 100644 --- a/src/mlpack/tests/knn_test.cpp +++ b/src/mlpack/tests/knn_test.cpp @@ -1357,6 +1357,34 @@ BOOST_AUTO_TEST_CASE(CopyConstructorAndOperatorCoverTreeTest) CheckMatrices(distances, distances3); } +/** + * Test the copy constructor and copy operator using the BinarySpaceTree. + */ +BOOST_AUTO_TEST_CASE(CopyConstructorAndOperatorBinarySpaceTreeTest) +{ + arma::mat dataset = arma::randu(5, 500); + typedef NeighborSearch NeighborSearchType; + NeighborSearchType knn(std::move(dataset)); + + // Copy constructor and operator. + NeighborSearchType knn2(knn); + NeighborSearchType knn3 = knn; + + // Get results. + arma::mat distances, distances2, distances3; + arma::Mat neighbors, neighbors2, neighbors3; + + knn.Search(3, neighbors, distances); + knn2.Search(3, neighbors2, distances2); + knn3.Search(3, neighbors3, distances3); + + CheckMatrices(neighbors, neighbors2); + CheckMatrices(neighbors, neighbors3); + CheckMatrices(distances, distances2); + CheckMatrices(distances, distances3); +} + /** * Test the copy constructor and copy operator using the Spill Tree. */ @@ -1385,6 +1413,34 @@ BOOST_AUTO_TEST_CASE(CopyConstructorAndOperatorSpillTreeTest) CheckMatrices(distances, distances3); } +/** + * Test the copy constructor and copy operator using the Octree. + */ +BOOST_AUTO_TEST_CASE(CopyConstructorAndOperatorOctreeTest) +{ + arma::mat dataset = arma::randu(5, 500); + typedef NeighborSearch NeighborSearchType; + NeighborSearchType knn(std::move(dataset)); + + // Copy constructor and operator. + NeighborSearchType knn2(knn); + NeighborSearchType knn3 = knn; + + // Get results. + arma::mat distances, distances2, distances3; + arma::Mat neighbors, neighbors2, neighbors3; + + knn.Search(3, neighbors, distances); + knn2.Search(3, neighbors2, distances2); + knn3.Search(3, neighbors3, distances3); + + CheckMatrices(neighbors, neighbors2); + CheckMatrices(neighbors, neighbors3); + CheckMatrices(distances, distances2); + CheckMatrices(distances, distances3); +} + /** * Test the move constructor. */ @@ -1411,7 +1467,7 @@ BOOST_AUTO_TEST_CASE(MoveConstructorTest) } /** - * Test the move constructor using R trees. + * Test the move constructor & move assignment using R trees. */ BOOST_AUTO_TEST_CASE(MoveConstructorRTreeTest) { @@ -1421,8 +1477,8 @@ BOOST_AUTO_TEST_CASE(MoveConstructorRTreeTest) NeighborSearchType* knn = new NeighborSearchType(std::move(dataset)); // Get predictions. - arma::mat distances, distances2; - arma::Mat neighbors, neighbors2; + arma::mat distances, distances2, distances3; + arma::Mat neighbors, neighbors2, neighbors3; knn->Search(3, neighbors, distances); @@ -1433,11 +1489,83 @@ BOOST_AUTO_TEST_CASE(MoveConstructorRTreeTest) knn2.Search(3, neighbors2, distances2); + // Use move assignment. + NeighborSearchType knn3 = std::move(knn2); + knn3.Search(3, neighbors3, distances3); + CheckMatrices(neighbors, neighbors2); + CheckMatrices(neighbors, neighbors3); CheckMatrices(distances, distances2); + CheckMatrices(distances, distances3); } +/** + * Test the move constructor & move assignment using Binary Space trees. + */ +BOOST_AUTO_TEST_CASE(MoveConstructorBinarySpaceTreeTest) +{ + arma::mat dataset = arma::randu(5, 500); + typedef NeighborSearch NeighborSearchType; + NeighborSearchType* knn = new NeighborSearchType(std::move(dataset)); + + // Get predictions. + arma::mat distances, distances2, distances3; + arma::Mat neighbors, neighbors2, neighbors3; + + knn->Search(3, neighbors, distances); + + // Use move constructor. + NeighborSearchType knn2(std::move(*knn)); + + delete knn; + + knn2.Search(3, neighbors2, distances2); + + // Use move assignment. + NeighborSearchType knn3 = std::move(knn2); + knn3.Search(3, neighbors3, distances3); + + CheckMatrices(neighbors, neighbors2); + CheckMatrices(neighbors, neighbors3); + CheckMatrices(distances, distances2); + CheckMatrices(distances, distances3); +} + +/** + * Test the move constructor & move assignment using Octree. + */ +BOOST_AUTO_TEST_CASE(MoveConstructorOctreeTest) +{ + arma::mat dataset = arma::randu(5, 500); + typedef NeighborSearch NeighborSearchType; + NeighborSearchType* knn = new NeighborSearchType(std::move(dataset)); + + // Get predictions. + arma::mat distances, distances2, distances3; + arma::Mat neighbors, neighbors2, neighbors3; + + knn->Search(3, neighbors, distances); + + // Use move constructor. + NeighborSearchType knn2(std::move(*knn)); + + delete knn; + + knn2.Search(3, neighbors2, distances2); + + // Use move assignment. + NeighborSearchType knn3 = std::move(knn2); + knn3.Search(3, neighbors3, distances3); + + CheckMatrices(neighbors, neighbors2); + CheckMatrices(neighbors, neighbors3); + CheckMatrices(distances, distances2); + CheckMatrices(distances, distances3); +} + /** * Test the move constructor & move assignment using Cover Tree. */ diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 2c109a8a7f..7988a961d0 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -3,6 +3,7 @@ * @author Dakshit Agrawal * @author Sourabh Varshney * @author Atharva Khandait + * @author Saksham Rastogi * * Tests for loss functions in mlpack::methods::ann:loss_functions. * @@ -21,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -431,4 +433,42 @@ BOOST_AUTO_TEST_CASE(DiceLossTest) BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols); } +/* + * Simple test for the mean bias error performance function. + */ +BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) +{ + arma::mat input, output, target; + MeanBiasError<> module; + + // Test the Forward function on a user generator input and compare it against + // the manually calculated result. + input = arma::mat("1.0 0.0 1.0 -1.0 -1.0 0.0 -1.0 0.0"); + target = arma::zeros(1, 8); + double error = module.Forward(std::move(input), std::move(target)); + BOOST_REQUIRE_EQUAL(error, 0.125); + + // Test the Backward function. + module.Backward(std::move(input), std::move(target), std::move(output)); + // We should get a vector with -1 everywhere. + for (double el : output) + { + BOOST_REQUIRE_EQUAL(el, -1); + } + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + + // Test the error function on a single input. + input = arma::mat("2"); + target = arma::mat("3"); + error = module.Forward(std::move(input), std::move(target)); + BOOST_REQUIRE_EQUAL(error, 1.0); + + // Test the Backward function on a single input. + module.Backward(std::move(input), std::move(target), std::move(output)); + // Test whether the output is negative. + BOOST_REQUIRE_EQUAL(arma::accu(output), -1); + BOOST_REQUIRE_EQUAL(output.n_elem, 1); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/nmf_test.cpp b/src/mlpack/tests/main_tests/nmf_test.cpp index f6ff018a0b..ffdda3001a 100644 --- a/src/mlpack/tests/main_tests/nmf_test.cpp +++ b/src/mlpack/tests/main_tests/nmf_test.cpp @@ -284,4 +284,83 @@ BOOST_AUTO_TEST_CASE(NMFMaxIterationTest) BOOST_REQUIRE_GT(arma::norm(h1 - h2), 1e-5); } +/** + * Test NMF with given initial_w and initial_h. + */ +BOOST_AUTO_TEST_CASE(NMFWHGivenInitTest) +{ + mat v = arma::randu(10, 10); + mat initialW = arma::randu(10, 5); + mat initialH = arma::randu(5, 10); + int r = 5; + + SetInputParam("input", v); + SetInputParam("rank", r); + SetInputParam("initial_w", initialW); + SetInputParam("initial_h", initialH); + + mlpackMain(); + + const mat w = CLI::GetParam("w"); + const mat h = CLI::GetParam("h"); + + // Check the shapes of W and H. + BOOST_REQUIRE_EQUAL(w.n_rows, 10); + BOOST_REQUIRE_EQUAL(w.n_cols, 5); + BOOST_REQUIRE_EQUAL(h.n_rows, 5); + BOOST_REQUIRE_EQUAL(h.n_cols, 10); +} + +/** + * Test NMF with given initial_w. + */ +BOOST_AUTO_TEST_CASE(NMFWGivenInitTest) +{ + mat v = arma::randu(10, 10); + mat initialW = arma::randu(10, 5); + int r = 5; + + SetInputParam("input", v); + SetInputParam("rank", r); + SetInputParam("initial_w", initialW); + + mlpackMain(); + + const mat w = CLI::GetParam("w"); + const mat h = CLI::GetParam("h"); + + // Check the shapes of W and H. + BOOST_REQUIRE_EQUAL(w.n_rows, 10); + BOOST_REQUIRE_EQUAL(w.n_cols, 5); + BOOST_REQUIRE_EQUAL(h.n_rows, 5); + BOOST_REQUIRE_EQUAL(h.n_cols, 10); +} + +/** + * Test NMF with given initial_h. + */ +BOOST_AUTO_TEST_CASE(NMFHGivenInitTest) +{ + mat v = arma::randu(10, 10); + mat initialH = arma::randu(5, 10); + int r = 5; + + SetInputParam("input", v); + SetInputParam("rank", r); + SetInputParam("initial_h", initialH); + + mlpackMain(); + + const mat w = CLI::GetParam("w"); + const mat h = CLI::GetParam("h"); + + // Check the shapes of W and H. + BOOST_REQUIRE_EQUAL(w.n_rows, 10); + BOOST_REQUIRE_EQUAL(w.n_cols, 5); + BOOST_REQUIRE_EQUAL(h.n_rows, 5); + BOOST_REQUIRE_EQUAL(h.n_cols, 10); +} + + BOOST_AUTO_TEST_SUITE_END(); + diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index ac2edddac7..da819f4ee4 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -666,9 +666,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionTest) labels[i] = 0; for (size_t i = 500; i < 1000; ++i) labels[i] = 1; - SoftmaxRegression sr(dataset, labels, 2); - SoftmaxRegression srXml(dataset.n_rows, 2); SoftmaxRegression srText(dataset.n_rows, 2); SoftmaxRegression srBinary(dataset.n_rows, 2); diff --git a/src/mlpack/tests/softmax_regression_test.cpp b/src/mlpack/tests/softmax_regression_test.cpp index eeafdfc85d..5d7833bdbf 100644 --- a/src/mlpack/tests/softmax_regression_test.cpp +++ b/src/mlpack/tests/softmax_regression_test.cpp @@ -360,8 +360,8 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionTrainTest) SoftmaxRegression sr(dataset.n_rows, 2); SoftmaxRegression sr2(dataset.n_rows, 2); sr.Parameters() = sr2.Parameters(); - sr.Train(dataset, labels, 2); ens::L_BFGS lbfgs; + sr.Train(dataset, labels, 2, std::move(lbfgs)); sr2.Train(dataset, labels, 2, std::move(lbfgs)); // Ensure that the parameters are the same.