apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into website-revamp

Browse Source
This commit is contained in:
Ryan Curtin
2024-05-15 18:32:34 -04:00
parent b7cb52e075 e851d0367e
commit 1b63fa0cec
5 changed files with 175 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
HISTORY.md
+2
View File
@@ -45,6 +45,8 @@ _????-??-??_
* Fix non-working `verbose` option for R bindings (#3691).
* Fix divide-by-zero edge case for LARS (#3701).
## mlpack 4.3.0
_2023-11-27_
src/mlpack/methods/ann/activation_functions/hyper_sinh_function.hpp
+18 -1
View File
@@ -2,7 +2,24 @@
* @file methods/ann/activation_functions/hyper_sinh_function.hpp
* @author Mayank Raj
*
* Definition and implementation of the Hyper-sinh function.
* Definition of the hyper-sinh function as described by
* L. Parisi et al. (2021) (full reference below).
*
* Original implementation in Python by Luca Parisi as per https://github.com/luca-parisi/hyper_sinh
*
* For more information, see the following paper.
*
* @code
* @article{parisi2021hyper,
* title={hyper-sinh: An accurate and reliable function from shallow to deep learning in TensorFlow and Keras},
* author={Parisi, Luca and Ma, Renfei and RaviChandran, Narrendar and Lanzillotta, Matteo},
* journal={Machine Learning with Applications},
* volume={6},
* pages={100112},
* year={2021},
* publisher={Elsevier}
* }
* @endcode
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
src/mlpack/methods/lars/lars_impl.hpp
+6 -25
View File
@@ -658,30 +658,6 @@ LARS<ModelMatType>::Train(const MatType& matX,
if (maxCorr < tolerance)
break;
// Floats require a really large tolerance for this condition.
const ElemType tol = (std::is_same<ElemType, double>::value) ? 1e-6 : 0.01;
if ((matGram != &matGramInternal) &&
((maxActiveCorr - minActiveCorr) / maxActiveCorr) > tol)
{
// Construct the error message to match the user's settings.
std::ostringstream oss;
oss << "LARS::Train(): correlation conditions violated; check that your "
<< "given Gram matrix is properly computed on ";
if (fitIntercept)
oss << "mean-centered ";
else
oss << "non-mean-centered ";
if (normalizeData)
oss << "unit-variance (normalized) ";
else
oss << "non-normalized ";
oss << "data";
if (lambda2 > 0.0)
oss << " with lambda2 = " << lambda2 << " added to the diagonal";
oss << "!";
throw std::runtime_error(oss.str());
}
// Add the variable to the active set and update the Gram matrix as
// necessary.
if (!lassocond)
@@ -706,7 +682,10 @@ LARS<ModelMatType>::Train(const MatType& matX,
// Compute signs of correlations.
arma::Col<ElemType> s(activeSet.size());
for (size_t i = 0; i < activeSet.size(); ++i)
s(i) = corr(activeSet[i]) / fabs(corr(activeSet[i]));
{
const size_t j = activeSet[i];
s[i] = (ElemType) (corr(j) == 0.0 ? 0.0 : (corr(j) > 0) ? 1.0 : -1.0);
}
// Compute the "equiangular" direction in parameter space (betaDirection).
// We use quotes because in the case of non-unit norm variables, this need
@@ -793,6 +772,8 @@ LARS<ModelMatType>::Train(const MatType& matX,
// need to take a step with the previous beta direction towards the next
// variable we will add.
s = s.subvec(0, activeSet.size() - 1); // Drop last element.
matGramActive = matGramActive.submat(0, 0, activeSet.size() - 1,
activeSet.size() - 1);
matS = s * ones<MatType>(1, activeSet.size());
// This worked last iteration, so there can't be a singularity.
solve(unnormalizedBetaDirection,
src/mlpack/tests/lars_test.cpp
+149 -18
View File
@@ -35,7 +35,7 @@ void LARSVerifyCorrectness(const VecType& beta,
size_t nDims = beta.n_elem;
// floats require a much larger tolerance.
const ElemType tol = (std::is_same<ElemType, double>::value) ? 1e-10 : 1e-3;
const ElemType tol = (std::is_same<ElemType, double>::value) ? 1e-8 : 5e-3;
for (size_t j = 0; j < nDims; ++j)
{
@@ -200,39 +200,35 @@ TEST_CASE("NoCholeskySingularityTest", "[LARSTest]")
}
// Make sure that Predict() provides reasonable enough solutions.
TEMPLATE_TEST_CASE("PredictTest", "[LARSTest]", arma::fmat, arma::mat)
TEST_CASE("PredictTest", "[LARSTest]")
{
typedef TestType MatType;
typedef typename MatType::elem_type ElemType;
for (size_t i = 0; i < 2; ++i)
{
// Run with both true and false.
bool useCholesky = bool(i);
MatType X;
arma::Row<ElemType> y;
arma::mat X;
arma::rowvec y;
GenerateProblem(X, y, 1000, 100);
for (ElemType lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.2)
for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.2)
{
for (ElemType lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2)
for (double lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2)
{
LARS<MatType> lars(useCholesky, lambda1, lambda2);
LARS<> lars(useCholesky, lambda1, lambda2);
lars.FitIntercept(false);
lars.NormalizeData(false);
lars.Train(X, y);
// Calculate what the actual error should be with these regression
// parameters.
arma::Col<ElemType> betaOptPred = (X * X.t()) * lars.Beta();
arma::Row<ElemType> predictions;
arma::vec betaOptPred = (X * X.t()) * lars.Beta();
arma::rowvec predictions;
lars.Predict(X, predictions);
arma::Col<ElemType> adjPred = X * predictions.t();
arma::vec adjPred = X * predictions.t();
const ElemType tol = (std::is_same<ElemType, double>::value) ? 1e-7 :
1e-3;
const double tol = 1e-7;
REQUIRE(predictions.n_elem == 1000);
for (size_t i = 0; i < betaOptPred.n_elem; ++i)
@@ -244,7 +240,8 @@ TEMPLATE_TEST_CASE("PredictTest", "[LARSTest]", arma::fmat, arma::mat)
}
// Now check with single-point Predict(), in two ways: we will pass
// different types into Predict() to test templating support.
// different types into Predict() to test templating support. We allow
// a looser tolerance for predictions.
for (size_t i = 0; i < X.n_cols; ++i)
predictions[i] = lars.Predict(X.col(i));
@@ -273,6 +270,140 @@ TEMPLATE_TEST_CASE("PredictTest", "[LARSTest]", arma::fmat, arma::mat)
}
}
// This is the same as PredictTest, but for arma::fmat, and it allows multiple
// trials for run to deal with the lower precision of floats.
TEST_CASE("PredictFloatTest", "[LARSTest]")
{
for (size_t i = 0; i < 2; ++i)
{
// Run with both true and false.
bool useCholesky = bool(i);
arma::fmat X;
arma::frowvec y;
for (float lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.2)
{
for (float lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2)
{
// For float data, sometimes the solutions are further away from the
// true solution due to precision issues, so we allow multiple trials.
bool success = false;
for (size_t trial = 0; trial < 3; ++trial)
{
// Generate a new problem so that we hopefully end up with a better
// fit.
GenerateProblem(X, y, 1000, 100);
LARS<arma::fmat> lars(useCholesky, lambda1, lambda2);
lars.FitIntercept(false);
lars.NormalizeData(false);
lars.Train(X, y);
// Calculate what the actual error should be with these regression
// parameters.
arma::fvec betaOptPred = (X * X.t()) * lars.Beta();
arma::frowvec predictions;
lars.Predict(X, predictions);
arma::fvec adjPred = X * predictions.t();
const float tol = 3e-5;
REQUIRE(predictions.n_elem == 1000);
bool trialSuccess = true;
for (size_t i = 0; i < betaOptPred.n_elem; ++i)
{
if (std::abs(betaOptPred[i]) < 1e-5)
{
if (adjPred[i] != Approx(0.0).margin(1e-5))
{
trialSuccess = false;
break;
}
}
else
{
if (adjPred[i] != Approx(betaOptPred[i]).epsilon(tol))
{
trialSuccess = false;
break;
}
}
}
// If this trial didn't succeed, skip to the next trial.
if (!trialSuccess)
continue;
// Now check with single-point Predict(), in two ways: we will pass
// different types into Predict() to test templating support. We allow
// a looser tolerance for predictions.
for (size_t i = 0; i < X.n_cols; ++i)
predictions[i] = lars.Predict(X.col(i));
adjPred = X * predictions.t();
for (size_t i = 0; i < betaOptPred.n_elem; ++i)
{
if (std::abs(betaOptPred[i]) < 1e-5)
{
if (adjPred[i] != Approx(0.0).margin(1e-5))
{
trialSuccess = false;
break;
}
}
else
{
if (adjPred[i] != Approx(betaOptPred[i]).epsilon(10 * tol))
{
trialSuccess = false;
break;
}
}
}
// If this trial didn't succeed, skip to the next trial.
if (!trialSuccess)
continue;
for (size_t i = 0; i < X.n_cols; ++i)
predictions[i] = lars.Predict(X.unsafe_col(i));
adjPred = X * predictions.t();
for (size_t i = 0; i < betaOptPred.n_elem; ++i)
{
if (std::abs(betaOptPred[i]) < 1e-5)
{
if (adjPred[i] != Approx(0.0).margin(1e-5))
{
trialSuccess = false;
break;
}
}
else
{
if (adjPred[i] != Approx(betaOptPred[i]).epsilon(10 * tol))
{
trialSuccess = false;
break;
}
}
}
// If this trial succeeded, we're done.
if (trialSuccess)
{
success = true;
break;
}
}
REQUIRE(success == true);
}
}
}
}
TEST_CASE("PredictRowMajorTest", "[LARSTest]")
{
arma::mat X;
@@ -719,7 +850,7 @@ void CheckKKT(const arma::vec& beta,
const arma::rowvec& y,
const double lambda)
{
const double epsilon = 1e-10; // For numerical precision.
const double epsilon = 1e-6; // For numerical precision.
arma::vec v = X.t() * X * beta - X.t() * y.t() + lambda * sign(beta);
// Active set indices with global numbering: could be empty.
@@ -1094,7 +1225,7 @@ TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat)
typedef TestType MatType;
typedef typename MatType::elem_type ElemType;
const ElemType tol = (std::is_same<ElemType, double>::value) ? 1e-5 : 1e-3;
const ElemType tol = (std::is_same<ElemType, double>::value) ? 1e-5 : 5e-3;
// Train a model on a randomly generated problem. Then, we will iterate
// through different selected lambda values, ensuring that the error on the
src/mlpack/tests/sparse_coding_test.cpp
-4
View File
@@ -8,10 +8,6 @@
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
// Note: We don't use BOOST_REQUIRE_CLOSE in the code below because we need
// to use FPC_WEAK, and it's not at all intuitive how to do that.
#include <mlpack/core.hpp>
#include <mlpack/methods/sparse_coding.hpp>