From 5921a73a71b4ec3dbb0d8a7867dae318e0ff0edc Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Wed, 6 Nov 2019 14:38:08 -0500 Subject: [PATCH 01/61] ability to calculate likelihood from pre-calculated emission probability for data stream --- src/mlpack/methods/hmm/hmm.hpp | 23 ++++++ src/mlpack/methods/hmm/hmm_impl.hpp | 119 +++++++++++++++++++++++----- 2 files changed, 120 insertions(+), 22 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 021ff74dad..b01223c3e2 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -292,6 +292,17 @@ class HMM */ double LogLikelihood(const arma::mat& dataSeq) const; + /** + * Compute the log-likelihood of the given emission probability up to time t + * + * @param dataSeq Data sequence to evaluate the likelihood of. + * @return Log-likelihood of the given sequence of emission up to time t. + */ + double LogLikelihood(size_t t, + const arma::vec& emissionLogProb, + double &logScale, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const; /** * HMM filtering. Computes the k-step-ahead expected emission at each time * conditioned only on prior observations. That is @@ -360,6 +371,18 @@ class HMM protected: + + void ForwardAtT0( + const arma::vec& emissionLogProb, + double& logScales, + arma::vec& forwardLogProb) const; + + void ForwardAtTn( + const arma::vec& emissionLogProb, + double& logScales, + const arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const; + // Helper functions. /** * The Forward algorithm (part of the Forward-Backward algorithm). Computes diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index ef857f4ee5..8245b472f6 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -517,6 +517,30 @@ double HMM::LogLikelihood(const arma::mat& dataSeq) const return accu(logScales); } +/** + * Compute the log-likelihood of the given emission probability. + */ +template +double HMM::LogLikelihood(size_t t, + const arma::vec& emissionLogProb, + double &logScale, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const +{ + if(t == 0){ + ForwardAtT0(emissionLogProb, logScale, forwardLogProb); + } + else{ + double curLogSacle; + ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); + logScale += curLogSacle; + } + + prevForwardLogProb = forwardLogProb; + + return logScale; +} + /** * HMM filtering. */ @@ -565,6 +589,68 @@ void HMM::Smooth(const arma::mat& dataSeq, smoothSeq += emission[i].Mean() * exp(stateLogProb.row(i)); } +/** + * The Forward procedure (part of the Forward-Backward algorithm). + */ +template +void HMM::ForwardAtT0(const arma::vec& emissionLogProb, + double& logScales, + arma::vec& forwardLogProb + ) const +{ + // Our goal is to calculate the forward probabilities: + // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. + + + ConvertToLogSpace(); + + forwardLogProb.resize(logTransition.n_rows); + forwardLogProb.fill(-std::numeric_limits::infinity()); + // The first entry in the forward algorithm uses the initial state + // probabilities. Note that MATLAB assumes that the starting state (at + // t = -1) is state 0; this is not our assumption here. To force that + // behavior, you could append a single starting state to every single data + // sequence and that should produce results in line with MATLAB. + for (size_t state = 0; state < logTransition.n_rows; state++) { + forwardLogProb(state) = logInitial(state) + emissionLogProb(state); + } + + // Normalize probability. + logScales = math::AccuLog(forwardLogProb); + if (std::isfinite(logScales)){ + forwardLogProb -= logScales; + } +} + +/** + * The Forward procedure (part of the Forward-Backward algorithm). + */ +template +void HMM::ForwardAtTn(const arma::vec& emissionLogProb, + double& logScales, + const arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb + ) const +{ + // Our goal is to calculate the forward probabilities: + // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. + + + // Now compute the probabilities for each successive observation. + for (size_t state = 0; state < logTransition.n_rows; state++) { + // The forward probability of state j at time t is the sum over all states + // of the probability of the previous state transitioning to the current + // state and emitting the given observation. + arma::vec tmp = prevForwardLogProb + logTransition.row(state).t(); + forwardLogProb(state) = math::AccuLog(tmp) + emissionLogProb(state); + } + // Normalize probability. + logScales = math::AccuLog(forwardLogProb); + if (std::isfinite(logScales)){ + forwardLogProb -= logScales; + } +} + /** * The Forward procedure (part of the Forward-Backward algorithm). */ @@ -578,43 +664,32 @@ void HMM::Forward(const arma::mat& dataSeq, forwardLogProb.resize(logTransition.n_rows, dataSeq.n_cols); forwardLogProb.fill(-std::numeric_limits::infinity()); logScales.resize(dataSeq.n_cols); - logScales.fill(-std::numeric_limits::infinity()); - - ConvertToLogSpace(); // The first entry in the forward algorithm uses the initial state // probabilities. Note that MATLAB assumes that the starting state (at // t = -1) is state 0; this is not our assumption here. To force that // behavior, you could append a single starting state to every single data // sequence and that should produce results in line with MATLAB. + + arma::vec emissionLogProb(logTransition.n_rows); for (size_t state = 0; state < logTransition.n_rows; state++) { - forwardLogProb(state, 0) = logInitial(state) + - emission[state].LogProbability(dataSeq.unsafe_col(0)); + emissionLogProb(state) = emission[state].LogProbability(dataSeq.unsafe_col(0)); } - - // Then normalize the column. - logScales[0] = math::AccuLog(forwardLogProb.col(0)); - if (std::isfinite(logScales[0])) - forwardLogProb.col(0) -= logScales[0]; + + arma::vec col0(forwardLogProb.colptr(0), logTransition.n_rows, false); + ForwardAtT0(emissionLogProb, logScales(0), col0); // Now compute the probabilities for each successive observation. for (size_t t = 1; t < dataSeq.n_cols; t++) { - for (size_t j = 0; j < logTransition.n_rows; j++) + for (size_t state = 0; state < logTransition.n_rows; state++) { - // The forward probability of state j at time t is the sum over all states - // of the probability of the previous state transitioning to the current - // state and emitting the given observation. - arma::vec tmp = forwardLogProb.col(t - 1) + logTransition.row(j).t(); - forwardLogProb(j, t) = math::AccuLog(tmp) + - emission[j].LogProbability(dataSeq.unsafe_col(t)); + emissionLogProb(state) = emission[state].LogProbability(dataSeq.unsafe_col(t)); } - - // Normalize probability. - logScales[t] = math::AccuLog(forwardLogProb.col(t)); - if (std::isfinite(logScales[t])) - forwardLogProb.col(t) -= logScales[t]; + + arma::vec colt(forwardLogProb.colptr(t), logTransition.n_rows, false); + ForwardAtTn(emissionLogProb, logScales(t), forwardLogProb.col(t-1), colt); } } From e123435aefedf941d4fb3b3d5d763cc6cf48db0c Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Mon, 9 Dec 2019 17:50:10 -0500 Subject: [PATCH 02/61] added logLikelihood() to calculate log likelihood for data stream at each point of time --- src/mlpack/methods/hmm/hmm.hpp | 33 +++++++++++-- src/mlpack/methods/hmm/hmm_impl.hpp | 77 +++++++++++++++++++---------- src/mlpack/tests/hmm_test.cpp | 13 ++++- 3 files changed, 90 insertions(+), 33 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 785458fc88..f7a20404be 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -295,13 +295,37 @@ class HMM /** * Compute the log-likelihood of the given emission probability up to time t * - * @param dataSeq Data sequence to evaluate the likelihood of. + * @param t time order + * @param log emission probability at time t. + * @param logScale Log-likelihood of the given sequence of emission + * probability up to time t-1 + * @param prevForwardProb Vector in which forward probabilities for time t-1 + * will be saved. + * @param forwardProb Vector in which forward probabilities for time t + * will be saved. * @return Log-likelihood of the given sequence of emission up to time t. */ - double LogLikelihood(size_t t, + double LogLikelihoodEmissionProb(size_t t, const arma::vec& emissionLogProb, - double &logScale, - arma::vec& prevForwardLogProb, + double &logScale, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const; + /** + * Compute the log-likelihood of the given data up to time t + * + * @param t time order + * @param data observation at time t. + * @param logScale Log-likelihood of the given sequence of data up to time t-1 + * @param prevForwardProb Vector in which forward probabilities for time t-1 + * will be saved. + * @param forwardProb Vector in which forward probabilities for time t + * will be saved. + * @return Log-likelihood of the given sequence of data up to time t. + */ + double LogLikelihood(size_t t, + const arma::vec &data, + double &logScale, + arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; /** * HMM filtering. Computes the k-step-ahead expected emission at each time @@ -382,7 +406,6 @@ class HMM protected: - void ForwardAtT0( const arma::vec& emissionLogProb, double& logScales, diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index e97dec16bc..a447d7501e 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -523,13 +523,13 @@ double HMM::LogLikelihood(const arma::mat& dataSeq) const } /** - * Compute the log-likelihood of the given emission probability. + * Compute the log-likelihood of the given emission probability up to time t */ template -double HMM::LogLikelihood(size_t t, +double HMM::LogLikelihoodEmissionProb(size_t t, const arma::vec& emissionLogProb, - double &logScale, - arma::vec& prevForwardLogProb, + double &logScale, + arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { if(t == 0){ @@ -537,15 +537,37 @@ double HMM::LogLikelihood(size_t t, } else{ double curLogSacle; - ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); + ForwardAtTn(emissionLogProb, curLogSacle, + prevForwardLogProb, forwardLogProb); logScale += curLogSacle; } - + prevForwardLogProb = forwardLogProb; - + return logScale; } +/** + * Compute the log-likelihood of the given data up to time t + */ +template +double HMM::LogLikelihood(size_t t, + const arma::vec &data, + double &logScale, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const +{ + arma::vec emissionLogProb(logTransition.n_rows); + + for (size_t state = 0; state < logTransition.n_rows; state++) + { + emissionLogProb(state) = emission[state].LogProbability(data); + } + + return LogLikelihoodEmissionProb(t, emissionLogProb, logScale, + prevForwardLogProb, forwardLogProb); +} + /** * HMM filtering. */ @@ -601,14 +623,14 @@ template void HMM::ForwardAtT0(const arma::vec& emissionLogProb, double& logScales, arma::vec& forwardLogProb - ) const -{ + ) const +{ // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. - - + + ConvertToLogSpace(); - + forwardLogProb.resize(logTransition.n_rows); forwardLogProb.fill(-std::numeric_limits::infinity()); // The first entry in the forward algorithm uses the initial state @@ -635,17 +657,17 @@ void HMM::ForwardAtTn(const arma::vec& emissionLogProb, double& logScales, const arma::vec& prevForwardLogProb, arma::vec& forwardLogProb - ) const -{ + ) const +{ // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. - - + + // Now compute the probabilities for each successive observation. for (size_t state = 0; state < logTransition.n_rows; state++) { - // The forward probability of state j at time t is the sum over all states - // of the probability of the previous state transitioning to the current - // state and emitting the given observation. + // The forward probability of state j at time t is the sum over all + // states of the probability of the previous state transitioning to + // the current state and emitting the given observation. arma::vec tmp = prevForwardLogProb + logTransition.row(state).t(); forwardLogProb(state) = math::AccuLog(tmp) + emissionLogProb(state); } @@ -678,13 +700,14 @@ void HMM::Forward(const arma::mat& dataSeq, // t = -1) is state 0; this is not our assumption here. To force that // behavior, you could append a single starting state to every single data // sequence and that should produce results in line with MATLAB. - + arma::vec emissionLogProb(logTransition.n_rows); for (size_t state = 0; state < logTransition.n_rows; state++) { - emissionLogProb(state) = emission[state].LogProbability(dataSeq.unsafe_col(0)); + emissionLogProb(state) = + emission[state].LogProbability(dataSeq.unsafe_col(0)); } - + arma::vec col0(forwardLogProb.colptr(0), logTransition.n_rows, false); ForwardAtT0(emissionLogProb, logScales(0), col0); @@ -693,11 +716,12 @@ void HMM::Forward(const arma::mat& dataSeq, { for (size_t state = 0; state < logTransition.n_rows; state++) { - emissionLogProb(state) = emission[state].LogProbability(dataSeq.unsafe_col(t)); + emissionLogProb(state) = + emission[state].LogProbability(dataSeq.unsafe_col(t)); } - + arma::vec colt(forwardLogProb.colptr(t), logTransition.n_rows, false); - ForwardAtTn(emissionLogProb, logScales(t), forwardLogProb.col(t-1), colt); + ForwardAtTn(emissionLogProb, logScales(t), forwardLogProb.col(t-1), colt); } } @@ -738,7 +762,8 @@ void HMM::Backward(const arma::mat& dataSeq, } /** - * Make sure the variables in log space are in sync with the linear counter parts + * Make sure the variables in log space are in sync with the linear + * counter parts */ template void HMM::ConvertToLogSpace() const diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index fd1c1946fc..6cda82d289 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -836,7 +836,15 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) }; arma::Row stateSeq; - auto likelihood = hmm.LogLikelihood(obs); + auto loglikelihood = hmm.LogLikelihood(obs); + + double loglikelihood2; + arma::vec prevForwardLogProb; + arma::vec forwardLogProb; + for(size_t t = 0; t stateSeqRef = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -845,7 +853,8 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; - BOOST_REQUIRE_CLOSE(likelihood, -2734.43, 1e-3); + BOOST_REQUIRE_CLOSE(loglikelihood, -2734.43, 1e-3); + BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihood2, 1e-5); for (size_t i = 0; i < stateSeqRef.n_cols; ++i) { From a28a8a1f6a5d0f8857d830a80e97ccf989d9b557 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Mon, 6 Jan 2020 11:45:03 -0500 Subject: [PATCH 03/61] fixed style issues --- src/mlpack/methods/hmm/hmm_impl.hpp | 12 ++++++------ src/mlpack/tests/hmm_test.cpp | 6 ++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index a447d7501e..e77ee5a115 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -532,10 +532,12 @@ double HMM::LogLikelihoodEmissionProb(size_t t, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { - if(t == 0){ + if (t == 0) + { ForwardAtT0(emissionLogProb, logScale, forwardLogProb); } - else{ + else + { double curLogSacle; ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); @@ -622,8 +624,7 @@ void HMM::Smooth(const arma::mat& dataSeq, template void HMM::ForwardAtT0(const arma::vec& emissionLogProb, double& logScales, - arma::vec& forwardLogProb - ) const + arma::vec& forwardLogProb) const { // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. @@ -656,8 +657,7 @@ template void HMM::ForwardAtTn(const arma::vec& emissionLogProb, double& logScales, const arma::vec& prevForwardLogProb, - arma::vec& forwardLogProb - ) const + arma::vec& forwardLogProb) const { // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 6cda82d289..d3cb56bff0 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -841,8 +841,10 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) double loglikelihood2; arma::vec prevForwardLogProb; arma::vec forwardLogProb; - for(size_t t = 0; t Date: Mon, 6 Jan 2020 11:49:47 -0500 Subject: [PATCH 04/61] fixed style issues. Removed tabs --- src/mlpack/methods/hmm/hmm_impl.hpp | 4 ++-- src/mlpack/tests/hmm_test.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index e77ee5a115..7ec65f7f0c 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -533,11 +533,11 @@ double HMM::LogLikelihoodEmissionProb(size_t t, arma::vec& forwardLogProb) const { if (t == 0) - { + { ForwardAtT0(emissionLogProb, logScale, forwardLogProb); } else - { + { double curLogSacle; ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index d3cb56bff0..72bebbc168 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -844,7 +844,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) for (size_t t = 0; t Date: Tue, 7 Jan 2020 11:29:40 -0500 Subject: [PATCH 05/61] no need to call ConvertToLogSpace() in Forward() since it's called inside ForwardAtT0() --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 7ec65f7f0c..0616099b69 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -693,8 +693,6 @@ void HMM::Forward(const arma::mat& dataSeq, logScales.resize(dataSeq.n_cols); logScales.fill(-std::numeric_limits::infinity()); - ConvertToLogSpace(); - // The first entry in the forward algorithm uses the initial state // probabilities. Note that MATLAB assumes that the starting state (at // t = -1) is state 0; this is not our assumption here. To force that From a0be23a6eb03180813318764aa74d111e1c8efff Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Wed, 12 Aug 2020 14:10:06 -0400 Subject: [PATCH 06/61] removed the t argument --- src/mlpack/methods/hmm/hmm.hpp | 22 ++++++++++------------ src/mlpack/methods/hmm/hmm_impl.hpp | 21 ++++++++++----------- src/mlpack/tests/hmm_test.cpp | 2 +- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index f7a20404be..d934a08ef2 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -295,36 +295,34 @@ class HMM /** * Compute the log-likelihood of the given emission probability up to time t * - * @param t time order * @param log emission probability at time t. - * @param logScale Log-likelihood of the given sequence of emission - * probability up to time t-1 + * @param logLikelihood Log-likelihood of the given sequence of emission + * probability up to time t-1 * @param prevForwardProb Vector in which forward probabilities for time t-1 - * will be saved. + * will be saved. Passing prevForwardProb as an empty vector indicates the + * start of sequence or time t=0 * @param forwardProb Vector in which forward probabilities for time t * will be saved. * @return Log-likelihood of the given sequence of emission up to time t. */ - double LogLikelihoodEmissionProb(size_t t, - const arma::vec& emissionLogProb, - double &logScale, + double LogLikelihoodEmissionProb(const arma::vec& emissionLogProb, + double &logLikelihood, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given data up to time t * - * @param t time order * @param data observation at time t. * @param logScale Log-likelihood of the given sequence of data up to time t-1 * @param prevForwardProb Vector in which forward probabilities for time t-1 - * will be saved. + * will be saved. Passing prevForwardProb as an empty vector indicates the + * start of sequence or time t=0 * @param forwardProb Vector in which forward probabilities for time t * will be saved. * @return Log-likelihood of the given sequence of data up to time t. */ - double LogLikelihood(size_t t, - const arma::vec &data, - double &logScale, + double LogLikelihood(const arma::vec &data, + double &logLikelihood, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; /** diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index a447d7501e..565db517db 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -526,34 +526,33 @@ double HMM::LogLikelihood(const arma::mat& dataSeq) const * Compute the log-likelihood of the given emission probability up to time t */ template -double HMM::LogLikelihoodEmissionProb(size_t t, - const arma::vec& emissionLogProb, - double &logScale, +double HMM::LogLikelihoodEmissionProb(const arma::vec& emissionLogProb, + double &logLikelihood, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { - if(t == 0){ - ForwardAtT0(emissionLogProb, logScale, forwardLogProb); + if(prevForwardLogProb.empty()){ + //start os sequence or time t=0 + ForwardAtT0(emissionLogProb, logLikelihood, forwardLogProb); } else{ double curLogSacle; ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); - logScale += curLogSacle; + logLikelihood += curLogSacle; } prevForwardLogProb = forwardLogProb; - return logScale; + return logLikelihood; } /** * Compute the log-likelihood of the given data up to time t */ template -double HMM::LogLikelihood(size_t t, - const arma::vec &data, - double &logScale, +double HMM::LogLikelihood(const arma::vec &data, + double &logLikelihood, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { @@ -564,7 +563,7 @@ double HMM::LogLikelihood(size_t t, emissionLogProb(state) = emission[state].LogProbability(data); } - return LogLikelihoodEmissionProb(t, emissionLogProb, logScale, + return LogLikelihoodEmissionProb(emissionLogProb, logLikelihood, prevForwardLogProb, forwardLogProb); } diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 6cda82d289..73b6866889 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -842,7 +842,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) arma::vec prevForwardLogProb; arma::vec forwardLogProb; for(size_t t = 0; t Date: Wed, 12 Aug 2020 18:38:08 -0400 Subject: [PATCH 07/61] added logScale() and LogScaleEmissionProb() --- src/mlpack/methods/hmm/hmm.hpp | 40 +++++++++++++++-- src/mlpack/methods/hmm/hmm_impl.hpp | 66 ++++++++++++++++++++++------- src/mlpack/tests/hmm_test.cpp | 38 ++++++++++++----- 3 files changed, 116 insertions(+), 28 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index d934a08ef2..8829768325 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -292,6 +292,23 @@ class HMM */ double LogLikelihood(const arma::mat& dataSeq) const; + /** + * Compute the log of the scaling factor of the given emission probability + * at time t. To calculate the log-likelihood for the whole sequence, + * accumulate log scale over the entire sequence + * + * @param log emission probability at time t. + * probability up to time t-1 + * @param prevForwardProb Vector in which forward probabilities for time t-1 + * will be saved. Passing prevForwardProb as an empty vector indicates the + * start of sequence or time t=0 + * @param forwardProb Vector in which forward probabilities for time t + * will be saved. + * @return Log scale factor of the given sequence of emission at time t. + */ + double LogScaleEmissionProb(const arma::vec& emissionLogProb, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given emission probability up to time t * @@ -309,11 +326,28 @@ class HMM double &logLikelihood, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; + /** + * Compute the log of the scaling factor of the given data at time t. + * To calculate the log-likelihood for the whole sequence, accumulate log + * scale over the entire sequence + * + * @param data observation at time t. + * @param prevForwardProb Vector in which forward probabilities for time t-1 + * will be saved. Passing prevForwardProb as an empty vector indicates the + * start of sequence or time t=0 + * @param forwardProb Vector in which forward probabilities for time t + * will be saved. + * @return Log scale factor of the given sequence of data up at time t. + */ + double LogScale(const arma::vec &data, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given data up to time t * * @param data observation at time t. - * @param logScale Log-likelihood of the given sequence of data up to time t-1 + * @param logLikelihood Log-likelihood of the given sequence of data + * up to time t-1 * @param prevForwardProb Vector in which forward probabilities for time t-1 * will be saved. Passing prevForwardProb as an empty vector indicates the * start of sequence or time t=0 @@ -423,7 +457,7 @@ class HMM * states and columns equal to the number of observations. * * @param dataSeq Data sequence to compute probabilities for. - * @param scales Vector in which scaling factors will be saved. + * @param logScales Vector in which the log of scaling factors will be saved. * @param forwardProb Matrix in which forward probabilities will be saved. */ void Forward(const arma::mat& dataSeq, @@ -438,7 +472,7 @@ class HMM * columns equal to the number of observations. * * @param dataSeq Data sequence to compute probabilities for. - * @param scales Vector of scaling factors. + * @param logScales Vector of log of scaling factors. * @param backwardProb Matrix in which backward probabilities will be saved. */ void Backward(const arma::mat& dataSeq, diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 565db517db..d998307912 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -523,30 +523,70 @@ double HMM::LogLikelihood(const arma::mat& dataSeq) const } /** - * Compute the log-likelihood of the given emission probability up to time t + * Compute the log of the scaling factor of the given emission probability + * at time t. To calculate the log-likelihood for the whole sequence, + * accumulate log scale over the entire sequence */ template -double HMM::LogLikelihoodEmissionProb(const arma::vec& emissionLogProb, - double &logLikelihood, +double HMM::LogScaleEmissionProb(const arma::vec& emissionLogProb, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { + double curLogSacle; if(prevForwardLogProb.empty()){ //start os sequence or time t=0 - ForwardAtT0(emissionLogProb, logLikelihood, forwardLogProb); + ForwardAtT0(emissionLogProb, curLogSacle, forwardLogProb); } else{ - double curLogSacle; ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); - logLikelihood += curLogSacle; } prevForwardLogProb = forwardLogProb; + return curLogSacle; +} + +/** + * Compute the log-likelihood of the given emission probability up to time t + */ +template +double HMM::LogLikelihoodEmissionProb( + const arma::vec& emissionLogProb, + double &logLikelihood, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const +{ + auto curLogScale = LogScaleEmissionProb(emissionLogProb, + prevForwardLogProb, forwardLogProb); + + logLikelihood = prevForwardLogProb.empty() + ? curLogScale : curLogScale + logLikelihood; + return logLikelihood; } +/** + * Compute the log of the scaling factor of the given data at time t. + * To calculate the log-likelihood for the whole sequence, accumulate log + * scale over the entire sequence + */ +template +double HMM::LogScale(const arma::vec &data, + arma::vec& prevForwardLogProb, + arma::vec& forwardLogProb) const +{ + arma::vec emissionLogProb(logTransition.n_rows); + + for (size_t state = 0; state < logTransition.n_rows; state++) + { + emissionLogProb(state) = emission[state].LogProbability(data); + } + + return LogScaleEmissionProb(emissionLogProb, + prevForwardLogProb, forwardLogProb); +} + /** * Compute the log-likelihood of the given data up to time t */ @@ -556,15 +596,11 @@ double HMM::LogLikelihood(const arma::vec &data, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { - arma::vec emissionLogProb(logTransition.n_rows); - - for (size_t state = 0; state < logTransition.n_rows; state++) - { - emissionLogProb(state) = emission[state].LogProbability(data); - } - - return LogLikelihoodEmissionProb(emissionLogProb, logLikelihood, - prevForwardLogProb, forwardLogProb); + auto curLogScale = LogScale(data, prevForwardLogProb, forwardLogProb); + + logLikelihood = prevForwardLogProb.empty() + ? curLogScale : curLogScale + logLikelihood; + return logLikelihood; } /** diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 73b6866889..49a8b067c3 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -835,16 +835,37 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) } }; - arma::Row stateSeq; - auto loglikelihood = hmm.LogLikelihood(obs); + const double loglikelihoodRef = -2734.43; - double loglikelihood2; - arma::vec prevForwardLogProb; - arma::vec forwardLogProb; - for(size_t t = 0; t stateSeq; hmm.Predict(obs, stateSeq); arma::Row stateSeqRef = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -853,9 +874,6 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; - BOOST_REQUIRE_CLOSE(loglikelihood, -2734.43, 1e-3); - BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihood2, 1e-5); - for (size_t i = 0; i < stateSeqRef.n_cols; ++i) { BOOST_REQUIRE_EQUAL(stateSeqRef.at(i), stateSeq.at(i)); From 51d210988a03d30f6c32540435c6e415269d5153 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Wed, 12 Aug 2020 19:06:32 -0400 Subject: [PATCH 08/61] fixed doxygen @param arguments --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 49ef018404..71904154cf 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -456,7 +456,7 @@ class HMM * * @param dataSeq Data sequence to compute probabilities for. * @param logScales Vector in which the log of scaling factors will be saved. - * @param forwardProb Matrix in which forward probabilities will be saved. + * @param forwardLogProb Matrix in which forward probabilities will be saved. */ void Forward(const arma::mat& dataSeq, arma::vec& logScales, @@ -471,7 +471,7 @@ class HMM * * @param dataSeq Data sequence to compute probabilities for. * @param logScales Vector of log of scaling factors. - * @param backwardProb Matrix in which backward probabilities will be saved. + * @param backwardLogProb Matrix in which backward probabilities will be saved. */ void Backward(const arma::mat& dataSeq, const arma::vec& logScales, From ffa6669c1195d2b8c89302045e164840763ca7fc Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Wed, 12 Aug 2020 19:11:27 -0400 Subject: [PATCH 09/61] fixed doxygen @param arguments --- src/mlpack/methods/hmm/hmm.hpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 71904154cf..ca732c7164 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -295,12 +295,12 @@ class HMM * at time t. To calculate the log-likelihood for the whole sequence, * accumulate log scale over the entire sequence * - * @param log emission probability at time t. + * @param emissionLogProb emission probability at time t. * probability up to time t-1 - * @param prevForwardProb Vector in which forward probabilities for time t-1 - * will be saved. Passing prevForwardProb as an empty vector indicates the + * @param prevForwardLogProb Vector in which forward probabilities for time + * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the * start of sequence or time t=0 - * @param forwardProb Vector in which forward probabilities for time t + * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log scale factor of the given sequence of emission at time t. */ @@ -310,13 +310,13 @@ class HMM /** * Compute the log-likelihood of the given emission probability up to time t * - * @param log emission probability at time t. + * @param emissionLogProb emission probability at time t. * @param logLikelihood Log-likelihood of the given sequence of emission * probability up to time t-1 - * @param prevForwardProb Vector in which forward probabilities for time t-1 - * will be saved. Passing prevForwardProb as an empty vector indicates the + * @param prevForwardLogProb Vector in which forward probabilities for time + * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the * start of sequence or time t=0 - * @param forwardProb Vector in which forward probabilities for time t + * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log-likelihood of the given sequence of emission up to time t. */ @@ -330,10 +330,10 @@ class HMM * scale over the entire sequence * * @param data observation at time t. - * @param prevForwardProb Vector in which forward probabilities for time t-1 - * will be saved. Passing prevForwardProb as an empty vector indicates the + * @param prevForwardLogProb Vector in which forward probabilities for time + * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the * start of sequence or time t=0 - * @param forwardProb Vector in which forward probabilities for time t + * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log scale factor of the given sequence of data up at time t. */ @@ -346,10 +346,10 @@ class HMM * @param data observation at time t. * @param logLikelihood Log-likelihood of the given sequence of data * up to time t-1 - * @param prevForwardProb Vector in which forward probabilities for time t-1 - * will be saved. Passing prevForwardProb as an empty vector indicates the + * @param prevForwardLogProb Vector in which forward probabilities for time + * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the * start of sequence or time t=0 - * @param forwardProb Vector in which forward probabilities for time t + * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log-likelihood of the given sequence of data up to time t. */ From e10669d130f31d0d86155af334c31c5dab7e1744 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Wed, 12 Aug 2020 19:22:09 -0400 Subject: [PATCH 10/61] fixed styles --- src/mlpack/methods/hmm/hmm.hpp | 8 ++++---- src/mlpack/methods/hmm/hmm_impl.hpp | 14 ++++++-------- src/mlpack/tests/hmm_test.cpp | 8 +++++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index ca732c7164..0229734dc5 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -299,7 +299,7 @@ class HMM * probability up to time t-1 * @param prevForwardLogProb Vector in which forward probabilities for time * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 + * start of sequence or time t=0 * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log scale factor of the given sequence of emission at time t. @@ -315,7 +315,7 @@ class HMM * probability up to time t-1 * @param prevForwardLogProb Vector in which forward probabilities for time * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 + * start of sequence or time t=0 * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log-likelihood of the given sequence of emission up to time t. @@ -332,7 +332,7 @@ class HMM * @param data observation at time t. * @param prevForwardLogProb Vector in which forward probabilities for time * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 + * start of sequence or time t=0 * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log scale factor of the given sequence of data up at time t. @@ -348,7 +348,7 @@ class HMM * up to time t-1 * @param prevForwardLogProb Vector in which forward probabilities for time * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 + * start of sequence or time t=0 * @param forwardLogProb Vector in which forward probabilities for time t * will be saved. * @return Log-likelihood of the given sequence of data up to time t. diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 62d9ee73d9..941d18e9ab 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -533,11 +533,12 @@ double HMM::LogScaleEmissionProb(const arma::vec& emissionLogProb, arma::vec& forwardLogProb) const { double curLogSacle; - if(prevForwardLogProb.empty()){ - //start os sequence or time t=0 + if (prevForwardLogProb.empty()){ + // start os sequence or time t=0 ForwardAtT0(emissionLogProb, curLogSacle, forwardLogProb); } - else{ + else + { ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); } @@ -559,10 +560,8 @@ double HMM::LogLikelihoodEmissionProb( { auto curLogScale = LogScaleEmissionProb(emissionLogProb, prevForwardLogProb, forwardLogProb); - logLikelihood = prevForwardLogProb.empty() ? curLogScale : curLogScale + logLikelihood; - return logLikelihood; } @@ -583,7 +582,7 @@ double HMM::LogScale(const arma::vec &data, emissionLogProb(state) = emission[state].LogProbability(data); } - return LogScaleEmissionProb(emissionLogProb, + return LogScaleEmissionProb(emissionLogProb, prevForwardLogProb, forwardLogProb); } @@ -597,10 +596,9 @@ double HMM::LogLikelihood(const arma::vec &data, arma::vec& forwardLogProb) const { auto curLogScale = LogScale(data, prevForwardLogProb, forwardLogProb); - logLikelihood = prevForwardLogProb.empty() ? curLogScale : curLogScale + logLikelihood; - return logLikelihood; + return logLikelihood; } /** diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 381b73e550..490d789c15 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -837,7 +837,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) const double loglikelihoodRef = -2734.43; - { + { auto loglikelihood = hmm.LogLikelihood(obs); BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihoodRef, 1e-3); } @@ -846,7 +846,8 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) double loglikelihood; arma::vec prevForwardLogProb; arma::vec forwardLogProb; - for(size_t t = 0; t Date: Wed, 12 Aug 2020 19:27:30 -0400 Subject: [PATCH 11/61] fixed styles --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 +- src/mlpack/tests/hmm_test.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 941d18e9ab..acc17dd542 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -538,7 +538,7 @@ double HMM::LogScaleEmissionProb(const arma::vec& emissionLogProb, ForwardAtT0(emissionLogProb, curLogSacle, forwardLogProb); } else - { + { ForwardAtTn(emissionLogProb, curLogSacle, prevForwardLogProb, forwardLogProb); } diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 490d789c15..fc4a6bf2b3 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -847,7 +847,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) arma::vec prevForwardLogProb; arma::vec forwardLogProb; for (size_t t = 0; t Date: Thu, 13 Aug 2020 15:46:09 -0400 Subject: [PATCH 12/61] fixed log-likelihood calculation --- src/mlpack/methods/hmm/hmm_impl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index acc17dd542..e2594ef8a5 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -558,10 +558,10 @@ double HMM::LogLikelihoodEmissionProb( arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { + auto isStartOfSeq = prevForwardLogProb.empty(); auto curLogScale = LogScaleEmissionProb(emissionLogProb, prevForwardLogProb, forwardLogProb); - logLikelihood = prevForwardLogProb.empty() - ? curLogScale : curLogScale + logLikelihood; + logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; return logLikelihood; } @@ -595,9 +595,9 @@ double HMM::LogLikelihood(const arma::vec &data, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { + auto isStartOfSeq = prevForwardLogProb.empty(); auto curLogScale = LogScale(data, prevForwardLogProb, forwardLogProb); - logLikelihood = prevForwardLogProb.empty() - ? curLogScale : curLogScale + logLikelihood; + logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; return logLikelihood; } From 4ab3ae1527f3ab27d7f7e7f2311832fa2cfacdc5 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Mon, 31 Aug 2020 13:31:53 -0400 Subject: [PATCH 13/61] fixed typo and style --- src/mlpack/methods/hmm/hmm_impl.hpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index e2594ef8a5..ffc8c4ede7 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -532,20 +532,21 @@ double HMM::LogScaleEmissionProb(const arma::vec& emissionLogProb, arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { - double curLogSacle; - if (prevForwardLogProb.empty()){ - // start os sequence or time t=0 - ForwardAtT0(emissionLogProb, curLogSacle, forwardLogProb); + double curLogScale; + if (prevForwardLogProb.empty()) + { + // start of sequence or time t=0 + ForwardAtT0(emissionLogProb, curLogScale, forwardLogProb); } else { - ForwardAtTn(emissionLogProb, curLogSacle, - prevForwardLogProb, forwardLogProb); + ForwardAtTn(emissionLogProb, curLogScale, + prevForwardLogProb, forwardLogProb); } prevForwardLogProb = forwardLogProb; - return curLogSacle; + return curLogScale; } /** @@ -558,8 +559,8 @@ double HMM::LogLikelihoodEmissionProb( arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { - auto isStartOfSeq = prevForwardLogProb.empty(); - auto curLogScale = LogScaleEmissionProb(emissionLogProb, + bool isStartOfSeq = prevForwardLogProb.empty(); + double curLogScale = LogScaleEmissionProb(emissionLogProb, prevForwardLogProb, forwardLogProb); logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; return logLikelihood; From 297169131c72868f8c68ad90a7dc433d93b6f648 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Mon, 31 Aug 2020 13:40:50 -0400 Subject: [PATCH 14/61] added comments for ForwardAtT0() and ForwardAtTn() --- src/mlpack/methods/hmm/hmm.hpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 0229734dc5..f2abbeb026 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -436,11 +436,31 @@ class HMM protected: + /** + * Given emission probabilities, computes forward probabilities at time t=0. + * The returned matrix has rows equal to the number of hidden + * states and columns equal to the number of observations. + * + * @param emissionLogProb emission probability at time t=0. + * @param logScales Vector in which the log of scaling factors will be saved. + * @param forwardLogProb Matrix in which forward probabilities will be saved. + */ void ForwardAtT0( const arma::vec& emissionLogProb, double& logScales, arma::vec& forwardLogProb) const; + /** + * Given emission probabilities, computes forward probabilities for time t>0. + * The returned matrix has rows equal to the number of hidden + * states and columns equal to the number of observations. + * + * @param emissionLogProb emission probability at time t>0. + * @param logScales Vector in which the log of scaling factors will be saved. + * @param prevForwardLogProb Vector in which forward probabilities for time + * t-1 will be saved. + * @param forwardLogProb Matrix in which forward probabilities will be saved. + */ void ForwardAtTn( const arma::vec& emissionLogProb, double& logScales, From ca6a46c8a4414f97fa03587a5397aa8143bdd7e6 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Mon, 31 Aug 2020 18:13:57 -0400 Subject: [PATCH 15/61] no need to keep the previous forward probabilities in a separate vector --- src/mlpack/methods/hmm/hmm.hpp | 75 +++++++++++++---------------- src/mlpack/methods/hmm/hmm_impl.hpp | 51 +++++++++----------- src/mlpack/tests/hmm_test.cpp | 7 +-- 3 files changed, 59 insertions(+), 74 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index f2abbeb026..5f39326018 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -294,68 +294,68 @@ class HMM * Compute the log of the scaling factor of the given emission probability * at time t. To calculate the log-likelihood for the whole sequence, * accumulate log scale over the entire sequence + * This is meant for incremental or streaming computation of the + * log-likelihood of a sequence. For the first data point, provide an empty + * forwardLogProb vector. * * @param emissionLogProb emission probability at time t. * probability up to time t-1 - * @param prevForwardLogProb Vector in which forward probabilities for time - * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 - * @param forwardLogProb Vector in which forward probabilities for time t - * will be saved. + * @param forwardLogProb Vector in which forward probabilities will be saved. + * Passing forwardLogProb as an empty vector indicates the start of sequence + * or time t=0 * @return Log scale factor of the given sequence of emission at time t. */ double LogScaleEmissionProb(const arma::vec& emissionLogProb, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given emission probability up to time t + * This is meant for incremental or streaming computation of the + * log-likelihood of a sequence. For the first data point, provide an empty + * forwardLogProb vector. * * @param emissionLogProb emission probability at time t. * @param logLikelihood Log-likelihood of the given sequence of emission * probability up to time t-1 - * @param prevForwardLogProb Vector in which forward probabilities for time - * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 - * @param forwardLogProb Vector in which forward probabilities for time t - * will be saved. + * @param forwardLogProb Vector in which forward probabilities will be saved. + * Passing forwardLogProb as an empty vector indicates the start of sequence + * or time t=0 * @return Log-likelihood of the given sequence of emission up to time t. */ double LogLikelihoodEmissionProb(const arma::vec& emissionLogProb, double &logLikelihood, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; /** * Compute the log of the scaling factor of the given data at time t. * To calculate the log-likelihood for the whole sequence, accumulate log - * scale over the entire sequence + * scale over the entire sequence. + * This is meant for incremental or streaming computation of the + * log-likelihood of a sequence. For the first data point, provide an empty + * forwardLogProb vector. * * @param data observation at time t. - * @param prevForwardLogProb Vector in which forward probabilities for time - * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 - * @param forwardLogProb Vector in which forward probabilities for time t - * will be saved. + * @param forwardLogProb Vector in which forward probabilities will be saved. + * Passing forwardLogProb as an empty vector indicates the start of sequence + * or time t=0 * @return Log scale factor of the given sequence of data up at time t. */ double LogScale(const arma::vec &data, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given data up to time t + * This is meant for incremental or streaming computation of the + * log-likelihood of a sequence. For the first data point, provide an empty + * forwardLogProb vector. * * @param data observation at time t. * @param logLikelihood Log-likelihood of the given sequence of data * up to time t-1 - * @param prevForwardLogProb Vector in which forward probabilities for time - * t-1 will be saved. Passing prevForwardProb as an empty vector indicates the - * start of sequence or time t=0 - * @param forwardLogProb Vector in which forward probabilities for time t - * will be saved. + * @param forwardLogProb Vector in which forward probabilities will be saved. + * Passing forwardLogProb as an empty vector indicates the start of sequence + * or time t=0 * @return Log-likelihood of the given sequence of data up to time t. */ double LogLikelihood(const arma::vec &data, double &logLikelihood, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const; /** * HMM filtering. Computes the k-step-ahead expected emission at each time @@ -438,34 +438,27 @@ class HMM protected: /** * Given emission probabilities, computes forward probabilities at time t=0. - * The returned matrix has rows equal to the number of hidden - * states and columns equal to the number of observations. * - * @param emissionLogProb emission probability at time t=0. + * @param emissionLogProb Emission probability at time t=0. * @param logScales Vector in which the log of scaling factors will be saved. - * @param forwardLogProb Matrix in which forward probabilities will be saved. + * @return Forward probabilities */ - void ForwardAtT0( + arma::vec ForwardAtT0( const arma::vec& emissionLogProb, - double& logScales, - arma::vec& forwardLogProb) const; + double& logScales) const; /** * Given emission probabilities, computes forward probabilities for time t>0. - * The returned matrix has rows equal to the number of hidden - * states and columns equal to the number of observations. * - * @param emissionLogProb emission probability at time t>0. + * @param emissionLogProb Emission probability at time t>0. * @param logScales Vector in which the log of scaling factors will be saved. - * @param prevForwardLogProb Vector in which forward probabilities for time - * t-1 will be saved. - * @param forwardLogProb Matrix in which forward probabilities will be saved. + * @param prevForwardLogProb Previous forward probabilities. + * @return Forward probabilities */ - void ForwardAtTn( + arma::vec ForwardAtTn( const arma::vec& emissionLogProb, double& logScales, - const arma::vec& prevForwardLogProb, - arma::vec& forwardLogProb) const; + const arma::vec& prevForwardLogProb) const; // Helper functions. /** diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index ffc8c4ede7..a4c2759377 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -529,23 +529,20 @@ double HMM::LogLikelihood(const arma::mat& dataSeq) const */ template double HMM::LogScaleEmissionProb(const arma::vec& emissionLogProb, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { double curLogScale; - if (prevForwardLogProb.empty()) + if (forwardLogProb.empty()) { // start of sequence or time t=0 - ForwardAtT0(emissionLogProb, curLogScale, forwardLogProb); + forwardLogProb = ForwardAtT0(emissionLogProb, curLogScale); } else { - ForwardAtTn(emissionLogProb, curLogScale, - prevForwardLogProb, forwardLogProb); + forwardLogProb = ForwardAtTn(emissionLogProb, curLogScale, + forwardLogProb); } - prevForwardLogProb = forwardLogProb; - return curLogScale; } @@ -556,12 +553,10 @@ template double HMM::LogLikelihoodEmissionProb( const arma::vec& emissionLogProb, double &logLikelihood, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { - bool isStartOfSeq = prevForwardLogProb.empty(); - double curLogScale = LogScaleEmissionProb(emissionLogProb, - prevForwardLogProb, forwardLogProb); + bool isStartOfSeq = forwardLogProb.empty(); + double curLogScale = LogScaleEmissionProb(emissionLogProb, forwardLogProb); logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; return logLikelihood; } @@ -573,7 +568,6 @@ double HMM::LogLikelihoodEmissionProb( */ template double HMM::LogScale(const arma::vec &data, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { arma::vec emissionLogProb(logTransition.n_rows); @@ -583,8 +577,7 @@ double HMM::LogScale(const arma::vec &data, emissionLogProb(state) = emission[state].LogProbability(data); } - return LogScaleEmissionProb(emissionLogProb, - prevForwardLogProb, forwardLogProb); + return LogScaleEmissionProb(emissionLogProb, forwardLogProb); } /** @@ -593,11 +586,10 @@ double HMM::LogScale(const arma::vec &data, template double HMM::LogLikelihood(const arma::vec &data, double &logLikelihood, - arma::vec& prevForwardLogProb, arma::vec& forwardLogProb) const { - auto isStartOfSeq = prevForwardLogProb.empty(); - auto curLogScale = LogScale(data, prevForwardLogProb, forwardLogProb); + auto isStartOfSeq = forwardLogProb.empty(); + auto curLogScale = LogScale(data, forwardLogProb); logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; return logLikelihood; } @@ -654,9 +646,8 @@ void HMM::Smooth(const arma::mat& dataSeq, * The Forward procedure (part of the Forward-Backward algorithm). */ template -void HMM::ForwardAtT0(const arma::vec& emissionLogProb, - double& logScales, - arma::vec& forwardLogProb) const +arma::vec HMM::ForwardAtT0(const arma::vec& emissionLogProb, + double& logScales) const { // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. @@ -664,7 +655,7 @@ void HMM::ForwardAtT0(const arma::vec& emissionLogProb, ConvertToLogSpace(); - forwardLogProb.resize(logTransition.n_rows); + arma::vec forwardLogProb(logTransition.n_rows); forwardLogProb.fill(-std::numeric_limits::infinity()); // The first entry in the forward algorithm uses the initial state // probabilities. Note that MATLAB assumes that the starting state (at @@ -680,21 +671,24 @@ void HMM::ForwardAtT0(const arma::vec& emissionLogProb, if (std::isfinite(logScales)){ forwardLogProb -= logScales; } + + return forwardLogProb; } /** * The Forward procedure (part of the Forward-Backward algorithm). */ template -void HMM::ForwardAtTn(const arma::vec& emissionLogProb, +arma::vec HMM::ForwardAtTn(const arma::vec& emissionLogProb, double& logScales, - const arma::vec& prevForwardLogProb, - arma::vec& forwardLogProb) const + const arma::vec& prevForwardLogProb) const { // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. + arma::vec forwardLogProb(logTransition.n_rows); + forwardLogProb.fill(-std::numeric_limits::infinity()); // Now compute the probabilities for each successive observation. for (size_t state = 0; state < logTransition.n_rows; state++) { // The forward probability of state j at time t is the sum over all @@ -708,6 +702,8 @@ void HMM::ForwardAtTn(const arma::vec& emissionLogProb, if (std::isfinite(logScales)){ forwardLogProb -= logScales; } + + return forwardLogProb; } /** @@ -738,8 +734,7 @@ void HMM::Forward(const arma::mat& dataSeq, emission[state].LogProbability(dataSeq.unsafe_col(0)); } - arma::vec col0(forwardLogProb.colptr(0), logTransition.n_rows, false); - ForwardAtT0(emissionLogProb, logScales(0), col0); + forwardLogProb.col(0) = ForwardAtT0(emissionLogProb, logScales(0)); // Now compute the probabilities for each successive observation. for (size_t t = 1; t < dataSeq.n_cols; t++) @@ -750,8 +745,8 @@ void HMM::Forward(const arma::mat& dataSeq, emission[state].LogProbability(dataSeq.unsafe_col(t)); } - arma::vec colt(forwardLogProb.colptr(t), logTransition.n_rows, false); - ForwardAtTn(emissionLogProb, logScales(t), forwardLogProb.col(t-1), colt); + forwardLogProb.col(t) = + ForwardAtTn(emissionLogProb, logScales(t), forwardLogProb.col(t-1)); } } diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index fc4a6bf2b3..c4aef1112b 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -844,24 +844,21 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) { double loglikelihood; - arma::vec prevForwardLogProb; arma::vec forwardLogProb; for (size_t t = 0; t Date: Mon, 31 Aug 2020 19:45:56 -0400 Subject: [PATCH 16/61] added comments for logLikelihood calculation test --- src/mlpack/tests/hmm_test.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index c4aef1112b..74fccae9bd 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -837,11 +837,14 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) const double loglikelihoodRef = -2734.43; + //test loglikelihood calculation for the whole data { auto loglikelihood = hmm.LogLikelihood(obs); BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihoodRef, 1e-3); } + //test loglikelihoosd calculation in incremental way. + //It simulates the case where we have a stream of data. { double loglikelihood; arma::vec forwardLogProb; @@ -853,6 +856,8 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihoodRef, 1e-3); } + //test loglikelihoosd calculation in incremental way. + //It simulates the case where we have a stream of data. { double loglikelihood = 0; arma::vec forwardLogProb; From a4ead122498b39a71f1d0e41127c75ec73af3b04 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:16:31 -0400 Subject: [PATCH 17/61] Update src/mlpack/methods/hmm/hmm.hpp Style fix Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 5f39326018..71d456a10a 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -444,8 +444,8 @@ class HMM * @return Forward probabilities */ arma::vec ForwardAtT0( - const arma::vec& emissionLogProb, - double& logScales) const; + const arma::vec& emissionLogProb, + double& logScales) const; /** * Given emission probabilities, computes forward probabilities for time t>0. From 797d245e2823b339e3ed74c1944edf8ddb350626 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:17:23 -0400 Subject: [PATCH 18/61] Update src/mlpack/methods/hmm/hmm.hpp style fix Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 71d456a10a..8baea018c6 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -456,9 +456,9 @@ class HMM * @return Forward probabilities */ arma::vec ForwardAtTn( - const arma::vec& emissionLogProb, - double& logScales, - const arma::vec& prevForwardLogProb) const; + const arma::vec& emissionLogProb, + double& logScales, + const arma::vec& prevForwardLogProb) const; // Helper functions. /** From e50ec28d3f77d86d9e9eace4c972bae3162d03c9 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:18:18 -0400 Subject: [PATCH 19/61] Update src/mlpack/methods/hmm/hmm_impl.hpp style fix Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index a4c2759377..ab64d65dc3 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -570,14 +570,14 @@ template double HMM::LogScale(const arma::vec &data, arma::vec& forwardLogProb) const { - arma::vec emissionLogProb(logTransition.n_rows); + arma::vec emissionLogProb(logTransition.n_rows); - for (size_t state = 0; state < logTransition.n_rows; state++) - { - emissionLogProb(state) = emission[state].LogProbability(data); - } + for (size_t state = 0; state < logTransition.n_rows; state++) + { + emissionLogProb(state) = emission[state].LogProbability(data); + } - return LogScaleEmissionProb(emissionLogProb, forwardLogProb); + return LogScaleEmissionProb(emissionLogProb, forwardLogProb); } /** From 5f7650a96bd0000ceb5754289c744650a866ad5b Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:20:01 -0400 Subject: [PATCH 20/61] Update src/mlpack/methods/hmm/hmm_impl.hpp fixed use of auto Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index ab64d65dc3..c3dd69f7a2 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -588,10 +588,10 @@ double HMM::LogLikelihood(const arma::vec &data, double &logLikelihood, arma::vec& forwardLogProb) const { - auto isStartOfSeq = forwardLogProb.empty(); - auto curLogScale = LogScale(data, forwardLogProb); - logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; - return logLikelihood; + bool isStartOfSeq = forwardLogProb.empty(); + double curLogScale = LogScale(data, forwardLogProb); + logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; + return logLikelihood; } /** From b66421248404d80fd8edb91aba2aca4c1291b012 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:21:15 -0400 Subject: [PATCH 21/61] Update src/mlpack/methods/hmm/hmm_impl.hpp use of Armadillo objects instead of looping to add Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index c3dd69f7a2..ed6a9d117f 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -662,9 +662,7 @@ arma::vec HMM::ForwardAtT0(const arma::vec& emissionLogProb, // t = -1) is state 0; this is not our assumption here. To force that // behavior, you could append a single starting state to every single data // sequence and that should produce results in line with MATLAB. - for (size_t state = 0; state < logTransition.n_rows; state++) { - forwardLogProb(state) = logInitial(state) + emissionLogProb(state); - } + forwardLogProb = logInitial + emissionLogProb; // Normalize probability. logScales = math::AccuLog(forwardLogProb); From 22a208597162b9b4e1565b00e7d9094f4fc0419e Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:21:38 -0400 Subject: [PATCH 22/61] Update src/mlpack/methods/hmm/hmm_impl.hpp style fix Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index ed6a9d117f..88efd691c7 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -666,9 +666,8 @@ arma::vec HMM::ForwardAtT0(const arma::vec& emissionLogProb, // Normalize probability. logScales = math::AccuLog(forwardLogProb); - if (std::isfinite(logScales)){ - forwardLogProb -= logScales; - } + if (std::isfinite(logScales)) + forwardLogProb -= logScales; return forwardLogProb; } From 8407aeee58d65657e3fee191ca7babe41e66a3b9 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:25:32 -0400 Subject: [PATCH 23/61] Update src/mlpack/methods/hmm/hmm_impl.hpp fixed a typo Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 88efd691c7..3cb5011402 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -785,7 +785,7 @@ void HMM::Backward(const arma::mat& dataSeq, /** * Make sure the variables in log space are in sync with the linear - * counter parts + * counterparts. */ template void HMM::ConvertToLogSpace() const From 1dffcea4f23e6011779ff1cd9d13b903b6cea275 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:26:19 -0400 Subject: [PATCH 24/61] Update src/mlpack/tests/hmm_test.cpp improved the comment Co-authored-by: Ryan Curtin --- src/mlpack/tests/hmm_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 74fccae9bd..f102f45f88 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -843,8 +843,8 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihoodRef, 1e-3); } - //test loglikelihoosd calculation in incremental way. - //It simulates the case where we have a stream of data. + // Test loglikelihood calculation in an incremental way. + // It simulates the case where we have a stream of data. { double loglikelihood; arma::vec forwardLogProb; From 2da39288a9240819240a4479cc58e749654f19da Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Wed, 16 Sep 2020 12:26:42 -0400 Subject: [PATCH 25/61] Update src/mlpack/tests/hmm_test.cpp improved the comment Co-authored-by: Ryan Curtin --- src/mlpack/tests/hmm_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index f102f45f88..2593c3f8a0 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -856,8 +856,8 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihoodRef, 1e-3); } - //test loglikelihoosd calculation in incremental way. - //It simulates the case where we have a stream of data. + // Test loglikelihood calculation in an incremental way. + // It simulates the case where we have a stream of data. { double loglikelihood = 0; arma::vec forwardLogProb; From e5c1f54abdfe64ec4dec8823ffdf6004b1f59196 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Wed, 16 Sep 2020 12:45:09 -0400 Subject: [PATCH 26/61] fixed style --- src/mlpack/methods/hmm/hmm_impl.hpp | 53 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 3cb5011402..56d915d9cf 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -653,23 +653,23 @@ arma::vec HMM::ForwardAtT0(const arma::vec& emissionLogProb, // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. - ConvertToLogSpace(); + ConvertToLogSpace(); - arma::vec forwardLogProb(logTransition.n_rows); - forwardLogProb.fill(-std::numeric_limits::infinity()); - // The first entry in the forward algorithm uses the initial state - // probabilities. Note that MATLAB assumes that the starting state (at - // t = -1) is state 0; this is not our assumption here. To force that - // behavior, you could append a single starting state to every single data - // sequence and that should produce results in line with MATLAB. + arma::vec forwardLogProb(logTransition.n_rows); + forwardLogProb.fill(-std::numeric_limits::infinity()); + // The first entry in the forward algorithm uses the initial state + // probabilities. Note that MATLAB assumes that the starting state (at + // t = -1) is state 0; this is not our assumption here. To force that + // behavior, you could append a single starting state to every single data + // sequence and that should produce results in line with MATLAB. forwardLogProb = logInitial + emissionLogProb; - // Normalize probability. - logScales = math::AccuLog(forwardLogProb); + // Normalize probability. + logScales = math::AccuLog(forwardLogProb); if (std::isfinite(logScales)) forwardLogProb -= logScales; - return forwardLogProb; + return forwardLogProb; } /** @@ -684,23 +684,22 @@ arma::vec HMM::ForwardAtTn(const arma::vec& emissionLogProb, // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. - arma::vec forwardLogProb(logTransition.n_rows); - forwardLogProb.fill(-std::numeric_limits::infinity()); - // Now compute the probabilities for each successive observation. - for (size_t state = 0; state < logTransition.n_rows; state++) { - // The forward probability of state j at time t is the sum over all - // states of the probability of the previous state transitioning to - // the current state and emitting the given observation. - arma::vec tmp = prevForwardLogProb + logTransition.row(state).t(); - forwardLogProb(state) = math::AccuLog(tmp) + emissionLogProb(state); - } - // Normalize probability. - logScales = math::AccuLog(forwardLogProb); - if (std::isfinite(logScales)){ - forwardLogProb -= logScales; - } + arma::vec forwardLogProb(logTransition.n_rows); + forwardLogProb.fill(-std::numeric_limits::infinity()); + // Now compute the probabilities for each successive observation. + for (size_t state = 0; state < logTransition.n_rows; state++) { + // The forward probability of state j at time t is the sum over all + // states of the probability of the previous state transitioning to + // the current state and emitting the given observation. + arma::vec tmp = prevForwardLogProb + logTransition.row(state).t(); + forwardLogProb(state) = math::AccuLog(tmp) + emissionLogProb(state); + } + // Normalize probability. + logScales = math::AccuLog(forwardLogProb); + if (std::isfinite(logScales)) + forwardLogProb -= logScales; - return forwardLogProb; + return forwardLogProb; } /** From a62c4fad2fb67091b3a462a80bb69c462a8473d0 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Fri, 25 Sep 2020 14:26:28 -0400 Subject: [PATCH 27/61] rename logScale() to logScaleFactor() --- src/mlpack/methods/hmm/hmm.hpp | 2 +- src/mlpack/methods/hmm/hmm_impl.hpp | 4 ++-- src/mlpack/tests/hmm_test.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 8baea018c6..37011eb249 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -338,7 +338,7 @@ class HMM * or time t=0 * @return Log scale factor of the given sequence of data up at time t. */ - double LogScale(const arma::vec &data, + double LogScaleFactor(const arma::vec &data, arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given data up to time t diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 56d915d9cf..6854923776 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -567,7 +567,7 @@ double HMM::LogLikelihoodEmissionProb( * scale over the entire sequence */ template -double HMM::LogScale(const arma::vec &data, +double HMM::LogScaleFactor(const arma::vec &data, arma::vec& forwardLogProb) const { arma::vec emissionLogProb(logTransition.n_rows); @@ -589,7 +589,7 @@ double HMM::LogLikelihood(const arma::vec &data, arma::vec& forwardLogProb) const { bool isStartOfSeq = forwardLogProb.empty(); - double curLogScale = LogScale(data, forwardLogProb); + double curLogScale = LogScaleFactor(data, forwardLogProb); logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; return logLikelihood; } diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 2593c3f8a0..8c1f11c233 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -863,7 +863,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) arma::vec forwardLogProb; for (size_t t = 0; t Date: Fri, 25 Sep 2020 14:35:10 -0400 Subject: [PATCH 28/61] renamed Emiision functions --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- src/mlpack/methods/hmm/hmm_impl.hpp | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 37011eb249..75f9dac519 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -305,7 +305,7 @@ class HMM * or time t=0 * @return Log scale factor of the given sequence of emission at time t. */ - double LogScaleEmissionProb(const arma::vec& emissionLogProb, + double EmissionLogScaleFactor(const arma::vec& emissionLogProb, arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given emission probability up to time t @@ -321,7 +321,7 @@ class HMM * or time t=0 * @return Log-likelihood of the given sequence of emission up to time t. */ - double LogLikelihoodEmissionProb(const arma::vec& emissionLogProb, + double EmissionLogLikelihood(const arma::vec& emissionLogProb, double &logLikelihood, arma::vec& forwardLogProb) const; /** diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 6854923776..92b3d3c16b 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -528,7 +528,8 @@ double HMM::LogLikelihood(const arma::mat& dataSeq) const * accumulate log scale over the entire sequence */ template -double HMM::LogScaleEmissionProb(const arma::vec& emissionLogProb, +double HMM::EmissionLogScaleFactor( + const arma::vec& emissionLogProb, arma::vec& forwardLogProb) const { double curLogScale; @@ -550,13 +551,14 @@ double HMM::LogScaleEmissionProb(const arma::vec& emissionLogProb, * Compute the log-likelihood of the given emission probability up to time t */ template -double HMM::LogLikelihoodEmissionProb( +double HMM::EmissionLogLikelihood( const arma::vec& emissionLogProb, double &logLikelihood, arma::vec& forwardLogProb) const { bool isStartOfSeq = forwardLogProb.empty(); - double curLogScale = LogScaleEmissionProb(emissionLogProb, forwardLogProb); + double curLogScale = EmissionLogScaleFactor(emissionLogProb, + forwardLogProb); logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; return logLikelihood; } @@ -577,7 +579,7 @@ double HMM::LogScaleFactor(const arma::vec &data, emissionLogProb(state) = emission[state].LogProbability(data); } - return LogScaleEmissionProb(emissionLogProb, forwardLogProb); + return EmissionLogScaleFactor(emissionLogProb, forwardLogProb); } /** From 337b3f9833aad15e3f9df569cb21c329a7a2fe2f Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Fri, 25 Sep 2020 16:24:20 -0400 Subject: [PATCH 29/61] added a test case for EmissionLogLikelihood() --- src/mlpack/tests/hmm_test.cpp | 220 +++++++++++++++++++++++++++++++++- 1 file changed, 219 insertions(+), 1 deletion(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 8c1f11c233..486b4b635f 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -802,6 +802,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) emission.Covariance(cov.at(i)); } + //100 2D observations arma::mat obs = { { -0.0424, -0.0395, -0.0336, -0.0294, -0.0299, -0.032, -0.0289, -0.0148, @@ -834,7 +835,211 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) 0.0521, 0.0313, 0.0188, 0.0113, 0.0068, 0.0042, 0.0026, 0.0018, 0.0014 } }; - + + //100 pre-calculated emission probabilities each for 10 states + std::vector emissionProb={ + { -2.7301e+03, 1.7874e+00, -1.9428e+00, -3.6365e+00, -4.0397e-01, + -1.5115e-01, -1.0328e+00, -1.1071e+00, 5.2876e-01, -1.0643e-01 }, + { -2.3684e+03, 1.8059e+00, -2.2058e+00, -4.0514e+00, -5.0935e-01, + -2.1126e-01, -1.1962e+00, -1.2567e+00, 4.1247e-01, -3.0199e-01 }, + { -1.7117e+03, 1.7981e+00, -2.5275e+00, -4.5634e+00, -6.4839e-01, + -2.9579e-01, -1.4000e+00, -1.4461e+00, 2.3795e-01, -5.5622e-01 }, + { -1.3089e+03, 1.7393e+00, -2.8685e+00, -5.0996e+00, -8.0288e-01, + -3.9863e-01, -1.6229e+00, -1.6478e+00, 2.3300e-02, -8.6617e-01 }, + { -1.3541e+03, 1.6414e+00, -3.1971e+00, -5.6043e+00, -9.5605e-01, + -5.1013e-01, -1.8460e+00, -1.8395e+00, -2.0603e-01, -1.2176e+00 }, + { -1.5521e+03, 1.5367e+00, -3.4806e+00, -6.0349e+00, -1.0924e+00, + -6.1426e-01, -2.0436e+00, -2.0051e+00, -4.2045e-01, -1.5500e+00 }, + { -1.2647e+03, 1.4680e+00, -3.6577e+00, -6.3144e+00, -1.1823e+00, + -6.8009e-01, -2.1646e+00, -2.1147e+00, -5.6512e-01, -1.7360e+00 }, + { -3.2650e+02, 1.4646e+00, -3.6693e+00, -6.3649e+00, -1.1957e+00, + -6.7711e-01, -2.1592e+00, -2.1377e+00, -5.8400e-01, -1.6543e+00 }, + { -1.3035e+02, 1.5123e+00, -3.5018e+00, -6.1593e+00, -1.1254e+00, + -6.0037e-01, -2.0181e+00, -2.0646e+00, -4.6413e-01, -1.3011e+00 }, + { -2.6279e+03, 1.5809e+00, -3.1559e+00, -5.6861e+00, -9.7699e-01, + -4.5903e-01, -1.7490e+00, -1.8956e+00, -2.2135e-01, -7.2772e-01 }, + { -9.6164e+03, 1.6193e+00, -2.6708e+00, -4.9944e+00, -7.8159e-01, + -2.8441e-01, -1.3909e+00, -1.6574e+00, 7.8411e-02, -5.0595e-02 }, + { -2.0944e+04, 1.5980e+00, -2.1094e+00, -4.1681e+00, -5.8143e-01, + -1.2105e-01, -1.0055e+00, -1.3879e+00, 3.4591e-01, 5.6464e-01 }, + { -3.3843e+04, 1.5241e+00, -1.5331e+00, -3.2977e+00, -4.1342e-01, + -8.0772e-03, -6.5026e-01, -1.1226e+00, 5.0244e-01, 9.8522e-01 }, + { -4.6678e+04, 1.3796e+00, -9.8223e-01, -2.4507e+00, -3.0368e-01, + 3.0609e-02, -3.5787e-01, -8.8913e-01, 4.9234e-01, 1.1530e+00 }, + { -6.0839e+04, 1.1013e+00, -4.8302e-01, -1.6712e+00, -2.7541e-01, + -2.2814e-02, -1.4691e-01, -7.1095e-01, 2.6698e-01, 1.0338e+00 }, + { -7.8940e+04, 6.2341e-01, -6.4826e-02, -1.0034e+00, -3.5198e-01, + -1.8517e-01, -3.7803e-02, -6.1240e-01, -2.1704e-01, 5.8353e-01 }, + { -1.0182e+05, -8.9362e-02, 2.5888e-01, -4.6429e-01, -5.5297e-01, + -4.7752e-01, -4.9871e-02, -6.0739e-01, -1.0089e+00, -2.6587e-01 }, + { -1.2437e+05, -9.8625e-01, 4.7236e-01, -8.1256e-02, -8.8097e-01, + -9.0979e-01, -2.0039e-01, -6.9837e-01, -2.1229e+00, -1.5424e+00 }, + { -1.3878e+05, -1.9546e+00, 5.6976e-01, 1.2361e-01, -1.3043e+00, + -1.4534e+00, -4.7690e-01, -8.6807e-01, -3.4831e+00, -3.1393e+00 }, + { -1.3979e+05, -2.8896e+00, 5.6962e-01, 1.6577e-01, -1.7631e+00, + -2.0456e+00, -8.3102e-01, -1.0792e+00, -4.9380e+00, -4.8430e+00 }, + { -1.2717e+05, -3.7474e+00, 5.0493e-01, 9.2444e-02, -2.1969e+00, + -2.6201e+00, -1.2028e+00, -1.2907e+00, -6.3319e+00, -6.4416e+00 }, + { -1.0548e+05, -4.5565e+00, 4.0397e-01, -4.5775e-02, -2.5711e+00, + -3.1354e+00, -1.5493e+00, -1.4771e+00, -7.5697e+00, -7.8170e+00 }, + { -8.0621e+04, -5.3691e+00, 2.8365e-01, -2.1252e-01, -2.8783e+00, + -3.5784e+00, -1.8523e+00, -1.6312e+00, -8.6245e+00, -8.9480e+00 }, + { -5.6310e+04, -6.2411e+00, 1.5008e-01, -3.9022e-01, -3.1294e+00, + -3.9597e+00, -2.1142e+00, -1.7569e+00, -9.5239e+00, -9.8785e+00 }, + { -3.4173e+04, -7.2306e+00, 3.0396e-03, -5.7242e-01, -3.3347e+00, + -4.2928e+00, -2.3417e+00, -1.8583e+00, -1.0301e+01, -1.0652e+01 }, + { -1.5877e+04, -8.3900e+00, -1.5871e-01, -7.5362e-01, -3.4959e+00, + -4.5816e+00, -2.5356e+00, -1.9353e+00, -1.0963e+01, -1.1284e+01 }, + { -3.3829e+03, -9.7572e+00, -3.3006e-01, -9.1554e-01, -3.5912e+00, + -4.8035e+00, -2.6770e+00, -1.9722e+00, -1.1452e+01, -1.1714e+01 }, + { -5.6088e+02, -1.1394e+01, -5.0305e-01, -1.0261e+00, -3.5777e+00, + -4.9138e+00, -2.7301e+00, -1.9403e+00, -1.1653e+01, -1.1829e+01 }, + { -1.4303e+04, -1.3346e+01, -6.7336e-01, -1.0564e+00, -3.4266e+00, + -4.8757e+00, -2.6690e+00, -1.8219e+00, -1.1470e+01, -1.1561e+01 }, + { -4.9066e+04, -1.5534e+01, -8.4176e-01, -1.0079e+00, -3.1636e+00, + -4.7028e+00, -2.5116e+00, -1.6369e+00, -1.0937e+01, -1.0995e+01 }, + { -9.9717e+04, -1.7702e+01, -1.0039e+00, -9.1443e-01, -2.8597e+00, + -4.4595e+00, -2.3138e+00, -1.4339e+00, -1.0224e+01, -1.0331e+01 }, + { -1.5886e+05, -1.9676e+01, -1.1535e+00, -7.9762e-01, -2.5479e+00, + -4.1805e+00, -2.1039e+00, -1.2332e+00, -9.4233e+00, -9.6530e+00 }, + { -2.2947e+05, -2.1635e+01, -1.3117e+00, -6.6325e-01, -2.2133e+00, + -3.8587e+00, -1.8780e+00, -1.0253e+00, -8.5051e+00, -8.9416e+00 }, + { -3.1968e+05, -2.3792e+01, -1.5095e+00, -5.1672e-01, -1.8381e+00, + -3.4770e+00, -1.6312e+00, -8.0190e-01, -7.4108e+00, -8.1836e+00 }, + { -4.3323e+05, -2.6183e+01, -1.7728e+00, -3.8390e-01, -1.4394e+00, + -3.0487e+00, -1.3857e+00, -5.7953e-01, -6.1647e+00, -7.4521e+00 }, + { -5.6473e+05, -2.8589e+01, -2.1061e+00, -3.0168e-01, -1.0547e+00, + -2.6054e+00, -1.1773e+00, -3.8708e-01, -4.8475e+00, -6.8476e+00 }, + { -6.9974e+05, -3.0612e+01, -2.4921e+00, -3.0913e-01, -7.2535e-01, + -2.1849e+00, -1.0419e+00, -2.5359e-01, -3.5677e+00, -6.4479e+00 }, + { -8.0655e+05, -3.1539e+01, -2.8524e+00, -4.2185e-01, -4.8692e-01, + -1.8260e+00, -9.9484e-01, -1.9514e-01, -2.4629e+00, -6.2373e+00 }, + { -8.5216e+05, -3.0655e+01, -3.0833e+00, -6.1881e-01, -3.3169e-01, + -1.5249e+00, -1.0091e+00, -1.9595e-01, -1.5717e+00, -6.0513e+00 }, + { -8.2392e+05, -2.7811e+01, -3.1362e+00, -8.7526e-01, -2.3459e-01, + -1.2631e+00, -1.0480e+00, -2.3278e-01, -8.7344e-01, -5.7341e+00 }, + { -7.3612e+05, -2.3582e+01, -3.0425e+00, -1.1744e+00, -1.7841e-01, + -1.0351e+00, -1.0893e+00, -2.9210e-01, -3.4780e-01, -5.2495e+00 }, + { -6.1397e+05, -1.8706e+01, -2.8744e+00, -1.5195e+00, -1.5516e-01, + -8.3816e-01, -1.1304e+00, -3.7330e-01, 3.7744e-02, -4.6424e+00 }, + { -4.8041e+05, -1.3799e+01, -2.7054e+00, -1.9262e+00, -1.6637e-01, + -6.7558e-01, -1.1810e+00, -4.8405e-01, 3.0197e-01, -3.9898e+00 }, + { -3.4790e+05, -9.2300e+00, -2.5518e+00, -2.3683e+00, -2.0524e-01, + -5.4582e-01, -1.2297e+00, -6.1666e-01, 4.5657e-01, -3.3063e+00 }, + { -2.2370e+05, -5.1887e+00, -2.3941e+00, -2.7911e+00, -2.5500e-01, + -4.3560e-01, -1.2487e+00, -7.5224e-01, 5.3161e-01, -2.5570e+00 }, + { -1.1273e+05, -1.7195e+00, -2.2258e+00, -3.1794e+00, -3.0915e-01, + -3.2974e-01, -1.2221e+00, -8.8867e-01, 5.5755e-01, -1.7017e+00 }, + { -2.8363e+04, 9.0588e-01, -2.0601e+00, -3.5233e+00, -3.7171e-01, + -2.2162e-01, -1.1370e+00, -1.0334e+00, 5.4434e-01, -7.3209e-01 }, + { -1.2122e+03, 1.9784e+00, -1.9455e+00, -3.7971e+00, -4.5862e-01, + -1.2081e-01, -9.8979e-01, -1.2000e+00, 4.7839e-01, 2.5783e-01 }, + { -7.1694e+04, 5.6327e-01, -2.0051e+00, -4.0345e+00, -6.1306e-01, + -6.6602e-02, -8.2833e-01, -1.4287e+00, 3.0684e-01, 1.0022e+00 }, + { -2.6198e+05, -3.8345e+00, -2.3396e+00, -4.2798e+00, -8.6900e-01, + -9.8822e-02, -7.1489e-01, -1.7486e+00, -1.6219e-02, 1.2358e+00 }, + { -5.5328e+05, -1.0687e+01, -2.9124e+00, -4.5058e+00, -1.2121e+00, + -2.2259e-01, -6.7273e-01, -2.1347e+00, -4.7585e-01, 8.8080e-01 }, + { -8.9436e+05, -1.8602e+01, -3.5518e+00, -4.6037e+00, -1.5911e+00, + -4.1140e-01, -6.7958e-01, -2.5173e+00, -1.0137e+00, 3.7886e-02 }, + { -1.2162e+06, -2.5781e+01, -4.0541e+00, -4.4848e+00, -1.9485e+00, + -6.3137e-01, -7.0903e-01, -2.8240e+00, -1.5699e+00, -1.1063e+00 }, + { -1.4436e+06, -3.0414e+01, -4.2395e+00, -4.1197e+00, -2.2265e+00, + -8.4654e-01, -7.3852e-01, -2.9921e+00, -2.0869e+00, -2.2970e+00 }, + { -1.5227e+06, -3.1337e+01, -3.9989e+00, -3.5197e+00, -2.3836e+00, + -1.0315e+00, -7.4887e-01, -2.9823e+00, -2.5313e+00, -3.3017e+00 }, + { -1.4386e+06, -2.8472e+01, -3.3472e+00, -2.7563e+00, -2.4087e+00, + -1.1801e+00, -7.3524e-01, -2.7971e+00, -2.9034e+00, -3.9803e+00 }, + { -1.2257e+06, -2.2958e+01, -2.4364e+00, -1.9521e+00, -2.3275e+00, + -1.3043e+00, -7.0875e-01, -2.4858e+00, -3.2295e+00, -4.3252e+00 }, + { -9.4813e+05, -1.6527e+01, -1.4675e+00, -1.2121e+00, -2.1965e+00, + -1.4367e+00, -6.9389e-01, -2.1228e+00, -3.5740e+00, -4.4844e+00 }, + { -6.6589e+05, -1.0680e+01, -6.1313e-01, -6.1440e-01, -2.0638e+00, + -1.5984e+00, -7.0917e-01, -1.7727e+00, -3.9726e+00, -4.5979e+00 }, + { -4.1809e+05, -6.2975e+00, 3.1651e-02, -1.8731e-01, -1.9586e+00, + -1.7982e+00, -7.6241e-01, -1.4730e+00, -4.4365e+00, -4.7645e+00 }, + { -2.2534e+05, -3.7546e+00, 4.3188e-01, 7.1872e-02, -1.8959e+00, + -2.0366e+00, -8.5455e-01, -1.2417e+00, -4.9637e+00, -5.0424e+00 }, + { -9.4330e+04, -3.0403e+00, 5.9517e-01, 1.8422e-01, -1.8702e+00, + -2.2952e+00, -9.7314e-01, -1.0776e+00, -5.5109e+00, -5.4155e+00 }, + { -2.1454e+04, -3.9202e+00, 5.5647e-01, 1.8381e-01, -1.8704e+00, + -2.5578e+00, -1.1056e+00, -9.6899e-01, -6.0419e+00, -5.8579e+00 }, + { -31.4830, -6.0953, 0.3567, 0.1044, -1.8840, -2.8086, -1.2397, + -0.9026, -6.5224, -6.3374 }, + { -2.2442e+04, -9.2735e+00, 3.4960e-02, -2.1605e-02, -1.8931e+00, + -3.0282e+00, -1.3611e+00, -8.6066e-01, -6.9076e+00, -6.8075e+00 }, + { -8.1676e+04, -1.3138e+01, -3.6831e-01, -1.6104e-01, -1.8763e+00, + -3.1905e+00, -1.4522e+00, -8.2511e-01, -7.1362e+00, -7.2081e+00 }, + { -1.6865e+05, -1.7287e+01, -8.0643e-01, -2.8264e-01, -1.8178e+00, + -3.2726e+00, -1.4987e+00, -7.8144e-01, -7.1585e+00, -7.4877e+00 }, + { -2.7001e+05, -2.1213e+01, -1.2247e+00, -3.6116e-01, -1.7095e+00, + -3.2596e+00, -1.4928e+00, -7.2002e-01, -6.9485e+00, -7.6058e+00 }, + { -3.7506e+05, -2.4628e+01, -1.5962e+00, -3.9394e-01, -1.5583e+00, + -3.1610e+00, -1.4428e+00, -6.4101e-01, -6.5350e+00, -7.5763e+00 }, + { -4.7871e+05, -2.7455e+01, -1.9194e+00, -3.9090e-01, -1.3720e+00, + -2.9900e+00, -1.3606e+00, -5.4763e-01, -5.9492e+00, -7.4279e+00 }, + { -5.7329e+05, -2.9501e+01, -2.1830e+00, -3.6323e-01, -1.1594e+00, + -2.7564e+00, -1.2564e+00, -4.4501e-01, -5.2194e+00, -7.1738e+00 }, + { -6.4968e+05, -3.0560e+01, -2.3747e+00, -3.2775e-01, -9.3375e-01, + -2.4742e+00, -1.1428e+00, -3.4141e-01, -4.3880e+00, -6.8281e+00 }, + { -6.9933e+05, -3.0501e+01, -2.4875e+00, -3.0631e-01, -7.1262e-01, + -2.1653e+00, -1.0343e+00, -2.4789e-01, -3.5174e+00, -6.4120e+00 }, + { -7.1802e+05, -2.9350e+01, -2.5271e+00, -3.2061e-01, -5.1194e-01, + -1.8521e+00, -9.4328e-01, -1.7450e-01, -2.6686e+00, -5.9486e+00 }, + { -7.0553e+05, -2.7236e+01, -2.5060e+00, -3.8730e-01, -3.4217e-01, + -1.5515e+00, -8.7707e-01, -1.2819e-01, -1.8857e+00, -5.4542e+00 }, + { -6.6569e+05, -2.4393e+01, -2.4435e+00, -5.1663e-01, -2.1031e-01, + -1.2775e+00, -8.3941e-01, -1.1339e-01, -1.2023e+00, -4.9470e+00 }, + { -6.1301e+05, -2.1269e+01, -2.3992e+00, -7.3370e-01, -1.2064e-01, + -1.0383e+00, -8.4300e-01, -1.3855e-01, -6.1878e-01, -4.4864e+00 }, + { -5.6195e+05, -1.8233e+01, -2.4507e+00, -1.0921e+00, -8.5743e-02, + -8.4467e-01, -9.1749e-01, -2.2378e-01, -1.3441e-01, -4.1677e+00 }, + { -5.0308e+05, -1.5078e+01, -2.5824e+00, -1.6122e+00, -1.1850e-01, + -7.0720e-01, -1.0690e+00, -3.7916e-01, 2.0948e-01, -3.9737e+00 }, + { -4.2417e+05, -1.1613e+01, -2.7333e+00, -2.2592e+00, -2.1392e-01, + -6.2529e-01, -1.2691e+00, -5.9232e-01, 3.8196e-01, -3.8021e+00 }, + { -3.4311e+05, -8.4490e+00, -2.9262e+00, -2.9840e+00, -3.6201e-01, + -6.0612e-01, -1.5036e+00, -8.4657e-01, 3.8172e-01, -3.6994e+00 }, + { -2.6553e+05, -5.7959e+00, -3.0657e+00, -3.6135e+00, -5.0450e-01, + -6.1056e-01, -1.6893e+00, -1.0726e+00, 2.9263e-01, -3.5310e+00 }, + { -1.6581e+05, -2.8806e+00, -2.9242e+00, -3.9480e+00, -5.5108e-01, + -5.3603e-01, -1.6743e+00, -1.1832e+00, 2.8121e-01, -2.8215e+00 }, + { -6.3112e+04, -4.4355e-02, -2.4673e+00, -3.8848e+00, -4.8010e-01, + -3.5415e-01, -1.4075e+00, -1.1547e+00, 4.0803e-01, -1.5227e+00 }, + { -5.4196e+03, 1.6750e+00, -1.9272e+00, -3.5655e+00, -3.8433e-01, + -1.5578e-01, -1.0312e+00, -1.0745e+00, 5.4628e-01, -1.8838e-01 }, + { -7.9742e+03, 1.9542e+00, -1.5297e+00, -3.2224e+00, -3.5023e-01, + -2.9557e-02, -7.1541e-01, -1.0340e+00, 5.7335e-01, 7.0234e-01 }, + { -4.6838e+04, 1.3383e+00, -1.2840e+00, -2.9202e+00, -3.6943e-01, + 2.1035e-02, -4.9879e-01, -1.0295e+00, 5.0388e-01, 1.1296e+00 }, + { -9.2965e+04, 5.0293e-01, -1.1033e+00, -2.6251e+00, -4.0461e-01, + 2.4992e-02, -3.5062e-01, -1.0246e+00, 3.8909e-01, 1.2595e+00 }, + { -1.3250e+05, -2.3738e-01, -9.9398e-01, -2.4136e+00, -4.4740e-01, + 6.0968e-03, -2.6559e-01, -1.0308e+00, 2.6684e-01, 1.2440e+00 }, + { -1.7149e+05, -9.7999e-01, -9.1698e-01, -2.2384e+00, -4.9912e-01, + -2.5475e-02, -2.0762e-01, -1.0468e+00, 1.3078e-01, 1.1599e+00 }, + { -2.2091e+05, -1.9350e+00, -8.5497e-01, -2.0582e+00, -5.7508e-01, + -7.7816e-02, -1.6138e-01, -1.0794e+00, -5.6045e-02, 9.8875e-01 }, + { -2.8140e+05, -3.1219e+00, -8.2568e-01, -1.8962e+00, -6.7862e-01, + -1.5242e-01, -1.3554e-01, -1.1353e+00, -2.9320e-01, 7.2082e-01 }, + { -3.4167e+05, -4.3171e+00, -8.2824e-01, -1.7733e+00, -7.8907e-01, + -2.3483e-01, -1.3167e-01, -1.2015e+00, -5.3627e-01, 4.0854e-01 }, + { -3.7868e+05, -5.0537e+00, -8.3691e-01, -1.7046e+00, -8.6035e-01, + -2.9036e-01, -1.3690e-01, -1.2447e+00, -6.9304e-01, 1.9260e-01 }, + { -3.7429e+05, -4.9406e+00, -7.8456e-01, -1.6323e+00, -8.6203e-01, + -3.0644e-01, -1.3159e-01, -1.2267e+00, -7.3358e-01, 1.3468e-01 }, + { -3.3293e+05, -4.0758e+00, -6.6873e-01, -1.5416e+00, -8.0032e-01, + -2.8877e-01, -1.1346e-01, -1.1498e+00, -6.7458e-01, 2.1365e-01 }, + { -2.7541e+05, -2.9085e+00, -5.3210e-01, -1.4470e+00, -7.0706e-01, + -2.5517e-01, -9.1435e-02, -1.0445e+00, -5.6402e-01, 3.5113e-01 }, + { -2.2010e+05, -1.8209e+00, -4.1116e-01, -1.3627e+00, -6.1220e-01, + -2.2144e-01, -7.3319e-02, -9.4005e-01, -4.4660e-01, 4.7992e-01 }, + { -1.7809e+05, -1.0242e+00, -3.2646e-01, -1.3011e+00, -5.3612e-01, + -1.9567e-01, -6.2291e-02, -8.5731e-01, -3.5032e-01, 5.7022e-01 }, + { -1.5426e+05, -5.8691e-01, -2.8121e-01, -1.2660e+00, -4.9111e-01, + -1.8141e-01, -5.7387e-02, -8.0842e-01, -2.9317e-01, 6.1601e-01 }, + }; + const double loglikelihoodRef = -2734.43; //test loglikelihood calculation for the whole data @@ -869,6 +1074,19 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihoodRef, 1e-3); } + // Test loglikelihood calculation in an incremental way. + // It simulates the case where we have emission probabilities pre-calculated. + { + double loglikelihood = 0; + arma::vec forwardLogProb; + for (size_t t = 0; t stateSeq; hmm.Predict(obs, stateSeq); From a9f23ad4798d84b9e5ff37dcf001d4d1526c3823 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Thu, 1 Oct 2020 10:26:10 -0400 Subject: [PATCH 30/61] updated HISTORY.md --- HISTORY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.md b/HISTORY.md index d7a2437e9e..d3fa680e73 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,6 @@ ### mlpack ?.?.? ###### ????-??-?? + * HMM: calculate likelihood for data stream with/without pre-calculated emission probability * Added Mean Absolute Percentage Error. ### mlpack 3.4.1 From 54b5172c064fce5c4d5046101bb52f0e2403b0a6 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Sun, 18 Oct 2020 11:16:31 -0400 Subject: [PATCH 31/61] made loglikelihood const Co-authored-by: Ryan Curtin --- src/mlpack/tests/hmm_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 486b4b635f..e9f8498837 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -1044,7 +1044,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMPredictTest) //test loglikelihood calculation for the whole data { - auto loglikelihood = hmm.LogLikelihood(obs); + const double loglikelihood = hmm.LogLikelihood(obs); BOOST_REQUIRE_CLOSE(loglikelihood, loglikelihoodRef, 1e-3); } From c59467e1a692e6bd976f2d4e8c380bc48fa2b979 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Tue, 27 Oct 2020 11:14:29 -0400 Subject: [PATCH 32/61] added more comment for LogScaleFactor() test case --- src/mlpack/tests/hmm_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 142b8c9bdc..69fe66e756 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -1061,6 +1061,8 @@ TEST_CASE("GaussianHMMPredictTest", "[HMMTest]") // Test loglikelihood calculation in an incremental way. // It simulates the case where we have a stream of data. + // In this case the accumulation of the log scales factor to calculate + // the logkielihood value is done outside of the loop { double loglikelihood = 0; arma::vec forwardLogProb; From af3bab25c6e005e74ee86c1c37c207ff5852bba8 Mon Sep 17 00:00:00 2001 From: Arash Abghari Date: Tue, 27 Oct 2020 11:42:04 -0400 Subject: [PATCH 33/61] updated the comments for EmissionLogLikelihood() and LogLikelihood() --- src/mlpack/methods/hmm/hmm.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 75f9dac519..726b60ca47 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -308,7 +308,8 @@ class HMM double EmissionLogScaleFactor(const arma::vec& emissionLogProb, arma::vec& forwardLogProb) const; /** - * Compute the log-likelihood of the given emission probability up to time t + * Compute the log-likelihood of the given emission probability up to time t, + * storing the result in logLikelihood. * This is meant for incremental or streaming computation of the * log-likelihood of a sequence. For the first data point, provide an empty * forwardLogProb vector. @@ -341,7 +342,8 @@ class HMM double LogScaleFactor(const arma::vec &data, arma::vec& forwardLogProb) const; /** - * Compute the log-likelihood of the given data up to time t + * Compute the log-likelihood of the given data up to time t, storing the + * result in logLikelihood. * This is meant for incremental or streaming computation of the * log-likelihood of a sequence. For the first data point, provide an empty * forwardLogProb vector. From dd4f2c3c9763f81b7117788f1e96683c86956727 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:11:51 -0500 Subject: [PATCH 34/61] Update HISTORY.md Co-authored-by: Ryan Curtin --- HISTORY.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index a1a8ae1580..6e2ee62e7c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? - * HMM: calculate likelihood for data stream with/without pre-calculated emission probability + * HMM: add functions to calculate likelihood for data stream with/without + pre-calculated emission probability (#2142). ### mlpack 3.4.2 ###### 2020-10-26 From 0cad47fb833f858794f2a2f689728e347586677b Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:12:11 -0500 Subject: [PATCH 35/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 726b60ca47..d0f57cb831 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -306,7 +306,7 @@ class HMM * @return Log scale factor of the given sequence of emission at time t. */ double EmissionLogScaleFactor(const arma::vec& emissionLogProb, - arma::vec& forwardLogProb) const; + arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given emission probability up to time t, * storing the result in logLikelihood. From ef29ca6db3782ad93820a1659a9f0cb167600a19 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:12:27 -0500 Subject: [PATCH 36/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index d0f57cb831..56e4cc0c0c 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -318,8 +318,8 @@ class HMM * @param logLikelihood Log-likelihood of the given sequence of emission * probability up to time t-1 * @param forwardLogProb Vector in which forward probabilities will be saved. - * Passing forwardLogProb as an empty vector indicates the start of sequence - * or time t=0 + * Passing forwardLogProb as an empty vector indicates the start of the + * sequence (i.e. time t=0). * @return Log-likelihood of the given sequence of emission up to time t. */ double EmissionLogLikelihood(const arma::vec& emissionLogProb, From 499a94bbc1a67dd385f78c1ce02ee7ef19a88b6f Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:12:43 -0500 Subject: [PATCH 37/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 56e4cc0c0c..f984f9d478 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -301,8 +301,8 @@ class HMM * @param emissionLogProb emission probability at time t. * probability up to time t-1 * @param forwardLogProb Vector in which forward probabilities will be saved. - * Passing forwardLogProb as an empty vector indicates the start of sequence - * or time t=0 + * Passing forwardLogProb as an empty vector indicates the start of the + * sequence (i.e. time t=0). * @return Log scale factor of the given sequence of emission at time t. */ double EmissionLogScaleFactor(const arma::vec& emissionLogProb, From 9b047b2769854f5f0bed8c67fdfa66a08ec8e3c6 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:13:00 -0500 Subject: [PATCH 38/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index f984f9d478..ef7b70be96 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -323,8 +323,8 @@ class HMM * @return Log-likelihood of the given sequence of emission up to time t. */ double EmissionLogLikelihood(const arma::vec& emissionLogProb, - double &logLikelihood, - arma::vec& forwardLogProb) const; + double &logLikelihood, + arma::vec& forwardLogProb) const; /** * Compute the log of the scaling factor of the given data at time t. * To calculate the log-likelihood for the whole sequence, accumulate log From 8b714a84e1f52912621123e78de6cb41c37e5e3d Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:13:26 -0500 Subject: [PATCH 39/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index ef7b70be96..abad722437 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -299,7 +299,6 @@ class HMM * forwardLogProb vector. * * @param emissionLogProb emission probability at time t. - * probability up to time t-1 * @param forwardLogProb Vector in which forward probabilities will be saved. * Passing forwardLogProb as an empty vector indicates the start of the * sequence (i.e. time t=0). From bfffcd3bd73de7a209e124d5186ce3124f6e9872 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:15:18 -0500 Subject: [PATCH 40/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index abad722437..aa4e58b60a 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -315,7 +315,8 @@ class HMM * * @param emissionLogProb emission probability at time t. * @param logLikelihood Log-likelihood of the given sequence of emission - * probability up to time t-1 + * probability up to time t-1. This will be overwritten with the log-likelihood + * of the given emission probability up to time t. * @param forwardLogProb Vector in which forward probabilities will be saved. * Passing forwardLogProb as an empty vector indicates the start of the * sequence (i.e. time t=0). From a384e51c37b7de91bc7bd30d28143fc7da78665b Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:16:07 -0500 Subject: [PATCH 41/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index aa4e58b60a..69d7fecce3 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -327,8 +327,9 @@ class HMM arma::vec& forwardLogProb) const; /** * Compute the log of the scaling factor of the given data at time t. - * To calculate the log-likelihood for the whole sequence, accumulate log - * scale over the entire sequence. + * To calculate the log-likelihood for the whole sequence, accumulate the + * log scale factor (the return value of this function) over the entire + * sequence. * This is meant for incremental or streaming computation of the * log-likelihood of a sequence. For the first data point, provide an empty * forwardLogProb vector. From b1154511d7e782657b62d1d4f9629b4557978304 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:16:24 -0500 Subject: [PATCH 42/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 69d7fecce3..10fe1cc080 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -341,7 +341,7 @@ class HMM * @return Log scale factor of the given sequence of data up at time t. */ double LogScaleFactor(const arma::vec &data, - arma::vec& forwardLogProb) const; + arma::vec& forwardLogProb) const; /** * Compute the log-likelihood of the given data up to time t, storing the * result in logLikelihood. From 8ef0206a92a6104bff99c99396c6d4e0b8816c41 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:16:42 -0500 Subject: [PATCH 43/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 10fe1cc080..50166fc103 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -336,8 +336,8 @@ class HMM * * @param data observation at time t. * @param forwardLogProb Vector in which forward probabilities will be saved. - * Passing forwardLogProb as an empty vector indicates the start of sequence - * or time t=0 + * Passing forwardLogProb as an empty vector indicates the start of the + * sequence (i.e. time t=0). * @return Log scale factor of the given sequence of data up at time t. */ double LogScaleFactor(const arma::vec &data, From df99ef8a4bf2b1411fa0bf672ccee6e573e416c5 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:17:00 -0500 Subject: [PATCH 44/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 50166fc103..a6df706620 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -351,7 +351,7 @@ class HMM * * @param data observation at time t. * @param logLikelihood Log-likelihood of the given sequence of data - * up to time t-1 + * up to time t-1. * @param forwardLogProb Vector in which forward probabilities will be saved. * Passing forwardLogProb as an empty vector indicates the start of sequence * or time t=0 From f7c09a3e6e44b2875414d8c57f8b009a0d977082 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:17:15 -0500 Subject: [PATCH 45/61] Update src/mlpack/methods/hmm/hmm.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index a6df706620..d0f555f1d7 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -353,8 +353,8 @@ class HMM * @param logLikelihood Log-likelihood of the given sequence of data * up to time t-1. * @param forwardLogProb Vector in which forward probabilities will be saved. - * Passing forwardLogProb as an empty vector indicates the start of sequence - * or time t=0 + * Passing forwardLogProb as an empty vector indicates the start of the + * sequence (i.e. time t=0). * @return Log-likelihood of the given sequence of data up to time t. */ double LogLikelihood(const arma::vec &data, From 0118cddbd2a57fad725be7b903997a7c0c88efe0 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:17:35 -0500 Subject: [PATCH 46/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 92b3d3c16b..b446f447ab 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -535,7 +535,7 @@ double HMM::EmissionLogScaleFactor( double curLogScale; if (forwardLogProb.empty()) { - // start of sequence or time t=0 + // We are at the start of the sequence (i.e. time t=0). forwardLogProb = ForwardAtT0(emissionLogProb, curLogScale); } else From 4d2d3b29d0a11695db9e9122b40102cefd013f38 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:17:53 -0500 Subject: [PATCH 47/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index b446f447ab..cf763128bf 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -529,8 +529,8 @@ double HMM::LogLikelihood(const arma::mat& dataSeq) const */ template double HMM::EmissionLogScaleFactor( - const arma::vec& emissionLogProb, - arma::vec& forwardLogProb) const + const arma::vec& emissionLogProb, + arma::vec& forwardLogProb) const { double curLogScale; if (forwardLogProb.empty()) From 62d193ada83b9b0009cfe4fb8f2acad3458e81c5 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:18:07 -0500 Subject: [PATCH 48/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index cf763128bf..6652f11aba 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -552,9 +552,9 @@ double HMM::EmissionLogScaleFactor( */ template double HMM::EmissionLogLikelihood( - const arma::vec& emissionLogProb, - double &logLikelihood, - arma::vec& forwardLogProb) const + const arma::vec& emissionLogProb, + double& logLikelihood, + arma::vec& forwardLogProb) const { bool isStartOfSeq = forwardLogProb.empty(); double curLogScale = EmissionLogScaleFactor(emissionLogProb, From 92ba668ebd8196955b5bfa4e989f727bd6bb0e01 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:18:28 -0500 Subject: [PATCH 49/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 6652f11aba..62bedf658d 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -570,7 +570,7 @@ double HMM::EmissionLogLikelihood( */ template double HMM::LogScaleFactor(const arma::vec &data, - arma::vec& forwardLogProb) const + arma::vec& forwardLogProb) const { arma::vec emissionLogProb(logTransition.n_rows); From 0e1168de6ff66b2fb97203cc6e8dcb2f3bd13551 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:18:47 -0500 Subject: [PATCH 50/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 62bedf658d..12fc62a6b4 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -586,8 +586,8 @@ double HMM::LogScaleFactor(const arma::vec &data, * Compute the log-likelihood of the given data up to time t */ template -double HMM::LogLikelihood(const arma::vec &data, - double &logLikelihood, +double HMM::LogLikelihood(const arma::vec& data, + double& logLikelihood, arma::vec& forwardLogProb) const { bool isStartOfSeq = forwardLogProb.empty(); From a11df25c24c85a4578d5d0aa7c4abe3e7fc8cacc Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:19:05 -0500 Subject: [PATCH 51/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 12fc62a6b4..a28a9370f2 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -649,7 +649,7 @@ void HMM::Smooth(const arma::mat& dataSeq, */ template arma::vec HMM::ForwardAtT0(const arma::vec& emissionLogProb, - double& logScales) const + double& logScales) const { // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. From ab7aa8749239f3adcb4e61b90d28db7a872d7a86 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:19:22 -0500 Subject: [PATCH 52/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index a28a9370f2..a056130179 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -653,8 +653,6 @@ arma::vec HMM::ForwardAtT0(const arma::vec& emissionLogProb, { // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. - - ConvertToLogSpace(); arma::vec forwardLogProb(logTransition.n_rows); From 5ba630aca305f0c28aa3c36b713fdbbf5ffda707 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:19:37 -0500 Subject: [PATCH 53/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index a056130179..800ce3fcbb 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -667,7 +667,7 @@ arma::vec HMM::ForwardAtT0(const arma::vec& emissionLogProb, // Normalize probability. logScales = math::AccuLog(forwardLogProb); if (std::isfinite(logScales)) - forwardLogProb -= logScales; + forwardLogProb -= logScales; return forwardLogProb; } From 493f8dbc38dfeec708cde684d2a046f06aa1c02f Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:19:52 -0500 Subject: [PATCH 54/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 800ce3fcbb..8d1f561d41 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -683,7 +683,6 @@ arma::vec HMM::ForwardAtTn(const arma::vec& emissionLogProb, // Our goal is to calculate the forward probabilities: // P(X_k | o_{1:k}) for all possible states X_k, for each time point k. - arma::vec forwardLogProb(logTransition.n_rows); forwardLogProb.fill(-std::numeric_limits::infinity()); // Now compute the probabilities for each successive observation. From 1bcea5a8918875e206f82db362547a20ca1fe809 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:20:06 -0500 Subject: [PATCH 55/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 8d1f561d41..9015fbfbad 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -696,7 +696,7 @@ arma::vec HMM::ForwardAtTn(const arma::vec& emissionLogProb, // Normalize probability. logScales = math::AccuLog(forwardLogProb); if (std::isfinite(logScales)) - forwardLogProb -= logScales; + forwardLogProb -= logScales; return forwardLogProb; } From b32328c720ab4b2fe3123ba81b4f953d336f19d4 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:20:25 -0500 Subject: [PATCH 56/61] Update src/mlpack/methods/hmm/hmm_impl.hpp Co-authored-by: Ryan Curtin --- src/mlpack/methods/hmm/hmm_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 9015fbfbad..4a42cd6d38 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -725,8 +725,8 @@ void HMM::Forward(const arma::mat& dataSeq, arma::vec emissionLogProb(logTransition.n_rows); for (size_t state = 0; state < logTransition.n_rows; state++) { - emissionLogProb(state) = - emission[state].LogProbability(dataSeq.unsafe_col(0)); + emissionLogProb(state) = + emission[state].LogProbability(dataSeq.unsafe_col(0)); } forwardLogProb.col(0) = ForwardAtT0(emissionLogProb, logScales(0)); From 34adc0f478c01bfe64cbfc4625a28af545527586 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:20:41 -0500 Subject: [PATCH 57/61] Update src/mlpack/tests/hmm_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/hmm_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 69fe66e756..b22912c3e6 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -800,7 +800,7 @@ TEST_CASE("GaussianHMMPredictTest", "[HMMTest]") emission.Covariance(cov.at(i)); } - //100 2D observations + // 100 2D observations. arma::mat obs = { { -0.0424, -0.0395, -0.0336, -0.0294, -0.0299, -0.032, -0.0289, -0.0148, From 0013b068a03a6f7b77aed94e605a5f1996dab241 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:20:59 -0500 Subject: [PATCH 58/61] Update src/mlpack/tests/hmm_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/hmm_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index b22912c3e6..181293116b 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -835,7 +835,7 @@ TEST_CASE("GaussianHMMPredictTest", "[HMMTest]") }; //100 pre-calculated emission probabilities each for 10 states - std::vector emissionProb={ + std::vector emissionProb = { { -2.7301e+03, 1.7874e+00, -1.9428e+00, -3.6365e+00, -4.0397e-01, -1.5115e-01, -1.0328e+00, -1.1071e+00, 5.2876e-01, -1.0643e-01 }, { -2.3684e+03, 1.8059e+00, -2.2058e+00, -4.0514e+00, -5.0935e-01, From abdfc9ea1d697a25c8ed974d6cbc4b31129b7408 Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:21:12 -0500 Subject: [PATCH 59/61] Update src/mlpack/tests/hmm_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/hmm_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 181293116b..64bebcf440 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -1040,7 +1040,7 @@ TEST_CASE("GaussianHMMPredictTest", "[HMMTest]") const double loglikelihoodRef = -2734.43; - //test loglikelihood calculation for the whole data + // Test log-likelihood calculation for the whole data. { const double loglikelihood = hmm.LogLikelihood(obs); REQUIRE(loglikelihood == Approx(loglikelihoodRef).epsilon(1e-3)); From 9d9c592d7edb78775f9fc59af81e049a04dfe6ff Mon Sep 17 00:00:00 2001 From: aabghari <44274379+aabghari@users.noreply.github.com> Date: Mon, 7 Dec 2020 10:21:22 -0500 Subject: [PATCH 60/61] Update src/mlpack/tests/hmm_test.cpp Co-authored-by: Ryan Curtin --- src/mlpack/tests/hmm_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 64bebcf440..21135acd9a 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -1062,7 +1062,7 @@ TEST_CASE("GaussianHMMPredictTest", "[HMMTest]") // Test loglikelihood calculation in an incremental way. // It simulates the case where we have a stream of data. // In this case the accumulation of the log scales factor to calculate - // the logkielihood value is done outside of the loop + // the log-likelihood value is done outside of the loop { double loglikelihood = 0; arma::vec forwardLogProb; From 0308fe001dd7142d3b87cabd204e6160541a8720 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Jan 2021 17:19:18 -0500 Subject: [PATCH 61/61] Apply suggestions from code review --- src/mlpack/methods/hmm/hmm.hpp | 5 ++++- src/mlpack/methods/hmm/hmm_impl.hpp | 34 ++++++++++++++--------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/mlpack/methods/hmm/hmm.hpp b/src/mlpack/methods/hmm/hmm.hpp index 3977c823eb..1821083a0e 100644 --- a/src/mlpack/methods/hmm/hmm.hpp +++ b/src/mlpack/methods/hmm/hmm.hpp @@ -305,7 +305,8 @@ class HMM * @return Log scale factor of the given sequence of emission at time t. */ double EmissionLogScaleFactor(const arma::vec& emissionLogProb, - arma::vec& forwardLogProb) const; + arma::vec& forwardLogProb) const; + /** * Compute the log-likelihood of the given emission probability up to time t, * storing the result in logLikelihood. @@ -325,6 +326,7 @@ class HMM double EmissionLogLikelihood(const arma::vec& emissionLogProb, double &logLikelihood, arma::vec& forwardLogProb) const; + /** * Compute the log of the scaling factor of the given data at time t. * To calculate the log-likelihood for the whole sequence, accumulate the @@ -342,6 +344,7 @@ class HMM */ double LogScaleFactor(const arma::vec &data, arma::vec& forwardLogProb) const; + /** * Compute the log-likelihood of the given data up to time t, storing the * result in logLikelihood. diff --git a/src/mlpack/methods/hmm/hmm_impl.hpp b/src/mlpack/methods/hmm/hmm_impl.hpp index 4ddee9895e..d3d1d72e61 100644 --- a/src/mlpack/methods/hmm/hmm_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_impl.hpp @@ -532,19 +532,19 @@ double HMM::EmissionLogScaleFactor( const arma::vec& emissionLogProb, arma::vec& forwardLogProb) const { - double curLogScale; - if (forwardLogProb.empty()) - { - // We are at the start of the sequence (i.e. time t=0). - forwardLogProb = ForwardAtT0(emissionLogProb, curLogScale); - } - else - { - forwardLogProb = ForwardAtTn(emissionLogProb, curLogScale, - forwardLogProb); - } + double curLogScale; + if (forwardLogProb.empty()) + { + // We are at the start of the sequence (i.e. time t=0). + forwardLogProb = ForwardAtT0(emissionLogProb, curLogScale); + } + else + { + forwardLogProb = ForwardAtTn(emissionLogProb, curLogScale, + forwardLogProb); + } - return curLogScale; + return curLogScale; } /** @@ -556,11 +556,11 @@ double HMM::EmissionLogLikelihood( double& logLikelihood, arma::vec& forwardLogProb) const { - bool isStartOfSeq = forwardLogProb.empty(); - double curLogScale = EmissionLogScaleFactor(emissionLogProb, - forwardLogProb); - logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; - return logLikelihood; + bool isStartOfSeq = forwardLogProb.empty(); + double curLogScale = EmissionLogScaleFactor(emissionLogProb, + forwardLogProb); + logLikelihood = isStartOfSeq ? curLogScale : curLogScale + logLikelihood; + return logLikelihood; } /**