From 36ca245f2d6cf609806e08bc4fc2eefd7704517e Mon Sep 17 00:00:00 2001
From: robotcator <problemset@163.com>
Date: Fri, 2 Aug 2019 09:58:56 +0800
Subject: [PATCH 01/13] add normal distribution

---
 src/mlpack/methods/ann/dists/CMakeLists.txt   |   2 +
 .../methods/ann/dists/normal_distribution.hpp | 132 ++++++++++++++++++
 .../ann/dists/normal_distribution_impl.hpp    |  51 +++++++
 3 files changed, 185 insertions(+)
 create mode 100644 src/mlpack/methods/ann/dists/normal_distribution.hpp
 create mode 100644 src/mlpack/methods/ann/dists/normal_distribution_impl.hpp

diff --git a/src/mlpack/methods/ann/dists/CMakeLists.txt b/src/mlpack/methods/ann/dists/CMakeLists.txt
index eea77c413c..fadec3c3c1 100644
--- a/src/mlpack/methods/ann/dists/CMakeLists.txt
+++ b/src/mlpack/methods/ann/dists/CMakeLists.txt
@@ -3,6 +3,8 @@
 set(SOURCES
   bernoulli_distribution.hpp
   bernoulli_distribution_impl.hpp
+  normal_distribution.hpp
+  normal_distribution_impl.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
new file mode 100644
index 0000000000..d1844bd829
--- /dev/null
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -0,0 +1,132 @@
+/**
+ * @file normal_distribution.hpp
+ * @author xiaohong ji
+ *
+ * Definition of the Normal distribution class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_DISTRIBUTIONS_NORMAL_DISTRIBUTION_HPP
+#define MLPACK_METHODS_ANN_DISTRIBUTIONS_NORMAL_DISTRIBUTION_HPP
+
+#include <mlpack/prereqs.hpp>
+#include "../activation_functions/logistic_function.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * Multiple independent Bernoulli distributions.
+ *
+ * Bernoulli distribution is the discrete probability distribution of a random
+ * variable which takes the value 1 with probability p and the value 0 with
+ * probability q = 1 - p.
+ * In this implementation, the p values of the distributions are given by the
+ * param matrix.
+ *
+ */
+class NormalDistribution
+{
+ private:
+  //! Mean of the distribution.
+  arma::vec mean;
+  //! Variance of the distribution.
+  arma::vec sigma;
+
+  // pi
+  static const constexpr double pi = 3.14159265358979323846264338327950288;
+
+ public:
+  /**
+   * Default constructor, which creates a Normal distribution with zero
+   * dimension.
+   */
+  NormalDistribution();
+
+  /**
+   * Create a Normal distribution with the given mean and sigma.
+   */
+  NormalDistribution(const arma::vec& mean, const arma::vec& sigma);
+
+  /**
+   * Return the probabilities of the given matrix of observations.
+   *
+   * @param observation The observation matrix.
+   */
+  arma::vec Probability(const arma::vec& observation) const
+  {
+    return arma::exp(LogProbability(observation));
+  }
+
+  /**
+   * Return the log probabilities of the given matrix of observations.
+   *
+   * @param observation The observation matrix.
+   */
+  arma::vec LogProbability(const arma::vec& observation) const;
+
+  /**
+   * Calculates the normal probability density function for each
+   * data point (column) in the given matrix.
+   *
+   * @param x List of observations.
+   * @param probabilities Output probabilities for each input observation.
+   */
+  void Probability(const arma::vec& x, arma::vec& probabilities) const
+  {
+    probabilities = Probability(x);
+  }
+
+  /**
+   * Return a randomly generated observation according to the probability
+   * distribution defined by this object.
+   *
+   * @return Random observation from this Normal distribution.
+   */
+  arma::vec Sample() const;
+
+  /**
+   * Return the mean.
+   */
+  const arma::vec& Mean() const { return mean; }
+
+  /**
+   * Return a modifiable copy of the mean.
+   */
+  arma::vec& Mean() { return mean; }
+
+  /**
+   * Return the variance.
+   */
+  const arma::vec& Sigma() const { return sigma; }
+
+  /**
+   * Return a modifiable copy of the variance.
+   */
+  arma::vec& Variance() { return sigma; }
+
+  //! Return the dimensionality of this distribution.
+  size_t Dimensionality() const { return mean.n_elem; }
+
+  /**
+   * Serialize the distribution.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */)
+  {
+    // We just need to serialize each of the members.
+    ar & BOOST_SERIALIZATION_NVP(mean);
+    ar & BOOST_SERIALIZATION_NVP(sigma);
+  }
+}; // class NormalDistribution
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "normal_distribution_impl.hpp"
+
+#endif
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
new file mode 100644
index 0000000000..aecc5cdf14
--- /dev/null
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -0,0 +1,51 @@
+/**
+ * @file normal_distribution_impl.hpp
+ * @author xiaohong ji
+ *
+ * Implementation of the Normal distribution class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_DISTRIBUTIONS_NORMAL_DISTRIBUTION_IMPL_HPP
+#define MLPACK_METHODS_ANN_DISTRIBUTIONS_NORMAL_DISTRIBUTION_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "normal_distribution.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+NormalDistribution::NormalDistribution()
+{
+  // Nothing to do here.
+}
+
+NormalDistribution::NormalDistribution(
+    const arma::vec& mean,
+    const arma::vec& sigma) :
+    mean(mean),
+    sigma(sigma)
+{
+}
+
+arma::vec NormalDistribution::Sample() const
+{
+  return sigma * arma::randn<arma::vec>(mean.n_elem) + mean;
+}
+
+arma::vec NormalDistribution::LogProbability(
+    const arma::vec& observation) const
+{
+  const arma::vec variance = arma::square(sigma);
+  arma::vec v1 = sigma + std::log(std::sqrt(2 * pi));
+  arma::vec v2 = arma::square(observation - sigma) / (2 * variance);
+  return  (-v1 - v2);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

From 8151d50cde440eeb0c5b68d5563441a2d68ffaa2 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nishantkr.ece18@itbhu.ac.in>
Date: Wed, 22 Apr 2020 17:37:14 +0530
Subject: [PATCH 02/13] bug fixes in logProbablity

---
 src/mlpack/methods/ann/dists/normal_distribution.hpp      | 8 ++++----
 src/mlpack/methods/ann/dists/normal_distribution_impl.hpp | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index d1844bd829..c4942cd957 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -33,7 +33,7 @@ class NormalDistribution
  private:
   //! Mean of the distribution.
   arma::vec mean;
-  //! Variance of the distribution.
+  //! Standard deviation of the distribution.
   arma::vec sigma;
 
   // pi
@@ -99,14 +99,14 @@ class NormalDistribution
   arma::vec& Mean() { return mean; }
 
   /**
-   * Return the variance.
+   * Return the standard deviation.
    */
   const arma::vec& Sigma() const { return sigma; }
 
   /**
-   * Return a modifiable copy of the variance.
+   * Return a modifiable copy of the standard deviation.
    */
-  arma::vec& Variance() { return sigma; }
+  arma::vec& StandardDeviation() { return sigma; }
 
   //! Return the dimensionality of this distribution.
   size_t Dimensionality() const { return mean.n_elem; }
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
index aecc5cdf14..c46538e4f8 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -40,8 +40,8 @@ arma::vec NormalDistribution::LogProbability(
     const arma::vec& observation) const
 {
   const arma::vec variance = arma::square(sigma);
-  arma::vec v1 = sigma + std::log(std::sqrt(2 * pi));
-  arma::vec v2 = arma::square(observation - sigma) / (2 * variance);
+  arma::vec v1 = arma::log(sigma) + std::log(std::sqrt(2 * pi));
+  arma::vec v2 = arma::square(observation - mu) / (2 * variance);
   return  (-v1 - v2);
 }
 

From ec750e0cf10dfc33ac7cccc7cc27da72fb0204ea Mon Sep 17 00:00:00 2001
From: nishantkr18 <nishantkr.ece18@itbhu.ac.in>
Date: Wed, 22 Apr 2020 18:42:18 +0530
Subject: [PATCH 03/13] added backward function in normal dist

---
 .../methods/ann/dists/normal_distribution.hpp | 23 +++++++++++++++++++
 .../ann/dists/normal_distribution_impl.hpp    | 13 ++++++++++-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index c4942cd957..874e45b448 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -68,6 +68,17 @@ class NormalDistribution
    */
   arma::vec LogProbability(const arma::vec& observation) const;
 
+  /**
+   * Stores the gradient of the probabilities of the observations
+   * with respect to mean and standard deviation.
+   *
+   * @param observation The observation matrix.
+   * @param dmu The gradient with respect to mean.
+   * @param dsigma The gradient with respect to standard deviation.
+   */
+  void ProbBackward(const arma::vec& observation, arma::vec& dmu,
+arma::vec& dsigma) const;
+
   /**
    * Calculates the normal probability density function for each
    * data point (column) in the given matrix.
@@ -80,6 +91,18 @@ class NormalDistribution
     probabilities = Probability(x);
   }
 
+  /**
+    * Calculates the log of normal probability density function for each
+    * data point (column) in the given matrix.
+    *
+    * @param x List of observations.
+    * @param log probabilities Output probabilities for each input observation.
+    */
+  void LogProbability(const arma::vec& x, arma::vec& probabilities) const
+  {
+    probabilities = LogProbability(x);
+  }
+
   /**
    * Return a randomly generated observation according to the probability
    * distribution defined by this object.
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
index c46538e4f8..d90438bb5e 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -41,10 +41,21 @@ arma::vec NormalDistribution::LogProbability(
 {
   const arma::vec variance = arma::square(sigma);
   arma::vec v1 = arma::log(sigma) + std::log(std::sqrt(2 * pi));
-  arma::vec v2 = arma::square(observation - mu) / (2 * variance);
+  arma::vec v2 = arma::square(observation - mean) / (2 * variance);
   return  (-v1 - v2);
 }
 
+void NormalDistribution::ProbBackward(
+    const arma::vec& observation,
+    arma::vec& dmu,
+    arma::vec& dsigma) const
+{
+    dmu = (observation - mean) / (arma::square(sigma)) % Probability(observation);
+    dsigma = (- 1.0 / sigma +
+              (arma::square(observation - mean) / arma::pow(sigma, 3)))
+              % Probability(observation);
+}
+
 } // namespace ann
 } // namespace mlpack
 

From 6cf26e0c4f47e4d549ae0195e8100bb3cee40418 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nishantkr.ece18@itbhu.ac.in>
Date: Wed, 22 Apr 2020 22:26:02 +0530
Subject: [PATCH 04/13] doc changes and tabs removal

---
 src/mlpack/methods/ann/dists/normal_distribution.hpp   | 10 +++-------
 .../methods/ann/dists/normal_distribution_impl.hpp     |  8 ++++----
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index 874e45b448..ca876b5c31 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -19,14 +19,10 @@ namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 /**
- * Multiple independent Bernoulli distributions.
- *
- * Bernoulli distribution is the discrete probability distribution of a random
- * variable which takes the value 1 with probability p and the value 0 with
- * probability q = 1 - p.
- * In this implementation, the p values of the distributions are given by the
- * param matrix.
+ * Implementation of the Normal Distribution function.
  *
+ * Normal distribution is a function which accepts a mean and a standard deviation
+ * term and creates a probablity distribution out of it.
  */
 class NormalDistribution
 {
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
index d90438bb5e..52f36bfc9e 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -50,10 +50,10 @@ void NormalDistribution::ProbBackward(
     arma::vec& dmu,
     arma::vec& dsigma) const
 {
-    dmu = (observation - mean) / (arma::square(sigma)) % Probability(observation);
-    dsigma = (- 1.0 / sigma +
-              (arma::square(observation - mean) / arma::pow(sigma, 3)))
-              % Probability(observation);
+  dmu = (observation - mean) / (arma::square(sigma)) % Probability(observation);
+  dsigma = (- 1.0 / sigma +
+            (arma::square(observation - mean) / arma::pow(sigma, 3)))
+            % Probability(observation);
 }
 
 } // namespace ann

From 6718255e9e12e94bbc70f1a2e4781ceffc910d19 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nishantkr.ece18@itbhu.ac.in>
Date: Wed, 22 Apr 2020 23:54:24 +0530
Subject: [PATCH 05/13] added test for normal-dist

---
 src/mlpack/tests/ann_dist_test.cpp | 37 ++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/src/mlpack/tests/ann_dist_test.cpp b/src/mlpack/tests/ann_dist_test.cpp
index 6c6dad1371..2f2bd3e513 100644
--- a/src/mlpack/tests/ann_dist_test.cpp
+++ b/src/mlpack/tests/ann_dist_test.cpp
@@ -12,6 +12,7 @@
 #include <mlpack/core.hpp>
 
 #include <mlpack/methods/ann/dists/bernoulli_distribution.hpp>
+#include <mlpack/methods/ann/dists/normal_distribution.hpp>
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
 
 #include <boost/test/unit_test.hpp>
@@ -127,4 +128,40 @@ BOOST_AUTO_TEST_CASE(JacobianBernoulliDistributionLogisticTest)
   }
 }
 
+/**
+ * Normal Distribution module test.
+ */
+BOOST_AUTO_TEST_CASE(NormalDistributionTest)
+{
+  arma::vec mu = {1.1, 1.2, 1.5, 1.7};
+  arma::vec sigma = {0.1, 0.11, 0.5, 0.23};
+
+  ann::NormalDistribution normalDist =
+    ann::NormalDistribution(mu, sigma);
+
+  arma::vec x = {1.05, 1.1, 1.7, 2.5};
+
+  arma::vec prob;
+  normalDist.LogProbability(x, prob);
+  
+  // testing output of log probablity for some random mu, sigma and x.
+  BOOST_REQUIRE_CLOSE(prob[0], 1.2586464, 1e-3);
+  BOOST_REQUIRE_CLOSE(prob[1], 0.8751131, 1e-3);
+  BOOST_REQUIRE_CLOSE(prob[2], -0.30579138, 1e-3);
+  BOOST_REQUIRE_CLOSE(prob[3], -5.498411, 1e-3);
+
+  arma::vec dmu, dsigma;
+  normalDist.ProbBackward(x, dmu, dsigma);
+
+  // testing output of dmu and dsigma for some random mu, sigma and x.
+  BOOST_REQUIRE_CLOSE(dmu[0], -17.603287, 1e-3);
+  BOOST_REQUIRE_CLOSE(dsigma[0], -26.40487, 1e-3);
+  BOOST_REQUIRE_CLOSE(dmu[1], -19.827663, 1e-3);
+  BOOST_REQUIRE_CLOSE(dsigma[1], -3.7852707, 1e-3);
+  BOOST_REQUIRE_CLOSE(dmu[2], 0.5892323, 1e-3);
+  BOOST_REQUIRE_CLOSE(dsigma[2], -1.2373875, 1e-3);
+  BOOST_REQUIRE_CLOSE(dmu[3], 0.061901994, 1e-3);
+  BOOST_REQUIRE_CLOSE(dsigma[3], 0.19751444, 1e-3);
+}
+
 BOOST_AUTO_TEST_SUITE_END();

From 144f57353dc6df008f7261b8a6586e46b263a9b7 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nishantkr.ece18@itbhu.ac.in>
Date: Thu, 23 Apr 2020 00:11:38 +0530
Subject: [PATCH 06/13] style bug fix

---
 src/mlpack/tests/ann_dist_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/ann_dist_test.cpp b/src/mlpack/tests/ann_dist_test.cpp
index 2f2bd3e513..d70bc6b4b7 100644
--- a/src/mlpack/tests/ann_dist_test.cpp
+++ b/src/mlpack/tests/ann_dist_test.cpp
@@ -143,7 +143,7 @@ BOOST_AUTO_TEST_CASE(NormalDistributionTest)
 
   arma::vec prob;
   normalDist.LogProbability(x, prob);
-  
+
   // testing output of log probablity for some random mu, sigma and x.
   BOOST_REQUIRE_CLOSE(prob[0], 1.2586464, 1e-3);
   BOOST_REQUIRE_CLOSE(prob[1], 0.8751131, 1e-3);

From 2353d107d3430b90a566607542c95fe236f82945 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nishantkr.ece18@itbhu.ac.in>
Date: Fri, 24 Apr 2020 02:26:36 +0530
Subject: [PATCH 07/13] added suggested changes

---
 .../methods/ann/dists/normal_distribution.hpp | 25 +++++++++++--------
 .../ann/dists/normal_distribution_impl.hpp    |  3 ++-
 src/mlpack/tests/ann_dist_test.cpp            |  4 +--
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index ca876b5c31..a8aa2059e0 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -1,6 +1,7 @@
 /**
  * @file normal_distribution.hpp
  * @author xiaohong ji
+ * @author Nishant Kumar
  *
  * Definition of the Normal distribution class.
  *
@@ -26,15 +27,6 @@ namespace ann /** Artificial Neural Network. */ {
  */
 class NormalDistribution
 {
- private:
-  //! Mean of the distribution.
-  arma::vec mean;
-  //! Standard deviation of the distribution.
-  arma::vec sigma;
-
-  // pi
-  static const constexpr double pi = 3.14159265358979323846264338327950288;
-
  public:
   /**
    * Default constructor, which creates a Normal distribution with zero
@@ -44,6 +36,9 @@ class NormalDistribution
 
   /**
    * Create a Normal distribution with the given mean and sigma.
+   *
+   * @param mean The mean of the normal distribution.
+   * @param sigma The standard deviation of the normal distribution.
    */
   NormalDistribution(const arma::vec& mean, const arma::vec& sigma);
 
@@ -72,8 +67,9 @@ class NormalDistribution
    * @param dmu The gradient with respect to mean.
    * @param dsigma The gradient with respect to standard deviation.
    */
-  void ProbBackward(const arma::vec& observation, arma::vec& dmu,
-arma::vec& dsigma) const;
+  void ProbBackward(const arma::vec& observation,
+                    arma::vec& dmu,
+                    arma::vec& dsigma) const;
 
   /**
    * Calculates the normal probability density function for each
@@ -140,6 +136,13 @@ arma::vec& dsigma) const;
     ar & BOOST_SERIALIZATION_NVP(mean);
     ar & BOOST_SERIALIZATION_NVP(sigma);
   }
+
+ private:
+  //! Mean of the distribution.
+  arma::vec mean;
+
+  //! Standard deviation of the distribution.
+  arma::vec sigma;
 }; // class NormalDistribution
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
index 52f36bfc9e..f81dcbbdaa 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -1,6 +1,7 @@
 /**
  * @file normal_distribution_impl.hpp
  * @author xiaohong ji
+ * @author Nishant Kumar
  *
  * Implementation of the Normal distribution class.
  *
@@ -40,7 +41,7 @@ arma::vec NormalDistribution::LogProbability(
     const arma::vec& observation) const
 {
   const arma::vec variance = arma::square(sigma);
-  arma::vec v1 = arma::log(sigma) + std::log(std::sqrt(2 * pi));
+  arma::vec v1 = arma::log(sigma) + std::log(std::sqrt(2 * M_PI));
   arma::vec v2 = arma::square(observation - mean) / (2 * variance);
   return  (-v1 - v2);
 }
diff --git a/src/mlpack/tests/ann_dist_test.cpp b/src/mlpack/tests/ann_dist_test.cpp
index d70bc6b4b7..f5f157f276 100644
--- a/src/mlpack/tests/ann_dist_test.cpp
+++ b/src/mlpack/tests/ann_dist_test.cpp
@@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(NormalDistributionTest)
   arma::vec prob;
   normalDist.LogProbability(x, prob);
 
-  // testing output of log probablity for some random mu, sigma and x.
+  // Testing output of log probability for some random mu, sigma and x.
   BOOST_REQUIRE_CLOSE(prob[0], 1.2586464, 1e-3);
   BOOST_REQUIRE_CLOSE(prob[1], 0.8751131, 1e-3);
   BOOST_REQUIRE_CLOSE(prob[2], -0.30579138, 1e-3);
@@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(NormalDistributionTest)
   arma::vec dmu, dsigma;
   normalDist.ProbBackward(x, dmu, dsigma);
 
-  // testing output of dmu and dsigma for some random mu, sigma and x.
+  // Testing output of dmu and dsigma for some random mu, sigma and x.
   BOOST_REQUIRE_CLOSE(dmu[0], -17.603287, 1e-3);
   BOOST_REQUIRE_CLOSE(dsigma[0], -26.40487, 1e-3);
   BOOST_REQUIRE_CLOSE(dmu[1], -19.827663, 1e-3);

From 12b68e7791176306f6a50d94aa312a42520e60f5 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nishantkr.ece18@itbhu.ac.in>
Date: Sun, 26 Apr 2020 17:11:55 +0530
Subject: [PATCH 08/13] added jacobian test and added template to normal dist

---
 .../methods/ann/dists/normal_distribution.hpp |  31 ++--
 .../ann/dists/normal_distribution_impl.hpp    |  35 +++--
 src/mlpack/tests/ann_dist_test.cpp            | 139 +++++++++++++++++-
 3 files changed, 173 insertions(+), 32 deletions(-)

diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index a8aa2059e0..c75396e788 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -25,6 +25,7 @@ namespace ann /** Artificial Neural Network. */ {
  * Normal distribution is a function which accepts a mean and a standard deviation
  * term and creates a probablity distribution out of it.
  */
+template <typename DataType = arma::mat>
 class NormalDistribution
 {
  public:
@@ -40,14 +41,14 @@ class NormalDistribution
    * @param mean The mean of the normal distribution.
    * @param sigma The standard deviation of the normal distribution.
    */
-  NormalDistribution(const arma::vec& mean, const arma::vec& sigma);
+  NormalDistribution(const DataType& mean, const DataType& sigma);
 
   /**
    * Return the probabilities of the given matrix of observations.
    *
    * @param observation The observation matrix.
    */
-  arma::vec Probability(const arma::vec& observation) const
+  DataType Probability(const DataType& observation) const
   {
     return arma::exp(LogProbability(observation));
   }
@@ -57,7 +58,7 @@ class NormalDistribution
    *
    * @param observation The observation matrix.
    */
-  arma::vec LogProbability(const arma::vec& observation) const;
+  DataType LogProbability(const DataType& observation) const;
 
   /**
    * Stores the gradient of the probabilities of the observations
@@ -67,9 +68,9 @@ class NormalDistribution
    * @param dmu The gradient with respect to mean.
    * @param dsigma The gradient with respect to standard deviation.
    */
-  void ProbBackward(const arma::vec& observation,
-                    arma::vec& dmu,
-                    arma::vec& dsigma) const;
+  void ProbBackward(const DataType& observation,
+                    DataType& dmu,
+                    DataType& dsigma) const;
 
   /**
    * Calculates the normal probability density function for each
@@ -78,7 +79,7 @@ class NormalDistribution
    * @param x List of observations.
    * @param probabilities Output probabilities for each input observation.
    */
-  void Probability(const arma::vec& x, arma::vec& probabilities) const
+  void Probability(const DataType& x, DataType& probabilities) const
   {
     probabilities = Probability(x);
   }
@@ -90,7 +91,7 @@ class NormalDistribution
     * @param x List of observations.
     * @param log probabilities Output probabilities for each input observation.
     */
-  void LogProbability(const arma::vec& x, arma::vec& probabilities) const
+  void LogProbability(const DataType& x, DataType& probabilities) const
   {
     probabilities = LogProbability(x);
   }
@@ -101,27 +102,27 @@ class NormalDistribution
    *
    * @return Random observation from this Normal distribution.
    */
-  arma::vec Sample() const;
+  DataType Sample() const;
 
   /**
    * Return the mean.
    */
-  const arma::vec& Mean() const { return mean; }
+  const DataType& Mean() const { return mean; }
 
   /**
    * Return a modifiable copy of the mean.
    */
-  arma::vec& Mean() { return mean; }
+  DataType& Mean() { return mean; }
 
   /**
    * Return the standard deviation.
    */
-  const arma::vec& Sigma() const { return sigma; }
+  const DataType& StandardDeviation() const { return sigma; }
 
   /**
    * Return a modifiable copy of the standard deviation.
    */
-  arma::vec& StandardDeviation() { return sigma; }
+  DataType& StandardDeviation() { return sigma; }
 
   //! Return the dimensionality of this distribution.
   size_t Dimensionality() const { return mean.n_elem; }
@@ -139,10 +140,10 @@ class NormalDistribution
 
  private:
   //! Mean of the distribution.
-  arma::vec mean;
+  DataType mean;
 
   //! Standard deviation of the distribution.
-  arma::vec sigma;
+  DataType sigma;
 }; // class NormalDistribution
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
index f81dcbbdaa..a5cd820190 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -19,37 +19,42 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
-NormalDistribution::NormalDistribution()
+template<typename DataType>
+NormalDistribution<DataType>::NormalDistribution()
 {
   // Nothing to do here.
 }
 
-NormalDistribution::NormalDistribution(
-    const arma::vec& mean,
-    const arma::vec& sigma) :
+template<typename DataType>
+NormalDistribution<DataType>::NormalDistribution(
+    const DataType& mean,
+    const DataType& sigma) :
     mean(mean),
     sigma(sigma)
 {
 }
 
-arma::vec NormalDistribution::Sample() const
+template<typename DataType>
+DataType NormalDistribution<DataType>::Sample() const
 {
-  return sigma * arma::randn<arma::vec>(mean.n_elem) + mean;
+  return sigma * arma::randn<DataType>(mean.n_elem) + mean;
 }
 
-arma::vec NormalDistribution::LogProbability(
-    const arma::vec& observation) const
+template<typename DataType>
+DataType NormalDistribution<DataType>::LogProbability(
+    const DataType& observation) const
 {
-  const arma::vec variance = arma::square(sigma);
-  arma::vec v1 = arma::log(sigma) + std::log(std::sqrt(2 * M_PI));
-  arma::vec v2 = arma::square(observation - mean) / (2 * variance);
+  const DataType variance = arma::square(sigma);
+  DataType v1 = arma::log(sigma) + std::log(std::sqrt(2 * M_PI));
+  DataType v2 = arma::square(observation - mean) / (2 * variance);
   return  (-v1 - v2);
 }
 
-void NormalDistribution::ProbBackward(
-    const arma::vec& observation,
-    arma::vec& dmu,
-    arma::vec& dsigma) const
+template<typename DataType>
+void NormalDistribution<DataType>::ProbBackward(
+    const DataType& observation,
+    DataType& dmu,
+    DataType& dsigma) const
 {
   dmu = (observation - mean) / (arma::square(sigma)) % Probability(observation);
   dsigma = (- 1.0 / sigma +
diff --git a/src/mlpack/tests/ann_dist_test.cpp b/src/mlpack/tests/ann_dist_test.cpp
index f5f157f276..18f2834007 100644
--- a/src/mlpack/tests/ann_dist_test.cpp
+++ b/src/mlpack/tests/ann_dist_test.cpp
@@ -1,6 +1,7 @@
 /**
  * @file ann_dist_test.cpp
  * @author Atharva Khandait
+ * @author Nishant Kumar
  *
  * Tests the ann distributions.
  *
@@ -136,8 +137,7 @@ BOOST_AUTO_TEST_CASE(NormalDistributionTest)
   arma::vec mu = {1.1, 1.2, 1.5, 1.7};
   arma::vec sigma = {0.1, 0.11, 0.5, 0.23};
 
-  ann::NormalDistribution normalDist =
-    ann::NormalDistribution(mu, sigma);
+  ann::NormalDistribution<> normalDist(mu, sigma);
 
   arma::vec x = {1.05, 1.1, 1.7, 2.5};
 
@@ -164,4 +164,139 @@ BOOST_AUTO_TEST_CASE(NormalDistributionTest)
   BOOST_REQUIRE_CLOSE(dsigma[3], 0.19751444, 1e-3);
 }
 
+/**
+ * Jacobian Normal distribution module test for mean.
+ */
+BOOST_AUTO_TEST_CASE(JacobianNormalDistributionMeanTest)
+{
+  for (size_t i = 0; i < 5; i++)
+  {
+    const size_t targetElements = math::RandInt(2, 1000);
+
+    arma::mat mu;
+    mu.randn(targetElements, 1);
+
+    arma::mat sigma;
+    sigma.randu(targetElements, 1);
+
+    arma::mat x;
+    x.randn(targetElements, 1);
+
+    NormalDistribution<> module(mu, sigma);
+
+    const double perturbation = 1e-6;
+    arma::mat output, outputA, outputB, jacobianA, jacobianB;
+
+    // Initialize the jacobian matrix.
+    module.Probability(x, output);
+    jacobianA = arma::zeros(x.n_elem, output.n_elem);
+
+    for (size_t j = 0; j < x.n_elem; ++j)
+    {
+      double original = module.Mean()(j);
+      module.Mean()(j) = original - perturbation;
+      module.Probability(x, outputA);
+      module.Mean()(j) = original + perturbation;
+      module.Probability(x, outputB);
+      module.Mean()(j) = original;
+
+      outputB -= outputA;
+      outputB /= 2 * perturbation;
+      jacobianA.row(j) = outputB.t();
+    }
+
+    // Initialize the derivative parameter.
+    arma::mat deriv = arma::zeros(output.n_rows, output.n_cols);
+
+    // Share the derivative parameter.
+    arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols,
+        false, false);
+
+    // Initialize the jacobian matrix.
+    jacobianB = arma::zeros(mu.n_elem, output.n_elem);
+
+    for (size_t k = 0; k < derivTemp.n_elem; ++k)
+    {
+      deriv.zeros();
+      derivTemp(k) = 1;
+
+      arma::mat deltaMu, deltaSigma;
+      module.ProbBackward(x, deltaMu, deltaSigma);
+
+      jacobianB.col(k) = deltaMu % deriv;
+    }
+
+    BOOST_REQUIRE_LE(arma::max(arma::max(arma::abs(jacobianA - jacobianB))),
+        5e-3);
+  }
+}
+
+/**
+ * Jacobian Normal distribution module test for standard deviation.
+ */
+BOOST_AUTO_TEST_CASE(JacobianNormalDistributionStandardDeviationTest)
+{
+  for (size_t i = 0; i < 5; i++)
+  {
+    const size_t targetElements = math::RandInt(2, 1000);
+
+    arma::mat mu;
+    mu.randn(targetElements, 1);
+
+    arma::mat sigma;
+    sigma.randu(targetElements, 1);
+
+    arma::mat x;
+    x.randn(targetElements, 1);
+
+    NormalDistribution<> module(mu, sigma);
+
+    const double perturbation = 1e-6;
+    arma::mat output, outputA, outputB, jacobianA, jacobianB;
+
+    // Initialize the jacobian matrix.
+    module.Probability(x, output);
+    jacobianA = arma::zeros(x.n_elem, output.n_elem);
+
+    for (size_t j = 0; j < x.n_elem; ++j)
+    {
+      double original = module.StandardDeviation()(j);
+      module.StandardDeviation()(j) = original - perturbation;
+      module.Probability(x, outputA);
+      module.StandardDeviation()(j) = original + perturbation;
+      module.Probability(x, outputB);
+      module.StandardDeviation()(j) = original;
+
+      outputB -= outputA;
+      outputB /= 2 * perturbation;
+      jacobianA.row(j) = outputB.t();
+    }
+
+    // Initialize the derivative parameter.
+    arma::mat deriv = arma::zeros(output.n_rows, output.n_cols);
+
+    // Share the derivative parameter.
+    arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols,
+        false, false);
+
+    // Initialize the jacobian matrix.
+    jacobianB = arma::zeros(sigma.n_elem, output.n_elem);
+
+    for (size_t k = 0; k < derivTemp.n_elem; ++k)
+    {
+      deriv.zeros();
+      derivTemp(k) = 1;
+
+      arma::mat deltaMu, deltaSigma;
+      module.ProbBackward(x, deltaMu, deltaSigma);
+
+      jacobianB.col(k) = deltaSigma % deriv;
+    }
+
+    BOOST_REQUIRE_LE(arma::max(arma::max(arma::abs(jacobianA - jacobianB))),
+        5e-3);
+  }
+}
+
+
 BOOST_AUTO_TEST_SUITE_END();

From 90c6177c9873741e843e5e29ec702c4ee7854572 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nniishantkumar@gmail.com>
Date: Sat, 9 May 2020 00:28:20 +0530
Subject: [PATCH 09/13] comment changes

---
 .../methods/ann/dists/normal_distribution.hpp | 21 ++++---------------
 .../ann/dists/normal_distribution_impl.hpp    |  9 ++++++++
 2 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index c75396e788..2b73d0f4ca 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -104,24 +104,16 @@ class NormalDistribution
    */
   DataType Sample() const;
 
-  /**
-   * Return the mean.
-   */
+  //! Get the mean.
   const DataType& Mean() const { return mean; }
 
-  /**
-   * Return a modifiable copy of the mean.
-   */
+  //! Modify the mean.
   DataType& Mean() { return mean; }
 
-  /**
-   * Return the standard deviation.
-   */
+  //! Get the standard deviation.
   const DataType& StandardDeviation() const { return sigma; }
 
-  /**
-   * Return a modifiable copy of the standard deviation.
-   */
+  //! Modify the standard deviation.
   DataType& StandardDeviation() { return sigma; }
 
   //! Return the dimensionality of this distribution.
@@ -132,11 +124,6 @@ class NormalDistribution
    */
   template<typename Archive>
   void serialize(Archive& ar, const unsigned int /* version */)
-  {
-    // We just need to serialize each of the members.
-    ar & BOOST_SERIALIZATION_NVP(mean);
-    ar & BOOST_SERIALIZATION_NVP(sigma);
-  }
 
  private:
   //! Mean of the distribution.
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
index a5cd820190..fcb02034ff 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -62,6 +62,15 @@ void NormalDistribution<DataType>::ProbBackward(
             % Probability(observation);
 }
 
+template<typename DataType>
+template<typename Archive>
+void NormalDistribution<DataType>::serialize(Archive& ar,
+                                             const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(mean);
+  ar & BOOST_SERIALIZATION_NVP(sigma);
+}
+
 } // namespace ann
 } // namespace mlpack
 

From 6e0973d255a271eaf179eef42277304670d2be58 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nniishantkumar@gmail.com>
Date: Sat, 9 May 2020 00:39:15 +0530
Subject: [PATCH 10/13] added template parameter discription

---
 src/mlpack/methods/ann/dists/normal_distribution.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index 2b73d0f4ca..906a92337c 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -24,6 +24,9 @@ namespace ann /** Artificial Neural Network. */ {
  *
  * Normal distribution is a function which accepts a mean and a standard deviation
  * term and creates a probablity distribution out of it.
+ * 
+ * @tparam DataType Type of the input data. (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
  */
 template <typename DataType = arma::mat>
 class NormalDistribution

From 6298b622e71d0519af0c1fd6bdb5cd6eaa0e623d Mon Sep 17 00:00:00 2001
From: nishantkr18 <nniishantkumar@gmail.com>
Date: Sat, 9 May 2020 02:37:31 +0530
Subject: [PATCH 11/13] suggested changes added

---
 src/mlpack/methods/.vscode/settings.json               |  6 ++++++
 src/mlpack/methods/ann/dists/normal_distribution.hpp   | 10 +++++-----
 .../methods/ann/dists/normal_distribution_impl.hpp     |  9 +++++----
 src/mlpack/tests/ann_dist_test.cpp                     |  4 ++--
 4 files changed, 18 insertions(+), 11 deletions(-)
 create mode 100644 src/mlpack/methods/.vscode/settings.json

diff --git a/src/mlpack/methods/.vscode/settings.json b/src/mlpack/methods/.vscode/settings.json
new file mode 100644
index 0000000000..c69cd43973
--- /dev/null
+++ b/src/mlpack/methods/.vscode/settings.json
@@ -0,0 +1,6 @@
+{
+    "files.associations": {
+        "*.txt": "cpp",
+        "armadillo": "cpp"
+    }
+}
\ No newline at end of file
diff --git a/src/mlpack/methods/ann/dists/normal_distribution.hpp b/src/mlpack/methods/ann/dists/normal_distribution.hpp
index 906a92337c..8f9844fd49 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution.hpp
@@ -69,7 +69,7 @@ class NormalDistribution
    *
    * @param observation The observation matrix.
    * @param dmu The gradient with respect to mean.
-   * @param dsigma The gradient with respect to standard deviation.
+   * @param dsigma The gradient with respect to the standard deviation.
    */
   void ProbBackward(const DataType& observation,
                     DataType& dmu,
@@ -79,7 +79,7 @@ class NormalDistribution
    * Calculates the normal probability density function for each
    * data point (column) in the given matrix.
    *
-   * @param x List of observations.
+   * @param x The observation matrix.
    * @param probabilities Output probabilities for each input observation.
    */
   void Probability(const DataType& x, DataType& probabilities) const
@@ -91,8 +91,8 @@ class NormalDistribution
     * Calculates the log of normal probability density function for each
     * data point (column) in the given matrix.
     *
-    * @param x List of observations.
-    * @param log probabilities Output probabilities for each input observation.
+    * @param x The observation matrix.
+    * @param probabilities Output log probabilities for each input observation.
     */
   void LogProbability(const DataType& x, DataType& probabilities) const
   {
@@ -126,7 +126,7 @@ class NormalDistribution
    * Serialize the distribution.
    */
   template<typename Archive>
-  void serialize(Archive& ar, const unsigned int /* version */)
+  void serialize(Archive& ar, const unsigned int /* version */);
 
  private:
   //! Mean of the distribution.
diff --git a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
index fcb02034ff..4212f30b70 100644
--- a/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
+++ b/src/mlpack/methods/ann/dists/normal_distribution_impl.hpp
@@ -3,7 +3,7 @@
  * @author xiaohong ji
  * @author Nishant Kumar
  *
- * Implementation of the Normal distribution class.
+ * Implementation of the Normal Distribution class.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -32,6 +32,7 @@ NormalDistribution<DataType>::NormalDistribution(
     mean(mean),
     sigma(sigma)
 {
+  // Nothing to do here.
 }
 
 template<typename DataType>
@@ -44,9 +45,9 @@ template<typename DataType>
 DataType NormalDistribution<DataType>::LogProbability(
     const DataType& observation) const
 {
-  const DataType variance = arma::square(sigma);
-  DataType v1 = arma::log(sigma) + std::log(std::sqrt(2 * M_PI));
-  DataType v2 = arma::square(observation - mean) / (2 * variance);
+  const DataType v1 = arma::log(sigma) + std::log(std::sqrt(2 * M_PI));
+  const DataType v2 = arma::square(observation - mean) /
+                      (2 * arma::square(sigma));
   return  (-v1 - v2);
 }
 
diff --git a/src/mlpack/tests/ann_dist_test.cpp b/src/mlpack/tests/ann_dist_test.cpp
index 18f2834007..face8565bd 100644
--- a/src/mlpack/tests/ann_dist_test.cpp
+++ b/src/mlpack/tests/ann_dist_test.cpp
@@ -165,7 +165,7 @@ BOOST_AUTO_TEST_CASE(NormalDistributionTest)
 }
 
 /**
- * Jacobian Normal distribution module test for mean.
+ * Jacobian Normal Distribution module test for mean.
  */
 BOOST_AUTO_TEST_CASE(JacobianNormalDistributionMeanTest)
 {
@@ -232,7 +232,7 @@ BOOST_AUTO_TEST_CASE(JacobianNormalDistributionMeanTest)
 }
 
 /**
- * Jacobian Normal distribution module test for standard deviation.
+ * Jacobian Normal Distribution module test for standard deviation.
  */
 BOOST_AUTO_TEST_CASE(JacobianNormalDistributionStandardDeviationTest)
 {

From db78bbb7ae71c1a429e6d753d57aeb09e5325c58 Mon Sep 17 00:00:00 2001
From: nishantkr18 <nniishantkumar@gmail.com>
Date: Sat, 9 May 2020 02:40:14 +0530
Subject: [PATCH 12/13] added .vscode by mistake..

---
 src/mlpack/methods/.vscode/settings.json | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100644 src/mlpack/methods/.vscode/settings.json

diff --git a/src/mlpack/methods/.vscode/settings.json b/src/mlpack/methods/.vscode/settings.json
deleted file mode 100644
index c69cd43973..0000000000
--- a/src/mlpack/methods/.vscode/settings.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-    "files.associations": {
-        "*.txt": "cpp",
-        "armadillo": "cpp"
-    }
-}
\ No newline at end of file

From 4293c98b9c6db8e6cc07b7e5d20000a00abc9440 Mon Sep 17 00:00:00 2001
From: Nishant Kumar <nniishantkumar@gmail.com>
Date: Wed, 13 May 2020 12:25:38 +0530
Subject: [PATCH 13/13] Update HISTORY.md

---
 HISTORY.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/HISTORY.md b/HISTORY.md
index e7e0b19770..368cc644d2 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -15,6 +15,8 @@
 
 ### mlpack 3.3.0
 ###### 2020-04-07
+  * Added `Normal Distribution` to `ann/dists` (#2382).
+
   * Templated return type of `Forward function` of loss functions (#2339).
 
   * Added `R2 Score` regression metric (#2323).