diff --git a/src/mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp b/src/mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp
index 321df63e37..68e58a99fe 100644
--- a/src/mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp
+++ b/src/mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp
@@ -25,7 +25,7 @@ namespace optimization {
  * 
  * @tparam UpdatePolicy A type of UpdatePolicy that sould be wrapped around.
  */
-template<typename UpdatePolicy>
+template<typename UpdatePolicyType>
 class GradientClipping
 {
  public:
@@ -34,12 +34,12 @@ class GradientClipping
    * 
    * @param minGradient Minimum possible value of gradient element.
    * @param maxGradient Maximum possible value of gradient element.
-   * @param updatePolicy An instance of the UpdatePolicy
+   * @param updatePolicy An instance of the UpdatePolicyType
    *                     used for actual optimization.
    */
   GradientClipping(const double minGradient,
                    const double maxGradient,
-                   UpdatePolicy updatePolicy) :
+                   UpdatePolicyType& updatePolicy) :
     minGradient(minGradient),
     maxGradient(maxGradient),
     updatePolicy(updatePolicy)
@@ -73,12 +73,25 @@ class GradientClipping
               const arma::mat& gradient)
   {
     // First, clip the gradient.
-    gradient.transform(
-        [&](double val)
-        { return std::min(std::max(val, minGradient), maxGradient); });
+    arma::mat clippedGradient = arma::clamp(gradient, minGradient, maxGradient);
     // And only then do the update.
-    updatePolicy.Update(iterate, stepSize, gradient);
+    updatePolicy.Update(iterate, stepSize, clippedGradient);
   }
+
+  //! Get the update policy.
+  UpdatePolicyType& UpdatePolicy() const { return updatePolicy; }
+  //! Modify the update policy.
+  UpdatePolicyType& UpdatePolicy() { return updatePolicy; }
+
+  //! Get the minimum gradient value.
+  double MinGradient() const { return minGradient; }
+  //! Modify the minimum gradient value.
+  double& MinGradient() { return minGradient; }
+
+  //! Get the maximum gradient value.
+  double MaxGradient() const { return maxGradient; }
+  //! Modify the maximum gradient value.
+  double& MaxGradient() { return maxGradient; }
  private:
   //! Minimum possible value of gradient element.
   double minGradient;
@@ -87,7 +100,7 @@ class GradientClipping
   double maxGradient;
 
   //! An instance of the UpdatePolicy used for actual optimization.
-  UpdatePolicy updatePolicy;
+  UpdatePolicyType updatePolicy;
 };
 
 } // namespace optimization
diff --git a/src/mlpack/methods/ann/layer/cross_entropy_error.hpp b/src/mlpack/methods/ann/layer/cross_entropy_error.hpp
index 086b731af9..c1f9e02256 100644
--- a/src/mlpack/methods/ann/layer/cross_entropy_error.hpp
+++ b/src/mlpack/methods/ann/layer/cross_entropy_error.hpp
@@ -36,8 +36,11 @@ class CrossEntropyError
  public:
   /**
    * Create the CrossEntropyError object.
+   * 
+   * @param eps The minimum value used for computing logarithms
+   *            and denominators in a numerically stable way.
    */
-  CrossEntropyError();
+  CrossEntropyError(double eps = 1e-10);
 
   /*
    * Computes the cross-entropy function.
@@ -74,6 +77,11 @@ class CrossEntropyError
   //! Modify the delta.
   OutputDataType& Delta() { return delta; }
 
+  //! Get the epsilon.
+  double Eps() const { return eps; }
+  //! Modify the epsilon.
+  double& Eps() { return eps; }
+
   /**
    * Serialize the layer
    */
@@ -89,6 +97,9 @@ class CrossEntropyError
 
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
+
+  //! The minimum value used for computing logarithms and denominators
+  double eps;
 }; // class CrossEntropyError
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp
index 82c95e8f04..f9ce79f738 100644
--- a/src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp
+++ b/src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp
@@ -19,7 +19,8 @@ namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType>
-CrossEntropyError<InputDataType, OutputDataType>::CrossEntropyError()
+CrossEntropyError<InputDataType, OutputDataType>::CrossEntropyError(double eps)
+  : eps(eps)
 {
   // Nothing to do here.
 }
@@ -29,8 +30,8 @@ template<typename eT>
 double CrossEntropyError<InputDataType, OutputDataType>::Forward(
     const arma::Mat<eT>&& input, const arma::Mat<eT>&& target)
 {
-  return -arma::accu(target % arma::trunc_log(input) +
-                     (1. - target) % arma::trunc_log(1. - input));
+  return -arma::accu(target % arma::log(input + eps) +
+                     (1. - target) % arma::log(1. - input + eps));
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -40,7 +41,7 @@ void CrossEntropyError<InputDataType, OutputDataType>::Backward(
     const arma::Mat<eT>&& target,
     arma::Mat<eT>&& output)
 {
-  output = (1. - target) / (1. - input + 1e-2) - target / (input + 1e-2);
+  output = (1. - target) / (1. - input + eps) - target / (input + eps);
 }
 
 template<typename InputDataType, typename OutputDataType>
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index d31aeed225..14a5311974 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -29,6 +29,7 @@ add_executable(mlpack_test
   fastmks_test.cpp
   feedforward_network_test.cpp
   gmm_test.cpp
+  gradient_clipping_test.cpp
   gradient_descent_test.cpp
   hmm_test.cpp
   hoeffding_tree_test.cpp
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 3a0e599b94..a1dd855b72 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -26,7 +26,7 @@ using namespace mlpack::ann;
 
 BOOST_AUTO_TEST_SUITE(ANNLayerTest);
 
-// Helper function whcih calls the Reset function of the given module.
+// Helper function which calls the Reset function of the given module.
 template<class T>
 void ResetFunction(
     T& layer,
@@ -531,7 +531,7 @@ BOOST_AUTO_TEST_CASE(JacobianLinearNoBiasLayerTest)
 /**
  * LinearNoBias layer numerically gradient test.
  */
-BOOST_AUTO_TEST_CASE(GradientLinearNoBiadLayerTest)
+BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest)
 {
   // LinearNoBias function gradient instantiation.
   struct GradientFunction
diff --git a/src/mlpack/tests/gradient_clipping_test.cpp b/src/mlpack/tests/gradient_clipping_test.cpp
new file mode 100644
index 0000000000..dbd2fe1242
--- /dev/null
+++ b/src/mlpack/tests/gradient_clipping_test.cpp
@@ -0,0 +1,73 @@
+/**
+ * @file gradient_clipping_test.cpp
+ * @author Konstantin Sidorov
+ *
+ * Test file for gradient clipping.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp>
+#include <mlpack/core/optimizers/lbfgs/test_functions.hpp>
+#include <mlpack/core/optimizers/sgd/test_function.hpp>
+
+#include <boost/test/unit_test.hpp>
+#include "test_tools.hpp"
+
+using namespace std;
+using namespace arma;
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+BOOST_AUTO_TEST_SUITE(GradientClippingTest);
+
+// Test checking that gradient clipping works with vanilla update.
+BOOST_AUTO_TEST_CASE(ClippedVanillaUpdateTest)
+{
+  VanillaUpdate vanillaUpdate;
+  GradientClipping<VanillaUpdate> update(-3.0, +3.0, vanillaUpdate);
+  update.Initialize(3, 3);
+
+  arma::mat coordinates = arma::zeros(3, 3);
+  // Setting step = 1 to make math easy.
+  double stepSize = 1.0;
+  arma::mat dummyGradient = { {-6, +6, 0}, {1, 2, 3}, {-3, 0, +4} };
+  update.Update(coordinates, stepSize, dummyGradient);
+  // After clipping, we should get the following coordinates:
+  arma::mat targetCoordinates = { { 3, -3, 0}, {-1, -2, -3}, {3, 0, -3} };
+  BOOST_REQUIRE_SMALL(arma::abs(coordinates - targetCoordinates).max(), 1e-7);
+}
+
+// Test checking that gradient clipping works with momentum update.
+BOOST_AUTO_TEST_CASE(ClippedMomentumUpdateTest)
+{
+  // Once again, setting momentum = 1 for easy math
+  // (now momentum = -stepSize * [sum of gradients])
+  MomentumUpdate momentumUpdate(1);
+  GradientClipping<MomentumUpdate> update(-3.0, +3.0, momentumUpdate);
+  update.Initialize(3, 3);
+
+  arma::mat coordinates = arma::zeros(3, 3);
+  double stepSize = 1.0;
+  arma::mat dummyGradient = { {-6, +6, 0}, {1, 2, 3}, {-3, 0, +4} };
+  update.Update(coordinates, stepSize, dummyGradient);
+  arma::mat targetCoordinates = { { 3, -3, 0}, {-1, -2, -3}, {3, 0, -3} };
+  // On the first Update() call the parameters
+  // should just be equal to (-gradient).
+  BOOST_REQUIRE_SMALL(arma::abs(coordinates - targetCoordinates).max(), 1e-7);
+  update.Update(coordinates, stepSize, dummyGradient);
+  // On the second Update() call the Momentum update will subtract
+  // the gradient from the momentum, which gives 2 * gradient value
+  // for the momentum on that step. Adding that to the gradient which
+  // was subtracted earlier yiels the 3 * gradient in the following check.
+  BOOST_REQUIRE_SMALL(arma::abs(coordinates - 3 * targetCoordinates).max(),
+                      1e-7);
+}
+
+BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp
index 451935a67c..a7c2d67561 100644
--- a/src/mlpack/tests/momentum_sgd_test.cpp
+++ b/src/mlpack/tests/momentum_sgd_test.cpp
@@ -57,24 +57,6 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction)
   BOOST_REQUIRE_LE(result, result1);
 }
 
-// Test checking that gradient clipping works with momentum SGD.
-BOOST_AUTO_TEST_CASE(ClippedSGDTestFunction)
-{
-  SGDTestFunction f;
-  MomentumUpdate momentumUpdate;
-  GradientClipping<MomentumUpdate> update(-3., +3., momentumUpdate);
-  StandardSGD s(0.0003, 5000000, 1e-9, true);
-
-  arma::mat coordinates = f.GetInitialPoint();
-  double result = s.Optimize(f, coordinates);
-
-  BOOST_REQUIRE_CLOSE(result, -1.0, 0.05);
-  BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
-  BOOST_REQUIRE_SMALL(coordinates[1], 1e-7);
-  BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
-}
-
-
 BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest)
 {
   // Loop over several variants.
diff --git a/src/mlpack/tests/sgd_test.cpp b/src/mlpack/tests/sgd_test.cpp
index 9c3c5dc143..77934c64a6 100644
--- a/src/mlpack/tests/sgd_test.cpp
+++ b/src/mlpack/tests/sgd_test.cpp
@@ -41,23 +41,6 @@ BOOST_AUTO_TEST_CASE(SimpleSGDTestFunction)
   BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
 }
 
-// Test checking that gradient clipping works with vanilla SGD.
-BOOST_AUTO_TEST_CASE(ClippedSGDTestFunction)
-{
-  SGDTestFunction f;
-  VanillaUpdate vanillaUpdate;
-  GradientClipping<VanillaUpdate> update(-3., +3., vanillaUpdate);
-  StandardSGD s(0.0003, 5000000, 1e-9, true);
-
-  arma::mat coordinates = f.GetInitialPoint();
-  double result = s.Optimize(f, coordinates);
-
-  BOOST_REQUIRE_CLOSE(result, -1.0, 0.05);
-  BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
-  BOOST_REQUIRE_SMALL(coordinates[1], 1e-7);
-  BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
-}
-
 BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest)
 {
   // Loop over several variants.