diff --git a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp
index b0873b5413..f36d0a1a0c 100644
--- a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp
+++ b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp
@@ -13,7 +13,7 @@
 #define MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_IMPL_HPP
 
 #include <limits>
-#include <mlpack/core/optimizers/function.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 namespace mlpack {
 namespace optimization {
@@ -61,7 +61,7 @@ void GridSearch::Optimize(
     size_t i)
 {
   // Make sure we have the methods that we need.
-  traits::CheckNonDifferentiableFunctionTypeAPI<FunctionType>();
+  ens::traits::CheckNonDifferentiableFunctionTypeAPI<FunctionType>();
 
   if (i < datasetInfo.Dimensionality())
   {
diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp
index 34b4c96660..8f9d8a5177 100644
--- a/src/mlpack/methods/ann/ffn.hpp
+++ b/src/mlpack/methods/ann/ffn.hpp
@@ -30,7 +30,7 @@
 #include <mlpack/methods/ann/layer/layer_types.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -106,7 +106,7 @@ class FFN
   /**
    * Train the feedforward network on the given input data. By default, the
    * RMSProp optimization algorithm is used, but others can be specified
-   * (such as mlpack::optimization::SGD).
+   * (such as ens::SGD).
    *
    * This will use the existing model parameters as a starting point for the
    * optimization. If this is not what you want, then you should access the
@@ -119,7 +119,7 @@ class FFN
    * @param predictors Input training variables.
    * @param responses Outputs results from input training variables.
    */
-  template<typename OptimizerType = mlpack::optimization::RMSProp>
+  template<typename OptimizerType = ens::RMSProp>
   void Train(arma::mat predictors, arma::mat responses);
 
   /**
diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp
index 3e5490e1ea..c8baea7698 100644
--- a/src/mlpack/methods/ann/rnn.hpp
+++ b/src/mlpack/methods/ann/rnn.hpp
@@ -24,7 +24,7 @@
 #include <mlpack/methods/ann/layer/layer_types.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -102,7 +102,7 @@ class RNN
   /**
    * Train the recurrent neural network on the given input data. By default, the
    * SGD optimization algorithm is used, but others can be specified
-   * (such as mlpack::optimization::RMSprop).
+   * (such as ens::RMSprop).
    *
    * This will use the existing model parameters as a starting point for the
    * optimization. If this is not what you want, then you should access the
@@ -122,7 +122,7 @@ class RNN
    * @param predictors Input training variables.
    * @param responses Outputs results from input training variables.
    */
-  template<typename OptimizerType = mlpack::optimization::StandardSGD>
+  template<typename OptimizerType = ens::StandardSGD>
   void Train(arma::cube predictors, arma::cube responses);
 
   /**
diff --git a/src/mlpack/methods/bias_svd/bias_svd.hpp b/src/mlpack/methods/bias_svd/bias_svd.hpp
index d39620382d..16208c67ad 100644
--- a/src/mlpack/methods/bias_svd/bias_svd.hpp
+++ b/src/mlpack/methods/bias_svd/bias_svd.hpp
@@ -15,7 +15,7 @@
 #define MLPACK_METHODS_BIAS_SVD_BIAS_SVD_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/cf/cf.hpp>
 
 #include "bias_svd_function.hpp"
@@ -32,7 +32,7 @@ namespace svd {
  * learning of large feature values by means of regularization.
  *
  * An example of how to use the interface is shown below:
- * 
+ *
  * @code
  * arma::mat data; // Rating data in the form of coordinate list.
  *
@@ -53,7 +53,7 @@ namespace svd {
  * @endcode
  *
  */
-template<typename OptimizerType = mlpack::optimization::StandardSGD>
+template<typename OptimizerType = ens::StandardSGD>
 class BiasSVD
 {
  public:
diff --git a/src/mlpack/methods/bias_svd/bias_svd_function.hpp b/src/mlpack/methods/bias_svd/bias_svd_function.hpp
index effd5bac9d..ed53ccfd8e 100644
--- a/src/mlpack/methods/bias_svd/bias_svd_function.hpp
+++ b/src/mlpack/methods/bias_svd/bias_svd_function.hpp
@@ -15,9 +15,7 @@
 #define MLPACK_METHODS_BIAS_SVD_BIAS_SVD_FUNCTION_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 namespace mlpack {
 namespace svd {
@@ -141,8 +139,7 @@ class BiasSVDFunction
 } // namespace svd
 } // namespace mlpack
 
-namespace mlpack {
-namespace optimization {
+namespace ens {
 
   /**
    * Template specialization for the SGD and parallel SGD optimizer. Used
@@ -162,8 +159,7 @@ namespace optimization {
       mlpack::svd::BiasSVDFunction<arma::mat>& function,
       arma::mat& parameters);
 
-} // namespace optimization
-} // namespace mlpack
+} // namespace ens
 
 #include "bias_svd_function_impl.hpp"
 
diff --git a/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp b/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp
index 7a153ecebd..e8ee71746c 100644
--- a/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp
+++ b/src/mlpack/methods/bias_svd/bias_svd_function_impl.hpp
@@ -182,8 +182,7 @@ void BiasSVDFunction<MatType>::Gradient(const arma::mat& parameters,
 } // namespace mlpack
 
 // Template specialization for the SGD optimizer.
-namespace mlpack {
-namespace optimization {
+namespace ens {
 
 template <>
 template <>
@@ -214,7 +213,7 @@ double StandardSGD::Optimize(
     if ((currentFunction % numFunctions) == 0)
     {
       const size_t epoch = i / numFunctions + 1;
-      Log::Info << "Epoch " << epoch << "; " << "objective "
+      mlpack::Log::Info << "Epoch " << epoch << "; " << "objective "
           << overallObjective << "." << std::endl;
 
       // Reset the counter variables.
@@ -295,21 +294,21 @@ inline double ParallelSGD<ExponentialBackoff>::Optimize(
     }
 
     // Output current objective function.
-    Log::Info << "Parallel SGD: iteration " << i << ", objective "
-      << overallObjective << "." << std::endl;
+    mlpack::Log::Info << "Parallel SGD: iteration " << i << ", objective "
+        << overallObjective << "." << std::endl;
 
     if (std::isnan(overallObjective) || std::isinf(overallObjective))
     {
-      Log::Warn << "Parallel SGD: converged to " << overallObjective
-        << "; terminating with failure. Try a smaller step size?"
-        << std::endl;
+      mlpack::Log::Warn << "Parallel SGD: converged to " << overallObjective
+          << "; terminating with failure. Try a smaller step size?"
+          << std::endl;
       return overallObjective;
     }
 
     if (std::abs(lastObjective - overallObjective) < tolerance)
     {
-      Log::Info << "SGD: minimized within tolerance " << tolerance << "; "
-        << "terminating optimization." << std::endl;
+      mlpack::Log::Info << "SGD: minimized within tolerance " << tolerance
+          << "; terminating optimization." << std::endl;
       return overallObjective;
     }
 
@@ -373,13 +372,12 @@ inline double ParallelSGD<ExponentialBackoff>::Optimize(
       }
     }
   }
-  Log::Info << "\n Parallel SGD terminated with objective : "
+  mlpack::Log::Info << "\n Parallel SGD terminated with objective : "
     << overallObjective << std::endl;
 
   return overallObjective;
 }
 
-} // namespace optimization
-} // namespace mlpack
+} // namespace ens
 
 #endif
diff --git a/src/mlpack/methods/bias_svd/bias_svd_impl.hpp b/src/mlpack/methods/bias_svd/bias_svd_impl.hpp
index 8c1ee5701b..587ac57850 100644
--- a/src/mlpack/methods/bias_svd/bias_svd_impl.hpp
+++ b/src/mlpack/methods/bias_svd/bias_svd_impl.hpp
@@ -44,7 +44,7 @@ void BiasSVD<OptimizerType>::Apply(const arma::mat& data,
 
   // Make the optimizer object using a BiasSVDFunction object.
   BiasSVDFunction<arma::mat> biasSVDFunc(data, rank, lambda);
-  mlpack::optimization::StandardSGD optimizer(alpha, batchSize,
+  ens::StandardSGD optimizer(alpha, batchSize,
       iterations * data.n_cols);
 
   // Get optimized parameters.
diff --git a/src/mlpack/methods/lmnn/lmnn.hpp b/src/mlpack/methods/lmnn/lmnn.hpp
index 7e169c888d..0cdc65a57b 100644
--- a/src/mlpack/methods/lmnn/lmnn.hpp
+++ b/src/mlpack/methods/lmnn/lmnn.hpp
@@ -14,7 +14,7 @@
 
 #include <mlpack/prereqs.hpp>
 #include <mlpack/core/metrics/lmetric.hpp>
-#include <mlpack/core/optimizers/adam/adam.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include "lmnn_function.hpp"
 
@@ -51,7 +51,7 @@ namespace lmnn /** Large Margin Nearest Neighbor. */ {
  * @tparam OptimizerType Optimizer to use for developing distance.
  */
 template<typename MetricType = metric::SquaredEuclideanDistance,
-         typename OptimizerType = optimization::AMSGrad>
+         typename OptimizerType = ens::AMSGrad>
 class LMNN
 {
  public:
diff --git a/src/mlpack/methods/lmnn/lmnn_function.hpp b/src/mlpack/methods/lmnn/lmnn_function.hpp
index 7f2e77a0a5..8e2d5bbcb2 100644
--- a/src/mlpack/methods/lmnn/lmnn_function.hpp
+++ b/src/mlpack/methods/lmnn/lmnn_function.hpp
@@ -40,7 +40,7 @@ namespace lmnn {
  * In addition to the standard Evaluate() and Gradient() functions which mlpack
  * optimizers use, overloads of Evaluate() and Gradient() are given which only
  * operate on one point in the dataset.  This is useful for optimizers like
- * stochastic gradient descent (see mlpack::optimization::SGD).
+ * stochastic gradient descent (see ens::SGD).
  */
 template<typename MetricType = metric::SquaredEuclideanDistance>
 class LMNNFunction
diff --git a/src/mlpack/methods/lmnn/lmnn_function_impl.hpp b/src/mlpack/methods/lmnn/lmnn_function_impl.hpp
index a7ae4d9f32..72b1b75a70 100644
--- a/src/mlpack/methods/lmnn/lmnn_function_impl.hpp
+++ b/src/mlpack/methods/lmnn/lmnn_function_impl.hpp
@@ -15,7 +15,6 @@
 #include "lmnn_function.hpp"
 
 #include <mlpack/core/math/make_alias.hpp>
-#include <mlpack/core/optimizers/function.hpp>
 
 namespace mlpack {
 namespace lmnn {
diff --git a/src/mlpack/methods/lmnn/lmnn_main.cpp b/src/mlpack/methods/lmnn/lmnn_main.cpp
index d4a7568f13..33e3195f86 100644
--- a/src/mlpack/methods/lmnn/lmnn_main.cpp
+++ b/src/mlpack/methods/lmnn/lmnn_main.cpp
@@ -19,9 +19,7 @@
 
 #include "lmnn.hpp"
 
-#include <mlpack/core/optimizers/bigbatch_sgd/bigbatch_sgd.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 // Define parameters.
 PROGRAM_INFO("Large Margin Nearest Neighbors (LMNN)",
@@ -163,7 +161,6 @@ PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 using namespace mlpack;
 using namespace mlpack::lmnn;
 using namespace mlpack::metric;
-using namespace mlpack::optimization;
 using namespace mlpack::util;
 using namespace std;
 
@@ -354,7 +351,7 @@ static void mlpackMain()
   }
   else if (optimizerType == "bbsgd")
   {
-    LMNN<LMetric<2>, BBS_BB> lmnn(data, labels, k);
+    LMNN<LMetric<2>, ens::BBS_BB> lmnn(data, labels, k);
     lmnn.Regularization() = regularization;
     lmnn.Range() = range;
     lmnn.Optimizer().StepSize() = stepSize;
@@ -369,7 +366,7 @@ static void mlpackMain()
   {
     // Using SGD is not recommended as the learning matrix can
     // diverge to inf causing serious memory problems.
-    LMNN<LMetric<2>, StandardSGD> lmnn(data, labels, k);
+    LMNN<LMetric<2>, ens::StandardSGD> lmnn(data, labels, k);
     lmnn.Regularization() = regularization;
     lmnn.Range() = range;
     lmnn.Optimizer().StepSize() = stepSize;
@@ -382,7 +379,7 @@ static void mlpackMain()
   }
   else if (optimizerType == "lbfgs")
   {
-    LMNN<LMetric<2>, L_BFGS> lmnn(data, labels, k);
+    LMNN<LMetric<2>, ens::L_BFGS> lmnn(data, labels, k);
     lmnn.Regularization() = regularization;
     lmnn.Range() = range;
     lmnn.Optimizer().MaxIterations() = maxIterations;
diff --git a/src/mlpack/methods/logistic_regression/logistic_regression.hpp b/src/mlpack/methods/logistic_regression/logistic_regression.hpp
index 11ea3ca41a..4e9782004e 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression.hpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression.hpp
@@ -15,7 +15,7 @@
 #define MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include "logistic_regression_function.hpp"
 
@@ -116,7 +116,7 @@ class LogisticRegression
   /**
    * Train the LogisticRegression model on the given input data.  By default,
    * the L-BFGS optimization algorithm is used, but others can be specified
-   * (such as mlpack::optimization::SGD).
+   * (such as ens::SGD).
    *
    * This will use the existing model parameters as a starting point for the
    * optimization.  If this is not what you want, then you should access the
@@ -127,7 +127,7 @@ class LogisticRegression
    * @param responses Outputs results from input training variables.
    * @return The final objective of the trained model (NaN or Inf on error)
    */
-  template<typename OptimizerType = mlpack::optimization::L_BFGS>
+  template<typename OptimizerType = ens::L_BFGS>
   double Train(const MatType& predictors,
              const arma::Row<size_t>& responses);
 
diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
index 2a0ed23a7b..14f946793c 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
@@ -15,12 +15,11 @@
 
 #include "logistic_regression.hpp"
 
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 using namespace std;
 using namespace mlpack;
 using namespace mlpack::regression;
-using namespace mlpack::optimization;
 using namespace mlpack::util;
 
 PROGRAM_INFO("L2-regularized Logistic Regression and Prediction",
@@ -279,7 +278,7 @@ static void mlpackMain()
 
     if (optimizerType == "sgd")
     {
-      SGD<> sgdOpt;
+      ens::SGD<> sgdOpt;
       sgdOpt.MaxIterations() = maxIterations;
       sgdOpt.Tolerance() = tolerance;
       sgdOpt.StepSize() = stepSize;
@@ -291,7 +290,7 @@ static void mlpackMain()
     }
     else if (optimizerType == "lbfgs")
     {
-      L_BFGS lbfgsOpt;
+      ens::L_BFGS lbfgsOpt;
       lbfgsOpt.MaxIterations() = maxIterations;
       lbfgsOpt.MinGradientNorm() = tolerance;
       Log::Info << "Training model with L-BFGS optimizer." << endl;
diff --git a/src/mlpack/methods/matrix_completion/matrix_completion.hpp b/src/mlpack/methods/matrix_completion/matrix_completion.hpp
index 6b0646080e..66c777b727 100644
--- a/src/mlpack/methods/matrix_completion/matrix_completion.hpp
+++ b/src/mlpack/methods/matrix_completion/matrix_completion.hpp
@@ -13,8 +13,7 @@
 #ifndef MLPACK_METHODS_MATRIX_COMPLETION_MATRIX_COMPLETION_HPP
 #define MLPACK_METHODS_MATRIX_COMPLETION_MATRIX_COMPLETION_HPP
 
-#include <mlpack/core/optimizers/sdp/sdp.hpp>
-#include <mlpack/core/optimizers/sdp/lrsdp.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 namespace mlpack {
 namespace matrix_completion {
@@ -112,12 +111,12 @@ class MatrixCompletion
   void Recover(arma::mat& recovered);
 
   //! Return the underlying SDP.
-  const optimization::LRSDP<optimization::SDP<arma::sp_mat>>& Sdp() const
+  const ens::LRSDP<ens::SDP<arma::sp_mat>>& Sdp() const
   {
     return sdp;
   }
   //! Modify the underlying SDP.
-  optimization::LRSDP<optimization::SDP<arma::sp_mat>>& Sdp() { return sdp; }
+  ens::LRSDP<ens::SDP<arma::sp_mat>>& Sdp() { return sdp; }
 
  private:
   //! Number of rows in original matrix.
@@ -130,7 +129,7 @@ class MatrixCompletion
   arma::mat values;
 
   //! The underlying SDP to be solved.
-  optimization::LRSDP<optimization::SDP<arma::sp_mat>> sdp;
+  ens::LRSDP<ens::SDP<arma::sp_mat>> sdp;
 
   //! Validate the input matrices.
   void CheckValues();
diff --git a/src/mlpack/methods/nca/nca.hpp b/src/mlpack/methods/nca/nca.hpp
index 22b8bce24f..75bf987890 100644
--- a/src/mlpack/methods/nca/nca.hpp
+++ b/src/mlpack/methods/nca/nca.hpp
@@ -14,7 +14,7 @@
 
 #include <mlpack/prereqs.hpp>
 #include <mlpack/core/metrics/lmetric.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include "nca_softmax_error_function.hpp"
 
@@ -45,7 +45,7 @@ namespace nca /** Neighborhood Components Analysis. */ {
  * @endcode
  */
 template<typename MetricType = metric::SquaredEuclideanDistance,
-         typename OptimizerType = optimization::StandardSGD>
+         typename OptimizerType = ens::StandardSGD>
 class NCA
 {
  public:
diff --git a/src/mlpack/methods/nca/nca_main.cpp b/src/mlpack/methods/nca/nca_main.cpp
index b1dd431aea..0cc9d3bfeb 100644
--- a/src/mlpack/methods/nca/nca_main.cpp
+++ b/src/mlpack/methods/nca/nca_main.cpp
@@ -18,7 +18,7 @@
 
 #include "nca.hpp"
 
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 // Define parameters.
 PROGRAM_INFO("Neighborhood Components Analysis (NCA)",
@@ -120,7 +120,6 @@ PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 using namespace mlpack;
 using namespace mlpack::nca;
 using namespace mlpack::metric;
-using namespace mlpack::optimization;
 using namespace mlpack::util;
 using namespace std;
 
@@ -232,7 +231,7 @@ static void mlpackMain()
   }
   else if (optimizerType == "lbfgs")
   {
-    NCA<LMetric<2>, L_BFGS> nca(data, labels);
+    NCA<LMetric<2>, ens::L_BFGS> nca(data, labels);
     nca.Optimizer().NumBasis() = numBasis;
     nca.Optimizer().MaxIterations() = maxIterations;
     nca.Optimizer().ArmijoConstant() = armijoConstant;
diff --git a/src/mlpack/methods/regularized_svd/regularized_svd.hpp b/src/mlpack/methods/regularized_svd/regularized_svd.hpp
index d329f77e14..cf6168a094 100644
--- a/src/mlpack/methods/regularized_svd/regularized_svd.hpp
+++ b/src/mlpack/methods/regularized_svd/regularized_svd.hpp
@@ -14,7 +14,7 @@
 #define MLPACK_METHODS_REGULARIZED_SVD_REGULARIZED_SVD_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/cf/cf.hpp>
 
 #include "regularized_svd_function.hpp"
@@ -54,7 +54,7 @@ namespace svd {
  * rSVD.Apply(data, rank, u, v);
  * @endcode
  */
-template<typename OptimizerType = mlpack::optimization::StandardSGD>
+template<typename OptimizerType = ens::StandardSGD>
 class RegularizedSVD
 {
  public:
diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp
index d46a7e2f6f..5ea4dad3c9 100644
--- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp
+++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp
@@ -14,9 +14,7 @@
 #define MLPACK_METHODS_REGULARIZED_SVD_REGULARIZED_FUNCTION_SVD_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 namespace mlpack {
 namespace svd {
@@ -135,8 +133,7 @@ class RegularizedSVDFunction
 } // namespace svd
 } // namespace mlpack
 
-namespace mlpack {
-namespace optimization {
+namespace ens {
 
   /**
    * Template specialization for the SGD and parallel SGD optimizer. Used
@@ -156,8 +153,7 @@ namespace optimization {
       mlpack::svd::RegularizedSVDFunction<arma::mat>& function,
       arma::mat& parameters);
 
-} // namespace optimization
-} // namespace mlpack
+} // namespace ens
 
 #include "regularized_svd_function_impl.hpp"
 
diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp
index 551992f8d8..bab98bf287 100644
--- a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp
+++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp
@@ -154,8 +154,7 @@ void RegularizedSVDFunction<MatType>::Gradient(const arma::mat& parameters,
 } // namespace mlpack
 
 // Template specialization for the SGD optimizer.
-namespace mlpack {
-namespace optimization {
+namespace ens {
 
 template <>
 template <>
@@ -183,7 +182,7 @@ double StandardSGD::Optimize(
     if ((currentFunction % numFunctions) == 0)
     {
       const size_t epoch = i / numFunctions + 1;
-      Log::Info << "Epoch " << epoch << "; " << "objective "
+      mlpack::Log::Info << "Epoch " << epoch << "; " << "objective "
           << overallObjective << "." << std::endl;
 
       // Reset the counter variables.
@@ -250,21 +249,21 @@ inline double ParallelSGD<ExponentialBackoff>::Optimize(
     }
 
     // Output current objective function.
-    Log::Info << "Parallel SGD: iteration " << i << ", objective "
-      << overallObjective << "." << std::endl;
+    mlpack::Log::Info << "Parallel SGD: iteration " << i << ", objective "
+        << overallObjective << "." << std::endl;
 
     if (std::isnan(overallObjective) || std::isinf(overallObjective))
     {
-      Log::Warn << "Parallel SGD: converged to " << overallObjective
-        << "; terminating with failure. Try a smaller step size?"
-        << std::endl;
+      mlpack::Log::Warn << "Parallel SGD: converged to " << overallObjective
+          << "; terminating with failure. Try a smaller step size?"
+          << std::endl;
       return overallObjective;
     }
 
     if (std::abs(lastObjective - overallObjective) < tolerance)
     {
-      Log::Info << "SGD: minimized within tolerance " << tolerance << "; "
-        << "terminating optimization." << std::endl;
+      mlpack::Log::Info << "SGD: minimized within tolerance " << tolerance
+          << "; terminating optimization." << std::endl;
       return overallObjective;
     }
 
@@ -318,13 +317,12 @@ inline double ParallelSGD<ExponentialBackoff>::Optimize(
       }
     }
   }
-  Log::Info << "\n Parallel SGD terminated with objective : "
-    << overallObjective << std::endl;
+  mlpack::Log::Info << "\n Parallel SGD terminated with objective : "
+      << overallObjective << std::endl;
 
   return overallObjective;
 }
 
-} // namespace optimization
-} // namespace mlpack
+} // namespace ens
 
 #endif
diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp
index 32e4b25ee3..622d19e35e 100644
--- a/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp
+++ b/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp
@@ -41,7 +41,7 @@ void RegularizedSVD<OptimizerType>::Apply(const arma::mat& data,
 
   // Make the optimizer object using a RegularizedSVDFunction object.
   RegularizedSVDFunction<arma::mat> rSVDFunc(data, rank, lambda);
-  mlpack::optimization::StandardSGD optimizer(alpha, batchSize,
+  ens::StandardSGD optimizer(alpha, batchSize,
       iterations * data.n_cols);
 
   // Get optimized parameters.
diff --git a/src/mlpack/methods/softmax_regression/softmax_regression.hpp b/src/mlpack/methods/softmax_regression/softmax_regression.hpp
index 56c7b1643b..f4679eec5d 100644
--- a/src/mlpack/methods/softmax_regression/softmax_regression.hpp
+++ b/src/mlpack/methods/softmax_regression/softmax_regression.hpp
@@ -13,7 +13,7 @@
 #define MLPACK_METHODS_SOFTMAX_REGRESSION_SOFTMAX_REGRESSION_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include "softmax_regression_function.hpp"
 
@@ -90,7 +90,7 @@ class SoftmaxRegression
    * @param lambda L2-regularization constant.
    * @param fitIntercept add intercept term or not.
    */
-  template<typename OptimizerType = mlpack::optimization::L_BFGS>
+  template<typename OptimizerType = ens::L_BFGS>
   SoftmaxRegression(const arma::mat& data,
                     const arma::Row<size_t>& labels,
                     const size_t numClasses,
@@ -165,7 +165,7 @@ class SoftmaxRegression
    * @param optimizer Desired optimizer.
    * @return Objective value of the final point.
    */
-  template<typename OptimizerType = mlpack::optimization::L_BFGS>
+  template<typename OptimizerType = ens::L_BFGS>
   double Train(const arma::mat& data,
                const arma::Row<size_t>& labels,
                const size_t numClasses,
diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp
index 97b54a2a54..855be1326a 100644
--- a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp
+++ b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp
@@ -13,7 +13,7 @@
 #include <mlpack/core/util/mlpack_main.hpp>
 
 #include <mlpack/methods/softmax_regression/softmax_regression.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <memory>
 #include <set>
@@ -256,7 +256,7 @@ Model* TrainSoftmax(const size_t maxIterations)
     const bool intercept = CLI::HasParam("no_intercept") ? false : true;
 
     const size_t numBasis = 5;
-    optimization::L_BFGS optimizer(numBasis, maxIterations);
+    ens::L_BFGS optimizer(numBasis, maxIterations);
     sm = new Model(trainData, trainLabels, numClasses,
         CLI::GetParam<double>("lambda"), intercept, std::move(optimizer));
   }
diff --git a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp
index 3999d6be00..4ce98333ca 100644
--- a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp
+++ b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder.hpp
@@ -13,7 +13,7 @@
 #define MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include "sparse_autoencoder_function.hpp"
 
@@ -77,7 +77,7 @@ class SparseAutoencoder
    * @param beta KL divergence parameter.
    * @param rho Sparsity parameter.
    */
-  template<typename OptimizerType = mlpack::optimization::L_BFGS>
+  template<typename OptimizerType = ens::L_BFGS>
   SparseAutoencoder(const arma::mat& data,
                     const size_t visibleSize,
                     const size_t hiddenSize,
diff --git a/src/mlpack/methods/svdplusplus/svdplusplus.hpp b/src/mlpack/methods/svdplusplus/svdplusplus.hpp
index 48f0c7e8be..83fac13472 100644
--- a/src/mlpack/methods/svdplusplus/svdplusplus.hpp
+++ b/src/mlpack/methods/svdplusplus/svdplusplus.hpp
@@ -15,7 +15,7 @@
 #define MLPACK_METHODS_SVDPLUSPLUS_SVDPLUSPLUS_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/cf/cf.hpp>
 
 #include "svdplusplus_function.hpp"
@@ -30,7 +30,7 @@ namespace svd {
  * vectors, user/item bias, and item vectors with regard to implicit feedback.
  * Parameters are optmized by Stochastic Gradient Desent(SGD). The updates also
  * penalize the learning of large feature values by means of regularization.
- * 
+ *
  * For more information, see the following paper:
  *
  * @inproceedings{koren2008factorization,
@@ -69,7 +69,7 @@ namespace svd {
  * svdPP.Apply(data, implicitData, rank, u, v, p, q, y);
  * @endcode
  */
-template<typename OptimizerType = mlpack::optimization::StandardSGD>
+template<typename OptimizerType = ens::StandardSGD>
 class SVDPlusPlus
 {
  public:
@@ -106,7 +106,7 @@ class SVDPlusPlus
              arma::vec& p,
              arma::vec& q,
              arma::mat& y);
-  
+
   /**
    * Trains the model and obtains user/item matrices, user/item bias, and
    * item implicit matrix. Whether a user rates an item is used as implicit
@@ -128,7 +128,7 @@ class SVDPlusPlus
              arma::vec& p,
              arma::vec& q,
              arma::mat& y);
-  
+
   /**
    * Converts the User, Item matrix of implicit data to Item-User Table.
    */
diff --git a/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp b/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp
index 209fc229bc..b5ebc78d19 100644
--- a/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp
+++ b/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp
@@ -15,9 +15,7 @@
 #define MLPACK_METHODS_SVDPLUSPLUS_SVDPLUSPLUS_FUNCTION_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 namespace mlpack {
 namespace svd {
@@ -149,8 +147,7 @@ class SVDPlusPlusFunction
 } // namespace svd
 } // namespace mlpack
 
-namespace mlpack {
-namespace optimization {
+namespace ens {
 
   /**
    * Template specialization for the SGD and parallel SGD optimizer. Used
@@ -170,8 +167,7 @@ namespace optimization {
       mlpack::svd::SVDPlusPlusFunction<arma::mat>& function,
       arma::mat& parameters);
 
-} // namespace optimization
-} // namespace mlpack
+} // namespace ens
 
 #include "svdplusplus_function_impl.hpp"
 
diff --git a/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp b/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp
index 57b179fe2e..4e8097e0ee 100644
--- a/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp
+++ b/src/mlpack/methods/svdplusplus/svdplusplus_function_impl.hpp
@@ -104,7 +104,7 @@ double SVDPlusPlusFunction<MatType>::Evaluate(const arma::mat& parameters,
     {
       userVec += parameters.col(implicitStart + it.row()).subvec(0, rank - 1);
       if (implicitVecsNormSquare(it.row()) < 0)
-      { 
+      {
         implicitVecsNormSquare(it.row()) = arma::dot(
             parameters.col(implicitStart + it.row()).subvec(0, rank - 1),
             parameters.col(implicitStart + it.row()).subvec(0, rank - 1));
@@ -288,8 +288,7 @@ void SVDPlusPlusFunction<MatType>::Gradient(const arma::mat& parameters,
 } // namespace mlpack
 
 // Template specialization for the SGD optimizer.
-namespace mlpack {
-namespace optimization {
+namespace ens {
 
 template <>
 template <>
@@ -324,7 +323,7 @@ double StandardSGD::Optimize(
     if ((currentFunction % numFunctions) == 0)
     {
       const size_t epoch = i / numFunctions + 1;
-      Log::Info << "Epoch " << epoch << "; " << "objective "
+      mlpack::Log::Info << "Epoch " << epoch << "; " << "objective "
           << overallObjective << "." << std::endl;
 
       // Reset the counter variables.
@@ -431,21 +430,21 @@ inline double ParallelSGD<ExponentialBackoff>::Optimize(
     }
 
     // Output current objective function.
-    Log::Info << "Parallel SGD: iteration " << i << ", objective "
-      << overallObjective << "." << std::endl;
+    mlpack::Log::Info << "Parallel SGD: iteration " << i << ", objective "
+        << overallObjective << "." << std::endl;
 
     if (std::isnan(overallObjective) || std::isinf(overallObjective))
     {
-      Log::Warn << "Parallel SGD: converged to " << overallObjective
-        << "; terminating with failure. Try a smaller step size?"
-        << std::endl;
+      mlpack::Log::Warn << "Parallel SGD: converged to " << overallObjective
+          << "; terminating with failure. Try a smaller step size?"
+          << std::endl;
       return overallObjective;
     }
 
     if (std::abs(lastObjective - overallObjective) < tolerance)
     {
-      Log::Info << "SGD: minimized within tolerance " << tolerance << "; "
-        << "terminating optimization." << std::endl;
+      mlpack::Log::Info << "SGD: minimized within tolerance " << tolerance
+          << "; terminating optimization." << std::endl;
       return overallObjective;
     }
 
@@ -548,13 +547,12 @@ inline double ParallelSGD<ExponentialBackoff>::Optimize(
       }
     }
   }
-  Log::Info << "\n Parallel SGD terminated with objective : "
-    << overallObjective << std::endl;
+  mlpack::Log::Info << "\n Parallel SGD terminated with objective : "
+      << overallObjective << std::endl;
 
   return overallObjective;
 }
 
-} // namespace optimization
-} // namespace mlpack
+} // namespace ens
 
 #endif
diff --git a/src/mlpack/methods/svdplusplus/svdplusplus_impl.hpp b/src/mlpack/methods/svdplusplus/svdplusplus_impl.hpp
index bdc532b14c..f4fbd75363 100644
--- a/src/mlpack/methods/svdplusplus/svdplusplus_impl.hpp
+++ b/src/mlpack/methods/svdplusplus/svdplusplus_impl.hpp
@@ -43,14 +43,14 @@ void SVDPlusPlus<OptimizerType>::Apply(const arma::mat& data,
   const int batchSize = 1;
   Log::Warn << "The batch size for optimizing SVDPlusPlus is 1."
       << std::endl;
-  
+
   // Converts implicitData to the form of sparse matrix.
   arma::sp_mat cleanedData;
   CleanData(implicitData, cleanedData, data);
 
   // Make the optimizer object using a SVDPlusPlusFunction object.
   SVDPlusPlusFunction<arma::mat> svdPPFunc(data, cleanedData, rank, lambda);
-  mlpack::optimization::StandardSGD optimizer(alpha, batchSize,
+  ens::StandardSGD optimizer(alpha, batchSize,
       iterations * data.n_cols);
 
   // Get optimized parameters.
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 770623dcaa..c763d4dbd8 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -765,7 +765,7 @@ BOOST_AUTO_TEST_CASE(LSTMRrhoTest)
   modelB.Add<LSTM<> >(10, 3);
   modelB.Add<LogSoftMax<> >();
 
-  optimization::StandardSGD opt(0.1, 1, 5, -100, false);
+  ens::StandardSGD opt(0.1, 1, 5, -100, false);
   modelA.Train(input, target, opt);
   modelB.Train(input, target, opt);
 
@@ -846,7 +846,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMRrhoTest)
   modelB.Add<FastLSTM<> >(10, 3);
   modelB.Add<LogSoftMax<> >();
 
-  optimization::StandardSGD opt(0.1, 1, 5, -100, false);
+  ens::StandardSGD opt(0.1, 1, 5, -100, false);
   modelA.Train(input, target, opt);
   modelB.Train(input, target, opt);
 
diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/async_learning_test.cpp
index 11cbe66900..7958cd0234 100644
--- a/src/mlpack/tests/async_learning_test.cpp
+++ b/src/mlpack/tests/async_learning_test.cpp
@@ -18,7 +18,7 @@
 #include <mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp>
 #include <mlpack/methods/reinforcement_learning/async_learning.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
-#include <mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/aggregated_policy.hpp>
 #include <mlpack/methods/reinforcement_learning/training_config.hpp>
@@ -28,7 +28,6 @@
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
 using namespace mlpack::rl;
 
 BOOST_AUTO_TEST_SUITE(AsyncLearningTest);
@@ -68,7 +67,8 @@ BOOST_AUTO_TEST_CASE(OneStepQLearningTest)
   config.StepLimit() = 200;
   config.TargetNetworkSyncInterval() = 200;
 
-  OneStepQLearning<CartPole, decltype(model), VanillaUpdate, decltype(policy)>
+  OneStepQLearning<
+      CartPole, decltype(model), ens::VanillaUpdate, decltype(policy)>
       agent(std::move(config), std::move(model), std::move(policy));
 
   arma::vec rewards(20, arma::fill::zeros);
@@ -130,7 +130,7 @@ BOOST_AUTO_TEST_CASE(OneStepSarsaTest)
   config.StepLimit() = 200;
   config.TargetNetworkSyncInterval() = 200;
 
-  OneStepSarsa<CartPole, decltype(model), VanillaUpdate, decltype(policy)>
+  OneStepSarsa<CartPole, decltype(model), ens::VanillaUpdate, decltype(policy)>
       agent(std::move(config), std::move(model), std::move(policy));
 
   arma::vec rewards(20, arma::fill::zeros);
@@ -192,7 +192,8 @@ BOOST_AUTO_TEST_CASE(NStepQLearningTest)
   config.StepLimit() = 200;
   config.TargetNetworkSyncInterval() = 200;
 
-  NStepQLearning<CartPole, decltype(model), VanillaUpdate, decltype(policy)>
+  NStepQLearning<
+      CartPole, decltype(model), ens::VanillaUpdate, decltype(policy)>
       agent(std::move(config), std::move(model), std::move(policy));
 
   arma::vec rewards(20, arma::fill::zeros);
diff --git a/src/mlpack/tests/bias_svd_test.cpp b/src/mlpack/tests/bias_svd_test.cpp
index ad78591d74..d4fe3b7841 100644
--- a/src/mlpack/tests/bias_svd_test.cpp
+++ b/src/mlpack/tests/bias_svd_test.cpp
@@ -12,15 +12,13 @@
  */
 #include <mlpack/core.hpp>
 #include <mlpack/methods/bias_svd/bias_svd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
 
 using namespace mlpack;
 using namespace mlpack::svd;
-using namespace mlpack::optimization;
 
 BOOST_AUTO_TEST_SUITE(BiasSVDTest);
 
@@ -276,7 +274,7 @@ BOOST_AUTO_TEST_CASE(BiasSVDFunctionOptimize)
 
   // Make the Bias SVD function and the optimizer.
   BiasSVDFunction<arma::mat> biasSVDFunc(data, rank, lambda);
-  mlpack::optimization::StandardSGD optimizer(alpha, iterations * numRatings);
+  ens::StandardSGD optimizer(alpha, iterations * numRatings);
 
   // Obtain optimized parameters after training.
   arma::mat optParameters = arma::randu(rank + 1, numUsers + numItems);
@@ -349,7 +347,7 @@ BOOST_AUTO_TEST_CASE(BiasSVDFunctionParallelOptimize)
 
   // Iterate till convergence.
   // The threadShareSize is chosen such that each function gets optimized.
-  ParallelSGD<ConstantStep> optimizer(0,
+  ens::ParallelSGD<ConstantStep> optimizer(0,
       std::ceil((float) biasSVDFunc.NumFunctions() / omp_get_max_threads()), 1e-5,
       true, decayPolicy);
 
diff --git a/src/mlpack/tests/convolutional_network_test.cpp b/src/mlpack/tests/convolutional_network_test.cpp
index 1f7c07078f..fe138afa19 100644
--- a/src/mlpack/tests/convolutional_network_test.cpp
+++ b/src/mlpack/tests/convolutional_network_test.cpp
@@ -12,7 +12,7 @@
  */
 #include <mlpack/core.hpp>
 
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/ffn.hpp>
 #include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
@@ -22,7 +22,6 @@
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
 
 BOOST_AUTO_TEST_SUITE(ConvolutionalNetworkTest);
 
@@ -97,7 +96,7 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest)
     model.Add<LogSoftMax<> >();
 
     // Train for only 8 epochs.
-    RMSProp opt(0.001, 1, 0.88, 1e-8, 8 * nPoints, -1);
+    ens::RMSProp opt(0.001, 1, 0.88, 1e-8, 8 * nPoints, -1);
 
     model.Train(X, Y, opt);
 
diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp
index 4878db37a5..1216e2d8b4 100644
--- a/src/mlpack/tests/cv_test.cpp
+++ b/src/mlpack/tests/cv_test.cpp
@@ -19,7 +19,7 @@
 #include <mlpack/core/cv/metrics/recall.hpp>
 #include <mlpack/core/cv/simple_cv.hpp>
 #include <mlpack/core/cv/k_fold_cv.hpp>
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/ann/ffn.hpp>
 #include <mlpack/methods/ann/init_rules/const_init.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
@@ -40,7 +40,6 @@ using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::cv;
 using namespace mlpack::naive_bayes;
-using namespace mlpack::optimization;
 using namespace mlpack::regression;
 using namespace mlpack::tree;
 
@@ -157,7 +156,7 @@ BOOST_AUTO_TEST_CASE(MSEMatResponsesTest)
   ffn.Add<Linear<>>(1, 2);
   ffn.Add<IdentityLayer<>>();
 
-  RMSProp opt(0.2);
+  ens::RMSProp opt(0.2);
   opt.BatchSize() = 1;
   opt.Shuffle() = false;
   ffn.Train(data, trainingResponses, opt);
diff --git a/src/mlpack/tests/dcgan_test.cpp b/src/mlpack/tests/dcgan_test.cpp
index 5779d3bf42..4044ff384d 100644
--- a/src/mlpack/tests/dcgan_test.cpp
+++ b/src/mlpack/tests/dcgan_test.cpp
@@ -17,7 +17,7 @@
 #include <mlpack/methods/ann/ffn.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/softmax_regression/softmax_regression.hpp>
-#include <mlpack/core/optimizers/adam/adam.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -25,7 +25,6 @@
 using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::math;
-using namespace mlpack::optimization;
 using namespace mlpack::regression;
 using namespace std::placeholders;
 
@@ -116,7 +115,7 @@ BOOST_AUTO_TEST_CASE(DCGANMNISTTest)
 
   // Create DCGAN
   GaussianInitialization gaussian(0, 1);
-  Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
+  ens::Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
       tolerance, shuffle);
   std::function<double()> noiseFunction = [] () {
       return math::RandNormal(0, 1);};
@@ -242,7 +241,7 @@ BOOST_AUTO_TEST_CASE(DCGANCelebATest)
 
   // Create DCGAN
   GaussianInitialization gaussian(0, 1);
-  Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
+  ens::Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
       tolerance, shuffle);
   std::function<double()> noiseFunction = [] () {
       return math::RandNormal(0, 1);};
diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp
index 0d691737ea..060a65e6b5 100644
--- a/src/mlpack/tests/feedforward_network_test.cpp
+++ b/src/mlpack/tests/feedforward_network_test.cpp
@@ -12,8 +12,7 @@
  */
 #include <mlpack/core.hpp>
 
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
-#include <mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
 #include <mlpack/methods/ann/ffn.hpp>
@@ -25,7 +24,6 @@
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
 
 BOOST_AUTO_TEST_SUITE(FeedForwardNetworkTest);
 
@@ -71,7 +69,7 @@ void BuildVanillaNetwork(MatType& trainData,
   model.Add<LogSoftMax<> >();
 
   // RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
-  RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
+  ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
   model.Train(trainData, trainLabels, opt);
 
   MatType predictionTemp;
@@ -157,7 +155,7 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest)
   model.Add<Linear<> >(50, 10);
   model.Add<LogSoftMax<> >();
 
-  VanillaUpdate opt;
+  ens::VanillaUpdate opt;
   model.ResetParameters();
   opt.Initialize(model.Parameters().n_rows, model.Parameters().n_cols);
   double stepSize = 0.01;
@@ -256,7 +254,7 @@ void BuildDropoutNetwork(MatType& trainData,
   model.Add<Linear<> >(hiddenLayerSize, outputSize);
   model.Add<LogSoftMax<> >();
 
-  RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
+  ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
 
   model.Train(trainData, trainLabels, opt);
 
@@ -368,7 +366,7 @@ void BuildDropConnectNetwork(MatType& trainData,
   model.Add<DropConnect<> >(hiddenLayerSize, outputSize);
   model.Add<LogSoftMax<> >();
 
-  RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
+  ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1);
 
   model.Train(trainData, trainLabels, opt);
 
@@ -480,7 +478,7 @@ BOOST_AUTO_TEST_CASE(SerializationTest)
   model.Add<Linear<> >(8, 3);
   model.Add<LogSoftMax<> >();
 
-  RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1);
+  ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1);
 
   model.Train(trainData, trainLabels, opt);
 
@@ -525,7 +523,7 @@ BOOST_AUTO_TEST_CASE(CustomLayerTest)
   model.Add<Linear<> >(8, 3);
   model.Add<LogSoftMax<> >();
 
-  RMSProp opt(0.01, 32, 0.88, 1e-8, 15, -1);
+  ens::RMSProp opt(0.01, 32, 0.88, 1e-8, 15, -1);
   model.Train(trainData, trainLabels, opt);
 
   arma::mat predictionTemp;
diff --git a/src/mlpack/tests/gan_test.cpp b/src/mlpack/tests/gan_test.cpp
index dce4be77bb..cb5f849f7c 100644
--- a/src/mlpack/tests/gan_test.cpp
+++ b/src/mlpack/tests/gan_test.cpp
@@ -17,7 +17,7 @@
 #include <mlpack/methods/ann/gan/gan.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/softmax_regression/softmax_regression.hpp>
-#include <mlpack/core/optimizers/adam/adam.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -25,7 +25,6 @@
 using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::math;
-using namespace mlpack::optimization;
 using namespace mlpack::regression;
 using namespace std::placeholders;
 
@@ -205,7 +204,7 @@ BOOST_AUTO_TEST_CASE(GANMNISTTest)
 
   // Create GAN
   GaussianInitialization gaussian(0, 1);
-  Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
+  ens::Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
       tolerance, shuffle);
   std::function<double()> noiseFunction = [] () {
       return math::RandNormal(0, 1);};
diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp
index eec9944dc4..e7283e37cc 100644
--- a/src/mlpack/tests/hpt_test.cpp
+++ b/src/mlpack/tests/hpt_test.cpp
@@ -15,8 +15,9 @@
 #include <mlpack/core/hpt/cv_function.hpp>
 #include <mlpack/core/hpt/fixed.hpp>
 #include <mlpack/core/hpt/hpt.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
+#include <mlpack/core/optimizers/gradient_descent/gradient_descent.hpp>
 #include <mlpack/core/optimizers/grid_search/grid_search.hpp>
-#include <mlpack/core/optimizers/gradient_descent/gradient_descent.cpp>
 #include <mlpack/methods/lars/lars.hpp>
 #include <mlpack/methods/logistic_regression/logistic_regression.hpp>
 
@@ -320,8 +321,8 @@ BOOST_AUTO_TEST_CASE(HPTGradientDescentTest)
   // We pass LARS just because some ML algorithm should be passed. We pass MSE
   // to tell HyperParameterTuner that the objective function (QuadraticFunction)
   // should be minimized.
-  HyperParameterTuner<LARS, MSE, QuadraticFunction, GradientDescent>
-      hpt(a, b, c, d, xMin, yMin, zMin);
+  HyperParameterTuner<LARS, MSE, QuadraticFunction,
+      mlpack::optimization::GradientDescent> hpt(a, b, c, d, xMin, yMin, zMin);
 
   // Setting GradientDescent to find more close solution to the optimal one.
   hpt.Optimizer().StepSize() = 0.1;
diff --git a/src/mlpack/tests/ksinit_test.cpp b/src/mlpack/tests/ksinit_test.cpp
index 2b7bb2da3d..1554955663 100644
--- a/src/mlpack/tests/ksinit_test.cpp
+++ b/src/mlpack/tests/ksinit_test.cpp
@@ -15,7 +15,7 @@
 
 #include <mlpack/core.hpp>
 
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
 #include <mlpack/methods/ann/ffn.hpp>
@@ -23,7 +23,6 @@
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
 
 BOOST_AUTO_TEST_SUITE(KSInitialization);
 
@@ -86,7 +85,7 @@ void BuildVanillaNetwork(MatType& trainData,
   model.Add<LeakyReLU<> >();
   model.Add<Linear<> >(hiddenLayerSize, outputSize);
 
-  RMSProp opt(0.01, 1, 0.88, 1e-8, maxEpochs * trainData.n_cols, 1e-18);
+  ens::RMSProp opt(0.01, 1, 0.88, 1e-8, maxEpochs * trainData.n_cols, 1e-18);
 
   model.Train(trainData, trainLabels, opt);
 
diff --git a/src/mlpack/tests/lmnn_test.cpp b/src/mlpack/tests/lmnn_test.cpp
index 288257ae8d..614d54196b 100644
--- a/src/mlpack/tests/lmnn_test.cpp
+++ b/src/mlpack/tests/lmnn_test.cpp
@@ -15,9 +15,8 @@
 #include <mlpack/core.hpp>
 #include <mlpack/core/metrics/lmetric.hpp>
 #include <mlpack/methods/lmnn/lmnn.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/neighbor_search/neighbor_search.hpp>
-#include <mlpack/core/optimizers/bigbatch_sgd/bigbatch_sgd.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -25,7 +24,7 @@
 using namespace mlpack;
 using namespace mlpack::metric;
 using namespace mlpack::lmnn;
-using namespace mlpack::optimization;
+using namespace ens;
 
 
 BOOST_AUTO_TEST_SUITE(LMNNTest);
diff --git a/src/mlpack/tests/logistic_regression_test.cpp b/src/mlpack/tests/logistic_regression_test.cpp
index 38c5e94634..af3b1be285 100644
--- a/src/mlpack/tests/logistic_regression_test.cpp
+++ b/src/mlpack/tests/logistic_regression_test.cpp
@@ -12,14 +12,13 @@
  */
 #include <mlpack/core.hpp>
 #include <mlpack/methods/logistic_regression/logistic_regression.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
 
 using namespace mlpack;
 using namespace mlpack::regression;
-using namespace mlpack::optimization;
 using namespace mlpack::distribution;
 
 BOOST_AUTO_TEST_SUITE(LogisticRegressionTest);
@@ -515,7 +514,7 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionSGDSimpleTest)
 
   // Create a logistic regression object using a custom SGD object with a much
   // smaller tolerance.
-  StandardSGD sgd(0.005, 1, 500000, 1e-10);
+  ens::StandardSGD sgd(0.005, 1, 500000, 1e-10);
   LogisticRegression<> lr(data, responses, sgd, 0.001);
 
   // Test sigmoid function.
@@ -563,7 +562,7 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionSGDRegularizationSimpleTest)
 
   // Create a logistic regression object using custom SGD with a much smaller
   // tolerance.
-  StandardSGD sgd(0.005, 32, 500000, 1e-10);
+  ens::StandardSGD sgd(0.005, 32, 500000, 1e-10);
   LogisticRegression<> lr(data, responses, sgd, 0.001);
 
   // Test sigmoid function.
@@ -601,7 +600,7 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionLBFGSGaussianTest)
 
   // Now train a logistic regression object on it.
   LogisticRegression<> lr(data.n_rows, 0.5);
-  lr.Train<L_BFGS>(data, responses);
+  lr.Train<ens::L_BFGS>(data, responses);
 
   // Ensure that the error is close to zero.
   const double acc = lr.ComputeAccuracy(data, responses);
@@ -648,7 +647,7 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionSGDGaussianTest)
 
   // Now train a logistic regression object on it.
   LogisticRegression<> lr(data.n_rows, 0.5);
-  lr.Train<StandardSGD>(data, responses);
+  lr.Train<ens::StandardSGD>(data, responses);
 
   // Ensure that the error is close to zero.
   const double acc = lr.ComputeAccuracy(data, responses);
@@ -684,7 +683,7 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionInstantiatedOptimizer)
   arma::Row<size_t> responses("1 1 0");
 
   // Create an optimizer and function.
-  L_BFGS lbfgsOpt;
+  ens::L_BFGS lbfgsOpt;
   lbfgsOpt.MinGradientNorm() = 1e-50;
   LogisticRegression<> lr(data, responses, lbfgsOpt, 0.0005);
 
@@ -698,7 +697,7 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionInstantiatedOptimizer)
   BOOST_REQUIRE_SMALL(sigmoids[2], 0.1);
 
   // Now do the same with SGD.
-  StandardSGD sgdOpt;
+  ens::StandardSGD sgdOpt;
   sgdOpt.StepSize() = 0.15;
   sgdOpt.Tolerance() = 1e-75;
   LogisticRegression<> lr2(data, responses, sgdOpt, 0.0005);
@@ -748,11 +747,11 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionSGDTrainTest)
   for (size_t i = 0; i < 800; ++i)
     labels[i] = math::RandInt(0, 2);
 
-  SGD<> sgd;
+  ens::SGD<> sgd;
   sgd.Shuffle() = false;
   LogisticRegression<> lr(dataset, labels, sgd, 0.3);
 
-  SGD<> sgd2;
+  ens::SGD<> sgd2;
   sgd2.Shuffle() = false;
   LogisticRegression<> lr2(dataset.n_rows, 0.3);
   lr2.Train(dataset, labels, sgd2);
@@ -799,12 +798,12 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionSparseSGDTest)
     labels[i] = math::RandInt(0, 2);
 
   LogisticRegression<> lr(10, 0.3);
-  SGD<> sgd;
+  ens::SGD<> sgd;
   sgd.Shuffle() = false;
   lr.Train(denseDataset, labels, sgd);
 
   LogisticRegression<arma::sp_mat> lrSparse(10, 0.3);
-  SGD<> sgdSparse;
+  ens::SGD<> sgdSparse;
   sgdSparse.Shuffle() = false;
   lrSparse.Train(dataset, labels, sgdSparse);
 
diff --git a/src/mlpack/tests/main_tests/nca_test.cpp b/src/mlpack/tests/main_tests/nca_test.cpp
index 479a8ffcd2..ee3709204f 100644
--- a/src/mlpack/tests/main_tests/nca_test.cpp
+++ b/src/mlpack/tests/main_tests/nca_test.cpp
@@ -20,7 +20,7 @@ static const std::string testName = "nca";
 #include <mlpack/core/metrics/lmetric.hpp>
 #include "test_helper.hpp"
 #include <mlpack/methods/nca/nca_main.cpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "../test_tools.hpp"
diff --git a/src/mlpack/tests/nca_test.cpp b/src/mlpack/tests/nca_test.cpp
index d6c6022200..e3d26dc59f 100644
--- a/src/mlpack/tests/nca_test.cpp
+++ b/src/mlpack/tests/nca_test.cpp
@@ -13,7 +13,7 @@
 #include <mlpack/core.hpp>
 #include <mlpack/core/metrics/lmetric.hpp>
 #include <mlpack/methods/nca/nca.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -21,7 +21,7 @@
 using namespace mlpack;
 using namespace mlpack::metric;
 using namespace mlpack::nca;
-using namespace mlpack::optimization;
+using namespace ens;
 
 //
 // Tests for the SoftmaxErrorFunction
diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp
index 165e7b3133..8bf2f3ffdf 100644
--- a/src/mlpack/tests/q_learning_test.cpp
+++ b/src/mlpack/tests/q_learning_test.cpp
@@ -22,8 +22,7 @@
 #include <mlpack/methods/reinforcement_learning/environment/acrobat.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
-#include <mlpack/core/optimizers/adam/adam_update.hpp>
-#include <mlpack/core/optimizers/rmsprop/rmsprop_update.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/reinforcement_learning/training_config.hpp>
 
 #include <boost/test/unit_test.hpp>
@@ -31,7 +30,7 @@
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
+using namespace ens;
 using namespace mlpack::rl;
 
 BOOST_AUTO_TEST_SUITE(QLearningTest);
diff --git a/src/mlpack/tests/rbm_network_test.cpp b/src/mlpack/tests/rbm_network_test.cpp
index 180ce91618..3af84af3e3 100644
--- a/src/mlpack/tests/rbm_network_test.cpp
+++ b/src/mlpack/tests/rbm_network_test.cpp
@@ -12,7 +12,7 @@
  * title = "{UCI} Machine Learning Repository",
  * url = "http://archive.ics.uci.edu/ml",
  * institution = "University of California,
- * Irvine, School of Information and Computer Sciences" } 
+ * Irvine, School of Information and Computer Sciences" }
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license. You should have received a copy of the
@@ -21,19 +21,17 @@
  */
 #include <mlpack/core.hpp>
 
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
 #include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
 #include <mlpack/methods/ann/rbm/rbm.hpp>
 #include <mlpack/methods/softmax_regression/softmax_regression.hpp>
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
+using namespace ens;
 using namespace mlpack::regression;
 
 BOOST_AUTO_TEST_SUITE(RBMNetworkTest);
@@ -77,7 +75,7 @@ BOOST_AUTO_TEST_CASE(BinaryRBMClassificationTest)
 
   size_t numRBMIterations = trainData.n_cols * numEpoches;
   numRBMIterations /= batchSize;
-  optimization::StandardSGD msgd(0.03, batchSize, numRBMIterations, 0, true);
+  ens::StandardSGD msgd(0.03, batchSize, numRBMIterations, 0, true);
   model.Reset();
   model.VisibleBias().ones();
   model.HiddenBias().ones();
@@ -174,7 +172,7 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest)
   size_t numRBMIterations = trainData.n_cols * numEpoches;
   numRBMIterations /= batchSize;
 
-  optimization::StandardSGD msgd(0.02, batchSize, numRBMIterations, 0, true);
+  ens::StandardSGD msgd(0.02, batchSize, numRBMIterations, 0, true);
   modelssRBM.Reset();
   modelssRBM.VisiblePenalty().fill(5);
   modelssRBM.SpikeBias().fill(1);
diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 8b55908379..8e92cbfe1a 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -11,8 +11,7 @@
  */
 #include <mlpack/core.hpp>
 
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
 #include <mlpack/methods/ann/rnn.hpp>
@@ -26,7 +25,7 @@
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
+using namespace ens;
 using namespace mlpack::math;
 
 BOOST_AUTO_TEST_SUITE(RecurrentNetworkTest);
diff --git a/src/mlpack/tests/regularized_svd_test.cpp b/src/mlpack/tests/regularized_svd_test.cpp
index db66a65e21..af8000d14f 100644
--- a/src/mlpack/tests/regularized_svd_test.cpp
+++ b/src/mlpack/tests/regularized_svd_test.cpp
@@ -11,15 +11,14 @@
  */
 #include <mlpack/core.hpp>
 #include <mlpack/methods/regularized_svd/regularized_svd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
 
 using namespace mlpack;
 using namespace mlpack::svd;
-using namespace mlpack::optimization;
+using namespace ens;
 
 BOOST_AUTO_TEST_SUITE(RegularizedSVDTest);
 
@@ -229,7 +228,7 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionOptimize)
 
   // Make the Reg SVD function and the optimizer.
   RegularizedSVDFunction<arma::mat> rSVDFunc(data, rank, lambda);
-  mlpack::optimization::StandardSGD optimizer(alpha, iterations * numRatings);
+  ens::StandardSGD optimizer(alpha, iterations * numRatings);
 
   // Obtain optimized parameters after training.
   arma::mat optParameters = arma::randu(rank, numUsers + numItems);
diff --git a/src/mlpack/tests/reward_clipping_test.cpp b/src/mlpack/tests/reward_clipping_test.cpp
index d5776d6852..379906aa9f 100644
--- a/src/mlpack/tests/reward_clipping_test.cpp
+++ b/src/mlpack/tests/reward_clipping_test.cpp
@@ -25,17 +25,15 @@
 #include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
 #include <mlpack/methods/reinforcement_learning/q_learning.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
-#include <mlpack/core/optimizers/adam/adam_update.hpp>
-#include <mlpack/core/optimizers/adam/adam.hpp>
-#include <mlpack/core/optimizers/rmsprop/rmsprop_update.hpp>
 #include <mlpack/methods/reinforcement_learning/training_config.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
 
 using namespace mlpack;
 using namespace mlpack::ann;
-using namespace mlpack::optimization;
+using namespace ens;
 using namespace mlpack::rl;
 
 BOOST_AUTO_TEST_SUITE(RewardClippingTest);
@@ -45,10 +43,10 @@ BOOST_AUTO_TEST_CASE(ClippedRewardTest)
 {
   Pendulum task;
   RewardClipping<Pendulum> rewardClipping(task, -2.0, +2.0);
-  
+
   RewardClipping<Pendulum>::State state = rewardClipping.InitialSample();
   RewardClipping<Pendulum>::Action action;
-  action.action[0] = math::Random(-1.0, 1.0);
+  action.action[0] = mlpack::math::Random(-1.0, 1.0);
   double reward = rewardClipping.Sample(state, action);
 
   BOOST_REQUIRE(reward <= 2.0);
diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp
index a02a82b085..be98ec7b45 100644
--- a/src/mlpack/tests/serialization_test.cpp
+++ b/src/mlpack/tests/serialization_test.cpp
@@ -1803,7 +1803,7 @@ void ANNLayerSerializationTest(LayerType& layer)
   model.Add<Linear<>>(10, output.n_rows);
   model.Add<LogSoftMax<>>();
 
-  optimization::StandardSGD opt(0.1, 1, 5, -100, false);
+  ens::StandardSGD opt(0.1, 1, 5, -100, false);
   model.Train(input, output, opt);
 
   arma::mat originalOutput;
diff --git a/src/mlpack/tests/softmax_regression_test.cpp b/src/mlpack/tests/softmax_regression_test.cpp
index 694c23175a..eeafdfc85d 100644
--- a/src/mlpack/tests/softmax_regression_test.cpp
+++ b/src/mlpack/tests/softmax_regression_test.cpp
@@ -18,7 +18,6 @@
 using namespace mlpack;
 using namespace mlpack::regression;
 using namespace mlpack::distribution;
-using namespace mlpack::optimization;
 
 BOOST_AUTO_TEST_SUITE(SoftmaxRegressionTest);
 
@@ -362,7 +361,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionTrainTest)
   SoftmaxRegression sr2(dataset.n_rows, 2);
   sr.Parameters() = sr2.Parameters();
   sr.Train(dataset, labels, 2);
-  L_BFGS lbfgs;
+  ens::L_BFGS lbfgs;
   sr2.Train(dataset, labels, 2, std::move(lbfgs));
 
   // Ensure that the parameters are the same.
@@ -387,10 +386,10 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionOptimizerTrainTest)
   for (size_t i = 500; i < 1000; ++i)
     labels[i] = size_t(1.0);
 
-  L_BFGS lbfgs;
+  ens::L_BFGS lbfgs;
   SoftmaxRegression sr(dataset.n_rows, 2, true);
 
-  L_BFGS lbfgs2;
+  ens::L_BFGS lbfgs2;
   SoftmaxRegression sr2(dataset.n_rows, 2, true);
 
   sr.Lambda() = sr2.Lambda() = 0.01;
diff --git a/src/mlpack/tests/svdplusplus_test.cpp b/src/mlpack/tests/svdplusplus_test.cpp
index 534b6be13b..42c55e6cac 100644
--- a/src/mlpack/tests/svdplusplus_test.cpp
+++ b/src/mlpack/tests/svdplusplus_test.cpp
@@ -12,14 +12,12 @@
  */
 #include <mlpack/core.hpp>
 #include <mlpack/methods/svdplusplus/svdplusplus.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp>
-#include <mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
 using namespace mlpack;
 using namespace mlpack::svd;
-using namespace mlpack::optimization;
 
 BOOST_AUTO_TEST_SUITE(SVDPlusPlusTest);
 
@@ -389,7 +387,7 @@ BOOST_AUTO_TEST_CASE(SVDPlusPlusFunctionOptimize)
 
   // Make the SVD++ function and the optimizer.
   SVDPlusPlusFunction<arma::mat> svdPPFunc(data, implicitData, rank, lambda);
-  mlpack::optimization::StandardSGD optimizer(alpha, iterations * numRatings);
+  ens::StandardSGD optimizer(alpha, iterations * numRatings);
 
   // Obtain optimized parameters after training.
   arma::mat optParameters = arma::randu(rank + 1, numUsers + 2 * numItems);
diff --git a/src/mlpack/tests/wgan_test.cpp b/src/mlpack/tests/wgan_test.cpp
index 9cf26dc8b8..f1ccc873cd 100644
--- a/src/mlpack/tests/wgan_test.cpp
+++ b/src/mlpack/tests/wgan_test.cpp
@@ -17,7 +17,7 @@
 #include <mlpack/methods/ann/ffn.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/softmax_regression/softmax_regression.hpp>
-#include <mlpack/core/optimizers/adam/adam.hpp>
+#include <mlpack/core/optimizers/ensmallen/ensmallen.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -25,7 +25,6 @@
 using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::math;
-using namespace mlpack::optimization;
 using namespace mlpack::regression;
 using namespace std::placeholders;
 
@@ -117,7 +116,7 @@ BOOST_AUTO_TEST_CASE(WGANMNISTTest)
 
   // Create WGAN
   GaussianInitialization gaussian(0, 1);
-  Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
+  ens::Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
       tolerance, shuffle);
   std::function<double()> noiseFunction = [] () {
       return math::RandNormal(0, 1);};
@@ -244,7 +243,7 @@ BOOST_AUTO_TEST_CASE(WGANGPMNISTTest)
 
   // Create WGANGP
   GaussianInitialization gaussian(0, 1);
-  Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
+  ens::Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations,
       tolerance, shuffle);
   std::function<double()> noiseFunction = [] () {
       return math::RandNormal(0, 1);};