diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
index 70e03de95f..2fa1c5dca6 100644
--- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
+++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp
@@ -43,10 +43,7 @@ template<typename InputType, typename OutputType>
 void FlexibleReLU<InputDataType, OutputDataType>::Forward(
     const InputType&& input, OutputType&& output)
 {
-  int i = -1;
-  output = arma::zeros<InputType>(input.n_rows, input.n_cols);
-  output.transform([input, &i, this](double val) { ++i;
-      return (std::max(input(i), 0.0) + alpha(0)); } );
+  output = arma::clamp(input, 0.0, DBL_MAX) + alpha(0);
 }
 
 template<typename InputDataType, typename OutputDataType>
@@ -56,11 +53,8 @@ void FlexibleReLU<InputDataType, OutputDataType>::Backward(
 {
   DataType derivative;
   //! Compute the first derivative of FlexibleReLU function.
-  derivative.set_size(input.n_rows, input.n_cols);
-  int i = -1;
-  derivative.transform([input, &i](double val) { ++i;
-    return (input(i) > 0? 1 : 0); } );
-
+  derivative = arma::sign(input);
+  derivative.elem(arma::find(derivative < 0.0)) += 1;
   g = gy % derivative;
 }
 
@@ -74,9 +68,8 @@ void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
   {
     gradient = arma::zeros<arma::Mat<eT>>(1, 1);
   }
-
-  arma::mat zeros = arma::zeros<arma::Mat<eT>>(input.n_rows, input.n_cols);
-  gradient(0) = arma::accu(error % arma::min(zeros, input)) / input.n_cols;
+  gradient(0) = arma::accu(error % arma::clamp(input, -DBL_MAX, 0.0))
+      / input.n_cols;
 }
 
 
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
index 6dd706a20c..57b71cb015 100644
--- a/src/mlpack/tests/ann_layer_test.cpp
+++ b/src/mlpack/tests/ann_layer_test.cpp
@@ -634,6 +634,48 @@ BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest)
   }
 }
 
+/**
+ * Flexible ReLU layer numerically gradient test.
+ */
+BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction()
+    {
+      input = arma::randn(10, 1);
+      target = arma::mat("1");
+
+      model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>(
+        input, target);
+      model->Add<Linear<> >(10, 2);
+      model->Add<FlexibleReLU<> >(0.05);
+      model->Add<LogSoftMax<> >();
+    }
+
+    ~GradientFunction()
+    {
+      delete model;
+    }
+
+    double Gradient(arma::mat& gradient) const
+    {
+      arma::mat output;
+      double error = model->Evaluate(model->Parameters(), 0, 1);
+      model->Gradient(model->Parameters(), 0, gradient, 1);
+      return error;
+    }
+
+    arma::mat& Parameters() { return model->Parameters(); }
+
+    FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>* model;
+    arma::mat input, target;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
+}
+
 /**
  * Jacobian MultiplyConstant module test.
  */