diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp
index 41326557bb..e78ac68a12 100644
--- a/src/mlpack/methods/ann/layer/highway.hpp
+++ b/src/mlpack/methods/ann/layer/highway.hpp
@@ -21,7 +21,7 @@ namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 /**
- * Implementation of the Highway layer. The Highway class can vary its behavior
+ * Implementation of the Highway layer.  The Highway class can vary its behavior
  * between that of feed-forward fully connected network container and that
  * of a layer which simply passes its inputs through depending on the transform
  * gate. Note that the size of the input and output matrices of this class
@@ -54,13 +54,6 @@ class HighwayType : public MultiLayer<InputType, OutputType>
   //! Create the HighwayTest object.
   HighwayType();
 
-  /**
-   * Create the HighwayTest object.
-   *
-   * @param inSize The number of input units.
-   */
-  HighwayType(const size_t inSize);
-
   //! Destroy the Highway object.
   ~HighwayType();
 
@@ -102,60 +95,20 @@ class HighwayType : public MultiLayer<InputType, OutputType>
                 const OutputType& error,
                 OutputType& gradient);
 
-  /**
-   * Add a new module to the model.
-   *
-   * @param args The layer parameter.
-   */
-  template <class LayerType, class... Args>
-  void Add(Args... args)
-  {
-    network.push_back(new LayerType(args...));
-    networkOwnerships.push_back(true);
-  }
-
-  /**
-   * Add a new module to the model.
-   *
-   * @param layer The Layer to be added to the model.
-   */
-  void Add(Layer<arma::mat, arma::mat>* layer)
-  {
-    network.push_back(layer);
-    networkOwnerships.push_back(false);
-  }
-
   //! Get the parameters.
   OutputType const& Parameters() const { return weights; }
   //! Modify the parameters.
   OutputType& Parameters() { return weights; }
 
-  //! Get the number of input units.
-  size_t InSize() const { return inSize; }
-
   //! Get the number of trainable weights.
-  const size_t WeightSize() const
+  size_t WeightSize() const
   {
-    size_t result = inSize * (inSize + 1);
-    for (size_t i = 0; i < network.size(); ++i)
-      result += network[i]->WeightSize();
+    size_t result = this->totalInputSize * (this->totalInputSize + 1);
+    for (size_t i = 0; i < this->network.size(); ++i)
+      result += this->network[i]->WeightSize();
     return result;
   }
 
-  //! Get the output dimensions.
-  const std::vector<size_t>& OutputDimensions() const
-  {
-    // Push the input dimensions through the layers in order to compute the
-    // output size.
-    network.front()->InputDimensions() = inputDimensions;
-    for (size_t i = 1; i < network.size(); ++i)
-    {
-      network[i]->InputDimensions() = network[i - 1]->OutputDimensions();
-    }
-
-    return network.back()->OutputDimensions();
-  }
-
   /**
    * Serialize the layer.
    */
@@ -163,33 +116,9 @@ class HighwayType : public MultiLayer<InputType, OutputType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
-  //! Locally-stored number of input units.
-  size_t inSize;
-
-  //! Parameter which indicates if the modules should be exposed.
-  bool model;
-
-  //! Indicator if we already initialized the model.
-  bool reset;
-
-  //! Locally-stored network modules.
-  std::vector<Layer<InputType, OutputType>*> network;
-
-  //! The list of network modules we are responsible for.
-  std::vector<bool> networkOwnerships;
-
-  //! Locally-stored empty list of modules.
-  std::vector<Layer<InputType, OutputType>*> empty;
-
   //! Locally-stored weight object.
   OutputType weights;
 
-  //! Locally-stored delta object.
-  OutputType delta;
-
-  //! Locally-stored gradient object.
-  OutputType gradient;
-
   //! Weights for transformation of output.
   OutputType transformWeight;
 
@@ -204,21 +133,6 @@ class HighwayType : public MultiLayer<InputType, OutputType>
 
   //! Locally-stored transform gate error.
   OutputType transformGateError;
-
-  //! Locally-stored input parameter object.
-  InputType inputParameter;
-
-  //! Locally-stored output parameter object.
-  OutputType outputParameter;
-
-  //! The input width.
-  size_t width;
-
-  //! The input height.
-  size_t height;
-
-  //! The normal output without highway network.
-  OutputType networkOutput;
 }; // class HighwayType
 
 // Standard Highway layer.
diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp
index 1a2e73c0cf..75d7c70a3b 100644
--- a/src/mlpack/methods/ann/layer/highway_impl.hpp
+++ b/src/mlpack/methods/ann/layer/highway_impl.hpp
@@ -20,49 +20,31 @@ namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputType, typename OutputType>
-HighwayType<InputType, OutputType>::HighwayType() :
-    inSize(0),
-    reset(false),
-    width(0),
-    height(0)
+HighwayType<InputType, OutputType>::HighwayType()
 {
   // Nothing to do here.
-}
-
-template<typename InputType, typename OutputType>
-HighwayType<InputType, OutputType>::HighwayType(
-    const size_t inSize) :
-    inSize(inSize),
-    reset(false),
-    width(0),
-    height(0)
-{
-  weights.set_size(inSize * inSize + inSize, 1);
+  // TODO: how do we add the child layers ?? (read paper ...)
 }
 
 template<typename InputType, typename OutputType>
 HighwayType<InputType, OutputType>::~HighwayType()
 {
-  for (size_t i = 0; i < network.size(); ++i)
-  {
-    if (networkOwnerships[i])
-      delete network[i];
-  }
 }
 
 template<typename InputType, typename OutputType>
 void HighwayType<InputType, OutputType>::SetWeights(
     typename OutputType::elem_type* weightsPtr)
 {
-  transformWeight = OutputType(weightsPtr, inSize, inSize, false, false);
+  transformWeight = OutputType(weightsPtr, this->inSize,
+      this->inSize, false, false);
   transformBias = OutputType(weightsPtr + transformWeight.n_elem,
-      inSize, 1, false, false);
+      this->inSize, 1, false, false);
 
   size_t start = transformWeight.n_elem + transformBias.n_elem;
-  for (size_t i = 0; i < network.size(); ++i)
+  for (size_t i = 0; i < this->network.size(); ++i)
   {
-    network[i]->SetWeights(weightsPtr + start);
-    start += network[i]->WeightSize();
+    this->network[i]->SetWeights(weightsPtr + start);
+    start += this->network[i]->WeightSize();
   }
 }
 
@@ -70,17 +52,18 @@ template<typename InputType, typename OutputType>
 void HighwayType<InputType, OutputType>::Forward(
     const InputType& input, OutputType& output)
 {
-  InitializeForwardPassMemory();
+  this->InitializeForwardPassMemory(input.n_cols);
 
-  network.front()->Forward(input, layerOutputs.front());
+  this->network.front()->Forward(input, this->layerOutputs.front());
 
-  for (size_t i = 1; i < network.size(); ++i)
+  for (size_t i = 1; i < this->network.size(); ++i)
   {
-    network[i]->Forward(layerOutputs[i - 1], layerOutputs[i]);
+    this->network[i]->Forward(this->layerOutputs[i - 1], this->layerOutputs[i]);
   }
 
-  output = network.back()->OutputParameter();
+  output = this->layerOutputs.back(); // TODO: can this be cleaned up?
 
+  // TODO: move to ComputeOutputDimensions()
   if (arma::size(output) != arma::size(input))
   {
     Log::Fatal << "The sizes of the output and input matrices of the Highway"
@@ -90,33 +73,33 @@ void HighwayType<InputType, OutputType>::Forward(
   transformGate = transformWeight * input;
   transformGate.each_col() += transformBias;
   transformGateActivation = 1.0 /(1 + arma::exp(-transformGate));
-  inputParameter = input;
-  networkOutput = output; // TODO: what is done with this?
-  output = (layerOutputs.back() % transformGateActivation) +
+  output = (this->layerOutputs.back() % transformGateActivation) +
       (input % (1 - transformGateActivation));
 }
 
 template<typename InputType, typename OutputType>
 void HighwayType<InputType, OutputType>::Backward(
-    const InputType& /* input */,
+    const InputType& input,
     const OutputType& gy,
     OutputType& g)
 {
-  InitializeBackwardPassMemory();
+  this->InitializeBackwardPassMemory(input.n_cols);
 
   OutputType gyTransform = gy % transformGateActivation;
-  network.back()->Backward(layerOutputs.back(), gyTransform,
-      layerDeltas.back());
+  this->network.back()->Backward(this->layerOutputs.back(), gyTransform,
+      this->layerDeltas.back());
 
-  for (size_t i = 2; i < network.size() + 1; ++i)
+  for (size_t i = 2; i < this->network.size() + 1; ++i)
   {
-    network[network.size() - i]->Backward(layerOutputs[network.size() - i],
-        layerDeltas[network.size() - i + 1], layerDeltas[network.size() - i]);
+    this->network[this->network.size() - i]->Backward(
+        this->layerOutputs[this->network.size() - i],
+        this->layerDeltas[this->network.size() - i + 1],
+        this->layerDeltas[this->network.size() - i]);
   }
 
-  transformGateError = gy % (networkOutput - inputParameter) %
+  transformGateError = gy % (gy - input) %
       transformGateActivation % (1.0 - transformGateActivation);
-  g = layerDeltas.front() + (transformWeight.t() * transformGateError) +
+  g = this->layerDeltas.front() + (transformWeight.t() * transformGateError) +
       (gy % (1 - transformGateActivation));
 }
 
@@ -126,32 +109,31 @@ void HighwayType<InputType, OutputType>::Gradient(
     const OutputType& error,
     OutputType& gradient)
 {
-  OutputType errorTransform = error % transformGateActivation;
-  size_t gradientStart = gradient.n_elem -
-      network[network.size() - 1].WeightSize();
-  network.back()->Gradient(
-      layerOutputs[network.size() - 2],
-      errorTransform,
-      OutputType(gradient.colptr(gradientStart), 1,
-          network[network.size() - 1].WeightSize(), false, true)
-    );
+  // Create an alias for the gradient that only refers to the elements in the
+  // network itself.
+  OutputType layerGradient(gradient.memptr() + (this->inSize *
+      (this->inSize + 1)), 1, gradient.n_elem - (this->inSize *
+      (this->inSize + 1)), false, true);
+  this->InitializeGradientPassMemory(layerGradient);
 
-  for (size_t i = 2; i < network.size(); ++i)
+  OutputType errorTransform = error % transformGateActivation;
+  this->network.back()->Gradient(
+      this->layerOutputs[this->network.size() - 2],
+      errorTransform,
+      this->layerGradients[this->network.size() - 1]);
+
+  for (size_t i = 2; i < this->network.size(); ++i)
   {
-    gradientStart -= network[network.size() - i]->WeightSize();
-    network[network.size() - i]->Gradient(
-        layerOutputs[network.size() - i - 1],
-        layerDeltas[network.size() - i],
-        OutputType(gradient.colptr(gradientStart), 1,
-            network[network.size() - i]->WeightSize(), false, true)
-      );
+    this->network[this->network.size() - i]->Gradient(
+        this->layerOutputs[this->network.size() - i - 1],
+        this->layerDeltas[this->network.size() - i],
+        this->layerGradients[this->network.size() - i]);
   }
 
-  network.front()->Gradient(
+  this->network.front()->Gradient(
       input,
-      layerDeltas[1],
-      layerDeltas.front()
-    );
+      this->layerDeltas[1],
+      this->layerGradients.front());
 
   gradient.submat(0, 0, transformWeight.n_elem - 1, 0) = arma::vectorise(
       transformGateError * input.t());
@@ -165,15 +147,6 @@ void HighwayType<InputType, OutputType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
   ar(cereal::base_class<Layer<InputType, OutputType>>(this));
-
-  ar(CEREAL_VECTOR_POINTER(network));
-
-  // Reset the memory.
-  if (Archive::is_loading::value)
-  {
-    networkOwnerships.clear();
-    networkOwnerships.resize(network.size(), true);
-  }
 }
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp
index c328f4c668..021ba28598 100644
--- a/src/mlpack/methods/ann/layer/layer_types.hpp
+++ b/src/mlpack/methods/ann/layer/layer_types.hpp
@@ -35,7 +35,7 @@
 //#include <mlpack/methods/ann/layer/glimpse.hpp>
 //#include <mlpack/methods/ann/layer/hardshrink.hpp>
 //#include <mlpack/methods/ann/layer/hard_tanh.hpp>
-//#include <mlpack/methods/ann/layer/highway.hpp>
+#include <mlpack/methods/ann/layer/highway.hpp>
 //#include <mlpack/methods/ann/layer/join.hpp>
 //#include <mlpack/methods/ann/layer/layer_norm.hpp>
 //#include <mlpack/methods/ann/layer/leaky_relu.hpp>
diff --git a/src/mlpack/methods/ann/layer/multi_layer.hpp b/src/mlpack/methods/ann/layer/multi_layer.hpp
index bdb3ba49bf..8a09a7b2ce 100644
--- a/src/mlpack/methods/ann/layer/multi_layer.hpp
+++ b/src/mlpack/methods/ann/layer/multi_layer.hpp
@@ -12,6 +12,8 @@
 #ifndef MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_HPP
 #define MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_HPP
 
+#include "../make_alias.hpp"
+
 namespace mlpack {
 namespace ann {
 
@@ -70,22 +72,19 @@ class MultiLayer : public Layer<InputType, OutputType>
                         const OutputType& error,
                         OutputType& gradient)
   {
-    // Pass gradients through each layer, creating an alias for the right
-    // elements of the gradient.
+    InitializeGradientPassMemory(gradient);
 
-    network.front()->Gradient(input, layerDeltas[1], OutputType(
-        gradient.memptr(), 1, network.front()->WeightSize(), false, true));
-    size_t start = network.front()->WeightSize();
+    // Pass gradients through each layer.
+    // TODO: do we need to go back to front?  I guess not?
+    network.front()->Gradient(input, layerDeltas[1], layerGradients.front());
     for (size_t i = 0; i < network.size() - 1; ++i)
     {
-      network[i]->Gradient(layerOutputs[i - 1], layerDeltas[i + 1], OutputType(
-          gradient.colptr(start), 1, network[i]->WeightSize(), false, true));
-      start += network[i]->WeightSize();
+      network[i]->Gradient(layerOutputs[i - 1], layerDeltas[i + 1],
+          layerGradients[i]);
     }
 
     network.back()->Gradient(layerOutputs[network.size() - 2], error,
-        OutputType(gradient.colptr(start), 1, network.back()->WeightSize(),
-                   false, true));
+        layerGradients[network.size() - 1]);
   }
 
   virtual void SetWeights(typename OutputType::elem_type* weightsPtr)
@@ -98,7 +97,7 @@ class MultiLayer : public Layer<InputType, OutputType>
     }
   }
 
-  virtual void OutputSize() const
+  virtual size_t OutputSize() const
   {
     // Return the output size of the last layer.
     return network.back()->OutputSize();
@@ -115,13 +114,15 @@ class MultiLayer : public Layer<InputType, OutputType>
 
   virtual void ComputeOutputDimensions()
   {
+    inSize = 0;
     totalInputSize = 0;
     totalOutputSize = 0;
 
     // Propagate the input dimensions forward to the output.
     network.front()->InputDimensions() = this->inputDimensions;
-    totalInputSize += std::accumulate(this->inputDimensions.begin(),
+    inSize = std::accumulate(this->inputDimensions.begin(),
         this->inputDimensions.end(), 0);
+    totalInputSize += inSize;
 
     for (size_t i = 1; i < network.size(); ++i)
     {
@@ -162,8 +163,11 @@ class MultiLayer : public Layer<InputType, OutputType>
     network.push_back(new LayerType(args...));
     layerOutputs.push_back(OutputType());
     layerDeltas.push_back(OutputType());
+    layerGradients.push_back(OutputType());
   }
 
+  // TODO: handle network ownership?
+
   /*
    * Add a new module to the model.
    *
@@ -174,6 +178,7 @@ class MultiLayer : public Layer<InputType, OutputType>
     network.push_back(layer);
     layerOutputs.push_back(OutputType());
     layerDeltas.push_back(OutputType());
+    layerGradients.push_back(OutputType());
   }
 
   const std::vector<Layer<InputType, OutputType>*> Network() const { return
@@ -195,6 +200,7 @@ network; }
       layerDeltaMatrix.clear();
       layerOutputs.resize(network.size(), OutputType());
       layerDeltas.resize(network.size(), OutputType());
+      layerGradients.resize(network.size(), OutputType());
     }
   }
 
@@ -219,9 +225,9 @@ network; }
     size_t start = 0;
     for (size_t i = 0; i < layerOutputs.size(); ++i)
     {
-      const size_t layerOutputSize = network[i].OutputSize();
-      layerOutputs[i] = OutputType(layerOutputMatrix.colptr(start),
-          layerOutputSize, batchSize, false, true);
+      const size_t layerOutputSize = network[i]->OutputSize();
+      MakeAlias(layerOutputs[i], layerOutputMatrix.colptr(start),
+          layerOutputSize, batchSize);
       start += batchSize * layerOutputSize;
     }
   }
@@ -229,9 +235,8 @@ network; }
   void InitializeBackwardPassMemory(const size_t batchSize)
   {
     // We need to initialize memory to store the output of each layer's
-    // Backward() and Gradient() calls.  We do this similarly to
-    // InitializeForwardPassMemory(), but we must store a matrix to use as the
-    // delta for each layer.
+    // Backward() call.  We do this similarly to InitializeForwardPassMemory(),
+    // but we must store a matrix to use as the delta for each layer.
     if (batchSize * totalInputSize > layerDeltaMatrix.n_elem ||
         batchSize * totalInputSize < std::floor(0.1 * layerOutputMatrix.n_elem))
     {
@@ -247,22 +252,41 @@ network; }
       const size_t layerInputSize = (i == 0) ?
           std::accumulate(this->inputDimensions.begin(),
               this->inputDimensions.end(), 0) :
-          network[i - 1].OutputSize();
-      layerDeltas[i] = OutputType(layerDeltaMatrix.colptr(start),
-          layerInputSize, batchSize, false, true);
+          network[i - 1]->OutputSize();
+      MakeAlias(layerDeltas[i], layerDeltaMatrix.colptr(start), layerInputSize,
+          batchSize);
       start += batchSize * layerInputSize;
     }
   }
 
+  void InitializeGradientPassMemory(OutputType& gradient)
+  {
+    // We need to initialize memory to store the gradients of each layer.
+    // To do this, we need to know the weight size of each layer.
+    size_t gradientStart = 0;
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      const size_t weightSize = network[i]->WeightSize();
+      MakeAlias(layerGradients[i], gradient.colptr(gradientStart), weightSize,
+          1);
+      gradientStart += weightSize;
+    }
+  }
+
   std::vector<Layer<InputType, OutputType>*> network;
 
+  // Total number of elements in the input, cached for convenience.
+  size_t inSize;
+  // Total number of input elements for *every* layer.
   size_t totalInputSize;
+  // Total number of output elements for *every* layer.
   size_t totalOutputSize;
 
-  arma::mat layerOutputMatrix;
-  std::vector<arma::mat> layerOutputs;
-  arma::mat layerDeltaMatrix;
-  std::vector<arma::mat> layerDeltas;
+  OutputType layerOutputMatrix;
+  std::vector<OutputType> layerOutputs;
+  OutputType layerDeltaMatrix;
+  std::vector<OutputType> layerDeltas;
+  std::vector<OutputType> layerGradients;
 };
 
 } // namespace ann
diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp
index 72136644af..5207b65c3d 100644
--- a/src/mlpack/methods/ann/layer/serialization.hpp
+++ b/src/mlpack/methods/ann/layer/serialization.hpp
@@ -126,6 +126,7 @@
     CEREAL_REGISTER_TYPE(mlpack::ann::ConcatenateType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::AddType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::RBF<__VA_ARGS__>); \
+    CEREAL_REGISTER_TYPE(mlpack::ann::HighwayType<__VA_ARGS__>); \
 
 // TODO: continue...
 
diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp
index 8ac3f9dda1..f7ab568f2c 100644
--- a/src/mlpack/tests/feedforward_network_test.cpp
+++ b/src/mlpack/tests/feedforward_network_test.cpp
@@ -560,7 +560,7 @@ TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]")
 
 /**
  * Train the highway network on a larger dataset.
- *
+ */
 TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]")
 {
   arma::mat dataset;
@@ -576,7 +576,7 @@ TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]")
 
   FFN<NegativeLogLikelihood<> > model;
   model.Add<Linear>(10);
-  Highway* highway = new Highway(10, true);
+  Highway* highway = new Highway();
   highway->Add<Linear>(10);
   highway->Add<Sigmoid>();
   model.Add(highway); // This takes ownership of the memory.